Exemple #1
0
import iftproto
import iftfile
import protocols
import threading
import time
import iftapi
import iftstats
import test_setup
import test_cleanup
import iftutil

import protocols.raven

iftapi.iftd_setup(iftutil.get_available_protocols(), "../iftd.xml")
test_setup.setup()
iftstats.startup(iftapi.list_protocols(), 1, "NaiveBayes")


http_connect_attrs = {iftproto.PROTO_PORTNUM: 8080}

iftsocket_connect_attrs = {iftproto.PROTO_PORTNUM: 4000}

src_file = "/home/jude/raven/tools/iftd/testfile"

job_attrs = {
    iftfile.JOB_ATTR_SRC_HOST: "localhost",
    iftfile.JOB_ATTR_SRC_NAME: "/home/jude/raven/tools/iftd/testfile",
    iftfile.JOB_ATTR_FILE_HASH: iftfile.get_hash(src_file),
    iftfile.JOB_ATTR_FILE_SIZE: iftfile.get_filesize(src_file),
    protocols.raven.TRANSFER_PROGRAM_PACKAGE: "transfer.arizonatransfer_http",
    protocols.raven.INIT_TRANSFER_PROGRAM_ARG1: None,
Exemple #2
0
    return
 
 if connect_args == None:
    # not found
    iftlog.log(5, "HTTPStorkServerHandler: connect_args not found for '" + get_filename + "'" )
    print tmp_connect_args
    self.send_response( 404 )
    return
 
 chunks_dir = job_attrs.get( iftfile.JOB_ATTR_DEST_CHUNK_DIR )
 file_name = job_attrs.get(iftfile.JOB_ATTR_DEST_NAME)
 
 # file request for cache miss!
 try:
    # get the file via iftd, but don't check the cache
    all_protolist = iftapi.list_protocols()
    protolist = []
    
    if "iftcache_receiver" in all_protolist:
       protolist = ["iftcache_receiver"]  # only use the cache protocol if it is available
    
    else:
       # don't use any nest_helper protocol
       for p in all_protolist:
          if p.find("nest_helper") == -1:
             protolist.append( p )
    
    # map each protocol to the given connection args
    proto_connect_args = {}
    for proto_name in protolist:
       proto_connect_args[proto_name] = connect_args
Exemple #3
0
   def do_GET(self):
      global cache_dir
      global tmp_connect_args
      global tmp_job_attrs
      
      # only pay attention to local requests
      if self.client_address[0] != "127.0.0.1" and self.client_address[0] != "localhost":
         iftlog.log(5, "HTTPCacheServerHandler: unauthorized GET from " + self.client_address[0])
         self.send_response( 403 )
         return

      get_filename = os.path.basename( self.path )
      
      connect_args = tmp_connect_args.get( get_filename )
      job_attrs = tmp_job_attrs.get( get_filename )
      
      if job_attrs == None:
         # not found...
         iftlog.log(5, "HTTPCacheServerHandler: job_attrs not found for '" + get_filename + "'" )
         self.send_response( 404 )
         return
      
      if connect_args == None:
         # not found
         iftlog.log(5, "HTTPCacheServerHandler: connect_args not found for '" + get_filename + "'" )
         print tmp_connect_args
         self.send_response( 404 )
         return
      
      chunks_dir = job_attrs.get( iftfile.JOB_ATTR_DEST_CHUNK_DIR )
      file_name = job_attrs.get(iftfile.JOB_ATTR_DEST_NAME)
      
      # file request for cache miss!
      try:
         # get the file via iftd, but don't check the cache
         all_protolist = iftapi.list_protocols()
         protolist = []
         
         # don't use any iftcache protocol
         for p in all_protolist:
            if p.find("iftcache") == -1:
               protolist.append( p )
         
         # map each protocol to the given connection args
         proto_connect_args = {}
         for proto_name in protolist:
            proto_connect_args[proto_name] = connect_args
         
         # put in a request to get the file
         iftlog.log(5, "iftcache: Squid cache miss, so trying all other available protocols")
         
         # receive to the cache directory
         cached_filepath = cache_dir.rstrip("/") + "/" + os.path.basename( job_attrs.get( iftfile.JOB_ATTR_DEST_NAME ))
         job_attrs[ iftfile.JOB_ATTR_DEST_NAME ] = cached_filepath
         job_attrs[ iftfile.JOB_ATTR_PROTOS ] = protolist
         rc = iftapi.begin_ift( job_attrs, proto_connect_args, False, True, connect_args[iftapi.CONNECT_ATTR_REMOTE_PORT], connect_args[iftapi.CONNECT_ATTR_REMOTE_RPC], connect_args[iftapi.CONNECT_ATTR_USER_TIMEOUT] )
         
         # success or failure?
         if rc != TRANSMIT_STATE_SUCCESS and rc != 0:
            iftlog.log(5, "iftcache: could not receive file " + file_name + " (rc = " + str(rc) + ")")
            self.send_response(500)
            return
         
         # open the file and write it back
         file_buff = []
         try:
            fd = open( cached_filepath, "rb" )
            file_buff = fd.read()
            fd.close()
         except Exception, inst:
            iftlog.exception("iftcache: received file to " + cached_filepath + ", but could not read it")
            self.send_response(500)
            return
         
         # reply the file
         self.send_response(200)
         self.send_header( 'Content-type', 'application/octet-stream' ) # force raw bytes
         self.send_header( 'Last-Modified', time.ctime( os.stat( cached_filepath ).st_mtime ) )
         self.end_headers()
         self.wfile.write( file_buff )
         
         # recreate the chunks directory, since we might have lost it...
         if chunks_dir != None and not os.path.exists( chunks_dir ):
            try:
               os.popen("mkdir -p " + chunks_dir).close()
            except:
               pass
            
         # done with this...
         tmp_connect_args[get_filename] = None
         tmp_job_attrs[get_filename] = None
         return