import iftproto import iftfile import protocols import threading import time import iftapi import iftstats import test_setup import test_cleanup import iftutil import protocols.raven iftapi.iftd_setup(iftutil.get_available_protocols(), "../iftd.xml") test_setup.setup() iftstats.startup(iftapi.list_protocols(), 1, "NaiveBayes") http_connect_attrs = {iftproto.PROTO_PORTNUM: 8080} iftsocket_connect_attrs = {iftproto.PROTO_PORTNUM: 4000} src_file = "/home/jude/raven/tools/iftd/testfile" job_attrs = { iftfile.JOB_ATTR_SRC_HOST: "localhost", iftfile.JOB_ATTR_SRC_NAME: "/home/jude/raven/tools/iftd/testfile", iftfile.JOB_ATTR_FILE_HASH: iftfile.get_hash(src_file), iftfile.JOB_ATTR_FILE_SIZE: iftfile.get_filesize(src_file), protocols.raven.TRANSFER_PROGRAM_PACKAGE: "transfer.arizonatransfer_http", protocols.raven.INIT_TRANSFER_PROGRAM_ARG1: None,
return if connect_args == None: # not found iftlog.log(5, "HTTPStorkServerHandler: connect_args not found for '" + get_filename + "'" ) print tmp_connect_args self.send_response( 404 ) return chunks_dir = job_attrs.get( iftfile.JOB_ATTR_DEST_CHUNK_DIR ) file_name = job_attrs.get(iftfile.JOB_ATTR_DEST_NAME) # file request for cache miss! try: # get the file via iftd, but don't check the cache all_protolist = iftapi.list_protocols() protolist = [] if "iftcache_receiver" in all_protolist: protolist = ["iftcache_receiver"] # only use the cache protocol if it is available else: # don't use any nest_helper protocol for p in all_protolist: if p.find("nest_helper") == -1: protolist.append( p ) # map each protocol to the given connection args proto_connect_args = {} for proto_name in protolist: proto_connect_args[proto_name] = connect_args
def do_GET(self): global cache_dir global tmp_connect_args global tmp_job_attrs # only pay attention to local requests if self.client_address[0] != "127.0.0.1" and self.client_address[0] != "localhost": iftlog.log(5, "HTTPCacheServerHandler: unauthorized GET from " + self.client_address[0]) self.send_response( 403 ) return get_filename = os.path.basename( self.path ) connect_args = tmp_connect_args.get( get_filename ) job_attrs = tmp_job_attrs.get( get_filename ) if job_attrs == None: # not found... iftlog.log(5, "HTTPCacheServerHandler: job_attrs not found for '" + get_filename + "'" ) self.send_response( 404 ) return if connect_args == None: # not found iftlog.log(5, "HTTPCacheServerHandler: connect_args not found for '" + get_filename + "'" ) print tmp_connect_args self.send_response( 404 ) return chunks_dir = job_attrs.get( iftfile.JOB_ATTR_DEST_CHUNK_DIR ) file_name = job_attrs.get(iftfile.JOB_ATTR_DEST_NAME) # file request for cache miss! try: # get the file via iftd, but don't check the cache all_protolist = iftapi.list_protocols() protolist = [] # don't use any iftcache protocol for p in all_protolist: if p.find("iftcache") == -1: protolist.append( p ) # map each protocol to the given connection args proto_connect_args = {} for proto_name in protolist: proto_connect_args[proto_name] = connect_args # put in a request to get the file iftlog.log(5, "iftcache: Squid cache miss, so trying all other available protocols") # receive to the cache directory cached_filepath = cache_dir.rstrip("/") + "/" + os.path.basename( job_attrs.get( iftfile.JOB_ATTR_DEST_NAME )) job_attrs[ iftfile.JOB_ATTR_DEST_NAME ] = cached_filepath job_attrs[ iftfile.JOB_ATTR_PROTOS ] = protolist rc = iftapi.begin_ift( job_attrs, proto_connect_args, False, True, connect_args[iftapi.CONNECT_ATTR_REMOTE_PORT], connect_args[iftapi.CONNECT_ATTR_REMOTE_RPC], connect_args[iftapi.CONNECT_ATTR_USER_TIMEOUT] ) # success or failure? if rc != TRANSMIT_STATE_SUCCESS and rc != 0: iftlog.log(5, "iftcache: could not receive file " + file_name + " (rc = " + str(rc) + ")") self.send_response(500) return # open the file and write it back file_buff = [] try: fd = open( cached_filepath, "rb" ) file_buff = fd.read() fd.close() except Exception, inst: iftlog.exception("iftcache: received file to " + cached_filepath + ", but could not read it") self.send_response(500) return # reply the file self.send_response(200) self.send_header( 'Content-type', 'application/octet-stream' ) # force raw bytes self.send_header( 'Last-Modified', time.ctime( os.stat( cached_filepath ).st_mtime ) ) self.end_headers() self.wfile.write( file_buff ) # recreate the chunks directory, since we might have lost it... if chunks_dir != None and not os.path.exists( chunks_dir ): try: os.popen("mkdir -p " + chunks_dir).close() except: pass # done with this... tmp_connect_args[get_filename] = None tmp_job_attrs[get_filename] = None return