def test_iftd( file, filehash, filesize, remote_host, tmpfs_dir ): ts = 0 te = 0 http_connect_attrs = { iftproto.PROTO_PORTNUM:8000, iftfile.JOB_ATTR_SRC_NAME:file, iftfile.JOB_ATTR_DEST_NAME:file, iftfile.JOB_ATTR_SRC_HOST:remote_host } job_attrs = { iftfile.JOB_ATTR_SRC_HOST:remote_host, iftfile.JOB_ATTR_SRC_NAME:file, iftfile.JOB_ATTR_FILE_SIZE:int(filesize), iftfile.JOB_ATTR_CHUNKSIZE:int(filesize), # get the whole file at once # iftfile.JOB_ATTR_FILE_HASH:filehash, iftfile.JOB_ATTR_DEST_NAME:file, iftfile.JOB_ATTR_DEST_HOST:"localhost", protocols.http.HTTP_SERVER_VERSION:11 } client = iftapi.make_XMLRPC_client() connects = { "http_receiver":http_connect_attrs, "http_sender":http_connect_attrs } for i in xrange(0, 10): # transfer the file with iftd 5 times ts = time.time() client.begin_ift( job_attrs, connects, False, True, 4001, "/RPC2", True, False, 60) te = time.time() iftlog.log(5, "iftd " + remote_host + file + " " + str(te - ts) )
def test_iftd( file, filehash, filesize, remote_host, tmpfs_dir ): ts = 0 te = 0 http3_connect_attrs = { iftproto.PROTO_PORTNUM:8000, iftfile.JOB_ATTR_SRC_NAME:file, iftfile.JOB_ATTR_DEST_NAME:file, iftfile.JOB_ATTR_SRC_HOST:remote_host } http4_connect_attrs = { iftproto.PROTO_PORTNUM:8000, iftfile.JOB_ATTR_SRC_NAME:file, iftfile.JOB_ATTR_DEST_NAME:file, iftfile.JOB_ATTR_SRC_HOST:remote_host } job_attrs = { iftfile.JOB_ATTR_SRC_HOST:remote_host, iftfile.JOB_ATTR_SRC_NAME:file, iftfile.JOB_ATTR_FILE_SIZE:int(filesize), iftfile.JOB_ATTR_FILE_HASH:filehash, iftfile.JOB_ATTR_DEST_NAME:file, iftfile.JOB_ATTR_CHUNKSIZE:int(filesize) / 20, iftfile.JOB_ATTR_DEST_HOST:"localhost" } client = iftapi.make_XMLRPC_client() connects = { "http3_receiver":http3_connect_attrs, "http3_sender":http3_connect_attrs, "http4_receiver":http4_connect_attrs, "http4_sender":http4_connect_attrs } client.clear_classifier() for i in xrange(0, 10): # transfer the file with iftd ts = time.time() client.begin_ift( job_attrs, connects, False, True, 4001, "/RPC2", True, False, 60) te = time.time() iftlog.log(5, "iftd " + remote_host + file + " " + str(te - ts) ) time.sleep(15) cls_data = client.get_proto_rankings( job_attrs, True ) iftlog.log(5, "") iftlog.log(5, "Protocol rankings") for (proto, prob) in cls_data: iftlog.log(5, str(proto) + ": " + str(prob)) iftlog.log(5, "")
def init_transfer_program(port, ignore2=None, ignore3=None, ignore4=None): """ <Purpose> This initializes a connection. It starts up iftd's filewriter thread <Arguments> None. <Exceptions> None. <Side Effects> None. <Returns> True. """ global iftd_server if port == None: port = 4000 iftd_server = iftapi.make_XMLRPC_client( "127.0.0.1", port, "RPC2", 60 ) return True
} src_file = "/X1Hi1.gif" job_attrs = { iftfile.JOB_ATTR_SRC_HOST:"i.imgur.com", iftfile.JOB_ATTR_SRC_NAME:src_file, protocols.raven.TRANSFER_PROGRAM_PACKAGE:"transfer.arizonatransfer_http", protocols.raven.INIT_TRANSFER_PROGRAM_ARG1:None, protocols.raven.INIT_TRANSFER_PROGRAM_ARG2:None, protocols.raven.INIT_TRANSFER_PROGRAM_ARG3:None, protocols.raven.INIT_TRANSFER_PROGRAM_ARG4:None, iftfile.JOB_ATTR_DEST_NAME:"/tmp/img.gif", iftfile.JOB_ATTR_DEST_HOST:"localhost" } client = iftapi.make_XMLRPC_client() connects = { "http_receiver":http_connect_attrs, "http_sender":http_connect_attrs, "iftsocket_receiver":iftsocket_connect_attrs, "iftsocket_sender":iftsocket_connect_attrs, "raven":None } rc = client.begin_ift( job_attrs, connects, False, True, 4001, "/RPC2", True, False, 60 ) print "called" test_cleanup.cleanup()
def getfiles1(host, filelist, destdir, prog_indicator=0, createhashfile=False, ignoreHash=False, prioritylist=None): """ <Purpose> This fetches files from given host, using prioritylist which holds transfer methods to fetch files. It tries to get files by one method, and if it fails it uses next possible method until it gets all files needed. <Arguments> host: 'host' holds two things, a server name and download directory. For example, if you want to retrieve files from '/tmp/' directory in 'quadrus.cs.arizona.edu' server, the 'host' will be 'quadrus.cs.arizona.edu/tmp'. 'host' should be a string. filelist: 'filelist' is a list of files which need to be retrieved. 'filelist' should be a list of dictionaties of the format: {"filename": filename, "hash": hash, "size": size, "hashfuncs": list of hashfuncs to try} the hash and size parameters can be None if unavailable destdir: 'destdir' is a destination directory where retrieved files will be placed. A user should have 'destdir' exist before retrieving files. 'destdir' should be a string. prog_indicator: DEPRICATED, unused. prioritylist: If it is not None, then this is a whitelist of protocols to use for transferring the given files. NOTE: the protocols here are base names (e.g. 'http', 'ftp', 'scp', etc., as opposed to 'http_receiver', 'ftp_receiver', 'scp_receiver', etc.). <Exceptions> None. <Side Effects> Messes with SIGALRM Set init_status <Returns> True or False to indicate success, and a list of downloaded files """ global init_status #global arizonafetch #global glo_prioritylist arizonareport.send_out(4, "[DEBUG] getfiles started") arizonareport.send_out(4, "host = " + str(host) + ", filelist = " + str(filelist)) # downloaded files list downloaded_files = [] # check if host is a string arizonageneral.check_type_simple(host, "host", str, "arizonatransfer.getfiles") # check if destdir is a string arizonageneral.check_type_simple(destdir, "destdir", str, "arizonatransfer.getfiles") # get username username = arizonageneral.getusername() # check that the destination directory exists if not os.path.isdir(destdir): arizonareport.send_syslog(arizonareport.ERR, "\ngetfiles(): The destination directory '" + destdir + "' does not exist... Aborting...") # return false and empty list return (False, downloaded_files) if prioritylist == None: # transfer method list set by arizonaconfig prioritylist = arizonaconfig.get_option("transfermethod") # check the method list # if prioritylist is None, there's something wrong with configuration if prioritylist == None : arizonareport.send_syslog(arizonareport.ERR, "getfiles(): No transfer method was given.") return (False, downloaded_files) # create a temporary directory for the transfer arizonareport.send_out(4, "[DEBUG] getfiles creating temp dir") try: temp_dir = arizonaconfig.get_option("transfertempdir") + str(securerandom.SecureRandom().random()) except TypeError: arizonareport.send_syslog(arizonareport.ERR, "getfiles(): No transfer temp dir is given.") return (False, downloaded_files) # in the case of destdir has '/' at the end # last '/' should go away to make result list(downloaded_files) match if len(destdir) > 1 and destdir.endswith('/'): destdir = destdir[:len(destdir) - 1] # if there are empty strings in the filelist, those will be taken away arizonareport.send_out(4, "[DEBUG] checking file list") filelist = __checkFileList(filelist) arizonareport.send_out(4, "[DEBUG] creating directories") for item in filelist: filename = item['filename'] dirname = os.path.dirname(filename) if dirname != "": arizonageneral.makedirs_existok(os.path.join(temp_dir, dirname)) arizonageneral.makedirs_existok(os.path.join(destdir, dirname)) filenames = [item['filename'] for item in filelist] # keep the number of the list to compare how many files are downloaded at the end. numoflist = len(filelist) # if there is no file needing to be fetched if filelist == []: arizonareport.send_syslog(arizonareport.ERR, "getfiles(): No files needed to be downloaded.") return (False, downloaded_files) if not os.path.exists(temp_dir): arizonageneral.makedirs_existok(temp_dir) failures = 0 for currFile in filelist: # attempt to get this file, with the data given common_attrs = {} # add attributes about this file that we know filename = currFile.get('filename',None) if filename == None or len(filename) == 0: # something's seriously wrong... continue if filename[0] != '/': filename = '/' + filename # need to be an absolute path common_attrs[iftfile.JOB_ATTR_SRC_NAME] = "/" + str(host.split("/")[1]) + filename common_attrs[iftfile.JOB_ATTR_DEST_NAME] = temp_dir + filename common_attrs[iftfile.JOB_ATTR_FILE_SIZE] = currFile.get('size',None) common_attrs[iftfile.JOB_ATTR_FILE_HASH] = currFile.get('hash',None) common_attrs[iftfile.JOB_ATTR_SRC_HOST] = str(host.split("/")[0]) common_attrs[iftfile.JOB_ATTR_DEST_HOST] = "localhost" job_attrs = copy.copy( common_attrs ) job_attrs[iftfile.JOB_ATTR_REMOTE_IFTD] = False # until we have a faster way of detecting it! connect_attrs = {} if prioritylist: job_attrs[iftfile.JOB_ATTR_PROTOS] = [proto + "_receiver" for proto in prioritylist] for proto in job_attrs[iftfile.JOB_ATTR_PROTOS]: connect_attrs[ proto ] = common_attrs __enable_timeout(__compute_timeout(filelist)) # attempt to get it with iftd iftd_server = iftapi.make_XMLRPC_client() try: # have iftd attempt to receive it rc = iftd_server.begin_ift( job_attrs, connect_attrs, False, True, 4001, "/RPC2", True, 3600 ) print "XMLRPC call completed" if rc != iftproto.ifttransmit.TRANSMIT_STATE_SUCCESS: arizonareport.send_syslog( arizonareport.ERR, 'retrieve_files: File transfer RC=' + str(rc)) print "XMLRPC call rc = " + str(rc) __disable_timeout() failures += 1 continue else: print "got the file " + filename except Exception, inst: arizonareport.send_syslog( arizonareport.ERR, 'getfiles1(): XMLRPC error, could not receive file ' + filename ) iftlog.exception("getfiles1(): XMLRPC error, could not receive file " + filename, inst) __disable_timeout() failures += 1 continue """ req = urllib2.Request("http://127.0.0.1:16387" + job_attrs.get( iftfile.JOB_ATTR_SRC_NAME ) ) req.add_header( "Pragma", iftapi.pack_attrs( job_attrs, connect_attrs ) ) resp = None try: # perform the request resp = urllib2.urlopen( req ) if resp.code != 200: arizonareport.send_syslog(arizonareport.ERR, "getfiles(): HTTP " + str(resp.code) + " from IFTD for file " + filename) failures += 1 __disable_timeout() continue except Exception, inst: arizonareport.send_out(3, "[" +username+"] "+str(element)+": error: "+ str( sys.exc_info()[0] ) ) failures += 1 __disable_timeout() continue """ __disable_timeout() # write this file to the destination dest_filename = destdir + filename try: shutil.move( common_attrs[ iftfile.JOB_ATTR_DEST_NAME ], dest_filename ) # success! downloaded_files.append( filename ) except Exception, inst: arizonareport.send_out(3, "[" + username + "] " + str(element) + ": error: " + str(sys.exc_info()[0]) ) failures += 1
def test_iftd( file, filehash, filesize, remote_host, tmpfs_dir ): ts = 0 te = 0 http_connect_attrs = { iftproto.PROTO_PORTNUM:8000, iftfile.JOB_ATTR_SRC_NAME:file, iftfile.JOB_ATTR_DEST_NAME:file, iftfile.JOB_ATTR_SRC_HOST:remote_host } job_attrs = { iftfile.JOB_ATTR_SRC_HOST:remote_host, iftfile.JOB_ATTR_SRC_NAME:file, iftfile.JOB_ATTR_FILE_SIZE:int(filesize), iftfile.JOB_ATTR_FILE_HASH:filehash, iftfile.JOB_ATTR_DEST_NAME:file, iftfile.JOB_ATTR_CHUNKSIZE:int(filesize) / 200, iftfile.JOB_ATTR_DEST_HOST:"localhost", protocols.bittorrent.IFTBITTORRENT_TORRENT_PATH:file + ".torrent", iftfile.JOB_ATTR_PROTOS:['http5_receiver', 'iftscp2_receiver', 'bittorrent2_receiver'] } bt_connect_attrs = { iftfile.JOB_ATTR_DEST_NAME:file, iftfile.JOB_ATTR_SRC_NAME:file, protocols.bittorrent.IFTBITTORRENT_PORTRANGE_LOW:1025, protocols.bittorrent.IFTBITTORRENT_PORTRANGE_HIGH:65534, protocols.bittorrent.IFTBITTORRENT_TORRENT_PATH:"/tmp/iftd-experiments/" + file + ".torrent" } client = iftapi.make_XMLRPC_client( timeout = 60 ) connects = { "http5_receiver":http_connect_attrs, "http5_sender":http_connect_attrs, "iftscp2_sender":None, "iftscp2_receiver":None, "bittorrent2_sender":bt_connect_attrs, "bittorrent2_receiver":bt_connect_attrs } client.clear_classifier() for i in xrange(0, 1): # transfer the file with iftd ts = time.time() rc = client.begin_ift( job_attrs, connects, False, True, 4001, "/RPC2", True, False, 60) te = time.time() iftlog.log(5, "iftd " + remote_host + file + " " + str(te - ts) ) if rc != iftproto.ifttransmit.TRANSMIT_STATE_SUCCESS: iftlog.log(5, "iftd failed to transfer " + str(file)) os.popen("rm -rf " + file).close() time.sleep(15) cls_data = client.get_proto_rankings( job_attrs, True ) iftlog.log(5, "") iftlog.log(5, "Protocol rankings") for (proto, prob) in cls_data: iftlog.log(5, str(proto) + ": " + str(prob)) iftlog.log(5, "")