def test_iftd( file, filehash, filesize, remote_host, tmpfs_dir ):
ts = 0
te = 0
http_connect_attrs = {
iftproto.PROTO_PORTNUM:8000,
iftfile.JOB_ATTR_SRC_NAME:file,
iftfile.JOB_ATTR_DEST_NAME:file,
iftfile.JOB_ATTR_SRC_HOST:remote_host
}
job_attrs = {
iftfile.JOB_ATTR_SRC_HOST:remote_host,
iftfile.JOB_ATTR_SRC_NAME:file,
iftfile.JOB_ATTR_FILE_SIZE:int(filesize),
iftfile.JOB_ATTR_CHUNKSIZE:int(filesize), # get the whole file at once
# iftfile.JOB_ATTR_FILE_HASH:filehash,
iftfile.JOB_ATTR_DEST_NAME:file,
iftfile.JOB_ATTR_DEST_HOST:"localhost",
protocols.http.HTTP_SERVER_VERSION:11
}
client = iftapi.make_XMLRPC_client()
connects = {
"http_receiver":http_connect_attrs,
"http_sender":http_connect_attrs
}
for i in xrange(0, 10):
# transfer the file with iftd 5 times
ts = time.time()
client.begin_ift( job_attrs, connects, False, True, 4001, "/RPC2", True, False, 60)
te = time.time()
iftlog.log(5, "iftd " + remote_host + file + " " + str(te - ts) )
def test_iftd( file, filehash, filesize, remote_host, tmpfs_dir ):
ts = 0
te = 0
http3_connect_attrs = {
iftproto.PROTO_PORTNUM:8000,
iftfile.JOB_ATTR_SRC_NAME:file,
iftfile.JOB_ATTR_DEST_NAME:file,
iftfile.JOB_ATTR_SRC_HOST:remote_host
}
http4_connect_attrs = {
iftproto.PROTO_PORTNUM:8000,
iftfile.JOB_ATTR_SRC_NAME:file,
iftfile.JOB_ATTR_DEST_NAME:file,
iftfile.JOB_ATTR_SRC_HOST:remote_host
}
job_attrs = {
iftfile.JOB_ATTR_SRC_HOST:remote_host,
iftfile.JOB_ATTR_SRC_NAME:file,
iftfile.JOB_ATTR_FILE_SIZE:int(filesize),
iftfile.JOB_ATTR_FILE_HASH:filehash,
iftfile.JOB_ATTR_DEST_NAME:file,
iftfile.JOB_ATTR_CHUNKSIZE:int(filesize) / 20,
iftfile.JOB_ATTR_DEST_HOST:"localhost"
}
client = iftapi.make_XMLRPC_client()
connects = {
"http3_receiver":http3_connect_attrs,
"http3_sender":http3_connect_attrs,
"http4_receiver":http4_connect_attrs,
"http4_sender":http4_connect_attrs
}
client.clear_classifier()
for i in xrange(0, 10):
# transfer the file with iftd
ts = time.time()
client.begin_ift( job_attrs, connects, False, True, 4001, "/RPC2", True, False, 60)
te = time.time()
iftlog.log(5, "iftd " + remote_host + file + " " + str(te - ts) )
time.sleep(15)
cls_data = client.get_proto_rankings( job_attrs, True )
iftlog.log(5, "")
iftlog.log(5, "Protocol rankings")
for (proto, prob) in cls_data:
iftlog.log(5, str(proto) + ": " + str(prob))
iftlog.log(5, "")
def init_transfer_program(port, ignore2=None, ignore3=None, ignore4=None):
"""
<Purpose>
This initializes a connection. It starts up iftd's filewriter thread
<Arguments>
None.
<Exceptions>
None.
<Side Effects>
None.
<Returns>
True.
"""
global iftd_server
if port == None:
port = 4000
iftd_server = iftapi.make_XMLRPC_client( "127.0.0.1", port, "RPC2", 60 )
return True
}
src_file = "/X1Hi1.gif"
job_attrs = {
iftfile.JOB_ATTR_SRC_HOST:"i.imgur.com",
iftfile.JOB_ATTR_SRC_NAME:src_file,
protocols.raven.TRANSFER_PROGRAM_PACKAGE:"transfer.arizonatransfer_http",
protocols.raven.INIT_TRANSFER_PROGRAM_ARG1:None,
protocols.raven.INIT_TRANSFER_PROGRAM_ARG2:None,
protocols.raven.INIT_TRANSFER_PROGRAM_ARG3:None,
protocols.raven.INIT_TRANSFER_PROGRAM_ARG4:None,
iftfile.JOB_ATTR_DEST_NAME:"/tmp/img.gif",
iftfile.JOB_ATTR_DEST_HOST:"localhost"
}
client = iftapi.make_XMLRPC_client()
connects = {
"http_receiver":http_connect_attrs,
"http_sender":http_connect_attrs,
"iftsocket_receiver":iftsocket_connect_attrs,
"iftsocket_sender":iftsocket_connect_attrs,
"raven":None
}
rc = client.begin_ift( job_attrs, connects, False, True, 4001, "/RPC2", True, False, 60 )
print "called"
test_cleanup.cleanup()
def getfiles1(host, filelist, destdir, prog_indicator=0, createhashfile=False, ignoreHash=False, prioritylist=None):
"""
<Purpose>
This fetches files from given host, using prioritylist which holds
transfer methods to fetch files.
It tries to get files by one method, and if it fails it uses next
possible method until it gets all files needed.
<Arguments>
host:
'host' holds two things, a server name and download directory.
For example, if you want to retrieve files from '/tmp/' directory
in 'quadrus.cs.arizona.edu' server, the 'host' will be
'quadrus.cs.arizona.edu/tmp'.
'host' should be a string.
filelist:
'filelist' is a list of files which need to be retrieved.
'filelist' should be a list of dictionaties of the format:
{"filename": filename,
"hash": hash,
"size": size,
"hashfuncs": list of hashfuncs to try}
the hash and size parameters can be None if unavailable
destdir:
'destdir' is a destination directory where retrieved files will
be placed. A user should have 'destdir' exist before retrieving
files. 'destdir' should be a string.
prog_indicator:
DEPRICATED, unused.
prioritylist:
If it is not None, then this is a whitelist of protocols to use
for transferring the given files. NOTE: the protocols here are
base names (e.g. 'http', 'ftp', 'scp', etc., as opposed to
'http_receiver', 'ftp_receiver', 'scp_receiver', etc.).
<Exceptions>
None.
<Side Effects>
Messes with SIGALRM
Set init_status
<Returns>
True or False to indicate success, and a list of downloaded files
"""
global init_status
#global arizonafetch
#global glo_prioritylist
arizonareport.send_out(4, "[DEBUG] getfiles started")
arizonareport.send_out(4, "host = " + str(host) + ", filelist = " + str(filelist))
# downloaded files list
downloaded_files = []
# check if host is a string
arizonageneral.check_type_simple(host, "host", str, "arizonatransfer.getfiles")
# check if destdir is a string
arizonageneral.check_type_simple(destdir, "destdir", str, "arizonatransfer.getfiles")
# get username
username = arizonageneral.getusername()
# check that the destination directory exists
if not os.path.isdir(destdir):
arizonareport.send_syslog(arizonareport.ERR, "\ngetfiles(): The destination directory '" + destdir + "' does not exist... Aborting...")
# return false and empty list
return (False, downloaded_files)
if prioritylist == None:
# transfer method list set by arizonaconfig
prioritylist = arizonaconfig.get_option("transfermethod")
# check the method list
# if prioritylist is None, there's something wrong with configuration
if prioritylist == None :
arizonareport.send_syslog(arizonareport.ERR, "getfiles(): No transfer method was given.")
return (False, downloaded_files)
# create a temporary directory for the transfer
arizonareport.send_out(4, "[DEBUG] getfiles creating temp dir")
try:
temp_dir = arizonaconfig.get_option("transfertempdir") + str(securerandom.SecureRandom().random())
except TypeError:
arizonareport.send_syslog(arizonareport.ERR, "getfiles(): No transfer temp dir is given.")
return (False, downloaded_files)
# in the case of destdir has '/' at the end
# last '/' should go away to make result list(downloaded_files) match
if len(destdir) > 1 and destdir.endswith('/'):
destdir = destdir[:len(destdir) - 1]
# if there are empty strings in the filelist, those will be taken away
arizonareport.send_out(4, "[DEBUG] checking file list")
filelist = __checkFileList(filelist)
arizonareport.send_out(4, "[DEBUG] creating directories")
for item in filelist:
filename = item['filename']
dirname = os.path.dirname(filename)
if dirname != "":
arizonageneral.makedirs_existok(os.path.join(temp_dir, dirname))
arizonageneral.makedirs_existok(os.path.join(destdir, dirname))
filenames = [item['filename'] for item in filelist]
# keep the number of the list to compare how many files are downloaded at the end.
numoflist = len(filelist)
# if there is no file needing to be fetched
if filelist == []:
arizonareport.send_syslog(arizonareport.ERR, "getfiles(): No files needed to be downloaded.")
return (False, downloaded_files)
if not os.path.exists(temp_dir):
arizonageneral.makedirs_existok(temp_dir)
failures = 0
for currFile in filelist:
# attempt to get this file, with the data given
common_attrs = {}
# add attributes about this file that we know
filename = currFile.get('filename',None)
if filename == None or len(filename) == 0:
# something's seriously wrong...
continue
if filename[0] != '/':
filename = '/' + filename # need to be an absolute path
common_attrs[iftfile.JOB_ATTR_SRC_NAME] = "/" + str(host.split("/")[1]) + filename
common_attrs[iftfile.JOB_ATTR_DEST_NAME] = temp_dir + filename
common_attrs[iftfile.JOB_ATTR_FILE_SIZE] = currFile.get('size',None)
common_attrs[iftfile.JOB_ATTR_FILE_HASH] = currFile.get('hash',None)
common_attrs[iftfile.JOB_ATTR_SRC_HOST] = str(host.split("/")[0])
common_attrs[iftfile.JOB_ATTR_DEST_HOST] = "localhost"
job_attrs = copy.copy( common_attrs )
job_attrs[iftfile.JOB_ATTR_REMOTE_IFTD] = False # until we have a faster way of detecting it!
connect_attrs = {}
if prioritylist:
job_attrs[iftfile.JOB_ATTR_PROTOS] = [proto + "_receiver" for proto in prioritylist]
for proto in job_attrs[iftfile.JOB_ATTR_PROTOS]:
connect_attrs[ proto ] = common_attrs
__enable_timeout(__compute_timeout(filelist))
# attempt to get it with iftd
iftd_server = iftapi.make_XMLRPC_client()
try:
# have iftd attempt to receive it
rc = iftd_server.begin_ift( job_attrs, connect_attrs, False, True, 4001, "/RPC2", True, 3600 )
print "XMLRPC call completed"
if rc != iftproto.ifttransmit.TRANSMIT_STATE_SUCCESS:
arizonareport.send_syslog( arizonareport.ERR, 'retrieve_files: File transfer RC=' + str(rc))
print "XMLRPC call rc = " + str(rc)
__disable_timeout()
failures += 1
continue
else:
print "got the file " + filename
except Exception, inst:
arizonareport.send_syslog( arizonareport.ERR, 'getfiles1(): XMLRPC error, could not receive file ' + filename )
iftlog.exception("getfiles1(): XMLRPC error, could not receive file " + filename, inst)
__disable_timeout()
failures += 1
continue
"""
req = urllib2.Request("http://127.0.0.1:16387" + job_attrs.get( iftfile.JOB_ATTR_SRC_NAME ) )
req.add_header( "Pragma", iftapi.pack_attrs( job_attrs, connect_attrs ) )
resp = None
try:
# perform the request
resp = urllib2.urlopen( req )
if resp.code != 200:
arizonareport.send_syslog(arizonareport.ERR, "getfiles(): HTTP " + str(resp.code) + " from IFTD for file " + filename)
failures += 1
__disable_timeout()
continue
except Exception, inst:
arizonareport.send_out(3, "[" +username+"] "+str(element)+": error: "+ str( sys.exc_info()[0] ) )
failures += 1
__disable_timeout()
continue
"""
__disable_timeout()
# write this file to the destination
dest_filename = destdir + filename
try:
shutil.move( common_attrs[ iftfile.JOB_ATTR_DEST_NAME ], dest_filename )
# success!
downloaded_files.append( filename )
except Exception, inst:
arizonareport.send_out(3, "[" + username + "] " + str(element) + ": error: " + str(sys.exc_info()[0]) )
failures += 1
def test_iftd( file, filehash, filesize, remote_host, tmpfs_dir ):
ts = 0
te = 0
http_connect_attrs = {
iftproto.PROTO_PORTNUM:8000,
iftfile.JOB_ATTR_SRC_NAME:file,
iftfile.JOB_ATTR_DEST_NAME:file,
iftfile.JOB_ATTR_SRC_HOST:remote_host
}
job_attrs = {
iftfile.JOB_ATTR_SRC_HOST:remote_host,
iftfile.JOB_ATTR_SRC_NAME:file,
iftfile.JOB_ATTR_FILE_SIZE:int(filesize),
iftfile.JOB_ATTR_FILE_HASH:filehash,
iftfile.JOB_ATTR_DEST_NAME:file,
iftfile.JOB_ATTR_CHUNKSIZE:int(filesize) / 200,
iftfile.JOB_ATTR_DEST_HOST:"localhost",
protocols.bittorrent.IFTBITTORRENT_TORRENT_PATH:file + ".torrent",
iftfile.JOB_ATTR_PROTOS:['http5_receiver', 'iftscp2_receiver', 'bittorrent2_receiver']
}
bt_connect_attrs = {
iftfile.JOB_ATTR_DEST_NAME:file,
iftfile.JOB_ATTR_SRC_NAME:file,
protocols.bittorrent.IFTBITTORRENT_PORTRANGE_LOW:1025,
protocols.bittorrent.IFTBITTORRENT_PORTRANGE_HIGH:65534,
protocols.bittorrent.IFTBITTORRENT_TORRENT_PATH:"/tmp/iftd-experiments/" + file + ".torrent"
}
client = iftapi.make_XMLRPC_client( timeout = 60 )
connects = {
"http5_receiver":http_connect_attrs,
"http5_sender":http_connect_attrs,
"iftscp2_sender":None,
"iftscp2_receiver":None,
"bittorrent2_sender":bt_connect_attrs,
"bittorrent2_receiver":bt_connect_attrs
}
client.clear_classifier()
for i in xrange(0, 1):
# transfer the file with iftd
ts = time.time()
rc = client.begin_ift( job_attrs, connects, False, True, 4001, "/RPC2", True, False, 60)
te = time.time()
iftlog.log(5, "iftd " + remote_host + file + " " + str(te - ts) )
if rc != iftproto.ifttransmit.TRANSMIT_STATE_SUCCESS:
iftlog.log(5, "iftd failed to transfer " + str(file))
os.popen("rm -rf " + file).close()
time.sleep(15)
cls_data = client.get_proto_rankings( job_attrs, True )
iftlog.log(5, "")
iftlog.log(5, "Protocol rankings")
for (proto, prob) in cls_data:
iftlog.log(5, str(proto) + ": " + str(prob))
iftlog.log(5, "")
