def send_iftd_receiver_choice( xmit_id, receiver_chunk_dir, best_proto, available_protos ):
"""
Called by receiver (remote) on the sender (local) to inform
the sender to begin actively sending, with the prefered protocol.
"""
# this had better be a valid call
if not TransferCore.sender_valid_xmit_id( xmit_id ):
iftlog.log(5, "iftapi.send_iftd_receiver_choice: invalid xmit ID " + str(xmit_id))
return None
connect_dict = TransferCore.get_connection_attrs( xmit_id )
user_job_attrs = TransferCore.get_job_attrs( xmit_id )
user_job_attrs[ iftfile.JOB_ATTR_DEST_CHUNK_DIR ] = receiver_chunk_dir
user_job = iftfile.iftjob( user_job_attrs )
if not available_protos:
available_protos = []
iftlog.log(1, "send_iftd_receiver_choice: will use " + str(available_protos) + ", where " + str(best_proto) + " is the best" )
#print "send_iftd_receiver_choice( " + str(xmit_id) + ", " + str(user_job_attrs) + ", " + str(best_proto) + ", " + str(available_protos) + ", " + str(connect_dict) + ")"
usable_protos = []
if not best_proto:
usable_protos = available_protos
else:
usable_protos = [best_proto] + available_protos
# just in case
if "unknown" in usable_protos:
usable_protos.remove("unknown")
if len(usable_protos) != 0:
# start up a transfer processor with the available active protocols
proto_instances = []
for proto in usable_protos:
proto = proto + "_sender" # if it's available, then there's a receiver available
if not PROTOCOLS[proto].isactive():
continue # only start up active protocols.
p = None
try:
p = copy.deepcopy( PROTOCOLS[proto] )
except Exception, inst:
iftlog.log(5, "ERROR: could not clone protocol " + proto)
continue
proto_instances.append( p )
# start up the active protocols
connected_protos = start_active_protos( user_job=user_job, connect_dict=connect_dict, protos=proto_instances, timeout=0.01 )
rc = iftutil.SenderThreadPool.start_new_thread( TransferCore.run_ift_send_active, (xmit_id, user_job, connected_protos, user_job.get_attr( iftfile.JOB_ATTR_TRANSFER_TIMEOUT ), best_proto == None ) )
if not rc:
iftlog.log(5, "send_iftd_receiver_choice: could not start new active sender thread")
xmit_id = None # error rc
fsize = os.stat( filename ).st_size
file_hash = os.popen("sha1sum " + filename).readlines()[0].split(" ")[0]
print "hash of " + filename + " is " + file_hash
job_attrs = {
iftfile.JOB_ATTR_SRC_NAME:filename,
iftfile.JOB_ATTR_PROTOS:"http",
iftfile.JOB_ATTR_SRC_HOST:"localhost",
iftfile.JOB_ATTR_CHUNKSIZE:4096,
iftfile.JOB_ATTR_FILE_SIZE:fsize,
iftfile.JOB_ATTR_FILE_HASH:file_hash
}
file_job = iftfile.iftjob( job_attrs )
connect_attrs = {
iftproto.PROTO_PORTNUM:8080,
iftfile.JOB_ATTR_SRC_NAME:filename
}
rc = sender.setup( connect_attrs )
assert rc == 0, "could not set up! (rc=" + str(rc) + ")"
sender.assign_job( file_job )
rc = sender.on_start( connect_attrs )
assert rc == 0, "could not start up! (rc=" + str(rc) + ")"
connect_attrs = {
iftproto.PROTO_PORTNUM:22,
iftfile.JOB_ATTR_SRC_NAME:filename,
iftfile.JOB_ATTR_SRC_HOST:"lectura.cs.arizona.edu",
protocols.iftscp.IFTSCP_REMOTE_LOGIN:"******",
iftfile.JOB_ATTR_DEST_NAME:"/home/jude/raven/tools/iftd/test/testfile-remote.txt",
iftfile.JOB_ATTR_REMOTE_IFTD:False,
iftfile.JOB_ATTR_SRC_CHUNK_DIR:"/home/jnelson",
iftfile.JOB_ATTR_NUM_CHUNKS: 1
}
rc = receiver.setup( connect_attrs )
assert rc == 0, "could not set up! (rc=" + str(rc) + ")"
rc = receiver.on_start( iftfile.iftjob( connect_attrs ), connect_attrs )
assert rc == 0, "could not start up! (rc=" + str(rc) + ")"
receiver.post_msg( iftproto.PROTO_MSG_USER, iftproto.PROTO_STATE_RUNNING )
receiver.run( -1 )
print "sleeping"
os.system("sleep 55")
print "done"
test_cleanup.cleanup()
def get_iftd_sender_data( xmit_id, job_attrs, available_protos, connect_dict ):
"""
Called by the receiver (remote) on the sender (local) to get
the sender's capabilities--specifically, which protocols it
has senders for, and where the chunks will be located.
Return the list of protocols usable to both
sender and receiver.
"""
global TransferCore
error_rc = (xmit_id, None, None, None, None, None, None, None)
file_name = job_attrs.get( iftfile.JOB_ATTR_SRC_NAME )
user_job = iftfile.iftjob( job_attrs )
# does the file exist?
if not os.path.exists( file_name ):
iftlog.log(5, "get_iftd_sender_data: file " + str(file_name) + " does not exist")
return error_rc # don't even bother
# is the file readable?
if not (stat.S_IWUSR & os.stat( file_name ).st_mode):
iftlog.log(5, "get_iftd_sender_data: file " + str(file_name) + " is not readable")
return error_rc # don't bother--can't read
# is the file accessible?
from iftdata import SEND_FILES_DIR
if SEND_FILES_DIR[-1] != '/':
SEND_FILES_DIR = SEND_FILES_DIR + "/"
if not os.path.abspath(file_name).startswith( SEND_FILES_DIR, 0, len(SEND_FILES_DIR)):
iftlog.log(5, "get_iftd_sender_data: will not send " + str(file_name) + ", it is not in " + SEND_FILES_DIR )
return error_rc # access control violation
# get our available protocols
my_protos = proto_names( senders( list_protocols() ) )
other_protos = []
if available_protos:
other_protos = proto_names( available_protos )
# calculate intersection between both available protos
my_protos_set = set( my_protos )
other_protos_set = set( other_protos )
usable_protos_set = my_protos_set.intersection( other_protos_set )
file_size = iftfile.get_filesize( file_name )
iftlog.log(1, "get_iftd_sender_data: file " + str(file_name) + ", size " + str(file_size))
# start my passive senders
sender_names = senders( list_protocols() )
proto_insts = []
for proto in sender_names:
if PROTOCOLS.get(proto) != None and not PROTOCOLS.get(proto).isactive():
p = None
# start this passive sender
try:
p = copy.deepcopy( PROTOCOLS.get(proto) )
except:
iftlog.log(5, "get_iftd_sender_data: could not start passive sender " + proto)
continue
proto_insts.append(p)
expected_fsize = user_job.get_attr( iftfile.JOB_ATTR_FILE_SIZE )
min_fsize = user_job.get_attr( iftfile.JOB_ATTR_FILE_MIN_SIZE )
max_fsize = user_job.get_attr( iftfile.JOB_ATTR_FILE_MAX_SIZE )
# do some sanity checking...
if min_fsize != None and max_fsize != None:
if file_size < min_fsize or file_size > max_fsize:
return error_rc # wrong size expectation
if expected_fsize != None and expected_fsize != file_size:
return error_rc # wrong size expectation
# set up
rc, file_hash, chunk_hashes, chunk_data = prepare_sender( file_name, user_job.get_attr( iftfile.JOB_ATTR_CHUNKSIZE ) )
if rc != 0:
iftlog.log(5, "get_iftd_sender_data: could not prepare to send")
return error_rc
user_job.supply_attr( iftfile.JOB_ATTR_FILE_SIZE, file_size )
user_job.supply_attr( iftfile.JOB_ATTR_FILE_HASH, file_hash )
user_job.supply_attr( iftfile.JOB_ATTR_FILE_TYPE, iftstats.filetype( file_name ) )
passive_protos = start_passive_protos( connect_dict, user_job, proto_insts, 1.0 )
# start passive protocol handling thread
TransferCore.begin_ift_send( xmit_id, user_job, chunk_data, user_job.get_attr( iftfile.JOB_ATTR_CHUNK_TIMEOUT ), connect_dict )
TransferCore.run_ift_send_passive( xmit_id, user_job, passive_protos, user_job.get_attr( iftfile.JOB_ATTR_CHUNK_TIMEOUT ))
proto_mask = [0] * len(sender_names)
for i in xrange(0, len(sender_names)):
p = sender_names[i]
if PROTOCOLS[p].isactive():
proto_mask[i] = True
else:
proto_mask[i] = False
return (xmit_id, iftfile.get_chunks_dir( file_name, file_hash, True), file_size, file_hash, iftstats.filetype(file_name), sender_names, proto_mask, chunk_hashes)
def begin_ift( job_attrs, connect_dict=None, sender=False, receiver=False, iftd_remote_port=USER_PORT+1, iftd_xmlrpc_path="/RPC2", user_timeout=60 ):
"""
Initiate an intelligent file transmission, using the local and remote iftd instances' previous knowledge
of which protocols work best for which files.
@arg job_attrs
This is a dictionary describing everything known about the file, the host with the file (the source host),
and the host to receive the file (the dest host)
@arg connect_dict
This is a dictionary that maps protocol names to dictionaries containing their connection arguments.
Protocol names must end in _sender or _receiver!
@arg sender
True if the file to transmit is on localhost (in which JOB_ATTR_SRC_NAME is a path on localhost's disk to the file)
@arg receiver
True if the file to transmit is on the remote host (in which JOB_ATTR_SRC_NAME is a path on the remote host's disk to the file)
@arg iftd_remote_port
Port number that the remote IFTD listens on for inter-IFTD communication.
@arg iftd_xmlrpc_path
Directory under which the remote IFTD listens for XMLRPC calls.
@arg user_timeout
Timeout in seconds before the transfer must complete to be a success.
@return
0 on success
-1 on bad data
< -1 on internal error (see iftdata.py for error constants)
11 (TRANSMIT_STATE_FAIULRE) on transmission failure
"""
iftlog.log(5, ">>> begin_ift entered <<<")
global PROTOCOLS
job = iftfile.iftjob( job_attrs )
# sanity check
if sender and receiver:
return -1 # cannot do
if not (sender or receiver):
return -1 # cannot do
# get the available protocols
available_protocols = job.get_attr( iftfile.JOB_ATTR_PROTOS )
if available_protocols == None or len(available_protocols) == 0:
available_protocols = list_protocols()
else:
absent = []
# make sure that they are all defined
for proto in available_protocols:
if proto not in PROTOCOLS.keys():
absent.append( proto )
for proto in absent:
iftlog.log(3, "begin_ift: unrecognized protocol " + proto + ", ignoring..." )
available_protocols.remove( proto )
# stuff the relevant data into connect_dict if needed
if connect_dict == None:
connect_dict = {}
for proto in list_protocols():
cd = connect_dict.get(proto)
if cd == None:
connect_dict[proto] = {}
cd = connect_dict[proto]
# am I the sender or receiver?
# Have the sender contact the remote iftd instance for protocol information.
if sender:
rc = iftsend( job, available_protocols, connect_dict, iftd_remote_port, iftd_xmlrpc_path, user_timeout )
iftlog.log(5, ">>> begin_ift rc=" + str(rc) + " <<<")
return rc
else:
rc = iftreceive( job, available_protocols, connect_dict, iftd_remote_port, iftd_xmlrpc_path, user_timeout )
iftlog.log(5, ">>> begin_ift rc=" + str(rc) + " <<<")
return rc
filename = "/tmp/test_iftfile"
file_d = open( filename, "w" )
for i in range(0, 500):
file_d.write("abcdefg\n")
file_d.close()
file_attrs = {
iftfile.JOB_ATTR_SRC_NAME:filename,
iftfile.JOB_ATTR_CHUNKSIZE:len("abcdefg\n")
}
ift_job = iftfile.iftjob( file_attrs )
ift_file = iftfile.iftfile( filename )
rc = ift_file.fopen( file_attrs, iftfile.MODE_READ )
assert rc != None, "Could not open file!"
assert ift_file.last_error() == 0, "Could not open file!"
for i in range(0, 500):
chunk, chunk_id = ift_file.read_chunk()
assert chunk_id == i, "Chunk ID incorrect (" + str(chunk_id) + ")"
assert chunk == "abcdefg\n", "Could not read chunk " + str(i) + ", got " + chunk
ift_file.fclose()
assert ift_file.last_error() == 0, "Could not close file!"
