import iftproto
import iftfile
import protocols
import threading
import time
import iftapi
import iftstats
import test_setup
import test_cleanup
import iftutil
import protocols.raven
iftapi.iftd_setup(iftutil.get_available_protocols(), "../iftd.xml")
test_setup.setup()
iftstats.startup(iftapi.list_protocols(), 1, "NaiveBayes")
http_connect_attrs = {iftproto.PROTO_PORTNUM: 8080}
iftsocket_connect_attrs = {iftproto.PROTO_PORTNUM: 4000}
src_file = "/home/jude/raven/tools/iftd/testfile"
job_attrs = {
iftfile.JOB_ATTR_SRC_HOST: "localhost",
iftfile.JOB_ATTR_SRC_NAME: "/home/jude/raven/tools/iftd/testfile",
iftfile.JOB_ATTR_FILE_HASH: iftfile.get_hash(src_file),
iftfile.JOB_ATTR_FILE_SIZE: iftfile.get_filesize(src_file),
protocols.raven.TRANSFER_PROGRAM_PACKAGE: "transfer.arizonatransfer_http",
protocols.raven.INIT_TRANSFER_PROGRAM_ARG1: None,
return
if connect_args == None:
# not found
iftlog.log(5, "HTTPStorkServerHandler: connect_args not found for '" + get_filename + "'" )
print tmp_connect_args
self.send_response( 404 )
return
chunks_dir = job_attrs.get( iftfile.JOB_ATTR_DEST_CHUNK_DIR )
file_name = job_attrs.get(iftfile.JOB_ATTR_DEST_NAME)
# file request for cache miss!
try:
# get the file via iftd, but don't check the cache
all_protolist = iftapi.list_protocols()
protolist = []
if "iftcache_receiver" in all_protolist:
protolist = ["iftcache_receiver"] # only use the cache protocol if it is available
else:
# don't use any nest_helper protocol
for p in all_protolist:
if p.find("nest_helper") == -1:
protolist.append( p )
# map each protocol to the given connection args
proto_connect_args = {}
for proto_name in protolist:
proto_connect_args[proto_name] = connect_args
def do_GET(self):
global cache_dir
global tmp_connect_args
global tmp_job_attrs
# only pay attention to local requests
if self.client_address[0] != "127.0.0.1" and self.client_address[0] != "localhost":
iftlog.log(5, "HTTPCacheServerHandler: unauthorized GET from " + self.client_address[0])
self.send_response( 403 )
return
get_filename = os.path.basename( self.path )
connect_args = tmp_connect_args.get( get_filename )
job_attrs = tmp_job_attrs.get( get_filename )
if job_attrs == None:
# not found...
iftlog.log(5, "HTTPCacheServerHandler: job_attrs not found for '" + get_filename + "'" )
self.send_response( 404 )
return
if connect_args == None:
# not found
iftlog.log(5, "HTTPCacheServerHandler: connect_args not found for '" + get_filename + "'" )
print tmp_connect_args
self.send_response( 404 )
return
chunks_dir = job_attrs.get( iftfile.JOB_ATTR_DEST_CHUNK_DIR )
file_name = job_attrs.get(iftfile.JOB_ATTR_DEST_NAME)
# file request for cache miss!
try:
# get the file via iftd, but don't check the cache
all_protolist = iftapi.list_protocols()
protolist = []
# don't use any iftcache protocol
for p in all_protolist:
if p.find("iftcache") == -1:
protolist.append( p )
# map each protocol to the given connection args
proto_connect_args = {}
for proto_name in protolist:
proto_connect_args[proto_name] = connect_args
# put in a request to get the file
iftlog.log(5, "iftcache: Squid cache miss, so trying all other available protocols")
# receive to the cache directory
cached_filepath = cache_dir.rstrip("/") + "/" + os.path.basename( job_attrs.get( iftfile.JOB_ATTR_DEST_NAME ))
job_attrs[ iftfile.JOB_ATTR_DEST_NAME ] = cached_filepath
job_attrs[ iftfile.JOB_ATTR_PROTOS ] = protolist
rc = iftapi.begin_ift( job_attrs, proto_connect_args, False, True, connect_args[iftapi.CONNECT_ATTR_REMOTE_PORT], connect_args[iftapi.CONNECT_ATTR_REMOTE_RPC], connect_args[iftapi.CONNECT_ATTR_USER_TIMEOUT] )
# success or failure?
if rc != TRANSMIT_STATE_SUCCESS and rc != 0:
iftlog.log(5, "iftcache: could not receive file " + file_name + " (rc = " + str(rc) + ")")
self.send_response(500)
return
# open the file and write it back
file_buff = []
try:
fd = open( cached_filepath, "rb" )
file_buff = fd.read()
fd.close()
except Exception, inst:
iftlog.exception("iftcache: received file to " + cached_filepath + ", but could not read it")
self.send_response(500)
return
# reply the file
self.send_response(200)
self.send_header( 'Content-type', 'application/octet-stream' ) # force raw bytes
self.send_header( 'Last-Modified', time.ctime( os.stat( cached_filepath ).st_mtime ) )
self.end_headers()
self.wfile.write( file_buff )
# recreate the chunks directory, since we might have lost it...
if chunks_dir != None and not os.path.exists( chunks_dir ):
try:
os.popen("mkdir -p " + chunks_dir).close()
except:
pass
# done with this...
tmp_connect_args[get_filename] = None
tmp_job_attrs[get_filename] = None
return
