def __send_file_chunks( self ): """ Call repeatedly in the ifttransmit main loop to send chunks. This will be called once prepare_transmit and possibly send_job have been called Return an event to be handled by ifttransmit """ if self.ready_to_send == False or self.transmit_state != TRANSMIT_STATE_CHUNKS: return (0,E_BAD_STATE) # can we do anything? if self.ift_job == None and self.ready_to_send == True: iftlog.log(5, self.name + ": No job to process! Use my assign_job() method and resume me") self.ready_to_send = False return (0,E_BAD_STATE) chunk = None chunk_id = -1 rc = 0 chunk, chunk_id, chunk_path, remote_chunk_path = self.__next_chunk() try: rc = self.send_one_chunk( chunk, chunk_id, chunk_path, remote_chunk_path ) except Exception, inst: iftlog.exception( self.name + ": could not send data", inst ) self.close_connection( TRANSMIT_STATE_FAILURE ) t = time.time() iftstats.log_chunk( self.ift_job, self.name, False, t, t, 0 ) return (PROTO_MSG_ERROR_FATAL, E_NO_DATA)
def get_reusable_data( proto_name, data_key, do_copy=True ): """ Get a (deep) copy of a piece of reusable transmitter data, given the name of the receiver. @arg proto_name Name of the protocol @arg data_key either REUSABLE_DATA_SETUP_ARGS or REUSABLE_DATA_RESPAWN Return the data on success, or None on error. """ if __reusable_data.has_key(proto_name) == False: return {} if __reusable_data[proto_name].has_key(data_key) == False: return {} try: if do_copy: return copy.deepcopy(__reusable_data[ proto_name ][ data_key ]) else: return __reusable_data[ proto_name ][ data_key ] except Exception, inst: iftlog.exception( "Could not get re-instantiation data for transmitter " + proto_name, inst ) return None
def recv_files( self, remote_file_paths, local_file_dir ): global cache_dir # get the file from the cache and write it to disk, if possible file_fd = cache_get_file( cache_path(self.file_to_recv), self.max_age, self.connect_args, self.job_attrs, self.squid_port, self.http_port ) if file_fd == None: iftlog.log(3, self.name + ": could not receive " + self.file_to_recv ) self.recv_finished( TRANSMIT_STATE_FAILURE ) return E_NO_DATA # not in cache ==> protocol failure data = file_fd.read() file_fd.close() tmp_file_name = "" try: tmp_file_name = local_file_dir + "/" + os.path.basename( self.job_attrs.get( iftfile.JOB_ATTR_DEST_NAME ) ) fd = open( tmp_file_name, "wb" ) fd.write( data ) fd.close() except Exception, inst: iftlog.exception(self.name + ".recv_files: failed to save " + self.file_to_recv + " to " + tmp_file_name, inst) return E_IOERROR
def get_unwritten_chunks(self): """ Get a list of the chunks that have not yet been received. Note: this list will possibly be outdated as soon as it is returned! """ try: # sanity check if self.__mode != MODE_WRITE: self.__error = E_BAD_MODE return E_BAD_MODE index = 0 ret = [] for i in xrange(0, len(self.__chunk_mask)): if self.__chunk_reservations[i] >= time.time(): continue try: chunk = self.__chunk_mask[i] if chunk == False: ret.append( i ) except: break if len(ret) == 0 and self.known_size == False: # if we don't know the size, add the chunks required to reach __bytes_max ret.append( len(self.__chunk_mask) ) return ret except Exception, inst: iftlog.exception("iftfile.get_unwritten_chunks", inst) return []
def make_chunks_dir( filename, filehash ): """ Make a directory from the filename and filehash to store incoming chunks into. """ if filename[0] == "/": filename = filename[1:] file_dir = os.path.basename( filename ) + "." + str(filehash) if os.path.exists( os.path.join(__file_chunks_dir, file_dir) ): iftlog.log(3, "WARNING: " + os.path.join(__file_chunks_dir, file_dir) + " exists!") try: os.popen("rm -rf " + os.path.join(__file_chunks_dir, file_dir) + "/*").close() except: pass return 0 # already done! chunk_dir = get_chunks_dir( filename, filehash ) try: rc = os.popen("mkdir -p " + chunk_dir ).close() if rc != None: iftlog.log(5, "iftfile: could not make chunk directory " + chunk_dir ) return E_IOERROR return 0 except Exception, inst: iftlog.exception( "iftfile: could not make chunk directory " + chunk_dir ) return E_IOERROR
def read_next_chunk( self ): """ Read the next chunk of this file Call fopen() before calling this. Return a chunk ID and a chunk with length 0 or more (0 indicates EOF) Return negative and None if the file isn't open """ if self.__mode != MODE_READ: # can't read if we aren't in READ mode self.__error = E_BAD_MODE return (None, E_BAD_MODE) if not os.path.exists(self.path): self.__error = E_IOERROR return (None, E_IOERROR) self.__read_lock.acquire() fd = None bytes = None try: fd = open(self.path, "r") fd.seek( self.__next_chunk * self.__chunk_size ) bytes = fd.read( self.__chunk_size ) fd.close() except Exception, inst: iftlog.exception( "iftfile: could not open " + self.path, inst) self.__error = E_IOERROR self.__read_lock.release() return (None, E_IOERROR)
def recv_job( self, job ): self.job_attrs = copy.deepcopy( job.attrs ) # get a reference so we can check out other data other protocols add # try to get sender information, such as server version try: self.remote_host = job.get_attr( iftfile.JOB_ATTR_SRC_HOST ).strip() # remove http:// and/or ftp:// and/or https:// if self.remote_host.find( "http://" ) == 0: self.remote_host = self.remote_host.lstrip( "http://" ) if self.remote_host.find( "ftp://" ) == 0: self.remote_host = self.remote_host.lstrip( "ftp://" ) if self.remote_host.find( "https://" ) == 0: self.remote_host = self.remote_host.lstrip( "http://" ) self.file_size = -1 if self.job_attrs and self.job_attrs.has_key( iftfile.JOB_ATTR_FILE_SIZE ): self.file_size = self.job_attrs[ iftfile.JOB_ATTR_FILE_SIZE ] # figure out how big the file is server_version = 10 if self.file_size == -1: self.file_size, server_version = self.get_remote_file_attrs( job.get_attr( iftfile.JOB_ATTR_SRC_HOST ), self.connect_args[ iftproto.PROTO_PORTNUM ], job.get_attr( iftfile.JOB_ATTR_SRC_NAME ) ) else: fs, server_version = self.get_remote_file_attrs( job.get_attr( iftfile.JOB_ATTR_SRC_HOST ), self.connect_args[ iftproto.PROTO_PORTNUM ], job.get_attr( iftfile.JOB_ATTR_SRC_NAME ) ) if int(fs) != int(self.file_size): # problem--the file doesn't have the right size! iftlog.log(5, self.name + ": ERROR: given file size (" + str(self.file_size) + ") does not match server's file size (" + str(fs) + "). -1 means got 404") return E_INVAL self.file_size = int(self.file_size) if self.file_size < 0: # error--couldn't stat remote file iftlog.log(5, self.name + ": WARNING: could not determine remote file size") else: if not job.get_attr( iftfile.JOB_ATTR_FILE_SIZE ): job.set_attr( iftfile.JOB_ATTR_FILE_SIZE, self.file_size ) self.job_attrs[ iftfile.JOB_ATTR_FILE_SIZE ] = self.file_size if server_version == 11: # we can do chunking internally self.set_chunking_mode( iftproto.PROTO_DETERMINISTIC_CHUNKING ) print self.name + ": deterministic chunking activated" else: # old HTTP server, no chunking self.set_chunking_mode( iftproto.PROTO_NO_CHUNKING ) print self.name + ": chunking deactivated" #if self.http_version == 11: # supports Range header # self.use_chunking = True iftlog.log(1, self.name + ": expected file size of " + str(self.file_size) ) return 0 except Exception, inst: iftlog.exception( "http: could not await_sender", inst) return E_UNHANDLED_EXCEPTION
def recv_file( self, remote_chunk_dir, desired_chunks ): print "recv_file" chunk_dict = {} # get each chunk from the remote host try: for chunk in desired_chunks: connection = urllib2.Request( "http://" + self.remote_host + ":" + str(self.connect_args[iftproto.PROTO_PORTNUM]) + os.path.join( remote_chunk_dir, str(chunk) ) ) response = urllib2.urlopen( connection ) if response.code == 200: # got chunk chunk_dict[chunk] = response.read() else: iftlog.log(3, self.name + ": WARNING: could not get chunk " + str(chunk) + ", status = " + str(response.code) ) response.close() if chunk_dict == {}: return (E_NO_DATA, None) return (0, chunk_dict) except Exception, inst: iftlog.exception(self.name + ": ERROR: could not get all chunks from " + str(self.remote_host) + " in " + str(remote_chunk_dir), inst) return (E_NO_CONNECT, None)
def await_sender( self, connection_attrs, timeout ): if connection_attrs != None: p = connection_attrs.get( PROTO_PORTNUM ) if p != None: try: self.port = int(p) except: pass # set up a server socket to the remote host self.soc = None self.connected = False if connection_attrs.get( IFTSOCKET_TIMEOUT ) != None: self.timeout = connection_attrs.get( IFTSOCKET_TIMEOUT ) else: self.timeout = 1 if self.port != None: try: self.soc = socket.socket( socket.AF_INET, socket.SOCK_STREAM ) self.soc.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.soc.settimeout( self.timeout ) # use the given timeout self.soc.bind( ("localhost", self.port) ) self.soc.listen(1) # only one remote host should talk to me iftlog.log(1, "iftsocket_receiver: Listening on localhost:" + str(self.port) ) except Exception, inst: iftlog.exception( "iftsocket_receiver: could not set up server socket", str(inst) ) return E_NO_CONNECT return 0
def do_GET(self): # file request? try: # which pieces were requested? if True or os.path.dirname( self.path.strip() ) in self.server.available_files: if self.path.endswith("done"): # receiver is done with this file #del self.server.available_files[ os.path.dirname( self.path.strip() ) ] self.send_response( 200 ) else: # file has been made available SimpleHTTPRequestHandler.do_GET(self) return else: self.send_response(404) # nothing to send return except Exception, inst: iftlog.exception( "Could not fully transmit " + self.path, inst) self.send_response(404) return
def unlock_chunk( self, owner, chunk_id ): """ Unlock a chunk. Only valid for MODE_WRITE Return 0 on success; negative on failure """ try: # sanity check if self.__mode != MODE_WRITE: self.__error = E_BAD_MODE return E_BAD_MODE if chunk_id < 0 or chunk_id >= self.__num_chunks: self.__error = E_INVAL return E_OVERFLOW if self.__chunk_owners[ chunk_id ] == owner: self.__chunk_reservations[ chunk_id ] = 0 self.__chunk_owners[ chunk_id ] = None self.__chunk_locks[ chunk_id ].release() return 0 return E_INVAL except Exception, inst: if self.__open == False: iftlog.log("iftfile.unlock_chunk: file is no longer open") iftlog.exception("iftfile.unlock_chunk: could not unlock " + str(chunk_id)) return E_BAD_STATE
def send_job( self, job ): # this file is to be made available try: self.http_server.available_files.append( job.get_attr( iftfile.JOB_ATTR_SRC_CHUNK_DIR ) ) return 0 except Exception, inst: iftlog.exception( self.name + ": could not make " + str(job.get_attr( iftfile.JOB_ATTR_SRC_CHUNK_DIR )) + " available", inst) return E_UNHANDLED_EXCEPTION
def import_package(name): try: __import__(name) iftlog.log(0, "iftloader: Loaded " + name) return 0 except Exception, inst: iftlog.exception("iftloader: could not load " + name, inst) return -1
def recv_chunks( self, remote_chunk_dir, desired_chunks ): # determine what has been received since the last time this was called chunk_list = self.get_chunk_list() active_set = set( chunk_list ) # wait until we actually receive something while len(active_set) - len(self.recv_prev) == 0: time.sleep(0.5) active_set = active_set | set( self.get_chunk_list() ) if self.torrent_handle.is_seed(): # we have all chunks active_set = set([i for i in xrange(0, self.torrent_handle.status().num_pieces)]) break s = self.torrent_handle.status() state_str = ['queued', 'checking', 'downloading metadata', \ 'downloading', 'finished', 'seeding', 'allocating'] print '%.2f%% complete (down: %.1f kb/s up: %.1f kB/s peers: %d) %s' % \ (s.progress * 100, s.download_rate / 1000, s.upload_rate / 1000, \ s.num_peers, state_str[s.state]) if not iftapi.is_alive(): return E_FAILURE continue # indicate what we have received new_chunks = active_set - self.recv_prev iftlog.log(3, self.name + ": received " + str(len(new_chunks)) + " more chunks") self.recv_prev = active_set print "have now received " + str(len(self.recv_prev)) + " chunks" # convert to dictionary bt_dir = self.torrent_handle.save_path() rc = 0 for chunk_id in new_chunks: file_slices = self.torrent_info.map_block( chunk_id, 0, self.chunksize ) # which file(s) did this chunk correspond to? for fs in file_slices: recv_file = self.torrent_info.file_at( fs.file_index ) try: chunk_fd = open( bt_dir + "/" + recv_file.path, "r") chunk_fd.seek( fs.offset ) chunk_data = chunk_fd.read( self.chunksize ) if chunk_data: self.add_chunk( chunk_id, chunk_data ) chunk_fd.close() print "copy chunk " + str(chunk_id) + " from " + str(recv_file.path) + " at offset " + str(fs.offset) + ", length " + str(len(chunk_table[chunk_id])) + " (chunksize is " + str(self.chunksize) + ")" except Exception, inst: iftlog.exception( self.name + ": could not get chunk " + str(chunk_id) + " from " + str(recv_file) + " at offset " + str(fs.offset), inst) rc = E_IOERROR continue
def __write_file( self, path, bytes ): try: fd = open( path, "wb" ) fd.write( bytes ) fd.close() return 0 except Exception, inst: iftlog.exception( self.name + ".__write_file: could not write to " + path, inst ) return E_IOERROR
def lock_chunk( self, owner, chunk_id, override=False, t=1.0 ): """ Lock a chunk for writing. No other threads can access it. Only valid for MODE_WRITE Blocks, and returns """ try: # sanity check if self.marked_complete: self.__error = E_COMPLETE return E_COMPLETE if self.__mode != MODE_WRITE: self.__error = E_BAD_MODE return E_BAD_MODE # if we know how many chunks there are, then lock it as usual if self.known_size == True: if chunk_id < 0 or chunk_id >= self.__num_chunks: self.__error = E_INVAL return E_INVAL else: # make a new entry if we need to and lock it if chunk_id < 0: self.__error = E_INVAL return E_INVAL self.__grow_metadata( chunk_id ) # can't lock the chunk if it already has data if self.__chunk_mask[ chunk_id ]: return E_DUPLICATE self.__chunk_locks[ chunk_id ].acquire() # we're takin' over if override: self.__chunk_reservations[ chunk_id ] = time.time() + t self.__chunk_owners[ chunk_id ] = owner # sanity check again (in case this was called after fclose()) if self.__mode != MODE_WRITE: self.__error = E_BAD_MODE self.__chunk_locks[ chunk_id ].release() return E_BAD_MODE # when we return, this thread holds the lock return 0 except Exception, inst: if self.__open == False: iftlog.log("iftfile.lock_chunk: file is no longer open") # should only happen if the file gets closed by another thread iftlog.exception("iftfile.lock_chunk: could not lock chunk " + str(chunk_id)) return E_BAD_STATE
def delete_proto_instance(handle): if __loaded_proto_instances.has_key(handle) == False: return E_NO_VALUE try: del __loaded_proto_instances[handle] return 0 except Exception, inst: iftlog.exception("iftloader: could not delete protocol instance", inst) return E_UNHANDLED_EXCEPTION
def fpurge(self): """ Erase all data we've retained. Do so atomically. """ self.__expand_lock.acquire() self.__open = False try: os.remove( self.path) except Exception, inst: iftlog.exception("iftfile: could not fpurge " + self.path, inst)
def setup( self, setup_attrs ): try: self.port = setup_attrs[ PROTO_PORTNUM ] server_addr = ('', self.port) self.http_server = ift_http_server( server_addr, ift_http_request_handler ) thread.start_new_thread( self.http_server.serve_forever, () ) return 0 except Exception, inst: iftlog.exception( "http_sender.setup: could not start HTTP server", inst ) return E_NO_CONNECT
def set_chunk( self, chunk, chunk_id, trunicate=True, strict=False ): """ Receive a chunk to be written. No trunication will happen unless trunicate = True Chunk does not need to be exactly the right size unless strict = True CALL ONLY FOR LOCKED CHUNKS Return 0 on success Return E_INVAL if there is already a chunk for this id, or if the id is out of range Return E_BAD_MODE if we're in READ mode Return E_OVERFLOW if the upper bound of the known file size has been reached Return E_UNDERFLOW if the chunk was too small and strict == True """ try: if self.marked_complete: self.__error = E_COMPLETE return E_COMPLETE if self.__mode != MODE_WRITE: # can't write if we aren't in WRITE mode self.__error = E_BAD_MODE return E_BAD_MODE if self.__bytes_max > 0 and self.__bytes_written + len(chunk) > self.__bytes_max: # problem--got too much data iftlog.log(3, "iftfile: attempted to write " + str(len(chunk)) + " more bytes beyond required maximum of " + str(self.__bytes_max) + ", will not write chunk " + str(chunk_id)) return E_OVERFLOW if trunicate and len(chunk) > self.__chunk_size: iftlog.log(3, "iftfile: got chunk of length " + str(len(chunk)) + ", trunicating to " + str(self.__chunk_size)) chunk = chunk[0 : self.__chunk_size-1] if len(chunk) < self.__chunk_size and strict: return E_UNDERFLOW # write to disk try: fd = open(self.path, "ab") fd.seek( self.__chunk_size * chunk_id ) fd.write( chunk ) fd.close() except Exception, inst: iftlog.exception("iftfile.set_chunk: could not write to " + self.path, inst) return E_IOERROR # do not receive this chunk again rc = self.mark_chunk( chunk_id, True ) if rc != 0: iftlog.log(3, "iftfile: will not write chunk " + str(chunk_id)) return rc return 0
def prepare_transmit( self, job ): # create the .torrent file from the given file fs = lt.file_storage() lt.add_files( fs, job.get_attr( iftfile.JOB_ATTR_SRC_NAME ) ) ct = lt.create_torrent( fs, job.get_attr( iftfile.JOB_ATTR_CHUNKSIZE ) ) ct.set_creator("iftd: " + self.name ) self.file_to_send = job.get_attr( iftfile.JOB_ATTR_SRC_NAME ) # if we were given a tracker or list of trackers, add them if job.get_attr( IFTBITTORRENT_TRACKER ) != None: if type(job.get_attr( IFTBITTORRENT_TRACKER )) == str: ct.add_tracker( job.get_attr( IFTBITTORRENT_TRACKER ), 0 ) if type(job.get_attr( IFTBITTORRENT_TRACKER )) == list: for tracker in job.get_attr( IFTBITTORRENT_TRACKER ): ct.add_tracker(tracker, 0) else: # add some default trackers ct.add_tracker("http://tracker.openbittorrent.com/announce", 0) ct.add_tracker("udp://tracker.openbittorrent.com:80/announce", 0) ct.add_tracker("http://tracker.publicbt.com/announce", 0) ct.add_tracker("udp://tracker.publicbt.com:80/announce", 0) # if we were given one or more http seeds, add them too if job.get_attr( IFTBITTORRENT_HTTP_SEEDS ) != None: if type(job.get_attr( IFTBITTORRENT_HTTP_SEEDS )) == str: ct.add_url_seed( job.get_attr( IFTBITTORRENT_HTTP_SEEDS ) ) if type(job.get_attr( IFTBITTORRENT_HTTP_SEEDS )) == list: for seed in job.get_attr( IFTBITTORRENT_HTTP_SEEDS ): ct.add_url_seed( seed ) lt.set_piece_hashes( ct, os.path.dirname( job.get_attr( iftfile.JOB_ATTR_SRC_NAME ) ) ) # encode the torrent into a .torrent buffer self.torrent_str = lt.bencode( ct.generate() ) # if given a torrent path, write out the torrent if job.get_attr( IFTBITTORRENT_TORRENT_PATH ) != None: try: fd = open( job.get_attr( IFTBITTORRENT_TORRENT_PATH ), "wb" ) fd.write( self.torrent_str ) fd.close() except Exception, inst: iftlog.exception( self.name + ": could not output torrent data to " + job.get_attr( IFTBITTORRENT_TORRENT_PATH ), inst) return E_IOERROR
def prepare_transmit( self, job ): # file should have remote hostname p = job.get_attr( PROTO_PORTNUM ) if p != None: self.port = p remote_host = None try: remote_host = job.attrs[ iftfile.JOB_ATTR_DEST_HOST] except Exception, inst: iftlog.exception( self.name + ": No remote host specified", inst ) return E_NO_CONNECT
def make_chunks( filename, chunksize ): """ Split the given file into chunks and store them in the chunks directory. Returns (0, file_hash, chunk_hashes, chunk_paths) on success; (nonzero, None, None, None) on error. """ # sanity check if not os.path.exists(filename): iftlog.log(3, "Skipping " + filename + " since it cannot be found") return (E_IOERROR, None, None, None) if not (stat.S_IWUSR & os.stat( filename ).st_mode): iftlog.log(3, "Skipping " + filename + " since I do not have read permission") return (E_IOERROR, None, None, None) # get file hash file_hash = get_hash( filename ) first_char = "" if filename[0] == "/": filename = filename[1:] first_char = "/" file_dir = os.path.basename(filename) + "." + str(file_hash) # does the directory exist? if os.path.exists( __file_chunks_dir + file_dir ): iftlog.log(3, "WARNING: " + __file_chunks_dir + file_dir + " exists! Removing...") rc = os.popen("rm -rf " + __file_chunks_dir + file_dir ).close() if rc != None: iftlog.log(5, "ERROR: could not make chunks for " + filename + "; " + __file_chunks_dir + file_dir + " could not be removed!") return (E_IOERROR, None, None, None) # make the directory and populate it with chunks rc = os.popen("mkdir -p " + __file_chunks_dir + file_dir ).close() if rc != None: iftlog.log(5, "ERROR: could not make chunk directory " + __file_chunks_dir + file_dir ) return (E_IOERROR, None, None, None) # open the file fd = None try: fd = open(first_char + filename, "rb") if not fd: iftlog.log(5, "ERROR: could not open " + filename + " for reading!") cleanup_chunks( filename, file_hash ) return (E_IOERROR, None, None, None) except Exception, inst: iftlog.exception("ERROR: could not open " + filename + " for reading!", inst) cleanup_chunks( filename, file_hash ) return (E_UNHANDLED_EXCEPTION, None, None, None)
def open_connection( self, job ): """ Get the job and validate receive attributes. If a job is given, then don't receive the job; just open the connection and use the given job. """ # prepare to receive recv_status = 0 try: recv_status = self.__prepare_receive( job ) except Exception, inst: iftlog.exception( self.name + ": could not prepare to receive", inst) return E_UNHANDLED_EXCEPTION
def recv_job( self, job ): self.file_to_recv = job.get_attr( iftfile.JOB_ATTR_DEST_NAME ) # begin to receive torrent_path = job.get_attr( IFTBITTORRENT_TORRENT_PATH ) torrent_bits = None try: torrent_fd = open( torrent_path, "r" ) torrent_bits = torrent_fd.read() except Exception, inst: iftlog.exception( self.name + ": could not read torrent file " + str(torrent_path), inst) return E_NO_DATA
def proto_clean(self): # let the remote host know to remove access to this file if self.job_attrs.get( iftfile.JOB_ATTR_REMOTE_IFTD ): try: connection = httplib.HTTPConnection( self.remote_host + ":" + str(self.connect_args[iftproto.PROTO_PORTNUM]) ) connection.request( "GET", os.path.join( self.job_attrs.get( iftfile.JOB_ATTR_SRC_CHUNK_DIR ), "done") ) response = connection.getresponse() if response.status != 200: iftlog.log(5, self.name + ": WARNING: could not tell sender to stop sending!") response.close() connection.close() except Exception, inst: iftlog.exception( self.name + ": WARNING: could not tell sender to stop sending! Exception encountered.", inst) pass
def calc_hash(self): """ Calculate our SHA-1 hash from all of our received chunks. Return a string of the hex digest """ try: m = hashlib.sha1() read_fd = open( self.path, "r") buff = read_fd.read() m.update( buff ) read_fd.close() return m.hexdigest() except Exception, inst: iftlog.exception("iftfile.calc_hash", inst) return ""
def cache_startup( cache_basedir, cache_server_portnum ): """ Start up the caching system if not running. Start the HTTP cache server and ensure that Squid is running. @return 0 on success, negative on failure """ global cache_ref global cache_sem global cache_server global cache_dir try: # are we running? cache_sem.acquire() if cache_ref != 0 or cache_server != None: iftlog.log(3, "iftcache: already running (refs = " + str(cache_ref) + ")") cache_sem.release() return 0 # already running # is Squid running? # TODO: is there a less kludgy way to do this? procs = os.popen( "ps -A | grep -i squid" ).readlines() if len(procs) <= 1: # only found our "grep" statement iftlog.log(5, "iftcache: Squid does not appear to be running...") cache_sem.release() return E_INVAL # attempt to make base directory, but fall back to a sensible default if that doesn't work try: if not os.path.exists( cache_basedir ): os.makedirs( cache_basedir ) else: iftlog.log(3, "iftcache: WARNING: using existing directory " + cache_basedir + " as cache directory") cache_dir = cache_basedir except Exception, inst: try: os.makedirs( "/tmp/iftcache_" + str(os.getpid())) cache_dir = "/tmp/iftcache_" + str(os.getpid()) iftlog.log(5, "iftcache: WARNING: " + cache_basedir + " is not valid, using /tmp/iftcache_" + str(os.getpid()) + " as base cache directory path") except Exception, inst: iftlog.exception("iftcache: Could not create cache directory!", inst) cache_sem.release() cache_shutdown() return E_UNHANDLED_EXCEPTION
def reserve_chunk( self, owner, chunk_id, t ): """ Reserve a chunk for a period of time. """ try: # if the file is closed by another thread, we might catch an exception # sanity check if self.marked_complete: self.__error = E_COMPLETE return E_COMPLETE if self.__mode != MODE_WRITE: self.__error = E_BAD_MODE return E_BAD_MODE if self.known_size == True: if chunk_id < 0 or chunk_id >= self.__num_chunks: self.__error = E_INVAL return E_INVAL else: if chunk_id < 0: self.__error = E_INVAL return E_INVAL # positive chunk id--do we need to add more locks? self.__grow_metadata( chunk_id ) # someone else had better not own this if self.__chunk_reservations[chunk_id] >= time.time() and self.__chunk_owners[ chunk_id ] != None: return E_TRY_AGAIN # can't reserve if someone else locked the chunk if not self.__chunk_locks[ chunk_id ].acquire(False): return E_TRY_AGAIN # reserve the chunk self.__chunk_reservations[ chunk_id ] = time.time() + t self.__chunk_owners[ chunk_id ] = owner self.__chunk_locks[ chunk_id ].release() except Exception, inst: # only happens if another thread closed the file iftlog.exception("iftfile.reserve_chunk: could not resrve chunk " + str(chunk_id) + " for " + str(owner), inst) return E_BAD_STATE
def mark_chunk( self, chunk_id, value ): """ Mark a chunk as either set (value == True) or unset (value == False) Return 0 on success, nonzero otherwise """ try: if self.marked_complete: self.__error = E_COMPLETE return E_COMPLETE if self.__mode != MODE_WRITE: # can't modify chunk mask if we aren't in WRITE mode self.__error = E_BAD_MODE return E_BAD_MODE # can we write? # don't overwrite what's already there if chunk_id < self.__num_chunks and chunk_id >= 0 and self.__chunk_mask[ chunk_id ] == True: self.__error = E_BAD_STATE return E_BAD_STATE if self.known_size == True: if chunk_id < 0 or chunk_id >= self.__num_chunks: self.__error = E_INVAL return E_INVAL else: if self.__bytes_max > 0 and self.__bytes_written + self.__chunk_size > self.__bytes_max: # problem--this would expand the file bigger than we have capped it at iftlog.log(3, "iftfile: attempted to write " + str(self.__chunk_size) + " more bytes beyond required maximum of " + str(self.__bytes_max) + ", will not mark chunk " + str(chunk_id) ) return E_OVERFLOW num_chunks = len(self.__chunk_mask) if num_chunks < chunk_id + 1: self.__chunk_mask = self.__chunk_mask + [False] * (chunk_id + 1 - num_chunks) self.__num_chunks = len( self.__chunk_mask ) self.__chunk_mask[ chunk_id ] = True return 0 except Exception, inst: if self.__open == False: iftlog.log(5, "iftfile.mark_chunk: file is not open") iftlog.exception( "iftfile.mark_chunk: could not mark chunk " + str(chunk_id) + " as " + str(value)) return E_BAD_STATE