class BitcasaDownload(object): def __init__(self, args, client, should_exit): log.debug("source dir: %s", args.src) log.debug("destination dir: %s", args.dst) log.debug("temp dir: %s", args.temp) log.debug("upload: %s", args.upload) if args.upload: log.debug("provider: %s", args.provider) log.debug("log dir: %s", args.logdir) log.debug("recursion: %s", args.rec) log.debug("depth: %s", args.depth) log.debug("max folder threads: %s", args.folderthreads) log.debug("max download threads: %s", args.threads) log.debug("progress: %s", args.progress) log.debug("silent queuer: %s", args.silentqueuer) log.debug("single: %s", args.single) #bittcasa base64 encdoded path self.basefolder = args.src if args.single: log.debug("Downloading single file. Setting max threads to 1") args.threads = 1 args.folderthreads = 1 self.args = args #Initialize self.should_exit = should_exit self.client = client self.session = requests.Session() self.results = results.Results(args.logdir, should_exit, args.nofilelog) # Threads self.download_threads = [] self.upload_threads = [] self.copy_threads = [] self.folder_threads = [] self.status = Status(should_exit) self.shutdown_sent = False def shutdown(self): if not self.shutdown_sent: self.shutdown_sent = True self.status.shutdown() def get_status(self): return self.status def process(self, base=None): log.debug("Getting base folder") if self.args.upload and self.args.local and base is None: base = BitcasaFolder(None, "root", self.basefolder) else: remainingtries = 3 apiratecount = 1 while base is None and remainingtries > 0 and not self.should_exit.is_set(): try: base = self.client.get_folder(self.basefolder) except BitcasaException as e: remainingtries -= 1 if e.code == 9006: apiratecount += 1 remainingtries += 1 log.warn("API rate limit reached. Will retry") else: log.warn("Couldn't get base folder %s. Will retry %s more times", e.code, remainingtries) if remainingtries > 0: time.sleep(10 * apiratecount) else: log.error("Error could not retrieve base folder") return if self.should_exit.is_set(): return log.debug("Queuing base folder") folder = { "folder": base, "path": "", "depth": 0 } if self.args.upload: folder["folder_id"] = self.args.dst step2_args = ( self.status, self.should_exit, self.results, self.args ) download_args = ( self.status, self.should_exit, self.session, self.results, self.args) folder_args = ( self.status, self.results, self.args, self.should_exit ) self.status.queue(folder) if not self.args.dryrun and not self.args.local: log.debug("Starting Downloaders") for qid in xrange(self.args.threads): qid += 1 download_thread = threading.Thread(target=DownloadThread, args=download_args, name="Download %s" % qid) download_thread.daemon = True download_thread.start() self.download_threads.append(download_thread) log.debug("Starting Queuers") for qid in xrange(self.args.folderthreads): qid += 1 folder_thread = threading.Thread(target=FolderThread, args=folder_args, name="Queuer %s" % qid) folder_thread.daemon = True folder_thread.start() self.folder_threads.append(folder_thread) if not self.args.dryrun and self.args.upload: log.debug("Starting Uploaders") for qid in xrange(self.args.threads): qid += 1 upload_thread = threading.Thread(target=UploadThread, args=step2_args, name="Upload %s" % qid) upload_thread.daemon = True upload_thread.start() self.upload_threads.append(upload_thread) elif not self.args.dryrun and self.args.temp: log.debug("Starting Movers") for qid in xrange(self.args.threads): qid += 1 copy_thread = threading.Thread(target=CopyThread, args=step2_args, name="Move %s" % qid) copy_thread.daemon = True copy_thread.start() self.copy_threads.append(copy_thread) if self.args.progress: self.status_thread = threading.Thread(target=self.status.StatusThread, args=(self.args.upload, self.args.temp, self.should_exit), name="Progress") self.status_thread.daemon = True self.status_thread.start() self.end_process() def process_single(self): log.debug("Getting file info") myfile = None if self.args.upload and self.args.local: size = 0 name = "" try: if os.path.isdir(self.basefolder): raise OSError("Incorrect file") size = os.path.getsize(self.basefolder) name = os.path.basename(self.basefolder) except OSError: log.exception("Error getting file") return myfile = BitcasaFile(None, self.basefolder, name, None, size) fold = BitcasaFolder(None, "root", "", items=[myfile]) else: remainingtries = 3 apiratecount = 1 while myfile is None and remainingtries > 0 and not self.should_exit.is_set(): try: myfile = self.client.get_file_meta(self.basefolder) except BitcasaException as e: remainingtries -= 1 if e.code == 9006: apiratecount += 1 remainingtries += 1 log.warn("API rate limit reached. Will retry") else: log.warn("Couldn't get file %s. Will retry %s more times", e.code, remainingtries) if remainingtries > 0: time.sleep(10 * apiratecount) else: log.error("Error could not retrieve file") return fold = BitcasaFolder(self.client, "root", "", items=[myfile]) log.debug("Got file info") self.process(fold) def end_process(self): # Give the queuers time to catch up try: if not self.should_exit.is_set(): time.sleep(10) except (KeyboardInterrupt, IOError): pass self.status.join_queues() log.debug("Finished waiting") #Log final speed and statistics if not self.args.dryrun: self.status.final_status(self.args.upload, self.args.temp, final=True)
class Peer( threading.Thread ): """ Peer is the iplementation the interface defined in the provided peer.h file. Peers run as their own thread, working through a queue of requests (rQueue) """ # ---------------- # PUBLIC INTERFACE # ---------------- def join( self ): """ Notifies all peers that this peer is joining in the network. """ if self.joined: return logging.info( 'Joining BitTorrent network' ) self.joined = True self.peerTCPServer = PeerTCPServer() self.peerTCPServer.init( self.host, self.port, self ) connectSucess = False for p in self.peers: result = self.sendStatus( p , True ) connectSucess = connectSucess or result if connectSucess: return RETURNCODES['errOk'] else: return RETURNCODES['errNoPeersFound'] def leave( self ): """ Notifies all peers that this peer no longer in the network. """ if not self.joined: return logging.info( 'Submitting request to leave network' ) package = { 'packageType': 'command', 'command': 'leave' } self.rQueue.put( package ) return RETURNCODES['errOk'] def query( self, status ): """ Copies this peer's status into the given status. Returns this peer's status (for good measure). """ self.status.copyStatus( status ) localStatus = self.status.getStatus() print '---PEER STATUS----' for f in localStatus: print f['filename'] print f print '' return RETURNCODES['errOk'] def insert( self, filename ): """ Inserts the provided filename into the bittorent network. """ try: filepath = os.path.join( os.getcwd(), self.filesDir, filename ) if not os.path.isfile( filepath ): logging.error( 'File Invalid or does not exist: ' + filename ) return RETURNCODES['errUnknownWarning'] else: logging.info( 'Submitting request to insert file: ' + filename ) filesize = os.path.getsize( filepath ) package = { 'packageType': 'command', 'command': 'insert', 'filename': filename, 'filepath': filepath, 'filesize': filesize } self.rQueue.put( package ) return RETURNCODES['errOk'] except Exception as inst: logging.error ( 'Exception while inserting file: ' + str( type( inst ) ) ) def remove( self, filename ): """Remove specified file from the network""" if not self.status.checkForFile( filename ): return RETURNCODES['errUnknownWarning'] package = { 'packageType': 'command', 'command': 'remove', 'filename': filename } self.rQueue.put( package ) return RETURNCODES['errOk'] # -------------------------------- # PRIVATE METHODS (IMPLEMENTATION) # -------------------------------- def run( self ): """ Run method called upon thread start Continuously processes requests or sends out file chunks """ i = 1 while True : try: if not self.rQueue.empty(): # Process request in queue package = self.rQueue.get() if package == 'shutdown': logging.info( 'Shutdown token found in rQueue' ) break self.processRequest( package ) elif self.joined and i == 0: # Periodically query peers to get up-to-date statuses self.massQuery() i = (i + 1) % QUERYRATE elif self.joined: # Attemp to share a chunk with all active peers self.shareChunks() i = (i + 1) % QUERYRATE time.sleep( 0.0005 ) except Exception as inst: logging.error ( 'Exception in Peer run: ' + str( type( inst ) ) ) def init( self, host, port, peers, dirName ): logging.info( 'Initializing new peer ' + self.getPeerKey( host, port ) ) self.filesDir = dirName self.host = host self.port = port self.peers = peers self.peerKey = self.getPeerKey( host, port ) self.activePeers = {} self.waitingQueries = [] self.joined = False self.status = Status( self.peerKey ) self.status.registerPeer( self.peerKey, {} ) self.sender = Sender( self ) self.rQueue = Queue() self.shutdownFlag = False self.start() def shutdown( self ): """Stop peer's thread to allow for program termination""" logging.info( 'Shutting down Peer' ) self.shutdownFlag = True self.leave() self.rQueue.put( 'shutdown' ) super( Peer, self ).join() # wait for thread to finish self.status.shutdown() def processRequest( self, package ): """Process request from peer's request queue""" if package == None: logging.error( 'Invalid request; ignored' ) return if package['packageType'] == 'message': # Handle message request context = package['context'] msgType = context['type'] source = ( context['host'], context['port'] ) sourceKey = self.getPeerKey( source[0], source[1] ) if 'data' in package: data = package['data'] if msgType == 'status': # Check if peer is registered if sourceKey in self.activePeers: self.status.updatePeer( sourceKey, data ) else: self.activePeers[ sourceKey ] = 0 self.status.registerPeer( sourceKey, data ) # Reply if peer requested if context['reply']: self.sendStatus( source, False ) # Remove from list of outstanding queries if sourceKey in self.waitingQueries: self.waitingQueries.remove( sourceKey ) elif msgType == 'chunk': filepath = os.path.join( os.getcwd(), self.filesDir, context['filename'] ) self.status.saveChunk( sourceKey, context['filename'], filepath,\ context['filesize'], context['chunkNum'], data ) elif msgType == 'query': self.sendStatus( source, False ) elif msgType == 'goodbye': logging.info( 'Processing goodbye request from: ' + sourceKey ) del self.activePeers[ sourceKey ] self.status.removePeer( sourceKey ) elif msgType == 'remove': logging.info( 'Processing remove file: ' + data + ' from: ' + sourceKey ) # Data is the filename if self.status.checkForFile( data ): self.status.removeFile( data ) # Remove actual file on filesystem filepath = os.path.join( os.getcwd(), self.filesDir, data ) if os.path.isfile( filepath ): os.remove( filepath ) else: logging.error( 'Invalid msgType: ' + msgType ) elif package['packageType'] == 'command': # Handle command from user input command = package['command'] if command == 'insert': logging.info( 'Processing insert file command: ' + package['filename'] ) self.status.insertFile( package['filename'], package['filepath'],\ package['filesize'] ) elif command == 'leave': logging.info( 'Processing leave request' ) self.peerTCPServer.shutdown() for p in self.activePeers: peer = self.getPeerTuple( p ) result = self.sender.sendMessage( peer[0], peer[1], 'goodbye' ) if not result: logging.error( 'Failed to send goodbye message to: ' + p ) self.joined = False # mark self as unjoined self.rQueue = Queue() # Create new Queue to throw away pending requets if self.shutdownFlag: self.rQueue.put( 'shutdown' ) for p in self.activePeers: # clear peer from status self.status.removePeer( p ) self.activePeers = {} elif command == 'remove': fName = package['filename'] context = { 'type': 'remove' } for p in self.activePeers: peer = self.getPeerTuple( p ) result = self.sender.sendData( peer[0], peer[1], context, fName ) if not result: logging.error( 'Failed to send delete message to: ' + p ) self.status.removeFile( fName ) else: logging.error( 'Invalid command: ' + command ) else: logging.error( 'Invalid packageType; package ignored' ) def shareChunks( self ): """Send out a chunk to each registered peer""" for p in self.activePeers: peer = self.getPeerTuple( p ) context, data = self.status.getChunkToSend( p ) if data is None: continue # no chunks to share with that peer context['type'] = 'chunk' if self.sender.sendData( peer[0], peer[1], context, data ): self.status.markChunkSent( p,context['filename'], context['chunkNum'] ) self.activePeers[ p ] = 0 else: logging.error( 'Failed to send packet to: ' + p ) # increment number of failures self.activePeers[ p ] += 1 # Remove any peers that failed too many times self.checkPeerFailures() def massQuery( self ): """Send out query message to all peers to get updated status""" for p in self.activePeers: if p in self.waitingQueries: # Still waiting for reply from last query; skip continue peer = self.getPeerTuple( p ) if self.sender.sendMessage( peer[0], peer[1], 'query' ): self.activePeers[ p ] = 0 self.waitingQueries.append( p ) else : logging.error( 'Failed to send query meessage to: ' + p ) self.activePeers[ p ] += 1 # Remove any peers that failed too many times self.checkPeerFailures() def sendStatus( self, peer, requestReply ): """Send out status message to given peer""" context = { 'type' : 'status', 'reply' : requestReply } data = self.status.getLocalFilesDict() return self.sender.sendData( peer[0], peer[1], context, data ) def checkPeerFailures( self ): """ Check the number of consecutive failures per peer and remove peers with > MAXFAILURES """ for p in self.activePeers.keys(): if self.activePeers[ p ] >= MAXFAILURES: logging.info( 'MAXFAILURES reached, removing peer : ' + p ) del self.activePeers[ p ] self.status.removePeer( p ) @staticmethod def getPeerKey( host, port ): """ Returns a string for use as a dictionary key They key is simply the host and port concatenated """ if not host or not port: logging.error('Invalid host or port provided to getPeerKey') raise TypeError('Invalid host or port') return host + ':' + str( port ) @staticmethod def getPeerTuple( peerKey ): """ Returns a host, port tuple extracted from a peerKey """ if not peerKey or not len(peerKey) > 0: logging.error('Invalid peerKey provided to getPeerTuple') raise TypeError('Invalid peerKey') vals = peerKey.split(':') return ( vals[0], int( vals[1] ) ) def getAllPeerKeys( self ): """ Returns a list of all peer keys in the form of a string of the format: 'peer:host' """ peerKeys = [] for p in self.peers: peerKeys.append( getPeerKey( p[0], p[1] ) ) return peerKeys