class BitcasaDownload(object):
def __init__(self, args, client, should_exit):
log.debug("source dir: %s", args.src)
log.debug("destination dir: %s", args.dst)
log.debug("temp dir: %s", args.temp)
log.debug("upload: %s", args.upload)
if args.upload:
log.debug("provider: %s", args.provider)
log.debug("log dir: %s", args.logdir)
log.debug("recursion: %s", args.rec)
log.debug("depth: %s", args.depth)
log.debug("max folder threads: %s", args.folderthreads)
log.debug("max download threads: %s", args.threads)
log.debug("progress: %s", args.progress)
log.debug("silent queuer: %s", args.silentqueuer)
log.debug("single: %s", args.single)
#bittcasa base64 encdoded path
self.basefolder = args.src
if args.single:
log.debug("Downloading single file. Setting max threads to 1")
args.threads = 1
args.folderthreads = 1
self.args = args
#Initialize
self.should_exit = should_exit
self.client = client
self.session = requests.Session()
self.results = results.Results(args.logdir, should_exit, args.nofilelog)
# Threads
self.download_threads = []
self.upload_threads = []
self.copy_threads = []
self.folder_threads = []
self.status = Status(should_exit)
self.shutdown_sent = False
def shutdown(self):
if not self.shutdown_sent:
self.shutdown_sent = True
self.status.shutdown()
def get_status(self):
return self.status
def process(self, base=None):
log.debug("Getting base folder")
if self.args.upload and self.args.local and base is None:
base = BitcasaFolder(None, "root", self.basefolder)
else:
remainingtries = 3
apiratecount = 1
while base is None and remainingtries > 0 and not self.should_exit.is_set():
try:
base = self.client.get_folder(self.basefolder)
except BitcasaException as e:
remainingtries -= 1
if e.code == 9006:
apiratecount += 1
remainingtries += 1
log.warn("API rate limit reached. Will retry")
else:
log.warn("Couldn't get base folder %s. Will retry %s more times", e.code, remainingtries)
if remainingtries > 0:
time.sleep(10 * apiratecount)
else:
log.error("Error could not retrieve base folder")
return
if self.should_exit.is_set():
return
log.debug("Queuing base folder")
folder = {
"folder": base,
"path": "",
"depth": 0
}
if self.args.upload:
folder["folder_id"] = self.args.dst
step2_args = ( self.status, self.should_exit, self.results, self.args )
download_args = ( self.status, self.should_exit, self.session, self.results, self.args)
folder_args = ( self.status, self.results, self.args, self.should_exit )
self.status.queue(folder)
if not self.args.dryrun and not self.args.local:
log.debug("Starting Downloaders")
for qid in xrange(self.args.threads):
qid += 1
download_thread = threading.Thread(target=DownloadThread, args=download_args, name="Download %s" % qid)
download_thread.daemon = True
download_thread.start()
self.download_threads.append(download_thread)
log.debug("Starting Queuers")
for qid in xrange(self.args.folderthreads):
qid += 1
folder_thread = threading.Thread(target=FolderThread, args=folder_args, name="Queuer %s" % qid)
folder_thread.daemon = True
folder_thread.start()
self.folder_threads.append(folder_thread)
if not self.args.dryrun and self.args.upload:
log.debug("Starting Uploaders")
for qid in xrange(self.args.threads):
qid += 1
upload_thread = threading.Thread(target=UploadThread, args=step2_args, name="Upload %s" % qid)
upload_thread.daemon = True
upload_thread.start()
self.upload_threads.append(upload_thread)
elif not self.args.dryrun and self.args.temp:
log.debug("Starting Movers")
for qid in xrange(self.args.threads):
qid += 1
copy_thread = threading.Thread(target=CopyThread, args=step2_args, name="Move %s" % qid)
copy_thread.daemon = True
copy_thread.start()
self.copy_threads.append(copy_thread)
if self.args.progress:
self.status_thread = threading.Thread(target=self.status.StatusThread, args=(self.args.upload, self.args.temp, self.should_exit), name="Progress")
self.status_thread.daemon = True
self.status_thread.start()
self.end_process()
def process_single(self):
log.debug("Getting file info")
myfile = None
if self.args.upload and self.args.local:
size = 0
name = ""
try:
if os.path.isdir(self.basefolder):
raise OSError("Incorrect file")
size = os.path.getsize(self.basefolder)
name = os.path.basename(self.basefolder)
except OSError:
log.exception("Error getting file")
return
myfile = BitcasaFile(None, self.basefolder, name, None, size)
fold = BitcasaFolder(None, "root", "", items=[myfile])
else:
remainingtries = 3
apiratecount = 1
while myfile is None and remainingtries > 0 and not self.should_exit.is_set():
try:
myfile = self.client.get_file_meta(self.basefolder)
except BitcasaException as e:
remainingtries -= 1
if e.code == 9006:
apiratecount += 1
remainingtries += 1
log.warn("API rate limit reached. Will retry")
else:
log.warn("Couldn't get file %s. Will retry %s more times", e.code, remainingtries)
if remainingtries > 0:
time.sleep(10 * apiratecount)
else:
log.error("Error could not retrieve file")
return
fold = BitcasaFolder(self.client, "root", "", items=[myfile])
log.debug("Got file info")
self.process(fold)
def end_process(self):
# Give the queuers time to catch up
try:
if not self.should_exit.is_set():
time.sleep(10)
except (KeyboardInterrupt, IOError):
pass
self.status.join_queues()
log.debug("Finished waiting")
#Log final speed and statistics
if not self.args.dryrun:
self.status.final_status(self.args.upload, self.args.temp, final=True)
class Peer( threading.Thread ):
"""
Peer is the iplementation the interface defined in the provided peer.h file.
Peers run as their own thread, working through a queue of requests (rQueue)
"""
# ----------------
# PUBLIC INTERFACE
# ----------------
def join( self ):
"""
Notifies all peers that this peer is joining in the network.
"""
if self.joined:
return
logging.info( 'Joining BitTorrent network' )
self.joined = True
self.peerTCPServer = PeerTCPServer()
self.peerTCPServer.init( self.host, self.port, self )
connectSucess = False
for p in self.peers:
result = self.sendStatus( p , True )
connectSucess = connectSucess or result
if connectSucess:
return RETURNCODES['errOk']
else:
return RETURNCODES['errNoPeersFound']
def leave( self ):
"""
Notifies all peers that this peer no longer in the network.
"""
if not self.joined:
return
logging.info( 'Submitting request to leave network' )
package = { 'packageType': 'command',
'command': 'leave' }
self.rQueue.put( package )
return RETURNCODES['errOk']
def query( self, status ):
"""
Copies this peer's status into the given status.
Returns this peer's status (for good measure).
"""
self.status.copyStatus( status )
localStatus = self.status.getStatus()
print '---PEER STATUS----'
for f in localStatus:
print f['filename']
print f
print ''
return RETURNCODES['errOk']
def insert( self, filename ):
"""
Inserts the provided filename into the bittorent network.
"""
try:
filepath = os.path.join( os.getcwd(), self.filesDir, filename )
if not os.path.isfile( filepath ):
logging.error( 'File Invalid or does not exist: ' + filename )
return RETURNCODES['errUnknownWarning']
else:
logging.info( 'Submitting request to insert file: ' + filename )
filesize = os.path.getsize( filepath )
package = { 'packageType': 'command',
'command': 'insert',
'filename': filename,
'filepath': filepath,
'filesize': filesize }
self.rQueue.put( package )
return RETURNCODES['errOk']
except Exception as inst:
logging.error ( 'Exception while inserting file: ' + str( type( inst ) ) )
def remove( self, filename ):
"""Remove specified file from the network"""
if not self.status.checkForFile( filename ):
return RETURNCODES['errUnknownWarning']
package = { 'packageType': 'command',
'command': 'remove',
'filename': filename }
self.rQueue.put( package )
return RETURNCODES['errOk']
# --------------------------------
# PRIVATE METHODS (IMPLEMENTATION)
# --------------------------------
def run( self ):
"""
Run method called upon thread start
Continuously processes requests or sends out file chunks
"""
i = 1
while True :
try:
if not self.rQueue.empty():
# Process request in queue
package = self.rQueue.get()
if package == 'shutdown':
logging.info( 'Shutdown token found in rQueue' )
break
self.processRequest( package )
elif self.joined and i == 0:
# Periodically query peers to get up-to-date statuses
self.massQuery()
i = (i + 1) % QUERYRATE
elif self.joined:
# Attemp to share a chunk with all active peers
self.shareChunks()
i = (i + 1) % QUERYRATE
time.sleep( 0.0005 )
except Exception as inst:
logging.error ( 'Exception in Peer run: ' + str( type( inst ) ) )
def init( self, host, port, peers, dirName ):
logging.info( 'Initializing new peer ' + self.getPeerKey( host, port ) )
self.filesDir = dirName
self.host = host
self.port = port
self.peers = peers
self.peerKey = self.getPeerKey( host, port )
self.activePeers = {}
self.waitingQueries = []
self.joined = False
self.status = Status( self.peerKey )
self.status.registerPeer( self.peerKey, {} )
self.sender = Sender( self )
self.rQueue = Queue()
self.shutdownFlag = False
self.start()
def shutdown( self ):
"""Stop peer's thread to allow for program termination"""
logging.info( 'Shutting down Peer' )
self.shutdownFlag = True
self.leave()
self.rQueue.put( 'shutdown' )
super( Peer, self ).join() # wait for thread to finish
self.status.shutdown()
def processRequest( self, package ):
"""Process request from peer's request queue"""
if package == None:
logging.error( 'Invalid request; ignored' )
return
if package['packageType'] == 'message':
# Handle message request
context = package['context']
msgType = context['type']
source = ( context['host'], context['port'] )
sourceKey = self.getPeerKey( source[0], source[1] )
if 'data' in package:
data = package['data']
if msgType == 'status':
# Check if peer is registered
if sourceKey in self.activePeers:
self.status.updatePeer( sourceKey, data )
else:
self.activePeers[ sourceKey ] = 0
self.status.registerPeer( sourceKey, data )
# Reply if peer requested
if context['reply']:
self.sendStatus( source, False )
# Remove from list of outstanding queries
if sourceKey in self.waitingQueries:
self.waitingQueries.remove( sourceKey )
elif msgType == 'chunk':
filepath = os.path.join( os.getcwd(), self.filesDir, context['filename'] )
self.status.saveChunk( sourceKey, context['filename'], filepath,\
context['filesize'], context['chunkNum'], data )
elif msgType == 'query':
self.sendStatus( source, False )
elif msgType == 'goodbye':
logging.info( 'Processing goodbye request from: ' + sourceKey )
del self.activePeers[ sourceKey ]
self.status.removePeer( sourceKey )
elif msgType == 'remove':
logging.info( 'Processing remove file: ' + data + ' from: ' + sourceKey )
# Data is the filename
if self.status.checkForFile( data ):
self.status.removeFile( data )
# Remove actual file on filesystem
filepath = os.path.join( os.getcwd(), self.filesDir, data )
if os.path.isfile( filepath ):
os.remove( filepath )
else:
logging.error( 'Invalid msgType: ' + msgType )
elif package['packageType'] == 'command':
# Handle command from user input
command = package['command']
if command == 'insert':
logging.info( 'Processing insert file command: ' + package['filename'] )
self.status.insertFile( package['filename'], package['filepath'],\
package['filesize'] )
elif command == 'leave':
logging.info( 'Processing leave request' )
self.peerTCPServer.shutdown()
for p in self.activePeers:
peer = self.getPeerTuple( p )
result = self.sender.sendMessage( peer[0], peer[1], 'goodbye' )
if not result:
logging.error( 'Failed to send goodbye message to: ' + p )
self.joined = False # mark self as unjoined
self.rQueue = Queue() # Create new Queue to throw away pending requets
if self.shutdownFlag:
self.rQueue.put( 'shutdown' )
for p in self.activePeers:
# clear peer from status
self.status.removePeer( p )
self.activePeers = {}
elif command == 'remove':
fName = package['filename']
context = { 'type': 'remove' }
for p in self.activePeers:
peer = self.getPeerTuple( p )
result = self.sender.sendData( peer[0], peer[1], context, fName )
if not result:
logging.error( 'Failed to send delete message to: ' + p )
self.status.removeFile( fName )
else:
logging.error( 'Invalid command: ' + command )
else:
logging.error( 'Invalid packageType; package ignored' )
def shareChunks( self ):
"""Send out a chunk to each registered peer"""
for p in self.activePeers:
peer = self.getPeerTuple( p )
context, data = self.status.getChunkToSend( p )
if data is None:
continue # no chunks to share with that peer
context['type'] = 'chunk'
if self.sender.sendData( peer[0], peer[1], context, data ):
self.status.markChunkSent( p,context['filename'], context['chunkNum'] )
self.activePeers[ p ] = 0
else:
logging.error( 'Failed to send packet to: ' + p )
# increment number of failures
self.activePeers[ p ] += 1
# Remove any peers that failed too many times
self.checkPeerFailures()
def massQuery( self ):
"""Send out query message to all peers to get updated status"""
for p in self.activePeers:
if p in self.waitingQueries:
# Still waiting for reply from last query; skip
continue
peer = self.getPeerTuple( p )
if self.sender.sendMessage( peer[0], peer[1], 'query' ):
self.activePeers[ p ] = 0
self.waitingQueries.append( p )
else :
logging.error( 'Failed to send query meessage to: ' + p )
self.activePeers[ p ] += 1
# Remove any peers that failed too many times
self.checkPeerFailures()
def sendStatus( self, peer, requestReply ):
"""Send out status message to given peer"""
context = { 'type' : 'status', 'reply' : requestReply }
data = self.status.getLocalFilesDict()
return self.sender.sendData( peer[0], peer[1], context, data )
def checkPeerFailures( self ):
"""
Check the number of consecutive failures per peer and remove
peers with > MAXFAILURES
"""
for p in self.activePeers.keys():
if self.activePeers[ p ] >= MAXFAILURES:
logging.info( 'MAXFAILURES reached, removing peer : ' + p )
del self.activePeers[ p ]
self.status.removePeer( p )
@staticmethod
def getPeerKey( host, port ):
"""
Returns a string for use as a dictionary key
They key is simply the host and port concatenated
"""
if not host or not port:
logging.error('Invalid host or port provided to getPeerKey')
raise TypeError('Invalid host or port')
return host + ':' + str( port )
@staticmethod
def getPeerTuple( peerKey ):
"""
Returns a host, port tuple extracted from a peerKey
"""
if not peerKey or not len(peerKey) > 0:
logging.error('Invalid peerKey provided to getPeerTuple')
raise TypeError('Invalid peerKey')
vals = peerKey.split(':')
return ( vals[0], int( vals[1] ) )
def getAllPeerKeys( self ):
"""
Returns a list of all peer keys in the form of
a string of the format: 'peer:host'
"""
peerKeys = []
for p in self.peers:
peerKeys.append( getPeerKey( p[0], p[1] ) )
return peerKeys
