def testTransferManagerSource(self): """ Test iface converter using a spec directly from the TransferManager """ tm = TransferManager() tm.addSource(None, "localhost", "localhost", 1) factory = EndpointFactory(0, False,False,False,False, '', [], False,False,False,False, '', []) ifs = factory.createIfaceList(tm.sources[0]) self.assertEqual(1, len(ifs)) self.assertEqual(('localhost', TEST_PORT, 1, ), ifs[0])
def createEndpoints(self): """Create local and remote transfer host endpoints""" assert not self.isCreated assert 0 < self.requestSize assert 0 < len(self.sources) assert 0 < len(self.sinks) assert self.sinkTarget assert self.sourceTarget self.factory = EndpointFactory(self.requestSize, self.sourceDIOFlag, self.sourceSerialFlag, self.sourceVerboseFlag, self.sourceTimestampFlag, self.sourceXddPath, self.sources, self.sinkDIOFlag, self.sinkSerialFlag, self.sinkVerboseFlag, self.sinkTimestampFlag, self.sinkXddPath, self.sinks) rc = self.factory.createEndpoints() if 0 == rc: self.isCreated = True rc = self.performPostCreateChecks() return rc
def createEndpoints(self): """Create local and remote transfer host endpoints""" assert not self.isCreated assert 0 < self.requestSize assert 0 < len(self.sources) assert 0 < len(self.sinks) assert self.sinkTarget assert self.sourceTarget self.factory = EndpointFactory( self.requestSize, self.sourceDIOFlag, self.sourceSerialFlag, self.sourceVerboseFlag, self.sourceTimestampFlag, self.sourceXddPath, self.sources, self.sinkDIOFlag, self.sinkSerialFlag, self.sinkVerboseFlag, self.sinkTimestampFlag, self.sinkXddPath, self.sinks) rc = self.factory.createEndpoints() if 0 == rc: self.isCreated = True rc = self.performPostCreateChecks() return rc
def testSimpleLoopback(self): """Test a simple loopback""" factory = EndpointFactory(1024, False, False, False, False, 'bin', [{'ip': 'localhost', 'hostname': 'localhost', 'threads':1, 'port':TEST_PORT, 'ifs':['localhost']}], False, False, False, False, 'bin', [{'ip': 'localhost', 'hostname': 'localhost', 'threads':1, 'port':TEST_PORT, 'ifs':['localhost']}]) factory.createEndpoints() self.assertEqual(1, len(factory.getSinkEndpoints())) self.assertEqual(1, len(factory.getSourceEndpoints())) self.assertEqual(2, len(factory.getEndpoints()))
def testTwoSources(self): """Test a loopback with 2 sources and 1 sink""" factory = EndpointFactory(1024, False, False, False, False, 'bin', [{'ip': 'localhost', 'hostname': 'localhost', 'threads':1, 'port':TEST_PORT, 'ifs':['localhost']}, {'ip': 'localhost', 'hostname': 'localhost', 'threads':1, 'port':TEST_PORT, 'ifs':['localhost']}], False, False, False, False, 'bin', [{'ip': 'localhost', 'hostname': 'localhost', 'threads':2, 'port':TEST_PORT, 'ifs':['localhost']}]) factory.createEndpoints() self.assertEqual(1, len(factory.getSinkEndpoints())) self.assertEqual(2, len(factory.getSourceEndpoints())) self.assertEqual(3, len(factory.getEndpoints()))
def testSameHost(self): """Test a source and sink on the same host""" # Get the hostname import platform host = platform.node() self.assertNotEqual(0, len(host)) factory = EndpointFactory(1024, False, False, False, False, 'bin', [{'ip': host, 'hostname': host, 'threads':1, 'port':TEST_PORT, 'ifs':[host]}], False, False, False, False, 'bin', [{'ip': host, 'hostname': host, 'threads':1, 'port':TEST_PORT, 'ifs':[host]}]) factory.createEndpoints() self.assertEqual(1, len(factory.getSinkEndpoints())) self.assertEqual(1, len(factory.getSourceEndpoints())) self.assertEqual(2, len(factory.getEndpoints())) source = factory.getSourceEndpoints()[0] sourceHost = source.hostname() self.assertEqual(host, sourceHost) sink = factory.getSinkEndpoints()[0] sinkHost = sink.hostname() self.assertEqual(host, sinkHost)
class TransferManager: """ Manager for creating, monitoring, and deallocating local and remote flows. """ def __init__(self): """Constructor""" self.isCreated = False self.isStarted = False self.isComplete = False self.isSuccess = False self.factory = None # Temporary variables used for showing progress self.beginTime = None self.currentSourceFile = None self.currentSinkFile = None self.currentFlowSize = 0 self.currentRestartOffset = 0 self.verboseLog = None self.requestSize = 0 self.transferSize = None self.restartFlag = False self.sinkDIOFlag = False self.sinkSerialFlag = False self.sinkTimestampFlag = False self.sinkVerboseFlag = False self.sinkTarget = None self.sinkXddPath = '' self.sinks = [] self.sourceDIOFlag = False self.sourceSerialFlag = False self.sourceVerboseFlag = False self.sourceTimestampFlag = False self.sourceTarget = None self.sourceXddPath = '' self.sources = [] def setRequestSize(self, reqSize): """Set the chunk size of data to move across the wire""" self.requestSize = reqSize def setTransferSize(self, transferSize): """Set the size of data to move across the wire""" self.transferSize = transferSize def setRestartFlag(self, restartFlag): """Set the amount of data previously sent across the wire""" self.restartFlag = restartFlag def setSinkTarget(self, target, dioFlag=False, serialFlag=False): """ Set the name of of the sink target file""" assert not self.isCreated self.sinkTarget = target self.sinkDIOFlag = dioFlag self.sinkSerialFlag = serialFlag def setSourceTarget(self, target, dioFlag=False, serialFlag=False): """Set the name of the source target file""" assert not self.isCreated self.sourceTarget = target self.sourceDIOFlag = dioFlag self.sourceSerialFlag = serialFlag def setVerbosity(self, verboseLevel, filename): """Set the level of verbosity""" if 1 <= verboseLevel: self.sinkVerboseFlag = True self.sourceVerboseFlag = True if 2 <= verboseLevel: self.sinkTimestampFlag = True self.sourceTimestampFlag = True self.verboseLog = filename def addSink(self, user, hostIP, hostname, threads, ifs = [], port = 40010): """Add a sink to the list of sinks""" assert not self.isCreated assert hostIP assert hostname assert 0 < threads sink = {'ip': hostIP, 'hostname': hostname, 'threads': threads, 'port': port, 'ifs': []} if 0 == len(ifs): sink['ifs'].append(hostIP) else: sink['ifs'].extend(ifs) self.sinks.append(sink) def addSource(self, user, hostIP, hostname, threads, ifs = [], port = 40010): """Add a source to the list of sources""" assert not self.isCreated assert hostIP assert hostname assert 0 < threads source = {'ip': hostIP, 'hostname': hostname, 'threads': threads, 'port': port, 'ifs': []} if 0 == len(ifs): source['ifs'].append(hostIP) else: source['ifs'].extend(ifs) self.sources.append(source) def setSinkXddPath(self, path): """Set the path sinks use to find xdd""" self.sinkXddPath = path def setSourceXddPath(self, path): """Set the path sources use to find xdd""" self.sourceXddPath = path def setXddPath(self, path): """Set the path used to find xdd""" self.sinkXddPath = path self.sourceXddPath = path def performPostCreateChecks(self): """@return 0 if all checks pass, otherwise non-zero""" rc = 0 # Check that all of the XDD's are the same version base = None current = None for f in self.factory.getEndpoints(): current = f.protocolVersion() if base is None: base = current if current is None or not current or current != base: print("ERROR: XDD Protocols do not match", file=sys.stderr) rc = 1 break # Print a generic warning if the sinks do not have pre-allocation # support for s in self.factory.getSinkEndpoints(): if not s.hasPreallocateAvailable(): print("WARNING: XDD preallocation support not available", file=sys.stderr) return rc def createEndpoints(self): """Create local and remote transfer host endpoints""" assert not self.isCreated assert 0 < self.requestSize assert 0 < len(self.sources) assert 0 < len(self.sinks) assert self.sinkTarget assert self.sourceTarget self.factory = EndpointFactory(self.requestSize, self.sourceDIOFlag, self.sourceSerialFlag, self.sourceVerboseFlag, self.sourceTimestampFlag, self.sourceXddPath, self.sources, self.sinkDIOFlag, self.sinkSerialFlag, self.sinkVerboseFlag, self.sinkTimestampFlag, self.sinkXddPath, self.sinks) rc = self.factory.createEndpoints() if 0 == rc: self.isCreated = True rc = self.performPostCreateChecks() return rc def createDir(self, sourceDir, targetDir): """Create the directory targetDir on the sink""" assert self.isCreated self.beginTime = time.time() self.currentFlowSize = 0 sink = self.factory.getSinkEndpoints()[0] rc = sink.createDirectory(targetDir) if 0 == rc: self.showProgress(sourceDir, 0) print() return rc def createSymlink(self, sourceLink, targetLink, targetTarget): """Create the directory targetDir on the sink""" assert self.isCreated self.beginTime = time.time() self.currentFlowSize = 0 sink = self.factory.getSinkEndpoints()[0] rc = sink.createSymlink(targetTarget, targetLink) if 0 == rc: self.showProgress(sourceLink, 0) print() return rc def startTransfer(self, sourceFile, sinkFile): """Start all flows in correct order""" assert self.isCreated rc = 0 self.currentSourceFile = sourceFile self.currentSinkFile = sinkFile self.currentRestartOffset = 0 # If needed get a transfer size if not self.transferSize: s = self.factory.getSourceEndpoints()[0] transferSize = s.getFileSize(sourceFile) else: transferSize = self.transferSize # Set a restart offset restartOffset = 0 if self.restartFlag: s = self.factory.getSinkEndpoints()[0] restartOffset = s.getRestartOffset(sinkFile) self.currentRestartOffset = restartOffset # Start sinks first sinks = self.factory.getSinkEndpoints() for s in sinks: r = s.startFlow(sinkFile, transferSize, self.restartFlag, restartOffset) if 0 != r: print('Sink flow failed during startup', s.errorString()) rc += 1 # Pause between sink and source startups time.sleep(XDD_SINK_TO_SOURCE_DELAY) # Then start sources sources = self.factory.getSourceEndpoints() for s in sources: r = s.startFlow(sourceFile, transferSize, self.restartFlag, restartOffset) if 0 != r: print('Source flow failed during startup', s.errorString()) rc += 1 # Update the log with the execute commands self.writeLog(isWriteCommand=True) # If all went well, set state and start timer if 0 == rc: self.isStarted = True self.beginTime = time.time() self.currentFlowSize = transferSize return rc def monitorTransfer(self, monitorInterval): """ Monitor the current flows. Print a progress report to the screen and update the logfile if necessary. """ assert self.isStarted endpoints = self.factory.getEndpoints() completedBytes = 0 completeCount = 0 failureCount = 0 while completeCount < len(endpoints) and 0 == failureCount: monBegin = time.time() completedBytes = 0 completeCount = 0 for e in endpoints: # Check for completions and failures if e.completionStatus() is not None: completeCount += 1 if 0 != e.completionStatus(): failureCount += 1 # Get the progress currentBytes = e.currentByte() if 0 < currentBytes: completedBytes += currentBytes # The flows don't really report exactly correct bytes yet, so # modify the completed bytes to be correct, this is due to a couple # things, sinks report the restart offset and multi-source transfers # use -startoffset rather than -restart offset correctedBytes = completedBytes - self.currentRestartOffset if len(self.sources) > 1: # Progress temporarily reports as len(sources) * restartOffset # But then shrinks as the actual heartbeats get reported initialVal = len(self.sources) * self.currentRestartOffset if correctedBytes == initialVal: correctedBytes = self.currentRestartOffset else: correctedBytes += self.currentRestartOffset # Print a status message self.showProgress(self.currentSourceFile, correctedBytes) self.writeLog() monEnd = time.time() # Sleep until its time to monitor again interval = monEnd - monBegin if interval < monitorInterval: time.sleep(monitorInterval - interval) # Get all of the remaining log output self.writeLog(flushAll=True) # Determine the outcome of transfer failures = 0 for e in endpoints: if e.completionStatus() is None: e.cancelFlow() elif 0 != e.completionStatus(): failures += 1 print('RC:', e.completionStatus()) print('ERROR:', e.errorString()) # If the completion status for all endpoints is success, print out # a final progress message and remove the restart cookie if 0 == failures: self.currentRestartOffset = 0 self.showProgress(self.currentSourceFile, self.currentFlowSize) print('') if self.restartFlag: for s in self.factory.getSinkEndpoints(): # Ignore the return code on purpose rc = s.removeRestartCookie(self.currentSinkFile) return failures def startAndMonitorTransfers(self, monitorInterval): """ Used to transfer and monitor a directory or multiple files as part of a single transfer """ # First determine if the sink path exists sink = self.factory.getSinkEndpoints()[0] sinkIsDir = sink.pathIsDir(self.sinkTarget) sinkExists = sink.pathExists(self.sinkTarget) # This is tricky, if the sink path exists from a previous # transfer attempt and this is a retry, then we need to lie about it if self.restartFlag and sinkExists and \ sink.transferIsComplete(self.sinkTarget, self.sinkTarget): sinkExists = False # Use one of the sources to build the transfer pairs # Note you have to do this on the source in case the requested # path is a directory or symlink source = self.factory.getSourceEndpoints()[0] (rc, dirs, files, links) = source.buildWalk(self.sourceTarget, self.sinkTarget, targetExists=sinkExists, targetIsDir=sinkIsDir) if 0 != rc: print('ERROR: Unable to access', self.sourceTarget) #print(rc) #print(dirs) #print(files) #print(links) # Build all of the remote directories with the first sink if 0 == rc: for (sdir, tdir) in dirs: rc = self.createDir(sdir, tdir) if 0 != rc: print('ERROR transferring', sdir) break sink.markTransferCompleted(self.sinkTarget, tdir) # Move all files if 0 == rc: for (sfile, tfile) in files: if self.restartFlag and \ sink.transferIsComplete(self.sinkTarget, tfile): print(sfile, 'previously completed') continue rc = self.startTransfer(sfile, tfile) if 0 != rc: print('ERROR starting transfer', sfile) break rc = self.monitorTransfer(monitorInterval) if 0 != rc: print('ERROR transferring', sfile) break if self.restartFlag and 0 == rc: sink.markTransferCompleted(self.sinkTarget, tfile) # Create all symlinks if 0 == rc: for (ssym, tsym, link) in links: rc = self.createSymlink(ssym, tsym, link) if 0 != rc: print('ERROR transferring', ssym) break sink.markTransferCompleted(self.sinkTarget, tsym) # Remove the progress file if 0 == rc: sink.removeTransferProgressFile(self.sinkTarget) return rc def cleanupEndpoints(self): """ Determine exit status for stopped flows, and kill any remaining flows """ rc = 0 if self.factory: endpoints = self.factory.getEndpoints() #for e in endpoints: # if True == e.pollFlow(): # f.cancelFlow() # print('Cancelling Flow') # rc = 1 # elif 0 != f.completionStatus(): # print('Flow exited with code:', e.completionStatus()) # print('Flow cmd:', e.executeString()) # rc = 2 self.factory.shutdown() del self.factory self.isCreate = False self.isStarted = False self.isCompleted = True return rc def completedSuccessfully(self): """@return true if the flows completed successfully""" return self.isComplete and self.isSuccess def showProgress(self, filename, completedBytes): """Print a status line to stderr without a linefeed""" # Calculate the status values try: currentBytes = completedBytes - self.currentRestartOffset pctProgress = (float(completedBytes) / self.currentFlowSize) * 100.0 transferTime = time.time() - self.beginTime bandwidth = float(currentBytes) / transferTime eta = '--:--:--' except ZeroDivisionError: # Zero size flows are completed by definition pctProgress = 100.0 transferTime = time.time() - self.beginTime bandwidth = 0.0 eta = '00:00:00' if (bandwidth > 0): eta_secs = round((self.currentFlowSize - completedBytes)/bandwidth) td = datetime.timedelta(seconds=eta_secs) eta = str(td) # Get the width of the terminal cols = 80 if sys.stdin.isatty(): stty_size = os.popen('stty size').read() if 0 != len(stty_size): (r, c) = stty_size.split() cols = int(c) # Generate the status string status = '' if 100.0 <= pctProgress: status += str(round(pctProgress)) + '% ' elif 10.0 <= pctProgress: status += str(pctProgress)[0:4] + '% ' else: status += str(pctProgress)[0:3] + '% ' if (1024*1024*1024) < bandwidth: status += str(completedBytes / (1024*1024*1024)) + 'GiB ' elif (1024*1024) < bandwidth: status += str(completedBytes / (1024*1024)) + 'MiB ' elif 1024 < bandwidth: status += str(completedBytes / 1024) + 'KiB ' else: status += str(completedBytes) + 'B ' status += str(round(bandwidth / 1024.0 / 1024.0)) + 'MiB/s ' status += eta + ' ETA' # Determine the padding length between the file name and status padding = ' ' if cols > (len(status) + len(filename)): padding *= cols - len(status) - len(filename) # Clear the existing status line with whitespace print('\r', ' ' * cols, sep='', end='', file=sys.stderr) # Write the new status line print('\r', filename, padding, status, sep='', end='', file=sys.stderr) def writeLog(self, isWriteCommand=False, flushAll=False): """ Log current status to the logfile. This isn't free. It sends alot of data over the RPC channels if this transfer is remote. """ if self.verboseLog is not None: f = open(self.verboseLog, 'a') # Write sink data if self.sinkVerboseFlag: sinks = self.factory.getSinkEndpoints() for s in sinks: prefix = s.hostname() + ': ' if isWriteCommand: f.write(prefix + s.executeString() + os.linesep) else: lines = s.output(flushAll).split(os.linesep) for l in lines: if l: f.write(prefix + l + os.linesep) # Write source data if self.sourceVerboseFlag: sources = self.factory.getSourceEndpoints() for s in sources: prefix = s.hostname() + ': ' if isWriteCommand: f.write(prefix + s.executeString() + os.linesep) else: lines = s.output(flushAll).split(os.linesep) for l in lines: if l: f.write(prefix + l + os.linesep) f.close()
class TransferManager: """ Manager for creating, monitoring, and deallocating local and remote flows. """ def __init__(self): """Constructor""" self.isCreated = False self.isStarted = False self.isComplete = False self.isSuccess = False self.factory = None # Temporary variables used for showing progress self.beginTime = None self.currentSourceFile = None self.currentSinkFile = None self.currentFlowSize = 0 self.currentRestartOffset = 0 self.verboseLog = None self.requestSize = 0 self.transferSize = None self.restartFlag = False self.sinkDIOFlag = False self.sinkSerialFlag = False self.sinkTimestampFlag = False self.sinkVerboseFlag = False self.sinkTarget = None self.sinkXddPath = '' self.sinks = [] self.sourceDIOFlag = False self.sourceSerialFlag = False self.sourceVerboseFlag = False self.sourceTimestampFlag = False self.sourceTarget = None self.sourceXddPath = '' self.sources = [] def setRequestSize(self, reqSize): """Set the chunk size of data to move across the wire""" self.requestSize = reqSize def setTransferSize(self, transferSize): """Set the size of data to move across the wire""" self.transferSize = transferSize def setRestartFlag(self, restartFlag): """Set the amount of data previously sent across the wire""" self.restartFlag = restartFlag def setSinkTarget(self, target, dioFlag=False, serialFlag=False): """ Set the name of of the sink target file""" assert not self.isCreated self.sinkTarget = target self.sinkDIOFlag = dioFlag self.sinkSerialFlag = serialFlag def setSourceTarget(self, target, dioFlag=False, serialFlag=False): """Set the name of the source target file""" assert not self.isCreated self.sourceTarget = target self.sourceDIOFlag = dioFlag self.sourceSerialFlag = serialFlag def setVerbosity(self, verboseLevel, filename): """Set the level of verbosity""" if 1 <= verboseLevel: self.sinkVerboseFlag = True self.sourceVerboseFlag = True if 2 <= verboseLevel: self.sinkTimestampFlag = True self.sourceTimestampFlag = True self.verboseLog = filename def addSink(self, user, hostIP, hostname, threads, ifs=[], port=40010): """Add a sink to the list of sinks""" assert not self.isCreated assert hostIP assert hostname assert 0 < threads sink = { 'ip': hostIP, 'hostname': hostname, 'threads': threads, 'port': port, 'ifs': [] } if 0 == len(ifs): sink['ifs'].append(hostIP) else: sink['ifs'].extend(ifs) self.sinks.append(sink) def addSource(self, user, hostIP, hostname, threads, ifs=[], port=40010): """Add a source to the list of sources""" assert not self.isCreated assert hostIP assert hostname assert 0 < threads source = { 'ip': hostIP, 'hostname': hostname, 'threads': threads, 'port': port, 'ifs': [] } if 0 == len(ifs): source['ifs'].append(hostIP) else: source['ifs'].extend(ifs) self.sources.append(source) def setSinkXddPath(self, path): """Set the path sinks use to find xdd""" self.sinkXddPath = path def setSourceXddPath(self, path): """Set the path sources use to find xdd""" self.sourceXddPath = path def setXddPath(self, path): """Set the path used to find xdd""" self.sinkXddPath = path self.sourceXddPath = path def performPostCreateChecks(self): """@return 0 if all checks pass, otherwise non-zero""" rc = 0 # Check that all of the XDD's are the same version base = None current = None for f in self.factory.getEndpoints(): current = f.protocolVersion() if base is None: base = current if current is None or not current or current != base: print("ERROR: XDD Protocols do not match", file=sys.stderr) rc = 1 break # Print a generic warning if the sinks do not have pre-allocation # support for s in self.factory.getSinkEndpoints(): if not s.hasPreallocateAvailable(): print("WARNING: XDD preallocation support not available", file=sys.stderr) return rc def createEndpoints(self): """Create local and remote transfer host endpoints""" assert not self.isCreated assert 0 < self.requestSize assert 0 < len(self.sources) assert 0 < len(self.sinks) assert self.sinkTarget assert self.sourceTarget self.factory = EndpointFactory( self.requestSize, self.sourceDIOFlag, self.sourceSerialFlag, self.sourceVerboseFlag, self.sourceTimestampFlag, self.sourceXddPath, self.sources, self.sinkDIOFlag, self.sinkSerialFlag, self.sinkVerboseFlag, self.sinkTimestampFlag, self.sinkXddPath, self.sinks) rc = self.factory.createEndpoints() if 0 == rc: self.isCreated = True rc = self.performPostCreateChecks() return rc def createDir(self, sourceDir, targetDir): """Create the directory targetDir on the sink""" assert self.isCreated self.beginTime = time.time() self.currentFlowSize = 0 sink = self.factory.getSinkEndpoints()[0] rc = sink.createDirectory(targetDir) if 0 == rc: self.showProgress(sourceDir, 0) print() return rc def createSymlink(self, sourceLink, targetLink, targetTarget): """Create the directory targetDir on the sink""" assert self.isCreated self.beginTime = time.time() self.currentFlowSize = 0 sink = self.factory.getSinkEndpoints()[0] rc = sink.createSymlink(targetTarget, targetLink) if 0 == rc: self.showProgress(sourceLink, 0) print() return rc def startTransfer(self, sourceFile, sinkFile): """Start all flows in correct order""" assert self.isCreated rc = 0 self.currentSourceFile = sourceFile self.currentSinkFile = sinkFile self.currentRestartOffset = 0 # If needed get a transfer size if not self.transferSize: s = self.factory.getSourceEndpoints()[0] transferSize = s.getFileSize(sourceFile) else: transferSize = self.transferSize # Set a restart offset restartOffset = 0 if self.restartFlag: s = self.factory.getSinkEndpoints()[0] restartOffset = s.getRestartOffset(sinkFile) self.currentRestartOffset = restartOffset # Start sinks first sinks = self.factory.getSinkEndpoints() for s in sinks: r = s.startFlow(sinkFile, transferSize, self.restartFlag, restartOffset) if 0 != r: print('Sink flow failed during startup', s.errorString()) rc += 1 # Pause between sink and source startups time.sleep(XDD_SINK_TO_SOURCE_DELAY) # Then start sources sources = self.factory.getSourceEndpoints() for s in sources: r = s.startFlow(sourceFile, transferSize, self.restartFlag, restartOffset) if 0 != r: print('Source flow failed during startup', s.errorString()) rc += 1 # Update the log with the execute commands self.writeLog(isWriteCommand=True) # If all went well, set state and start timer if 0 == rc: self.isStarted = True self.beginTime = time.time() self.currentFlowSize = transferSize return rc def monitorTransfer(self, monitorInterval): """ Monitor the current flows. Print a progress report to the screen and update the logfile if necessary. """ assert self.isStarted endpoints = self.factory.getEndpoints() completedBytes = 0 completeCount = 0 failureCount = 0 while completeCount < len(endpoints) and 0 == failureCount: monBegin = time.time() completedBytes = 0 completeCount = 0 for e in endpoints: # Check for completions and failures if e.completionStatus() is not None: completeCount += 1 if 0 != e.completionStatus(): failureCount += 1 # Get the progress currentBytes = e.currentByte() if 0 < currentBytes: completedBytes += currentBytes # The flows don't really report exactly correct bytes yet, so # modify the completed bytes to be correct, this is due to a couple # things, sinks report the restart offset and multi-source transfers # use -startoffset rather than -restart offset correctedBytes = completedBytes - self.currentRestartOffset if len(self.sources) > 1: # Progress temporarily reports as len(sources) * restartOffset # But then shrinks as the actual heartbeats get reported initialVal = len(self.sources) * self.currentRestartOffset if correctedBytes == initialVal: correctedBytes = self.currentRestartOffset else: correctedBytes += self.currentRestartOffset # Print a status message self.showProgress(self.currentSourceFile, correctedBytes) self.writeLog() monEnd = time.time() # Sleep until its time to monitor again interval = monEnd - monBegin if interval < monitorInterval: time.sleep(monitorInterval - interval) # Get all of the remaining log output self.writeLog(flushAll=True) # Determine the outcome of transfer failures = 0 for e in endpoints: if e.completionStatus() is None: e.cancelFlow() elif 0 != e.completionStatus(): failures += 1 print('RC:', e.completionStatus()) print('ERROR:', e.errorString()) # If the completion status for all endpoints is success, print out # a final progress message and remove the restart cookie if 0 == failures: self.currentRestartOffset = 0 self.showProgress(self.currentSourceFile, self.currentFlowSize) print('') if self.restartFlag: for s in self.factory.getSinkEndpoints(): # Ignore the return code on purpose rc = s.removeRestartCookie(self.currentSinkFile) return failures def startAndMonitorTransfers(self, monitorInterval): """ Used to transfer and monitor a directory or multiple files as part of a single transfer """ # First determine if the sink path exists sink = self.factory.getSinkEndpoints()[0] sinkIsDir = sink.pathIsDir(self.sinkTarget) sinkExists = sink.pathExists(self.sinkTarget) # This is tricky, if the sink path exists from a previous # transfer attempt and this is a retry, then we need to lie about it if self.restartFlag and sinkExists and \ sink.transferIsComplete(self.sinkTarget, self.sinkTarget): sinkExists = False # Use one of the sources to build the transfer pairs # Note you have to do this on the source in case the requested # path is a directory or symlink source = self.factory.getSourceEndpoints()[0] (rc, dirs, files, links) = source.buildWalk(self.sourceTarget, self.sinkTarget, targetExists=sinkExists, targetIsDir=sinkIsDir) if 0 != rc: print('ERROR: Unable to access', self.sourceTarget) #print(rc) #print(dirs) #print(files) #print(links) # Build all of the remote directories with the first sink if 0 == rc: for (sdir, tdir) in dirs: rc = self.createDir(sdir, tdir) if 0 != rc: print('ERROR transferring', sdir) break sink.markTransferCompleted(self.sinkTarget, tdir) # Move all files if 0 == rc: for (sfile, tfile) in files: if self.restartFlag and \ sink.transferIsComplete(self.sinkTarget, tfile): print(sfile, 'previously completed') continue rc = self.startTransfer(sfile, tfile) if 0 != rc: print('ERROR starting transfer', sfile) break rc = self.monitorTransfer(monitorInterval) if 0 != rc: print('ERROR transferring', sfile) break if self.restartFlag and 0 == rc: sink.markTransferCompleted(self.sinkTarget, tfile) # Create all symlinks if 0 == rc: for (ssym, tsym, link) in links: rc = self.createSymlink(ssym, tsym, link) if 0 != rc: print('ERROR transferring', ssym) break sink.markTransferCompleted(self.sinkTarget, tsym) # Remove the progress file if 0 == rc: sink.removeTransferProgressFile(self.sinkTarget) return rc def cleanupEndpoints(self): """ Determine exit status for stopped flows, and kill any remaining flows """ rc = 0 if self.factory: endpoints = self.factory.getEndpoints() #for e in endpoints: # if True == e.pollFlow(): # f.cancelFlow() # print('Cancelling Flow') # rc = 1 # elif 0 != f.completionStatus(): # print('Flow exited with code:', e.completionStatus()) # print('Flow cmd:', e.executeString()) # rc = 2 self.factory.shutdown() del self.factory self.isCreate = False self.isStarted = False self.isCompleted = True return rc def completedSuccessfully(self): """@return true if the flows completed successfully""" return self.isComplete and self.isSuccess def showProgress(self, filename, completedBytes): """Print a status line to stderr without a linefeed""" # Calculate the status values try: currentBytes = completedBytes - self.currentRestartOffset pctProgress = (float(completedBytes) / self.currentFlowSize) * 100.0 transferTime = time.time() - self.beginTime bandwidth = float(currentBytes) / transferTime eta = '--:--:--' except ZeroDivisionError: # Zero size flows are completed by definition pctProgress = 100.0 transferTime = time.time() - self.beginTime bandwidth = 0.0 eta = '00:00:00' if (bandwidth > 0): eta_secs = round( (self.currentFlowSize - completedBytes) / bandwidth) td = datetime.timedelta(seconds=eta_secs) eta = str(td) # Get the width of the terminal cols = 80 if sys.stdin.isatty(): stty_size = os.popen('stty size').read() if 0 != len(stty_size): (r, c) = stty_size.split() cols = int(c) # Generate the status string status = '' if 100.0 <= pctProgress: status += str(round(pctProgress)) + '% ' elif 10.0 <= pctProgress: status += str(pctProgress)[0:4] + '% ' else: status += str(pctProgress)[0:3] + '% ' if (1024 * 1024 * 1024) < bandwidth: status += str(completedBytes / (1024 * 1024 * 1024)) + 'GiB ' elif (1024 * 1024) < bandwidth: status += str(completedBytes / (1024 * 1024)) + 'MiB ' elif 1024 < bandwidth: status += str(completedBytes / 1024) + 'KiB ' else: status += str(completedBytes) + 'B ' status += str(round(bandwidth / 1024.0 / 1024.0)) + 'MiB/s ' status += eta + ' ETA' # Determine the padding length between the file name and status padding = ' ' if cols > (len(status) + len(filename)): padding *= cols - len(status) - len(filename) # Clear the existing status line with whitespace print('\r', ' ' * cols, sep='', end='', file=sys.stderr) # Write the new status line print('\r', filename, padding, status, sep='', end='', file=sys.stderr) def writeLog(self, isWriteCommand=False, flushAll=False): """ Log current status to the logfile. This isn't free. It sends alot of data over the RPC channels if this transfer is remote. """ if self.verboseLog is not None: f = open(self.verboseLog, 'a') # Write sink data if self.sinkVerboseFlag: sinks = self.factory.getSinkEndpoints() for s in sinks: prefix = s.hostname() + ': ' if isWriteCommand: f.write(prefix + s.executeString() + os.linesep) else: lines = s.output(flushAll).split(os.linesep) for l in lines: if l: f.write(prefix + l + os.linesep) # Write source data if self.sourceVerboseFlag: sources = self.factory.getSourceEndpoints() for s in sources: prefix = s.hostname() + ': ' if isWriteCommand: f.write(prefix + s.executeString() + os.linesep) else: lines = s.output(flushAll).split(os.linesep) for l in lines: if l: f.write(prefix + l + os.linesep) f.close()