def waitFor(func, maxWait=None, message=None): r"""Waits for the given function to return a truthy or until a set time. """ waited = 0 maxWait = maxWait or defaultDelay if message: log(message) while True: ret = func() if ret: return ret waited += tickTime if waited > maxWait: raise Exception('Maximum wait time exceded.' if not message else 'Failed waiting for: %s' % message) if message: debug(message) debug('Waited: %d, MaxWait: %d.' % (waited, maxWait)) sleep(tickTime)
def setup(Project): global fsRoot _temp = Project.fsRoot fsRoot = abspath(_temp) debug('fsRoot: %s' % _temp)
def forgive(func): try: func() except Exception as e: #pylint: disable=W0703 debug(e) return e
def launch(command, **KWArgs): r"""Launches a process and quits without waiting for its completion. """ debug(command) dump(getPretty(KWArgs)) return Popen(split(command), stdout=PIPE, stderr=PIPE, **KWArgs)
def download(self, remotePath, localPath=''): debug('downloading %s to %s' % (remotePath, localPath)) SFTP = self.open_sftp() localPath = getTgtName(localPath, remotePath) SFTP.get(remotePath, localPath) SFTP.close() return localPath
def rename(srcPath, tgtPath, autoClean=True): r"""Renames any given file / dir / junction. """ if autoClean: removePath(tgtPath) debug('rename: %s => %s' % (srcPath, tgtPath)) os.rename(srcPath, tgtPath)
def execute(self, command): debug(command) dummy, stdout, stderr = self.exec_command(command) return { 'code': stdout.channel.recv_exit_status(), 'out': stdout.read(), 'err': stderr.read(), }
def callScript(self, ecCommand): out = assertShell(call(self.mockScriptTpl % ecCommand, shell=True)) if out: Out = rob(lambda: json.loads(out)) if Out is None: raise Exception(out) debug(getPretty(Out)) return Out
def run(command, **KWArgs): r"""Starts a process, waits till the process completes and returns the return-code. #Tip: Use this method to live stream output from the command. """ debug(command) dump(getPretty(KWArgs)) p = Popen(split(command), **KWArgs) p.wait() return p.returncode
def call(command, **KWArgs): # from gitapi.py r"""Starts a process, waits till the process completes and returns a dictionary with the return-code, stdout and stderr. #Tip: Use this method when there's a need to process stdout or stderr. """ debug(command) dump(getPretty(KWArgs)) p = Popen(split(command), stdout=PIPE, stderr=PIPE, **KWArgs) out, err = [x.decode('utf-8') for x in p.communicate()] return {'out': out, 'err': err, 'code': p.returncode}
def debugCall(command, **KWArgs): r"""Starts a process, waits till the process completes and returns a dictionary with the return-code, stdout and stderr. #Tip: Use this method call scripts during development, errors would be logged to the live stderr, at the same time stdout could be buffered for processing. #Tip: A modified pdb like, modPdb = pdb.Pdb(stdout=sys.__stderr__), could be used to debug scripts in stderr. """ debug(command) dump(getPretty(KWArgs)) p = Popen(split(command), stdout=PIPE, **KWArgs) return getProcessData(p)
def removePath(tgtPath, requiredAncestor=None, forced=False): r"""Removes any given file / dir / junction. Args: tgtPath (path): The path to remove. requiredAncestor (path): The target will only be removed if it's a descendant of this dir. Defaults to the global attr fsRoot. forced (bool): When set to true, an ancestor won't be required. """ if not forced: requireAncestor(tgtPath, requiredAncestor) debug('remove: %s' % tgtPath) return _removePath(tgtPath)
def copyContent(srcPath, tgtPath, autoClean=True): r""" Copies the content of one file to another. # #Note: Unlike shutil.copy attributes aren't copied. """ if (removePath(tgtPath) == 1 if autoClean else True): # Ensure parent only if the path is not removed. ensureParent(tgtPath) debug('copy: %s => %s' % (srcPath, tgtPath)) with open(tgtPath, 'wb') as tgt: for chunk in iterateContent(srcPath): tgt.write(chunk)
def writable(command, data, **KWArgs): r"""Opens a process and writes the given data to its STDIN. The newline character could be used to separate multiple lines. """ debug(command) dump(getPretty(KWArgs)) # #Note: data isn't dumped, to keep it secure. p = Popen(split(command), stdout=PIPE, stderr=PIPE, stdin=PIPE, **KWArgs) out, err = p.communicate(data) code = p.returncode return {'out': out, 'err': err, 'code': code}
def linkTree(srcPath, tgtPath, pattern='**', regex=False, autoClean=True, hardLink=False): r"""Re-creates the structure of the source at the target by creating dirs and linking files. """ _srcPath = abspath(srcPath) # Link sources should be abs-paths. dirMaker = makeDir if autoClean else makeMissingDir linkWorker = link if hardLink else symlink # Hard links aren't the default, as they can't work across drives. linker = linkWorker if autoClean or not exists(tgtPath) else lambda srcPath, tgtPath: (removePath(tgtPath), linkWorker(srcPath, tgtPath)) # File safety isn't a concern inside a missing dir. parentMaker = ensureParent if pattern != '**' and not regex else doNoting (ensureCleanDir if autoClean else ensureDir)(tgtPath) for Path in collectPaths(srcPath, pattern, regex): if Path[1] != 1: dirMaker(joinPaths(tgtPath, Path[0])) else: path = Path[0] _tgtPath = joinPaths(tgtPath, path) debug('link: %s => %s' % (joinPaths(srcPath, path), _tgtPath)) parentMaker(_tgtPath) linker(joinPaths(_srcPath, path), _tgtPath)
def _upload(SFTP, localPath, remotePath): debug('uploading %s to %s' % (localPath, remotePath)) pathType = getPathType(localPath) if pathType == 1: # file retry(lambda: SFTP.put(localPath, remotePath) or 1) # #Note: 1 is returned to notify retry of a success. elif pathType > 1: # dir / junction err = forgive(lambda: mkdirs(SFTP, remotePath)) if err and not isinstance(err, IOError): raise err for item in listdir(localPath): # #Note: With dir uploads, the uploads are merged (added / overwritten) with existing paths. retry(lambda: _upload(SFTP, *pair(localPath, remotePath, item))) #pylint: disable=cell-var-from-loop else: # Path doesn't exist. raise Exception('Invalid source path: %s' % localPath) return remotePath
def test_linkTree(self): # #Pending: Test all the argument combinations. rebuildStructures() debug('Testing with default arguments...') target = '%s/base1' % tempDir linkTree(baseDir, target) assert cmp( getStructureDict(baseDir), getStructureDict(target)) == 0, 'The structures do not match.' removePath(target) debug('Testing patterns...') def testPattern(pattern, excludeDirs=False): linkTree(baseDir, target, pattern) assert cmp( getStructureDict(baseDir, pattern), getStructureDict(target, excludeDirs=excludeDirs) ) == 0, 'The structures do not match.' # #Note: pattern isn't fed into getStructureDict(target) to avoid mirror bias (those calls that go wrong on both sides, thus equate and pass). testPattern('**!**.txt') testPattern('**.txt', True)
def piped(*Commands, **KWArgs): r"""Emulates piped commands in *nix systems. Returns a dictionary with the final return-code, stdout and a stderr. """ dump(getPretty(KWArgs)) out = None err = None code = 0 for command in Commands: debug(command) p = Popen(split(command), stdout=PIPE, stderr=PIPE, stdin=PIPE, **KWArgs) out, err = p.communicate(out) code = p.returncode if code: break return {'out': out, 'err': err, 'code': code}
def test_collectPaths(self): r"""Tests the function collectPaths. """ rebuildStructures() def ensureValidPathCollection(collectPathsPattern, regexValidator, invert=False): assert cmp( { Pair[0]: Pair[1] for Pair in collectPaths(baseDir, collectPathsPattern) }, { k: v for k, v in FSDict.iteritems() if bool(re.match('^%s$' % regexValidator, k)) == (not invert) }, ) == 0, 'collectPaths failed on: %s' % collectPathsPattern debug('Testing proper collections...') testScenarios( ensureValidPathCollection, ('**', r'.*'), ('*', r'[^\/]*'), ('*d*', r'[^\/]*d[^\/]*'), ('*.txt', r'[^\/]*\.txt'), ('**.txt', r'.*\.txt'), ('.*', r'\.[^\/]*'), ('*.py*', r'[^\/]*\.py[^\/]*'), ('**/dir1/**', r'.*\/dir1\/.*'), ('*ir1/**', r'[^\/]*ir1\/.*'), ('**/*ir1/**', r'.*\/[^\/]*ir1\/.*'), ) debug('Testing empty collections...') testScenarios( ensureValidPathCollection, ('**/*2/**', ''), ('dir', ''), ) debug('Testing exclusions...') testScenarios( ensureValidPathCollection, ('**!**', r'.*', True), ('**!*.txt', r'[^\/]*\.txt', True), ('*!*fi*.tx*', r'.*\/.*|[^\/]*fi[^\/]*\.tx[^\/]*', True), )
def makeDir(path): debug('making: %s' % path) mkdir(path)
def _makeLink(srcPath, tgtPath, pathType, hardLink): debug('link: %s => %s' % (srcPath, tgtPath)) (link if hardLink and pathType == 1 else symlink)(abspath(srcPath), tgtPath) # #Note: Dirs can't be hard-linked.
def rebuildStructures(): debug('Building Structures...') ensureCleanDir(baseDir) buildStructure(baseDir, FSDict)
def upload(self, srcPath, tgtPath): # #Note: Uploads are done always to the temp dir. debug('uploading %s -> %s' % (srcPath, tgtPath)) ensureParent(tgtPath) return copy(srcPath, tgtPath)