def waitFor(func, maxWait=None, message=None):
r"""Waits for the given function to return a truthy or until a set time.
"""
waited = 0
maxWait = maxWait or defaultDelay
if message:
log(message)
while True:
ret = func()
if ret:
return ret
waited += tickTime
if waited > maxWait:
raise Exception('Maximum wait time exceded.' if not message else
'Failed waiting for: %s' % message)
if message:
debug(message)
debug('Waited: %d, MaxWait: %d.' % (waited, maxWait))
sleep(tickTime)
def setup(Project):
global fsRoot
_temp = Project.fsRoot
fsRoot = abspath(_temp)
debug('fsRoot: %s' % _temp)
def forgive(func):
try:
func()
except Exception as e: #pylint: disable=W0703
debug(e)
return e
def launch(command, **KWArgs):
r"""Launches a process and quits without waiting for its completion.
"""
debug(command)
dump(getPretty(KWArgs))
return Popen(split(command), stdout=PIPE, stderr=PIPE, **KWArgs)
def download(self, remotePath, localPath=''):
debug('downloading %s to %s' % (remotePath, localPath))
SFTP = self.open_sftp()
localPath = getTgtName(localPath, remotePath)
SFTP.get(remotePath, localPath)
SFTP.close()
return localPath
def rename(srcPath, tgtPath, autoClean=True):
r"""Renames any given file / dir / junction.
"""
if autoClean:
removePath(tgtPath)
debug('rename: %s => %s' % (srcPath, tgtPath))
os.rename(srcPath, tgtPath)
def execute(self, command):
debug(command)
dummy, stdout, stderr = self.exec_command(command)
return {
'code': stdout.channel.recv_exit_status(),
'out': stdout.read(),
'err': stderr.read(),
}
def callScript(self, ecCommand):
out = assertShell(call(self.mockScriptTpl % ecCommand, shell=True))
if out:
Out = rob(lambda: json.loads(out))
if Out is None:
raise Exception(out)
debug(getPretty(Out))
return Out
def run(command, **KWArgs):
r"""Starts a process, waits till the process completes and returns the return-code.
#Tip: Use this method to live stream output from the command.
"""
debug(command)
dump(getPretty(KWArgs))
p = Popen(split(command), **KWArgs)
p.wait()
return p.returncode
def call(command, **KWArgs): # from gitapi.py
r"""Starts a process, waits till the process completes and returns a dictionary with the return-code, stdout and stderr.
#Tip: Use this method when there's a need to process stdout or stderr.
"""
debug(command)
dump(getPretty(KWArgs))
p = Popen(split(command), stdout=PIPE, stderr=PIPE, **KWArgs)
out, err = [x.decode('utf-8') for x in p.communicate()]
return {'out': out, 'err': err, 'code': p.returncode}
def debugCall(command, **KWArgs):
r"""Starts a process, waits till the process completes and returns a dictionary with the return-code, stdout and stderr.
#Tip: Use this method call scripts during development, errors would be logged to the live stderr, at the same time stdout could be buffered for processing.
#Tip: A modified pdb like, modPdb = pdb.Pdb(stdout=sys.__stderr__), could be used to debug scripts in stderr.
"""
debug(command)
dump(getPretty(KWArgs))
p = Popen(split(command), stdout=PIPE, **KWArgs)
return getProcessData(p)
def removePath(tgtPath, requiredAncestor=None, forced=False):
r"""Removes any given file / dir / junction.
Args:
tgtPath (path): The path to remove.
requiredAncestor (path): The target will only be removed if it's a descendant of this dir. Defaults to the global attr fsRoot.
forced (bool): When set to true, an ancestor won't be required.
"""
if not forced:
requireAncestor(tgtPath, requiredAncestor)
debug('remove: %s' % tgtPath)
return _removePath(tgtPath)
def copyContent(srcPath, tgtPath, autoClean=True):
r"""
Copies the content of one file to another.
# #Note: Unlike shutil.copy attributes aren't copied.
"""
if (removePath(tgtPath) == 1 if autoClean else True): # Ensure parent only if the path is not removed.
ensureParent(tgtPath)
debug('copy: %s => %s' % (srcPath, tgtPath))
with open(tgtPath, 'wb') as tgt:
for chunk in iterateContent(srcPath):
tgt.write(chunk)
def writable(command, data, **KWArgs):
r"""Opens a process and writes the given data to its STDIN.
The newline character could be used to separate multiple lines.
"""
debug(command)
dump(getPretty(KWArgs))
# #Note: data isn't dumped, to keep it secure.
p = Popen(split(command), stdout=PIPE, stderr=PIPE, stdin=PIPE, **KWArgs)
out, err = p.communicate(data)
code = p.returncode
return {'out': out, 'err': err, 'code': code}
def linkTree(srcPath, tgtPath, pattern='**', regex=False, autoClean=True, hardLink=False):
r"""Re-creates the structure of the source at the target by creating dirs and linking files.
"""
_srcPath = abspath(srcPath) # Link sources should be abs-paths.
dirMaker = makeDir if autoClean else makeMissingDir
linkWorker = link if hardLink else symlink # Hard links aren't the default, as they can't work across drives.
linker = linkWorker if autoClean or not exists(tgtPath) else lambda srcPath, tgtPath: (removePath(tgtPath), linkWorker(srcPath, tgtPath)) # File safety isn't a concern inside a missing dir.
parentMaker = ensureParent if pattern != '**' and not regex else doNoting
(ensureCleanDir if autoClean else ensureDir)(tgtPath)
for Path in collectPaths(srcPath, pattern, regex):
if Path[1] != 1:
dirMaker(joinPaths(tgtPath, Path[0]))
else:
path = Path[0]
_tgtPath = joinPaths(tgtPath, path)
debug('link: %s => %s' % (joinPaths(srcPath, path), _tgtPath))
parentMaker(_tgtPath)
linker(joinPaths(_srcPath, path), _tgtPath)
def _upload(SFTP, localPath, remotePath):
debug('uploading %s to %s' % (localPath, remotePath))
pathType = getPathType(localPath)
if pathType == 1: # file
retry(lambda: SFTP.put(localPath, remotePath) or 1) # #Note: 1 is returned to notify retry of a success.
elif pathType > 1: # dir / junction
err = forgive(lambda: mkdirs(SFTP, remotePath))
if err and not isinstance(err, IOError):
raise err
for item in listdir(localPath): # #Note: With dir uploads, the uploads are merged (added / overwritten) with existing paths.
retry(lambda: _upload(SFTP, *pair(localPath, remotePath, item))) #pylint: disable=cell-var-from-loop
else: # Path doesn't exist.
raise Exception('Invalid source path: %s' % localPath)
return remotePath
def test_linkTree(self): # #Pending: Test all the argument combinations.
rebuildStructures()
debug('Testing with default arguments...')
target = '%s/base1' % tempDir
linkTree(baseDir, target)
assert cmp(
getStructureDict(baseDir),
getStructureDict(target)) == 0, 'The structures do not match.'
removePath(target)
debug('Testing patterns...')
def testPattern(pattern, excludeDirs=False):
linkTree(baseDir, target, pattern)
assert cmp(
getStructureDict(baseDir, pattern),
getStructureDict(target, excludeDirs=excludeDirs)
) == 0, 'The structures do not match.' # #Note: pattern isn't fed into getStructureDict(target) to avoid mirror bias (those calls that go wrong on both sides, thus equate and pass).
testPattern('**!**.txt')
testPattern('**.txt', True)
def piped(*Commands, **KWArgs):
r"""Emulates piped commands in *nix systems. Returns a dictionary with the final return-code, stdout and a stderr.
"""
dump(getPretty(KWArgs))
out = None
err = None
code = 0
for command in Commands:
debug(command)
p = Popen(split(command),
stdout=PIPE,
stderr=PIPE,
stdin=PIPE,
**KWArgs)
out, err = p.communicate(out)
code = p.returncode
if code:
break
return {'out': out, 'err': err, 'code': code}
def test_collectPaths(self):
r"""Tests the function collectPaths.
"""
rebuildStructures()
def ensureValidPathCollection(collectPathsPattern,
regexValidator,
invert=False):
assert cmp(
{
Pair[0]: Pair[1]
for Pair in collectPaths(baseDir, collectPathsPattern)
},
{
k: v
for k, v in FSDict.iteritems()
if bool(re.match('^%s$' %
regexValidator, k)) == (not invert)
},
) == 0, 'collectPaths failed on: %s' % collectPathsPattern
debug('Testing proper collections...')
testScenarios(
ensureValidPathCollection,
('**', r'.*'),
('*', r'[^\/]*'),
('*d*', r'[^\/]*d[^\/]*'),
('*.txt', r'[^\/]*\.txt'),
('**.txt', r'.*\.txt'),
('.*', r'\.[^\/]*'),
('*.py*', r'[^\/]*\.py[^\/]*'),
('**/dir1/**', r'.*\/dir1\/.*'),
('*ir1/**', r'[^\/]*ir1\/.*'),
('**/*ir1/**', r'.*\/[^\/]*ir1\/.*'),
)
debug('Testing empty collections...')
testScenarios(
ensureValidPathCollection,
('**/*2/**', ''),
('dir', ''),
)
debug('Testing exclusions...')
testScenarios(
ensureValidPathCollection,
('**!**', r'.*', True),
('**!*.txt', r'[^\/]*\.txt', True),
('*!*fi*.tx*', r'.*\/.*|[^\/]*fi[^\/]*\.tx[^\/]*', True),
)
def makeDir(path):
debug('making: %s' % path)
mkdir(path)
def _makeLink(srcPath, tgtPath, pathType, hardLink):
debug('link: %s => %s' % (srcPath, tgtPath))
(link if hardLink and pathType == 1 else symlink)(abspath(srcPath), tgtPath) # #Note: Dirs can't be hard-linked.
def rebuildStructures():
debug('Building Structures...')
ensureCleanDir(baseDir)
buildStructure(baseDir, FSDict)
def upload(self, srcPath,
tgtPath): # #Note: Uploads are done always to the temp dir.
debug('uploading %s -> %s' % (srcPath, tgtPath))
ensureParent(tgtPath)
return copy(srcPath, tgtPath)