def uploadFiles(instance, conf, srcFiles, outDir, log=False):
"""
This uploads srcFiles to the outDir, this is very low level and meant to allow the user to upload anywhere
instance - the instance to upload to
conf - contains various config data, the following options need to be present:
ssh.options - options to pass to ssh
ssh.user - user to ssh as
srcFiles - a list of files to upload
outDir - the directory to put them in
"""
##
# This is cheap, but we'll have the mkdir print its error out to screen if logging is on, in reality if this fails
# we should wrap it up somehow in an exception
runSystemInstanceEx(instance, 'mkdir -p ' + outDir, None, (log and errorPrintS or None), user=conf('ssh.user'), options=conf('ssh.options'), log=log)
for f in srcFiles:
scpToEx(instance['public_dns'], f, outDir, user=conf('ssh.user'), options=conf('ssh.options'), log=log)
def downloadPipeline(instance, conf, pipelineId, outDir, outBaseName, overwrite=False, log=False):
"""
Downloads a pipeline from an instance.
conf - contains various config data, the following options need to be present:
ssh.options - options to pass to ssh
ssh.user - user to ssh as
dirs.clovr_project - the directory on the imave clovr project is expected to live
pipelineId - the id of the pipeline to download
outDir - local directory to download to
outBaseName - The basename for the output file
overwrite - if the downloaded file exists locally already, should it be downloaded (default No)
"""
outF = '/mnt/%s.tar.gz' % outBaseName
cmd = ['cd %s;' % conf('dirs.clovr_project'),
'tar',
'-zcf',
outF,
'output_repository/*/%s_*' % pipelineId,
'workflow/runtime/*/%s_*' % pipelineId]
runSystemInstanceEx(instance, ' '.join(cmd), None, (log and errorPrintS or None), user=conf('ssh.user'), options=conf('ssh.options'), log=log)
outFilename = os.path.join(outDir, os.path.basename(outF))
fileExists = os.path.exists(outFilename)
if fileExists and overwrite or not fileExists:
runSystemEx('mkdir -p ' + outDir)
scpFromEx(instance['public_dns'], outF, outDir, user=conf('ssh.user'), options=conf('ssh.options'), log=log)
runSystemInstanceEx(instance, 'rm ' + outF, None, (log and errorPrintS or None), user=conf('ssh.user'), options=conf('ssh.options'), log=log)
return outFilename
else:
runSystemInstanceEx(instance, 'rm ' + outF, None, (log and errorPrintS or None), user=conf('ssh.user'), options=conf('ssh.options'), log=log)
raise DownloadPipelineOverwriteError(outFilename)
def makeDirsOnCluster(cluster, dirNames):
"""
Creates a series of directories on a cluster
"""
for d in dirNames:
runSystemInstanceEx(cluster['master'],
'mkdir -p ' + d,
None,
errorPrintS,
user=cluster['config']['ssh.user'],
options=cluster['config']['ssh.options'],
log=True)
try:
runSystemInstanceEx(cluster['master'],
'chown -R %s %s' % (cluster['config']['vappio.user'], d),
None,
errorPrintS,
user=cluster['config']['ssh.user'],
options=cluster['config']['ssh.options'],
log=True)
except ProgramRunError, err:
pass
def uploadTag(instance, conf, tagDir, tagName, srcFiles, outDir, log=False):
"""
This generates a tag (file list) from srcFiles and uploads the
tag to the tag directory.
instance - instance to upload to
conf - contains various config data, the following options need to be present:
ssh.options - options to pass to ssh
ssh.user - user to ssh as
tagDir - directory to put the tag in
tagName - name of the tag
srcFiles - list of files that the tag will be generated from
outDir - the directory the srcFiles were put into
"""
tempFName = os.path.join('/tmp', str(time.time()))
fout = open(tempFName, 'w')
for f in srcFiles:
fout.write(os.path.join(outDir, os.path.basename(f)) + '\n')
fout.close()
runSystemInstanceEx(instance, 'mkdir -p ' + tagDir, None, (log and errorPrintS or None), user=conf('ssh.user'), options=conf('ssh.options'), log=log)
scpToEx(instance['public_dns'], tempFName, os.path.join(tagDir, tagName), user=conf('ssh.user'), options=conf('ssh.options'), log=log)
os.remove(tempFName)
def makeDirsOnCluster(cluster, dirNames):
"""
Creates a series of directories on a cluster
"""
for d in dirNames:
runSystemInstanceEx(cluster['master'],
'mkdir -p ' + d,
None,
errorPrintS,
user=cluster['config']['ssh.user'],
options=cluster['config']['ssh.options'],
log=True)
try:
runSystemInstanceEx(cluster['master'],
'chown -R %s %s' %
(cluster['config']['vappio.user'], d),
None,
errorPrintS,
user=cluster['config']['ssh.user'],
options=cluster['config']['ssh.options'],
log=True)
except ProgramRunError, err:
pass
try:
makeDirsOnCluster(dstCluster, dirNames)
except TryError, err:
errorPrint('Caught TryError, ignoring for now: %s - %s ' % (err.msg, str(err.result)))
##
# Now, copy up all of the files
for l, d in dstFileNames:
scpToEx(dstCluster['master']['public_dns'], l, d, user=srcCluster['config']['ssh.user'], options=srcCluster['config']['ssh.options'], log=True)
##
# We are uploading as root, so chown everything to the user that everything in vappio will be done under
try:
runSystemInstanceEx(dstCluster['master'],
'chown %s %s' % (dstCluster['config']['vappio.user'], d),
None,
errorPrintS,
user=dstCluster['config']['ssh.user'],
options=dstCluster['config']['ssh.options'],
log=True)
except ProgramRunError:
errorPrint('Chown failed on ' + d)
##
# return the list of uploaded filenames
return [d for l, d in dstFileNames]
def partitionFiles(files, baseDir):
if baseDir:
baseDirFiles = [f.replace(baseDir, '')
for f in files
##
# Now, copy up all of the files
for l, d in dstFileNames:
scpToEx(dstCluster['master']['public_dns'],
l,
d,
user=srcCluster['config']['ssh.user'],
options=srcCluster['config']['ssh.options'],
log=True)
##
# We are uploading as root, so chown everything to the user that everything in vappio will be done under
try:
runSystemInstanceEx(dstCluster['master'],
'chown %s %s' %
(dstCluster['config']['vappio.user'], d),
None,
errorPrintS,
user=dstCluster['config']['ssh.user'],
options=dstCluster['config']['ssh.options'],
log=True)
except ProgramRunError:
errorPrint('Chown failed on ' + d)
##
# return the list of uploaded filenames
return [d for l, d in dstFileNames]
def partitionFiles(files, baseDir):
if baseDir:
baseDirFiles = [
f.replace(baseDir, '') for f in files if f.startswith(baseDir)
