def main(options, files):
metadata = dict([s.split('=', 1) for s in options('general.metadata')])
if options('general.tag_base_dir'):
metadata['tag_base_dir'] = options('general.tag_base_dir')
if options('general.append') and not options('general.overwrite'):
action = 'append'
elif not options('general.append') and options('general.overwrite'):
action = 'overwrite'
elif not options('general.append') and not options('general.overwrite'):
action = 'create'
else:
raise Exception('--append and --overwrite are mutually exclusive')
if options('general.stdin'):
files = [f.strip() for f in sys.stdin.readlines()]
tag = tagData(
options('general.host'), options('general.cluster'), action,
options('general.tag_name'), [_makeAbsolute(f) for f in files],
options('general.urls'), metadata, options('general.recursive'),
options('general.expand'),
options('general.compress')
and _makeAbsolute(options('general.compress')) or None)
if options('general.print_task_name'):
print tag['task_name']
else:
runTaskStatus(tag['task_name'], options('general.cluster'))
def main(options, args):
if not options('general.pipeline_resume'):
conf = config.configFromStream(open(options('general.pipeline_config')), lazy=True)
ret = pipeline.validatePipelineConfig(options('general.host'),
options('general.cluster'),
options('general.bare_run'),
conf)
if ret['errors']:
for e in ret['errors']:
print '\t'.join(['ERROR', ','.join(e['keys']), e['message']])
# Exit with an error
sys.exit(1)
if not options('general.validate'):
p = pipeline.runPipeline(options('general.host'),
options('general.cluster'),
options('general.pipeline_parent'),
options('general.bare_run'),
conf,
options('general.pipeline_queue'),
options('general.overwrite'))
if options('general.print_task_name'):
print p['task_name']
else:
runTaskStatus(p['task_name'])
else:
p = pipeline.resumePipeline(options('general.host'),
options('general.cluster'),
options('general.pipeline_resume'))
if options('general.print_task_name'):
print p['task_name']
else:
runTaskStatus(p['task_name'])
def main(options, files):
metadata = dict([s.split('=', 1) for s in options('general.metadata')])
if options('general.tag_base_dir'):
metadata['tag_base_dir'] = options('general.tag_base_dir')
if options('general.append') and not options('general.overwrite'):
action = 'append'
elif not options('general.append') and options('general.overwrite'):
action = 'overwrite'
elif not options('general.append') and not options('general.overwrite'):
action = 'create'
else:
raise Exception('--append and --overwrite are mutually exclusive')
if options('general.stdin'):
files = [f.strip() for f in sys.stdin.readlines()]
tag = tagData(options('general.host'),
options('general.cluster'),
action,
options('general.tag_name'),
[_makeAbsolute(f) for f in files],
options('general.urls'),
metadata,
options('general.recursive'),
options('general.expand'),
options('general.compress') and _makeAbsolute(options('general.compress')) or None)
if options('general.print_task_name'):
print tag['task_name']
else:
runTaskStatus(tag['task_name'], options('general.cluster'))
def main(options, _args):
taskName = addInstances(options('general.host'),
options('general.cluster'),
options('general.num_exec'), 0,
options('general.exec_type'))
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName, clusterName=options('general.cluster'))
def main(options, _args):
taskName = terminateCluster(options('general.host'),
options('general.cluster'),
options('general.auth_token'))
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName,
clusterName='local',
hostName=options('general.host'))
def main(options, _args):
taskName = terminateCluster(options('general.host'),
options('general.cluster'),
options('general.auth_token'))
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName,
clusterName='local',
hostName=options('general.host'))
def main(options, _args):
taskName = downloadPipelineOutput(options('general.host'),
options('general.name'),
options('general.pipeline'),
options('general.output_dir'),
options('general.overwrite'))
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName)
def main(options, _args):
taskName = addInstances(options('general.host'),
options('general.cluster'),
options('general.num_exec'),
0,
options('general.exec_type'))
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName, clusterName=options('general.cluster'))
def main(options, _args):
taskName = downloadPipelineOutput(options('general.host'),
options('general.name'),
options('general.pipeline'),
options('general.output_dir'),
options('general.overwrite'))
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName)
def main(options, _args):
taskName = importCluster(options('general.host'),
options('general.src_cluster'),
options('general.dst_cluster'),
options('general.cred'))
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName)
def main(options, files):
taskName = downloadTag(options('general.host'),
options('general.tag_name'),
options('general.src_cluster'),
options('general.dst_cluster'),
options('general.output_dir'),
options('general.expand'),
options('general.compress'))
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName)
def main(options, files):
taskName = downloadTag(options('general.host'),
options('general.tag_name'),
options('general.src_cluster'),
options('general.dst_cluster'),
options('general.output_dir'),
options('general.expand'),
options('general.compress'))
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName)
def main(options, _files):
if options('general.transfer_type') == 'cluster':
taskName = transferBetweenClusters(options)
elif options('general.transfer_type') == 's3':
if options('general.dst_cluster') != 'local':
raise Exception('S3 is only for pushing to S3 from a cluster')
taskName = transferToS3(options)
else:
raise Exception('Unknown transfer type: ' + options('general.transfer_type'))
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName)
def main(options, _args):
# Setup the cert if the user gave the options
if options('general.cert') and options('general.pkey'):
taskName = credential.saveCredential(
'localhost', 'local', options('general.cred_name'),
options('general.cred_desc'), options('general.ctype'),
(options('general.cert') and open(options('general.cert')).read()),
(options('general.pkey') and open(options('general.pkey')).read()),
dict([s.split('=', 1) for s in options('general.metadata')]))
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName)
def main(options, args):
tasklet = args[0]
conf = dict([v.split('=', 1) for v in options('general.config', default=[])])
if options('general.pipeline'):
if tasklet:
tasklet = 'get-pipeline-conf %s | %s | set-pipeline-conf %s' % (options('general.pipeline'), tasklet, options('general.pipeline'))
else:
tasklet = 'get-pipeline-conf %s | set-pipeline-conf %s' % (options('general.pipeline'), options('general.pipeline'))
taskName = pipeline.runTasklets(options('general.host'), options('general.cluster'), conf, tasklet)
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName)
def main(options, _files):
if options('general.transfer_type') == 'cluster':
taskName = transferBetweenClusters(options)
elif options('general.transfer_type') == 's3':
if options('general.dst_cluster') != 'local':
raise Exception('S3 is only for pushing to S3 from a cluster')
taskName = transferToS3(options)
else:
raise Exception('Unknown transfer type: ' +
options('general.transfer_type'))
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName)
def main(options, _args):
# Setup the cert if the user gave the options
if options('general.cert') and options('general.pkey'):
taskName = credential.saveCredential('localhost',
'local',
options('general.cred_name'),
options('general.cred_desc'),
options('general.ctype'),
(options('general.cert') and
open(options('general.cert')).read()),
(options('general.pkey') and
open(options('general.pkey')).read()),
dict([s.split('=', 1)
for s in options('general.metadata')]))
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName)
def main(options, _args):
conf = func.updateDict(dict([c.split('=') for c in options('general.config_options')]),
{'cluster.master_type': options('general.master_instance_type'),
'cluster.exec_type': options('general.exec_instance_type'),
'cluster.master_bid_price': options('general.master_bid_price'),
'cluster.exec_bid_price': options('general.exec_bid_price')})
taskName = startCluster(options('general.host'),
options('general.cluster'),
options('general.num_exec'),
0,
options('general.cred'),
conf)
if options('general.print_task_name'):
print taskName
else:
runTaskStatus(taskName)