def main(options, files): metadata = dict([s.split('=', 1) for s in options('general.metadata')]) if options('general.tag_base_dir'): metadata['tag_base_dir'] = options('general.tag_base_dir') if options('general.append') and not options('general.overwrite'): action = 'append' elif not options('general.append') and options('general.overwrite'): action = 'overwrite' elif not options('general.append') and not options('general.overwrite'): action = 'create' else: raise Exception('--append and --overwrite are mutually exclusive') if options('general.stdin'): files = [f.strip() for f in sys.stdin.readlines()] tag = tagData( options('general.host'), options('general.cluster'), action, options('general.tag_name'), [_makeAbsolute(f) for f in files], options('general.urls'), metadata, options('general.recursive'), options('general.expand'), options('general.compress') and _makeAbsolute(options('general.compress')) or None) if options('general.print_task_name'): print tag['task_name'] else: runTaskStatus(tag['task_name'], options('general.cluster'))
def main(options, args): if not options('general.pipeline_resume'): conf = config.configFromStream(open(options('general.pipeline_config')), lazy=True) ret = pipeline.validatePipelineConfig(options('general.host'), options('general.cluster'), options('general.bare_run'), conf) if ret['errors']: for e in ret['errors']: print '\t'.join(['ERROR', ','.join(e['keys']), e['message']]) # Exit with an error sys.exit(1) if not options('general.validate'): p = pipeline.runPipeline(options('general.host'), options('general.cluster'), options('general.pipeline_parent'), options('general.bare_run'), conf, options('general.pipeline_queue'), options('general.overwrite')) if options('general.print_task_name'): print p['task_name'] else: runTaskStatus(p['task_name']) else: p = pipeline.resumePipeline(options('general.host'), options('general.cluster'), options('general.pipeline_resume')) if options('general.print_task_name'): print p['task_name'] else: runTaskStatus(p['task_name'])
def main(options, files): metadata = dict([s.split('=', 1) for s in options('general.metadata')]) if options('general.tag_base_dir'): metadata['tag_base_dir'] = options('general.tag_base_dir') if options('general.append') and not options('general.overwrite'): action = 'append' elif not options('general.append') and options('general.overwrite'): action = 'overwrite' elif not options('general.append') and not options('general.overwrite'): action = 'create' else: raise Exception('--append and --overwrite are mutually exclusive') if options('general.stdin'): files = [f.strip() for f in sys.stdin.readlines()] tag = tagData(options('general.host'), options('general.cluster'), action, options('general.tag_name'), [_makeAbsolute(f) for f in files], options('general.urls'), metadata, options('general.recursive'), options('general.expand'), options('general.compress') and _makeAbsolute(options('general.compress')) or None) if options('general.print_task_name'): print tag['task_name'] else: runTaskStatus(tag['task_name'], options('general.cluster'))
def main(options, _args): taskName = addInstances(options('general.host'), options('general.cluster'), options('general.num_exec'), 0, options('general.exec_type')) if options('general.print_task_name'): print taskName else: runTaskStatus(taskName, clusterName=options('general.cluster'))
def main(options, _args): taskName = terminateCluster(options('general.host'), options('general.cluster'), options('general.auth_token')) if options('general.print_task_name'): print taskName else: runTaskStatus(taskName, clusterName='local', hostName=options('general.host'))
def main(options, _args): taskName = downloadPipelineOutput(options('general.host'), options('general.name'), options('general.pipeline'), options('general.output_dir'), options('general.overwrite')) if options('general.print_task_name'): print taskName else: runTaskStatus(taskName)
def main(options, _args): taskName = importCluster(options('general.host'), options('general.src_cluster'), options('general.dst_cluster'), options('general.cred')) if options('general.print_task_name'): print taskName else: runTaskStatus(taskName)
def main(options, files): taskName = downloadTag(options('general.host'), options('general.tag_name'), options('general.src_cluster'), options('general.dst_cluster'), options('general.output_dir'), options('general.expand'), options('general.compress')) if options('general.print_task_name'): print taskName else: runTaskStatus(taskName)
def main(options, _files): if options('general.transfer_type') == 'cluster': taskName = transferBetweenClusters(options) elif options('general.transfer_type') == 's3': if options('general.dst_cluster') != 'local': raise Exception('S3 is only for pushing to S3 from a cluster') taskName = transferToS3(options) else: raise Exception('Unknown transfer type: ' + options('general.transfer_type')) if options('general.print_task_name'): print taskName else: runTaskStatus(taskName)
def main(options, _args): # Setup the cert if the user gave the options if options('general.cert') and options('general.pkey'): taskName = credential.saveCredential( 'localhost', 'local', options('general.cred_name'), options('general.cred_desc'), options('general.ctype'), (options('general.cert') and open(options('general.cert')).read()), (options('general.pkey') and open(options('general.pkey')).read()), dict([s.split('=', 1) for s in options('general.metadata')])) if options('general.print_task_name'): print taskName else: runTaskStatus(taskName)
def main(options, args): tasklet = args[0] conf = dict([v.split('=', 1) for v in options('general.config', default=[])]) if options('general.pipeline'): if tasklet: tasklet = 'get-pipeline-conf %s | %s | set-pipeline-conf %s' % (options('general.pipeline'), tasklet, options('general.pipeline')) else: tasklet = 'get-pipeline-conf %s | set-pipeline-conf %s' % (options('general.pipeline'), options('general.pipeline')) taskName = pipeline.runTasklets(options('general.host'), options('general.cluster'), conf, tasklet) if options('general.print_task_name'): print taskName else: runTaskStatus(taskName)
def main(options, _args): # Setup the cert if the user gave the options if options('general.cert') and options('general.pkey'): taskName = credential.saveCredential('localhost', 'local', options('general.cred_name'), options('general.cred_desc'), options('general.ctype'), (options('general.cert') and open(options('general.cert')).read()), (options('general.pkey') and open(options('general.pkey')).read()), dict([s.split('=', 1) for s in options('general.metadata')])) if options('general.print_task_name'): print taskName else: runTaskStatus(taskName)
def main(options, _args): conf = func.updateDict(dict([c.split('=') for c in options('general.config_options')]), {'cluster.master_type': options('general.master_instance_type'), 'cluster.exec_type': options('general.exec_instance_type'), 'cluster.master_bid_price': options('general.master_bid_price'), 'cluster.exec_bid_price': options('general.exec_bid_price')}) taskName = startCluster(options('general.host'), options('general.cluster'), options('general.num_exec'), 0, options('general.cred'), conf) if options('general.print_task_name'): print taskName else: runTaskStatus(taskName)