def api_for_instance(instance_name): if not instance_name: # Use environment return arvados.api('v1', model=OrderedJsonModel()) if '/' in instance_name: config_file = instance_name else: config_file = os.path.join(os.environ['HOME'], '.config', 'arvados', "{}.conf".format(instance_name)) try: cfg = arvados.config.load(config_file) except (IOError, OSError) as e: abort(("Could not open config file {}: {}\n" + "You must make sure that your configuration tokens\n" + "for Arvados instance {} are in {} and that this\n" + "file is readable.").format( config_file, e, instance_name, config_file)) if 'ARVADOS_API_HOST' in cfg and 'ARVADOS_API_TOKEN' in cfg: api_is_insecure = ( cfg.get('ARVADOS_API_HOST_INSECURE', '').lower() in set( ['1', 't', 'true', 'y', 'yes'])) client = arvados.api('v1', host=cfg['ARVADOS_API_HOST'], token=cfg['ARVADOS_API_TOKEN'], insecure=api_is_insecure, model=OrderedJsonModel()) else: abort('need ARVADOS_API_HOST and ARVADOS_API_TOKEN for {}'.format(instance_name)) return client
def NewSummarizer(process_or_uuid, **kwargs): """Construct with the appropriate subclass for this uuid/object.""" if isinstance(process_or_uuid, dict): process = process_or_uuid uuid = process['uuid'] else: uuid = process_or_uuid process = None arv = arvados.api('v1', model=OrderedJsonModel()) if '-dz642-' in uuid: if process is None: process = arv.containers().get(uuid=uuid).execute() klass = ContainerTreeSummarizer elif '-xvhdp-' in uuid: if process is None: process = arv.container_requests().get(uuid=uuid).execute() klass = ContainerTreeSummarizer elif '-8i9sb-' in uuid: if process is None: process = arv.jobs().get(uuid=uuid).execute() klass = JobTreeSummarizer elif '-d1hrv-' in uuid: if process is None: process = arv.pipeline_instances().get(uuid=uuid).execute() klass = PipelineSummarizer elif '-4zz18-' in uuid: return CollectionSummarizer(collection_id=uuid) else: raise ArgumentError("Unrecognized uuid %s", uuid) return klass(process, uuid=uuid, **kwargs)
def main(args, stdout, stderr, api_client=None): parser = arg_parser() job_order_object = None arvargs = parser.parse_args(args) if arvargs.create_template and not arvargs.job_order: job_order_object = ({}, "") try: if api_client is None: api_client = arvados.api('v1', model=OrderedJsonModel()) runner = ArvCwlRunner(api_client, work_api=arvargs.work_api) except Exception as e: logger.error(e) return 1 arvargs.conformance_test = None arvargs.use_container = True return cwltool.main.main(args=arvargs, stdout=stdout, stderr=stderr, executor=runner.arvExecutor, makeTool=runner.arvMakeTool, versionfunc=versionstring, job_order_object=job_order_object, make_fs_access=partial(CollectionFsAccess, api_client=api_client))
def test_ordered_json_model(self): mock_responses = { 'arvados.humans.get': (None, json.dumps( collections.OrderedDict( (c, int(c, 16)) for c in string.hexdigits))), } req_builder = apiclient_http.RequestMockBuilder(mock_responses) api = arvados.api('v1', requestBuilder=req_builder, model=OrderedJsonModel()) result = api.humans().get(uuid='test').execute() self.assertEqual(string.hexdigits, ''.join(result.keys()))
def __init__(self, root, skip_child_jobs=False, **kwargs): arv = arvados.api('v1', model=OrderedJsonModel()) label = kwargs.pop('label', None) or root.get('name') or root['uuid'] root['name'] = label children = collections.OrderedDict() todo = collections.deque((root, )) while len(todo) > 0: current = todo.popleft() label = current['name'] sort_key = current['created_at'] if current['uuid'].find('-xvhdp-') > 0: current = arv.containers().get( uuid=current['container_uuid']).execute() summer = ContainerSummarizer(current, label=label, **kwargs) summer.sort_key = sort_key children[current['uuid']] = summer page_filters = [] while True: child_crs = arv.container_requests().index( order=['uuid asc'], filters=page_filters + [['requesting_container_uuid', '=', current['uuid']]], ).execute() if not child_crs['items']: break elif skip_child_jobs: logger.warning( '%s: omitting stats from %d child containers' ' because --skip-child-jobs flag is on', label, child_crs['items_available']) break page_filters = [['uuid', '>', child_crs['items'][-1]['uuid']]] for cr in child_crs['items']: if cr['container_uuid']: logger.debug('%s: container req %s', current['uuid'], cr['uuid']) cr['name'] = cr.get('name') or cr['uuid'] todo.append(cr) sorted_children = collections.OrderedDict() for uuid in sorted(list(children.keys()), key=lambda uuid: children[uuid].sort_key): sorted_children[uuid] = children[uuid] super(ContainerTreeSummarizer, self).__init__(children=sorted_children, label=root['name'], **kwargs)
def __init__(self, pipeline_instance_uuid, **kwargs): arv = arvados.api('v1', model=OrderedJsonModel()) instance = arv.pipeline_instances().get( uuid=pipeline_instance_uuid).execute() self.summarizers = collections.OrderedDict() for cname, component in instance['components'].iteritems(): if 'job' not in component: logger.warning("%s: skipping component with no job assigned", cname) else: logger.info("%s: job %s", cname, component['job']['uuid']) summarizer = JobSummarizer(component['job'], **kwargs) summarizer.label = '{} {}'.format(cname, component['job']['uuid']) self.summarizers[cname] = summarizer self.label = pipeline_instance_uuid
def test_ordered_json_model(self): mock_responses = { 'arvados.humans.get': (None, json.dumps( collections.OrderedDict( (c, int(c, 16)) for c in string.hexdigits))), } req_builder = apiclient_http.RequestMockBuilder(mock_responses) api = arvados.api('v1', host=os.environ['ARVADOS_API_HOST'], token='discovery-doc-only-no-token-needed', insecure=True, requestBuilder=req_builder, model=OrderedJsonModel()) result = api.humans().get(uuid='test').execute() self.assertEqual(string.hexdigits, ''.join(result.keys()))
def __init__(self, job, label=None, **kwargs): arv = arvados.api('v1', model=OrderedJsonModel()) label = label or job.get('name', job['uuid']) children = collections.OrderedDict() children[job['uuid']] = JobSummarizer(job, label=label, **kwargs) if job.get('components', None): preloaded = {} for j in arv.jobs().index( limit=len(job['components']), filters=[['uuid','in',job['components'].values()]]).execute()['items']: preloaded[j['uuid']] = j for cname in sorted(job['components'].keys()): child_uuid = job['components'][cname] j = (preloaded.get(child_uuid) or arv.jobs().get(uuid=child_uuid).execute()) children[child_uuid] = JobTreeSummarizer(job=j, label=cname, **kwargs) super(JobTreeSummarizer, self).__init__( children=children, label=label, **kwargs)
def main(args, stdout, stderr, api_client=None, keep_client=None): parser = arg_parser() job_order_object = None arvargs = parser.parse_args(args) if (arvargs.create_template or arvargs.create_workflow or arvargs.update_workflow) and not arvargs.job_order: job_order_object = ({}, "") add_arv_hints() try: if api_client is None: api_client=arvados.api('v1', model=OrderedJsonModel()) runner = ArvCwlRunner(api_client, work_api=arvargs.work_api, keep_client=keep_client, output_name=arvargs.output_name) except Exception as e: logger.error(e) return 1 if arvargs.debug: logger.setLevel(logging.DEBUG) if arvargs.quiet: logger.setLevel(logging.WARN) logging.getLogger('arvados.arv-run').setLevel(logging.WARN) if arvargs.metrics: metrics.setLevel(logging.DEBUG) logging.getLogger("cwltool.metrics").setLevel(logging.DEBUG) arvargs.conformance_test = None arvargs.use_container = True return cwltool.main.main(args=arvargs, stdout=stdout, stderr=stderr, executor=runner.arv_executor, makeTool=runner.arv_make_tool, versionfunc=versionstring, job_order_object=job_order_object, make_fs_access=partial(CollectionFsAccess, api_client=api_client))
def main(stdin, stdout, stderr, arglist): args = parse_arguments(arglist) prompter = Prompter(stdin if args.interactive else None, stdout, stderr) git_conf_dir = get_git_dir('--git-dir') git_work_dir = get_git_dir('--show-toplevel') pipeline_config = load_pipeline_config(git_conf_dir, args.dest) cluster = Cluster(args.dest, pipeline_config.items(args.dest)) if not os.path.exists(cluster.arv_conf_path): print("Error: No Arvados configuration at {}.".format( cluster.arv_conf_path), file=stderr) sys.exit(1) arvados.config.initialize(cluster.arv_conf_path) rel_template_path = pipeline_config.get_or_ask('DEFAULT', 'template_path', ask_template_path, prompter, stdout, git_work_dir) if cluster.template_uuid is None: cluster.template_uuid = prompter.ask( "Pipeline template UUID on " + args.dest, "No pipeline template UUID known on " + args.dest) setup_git(cluster, prompter) with open(os.path.join(git_work_dir, rel_template_path)) as pt_file: pipeline_template = json.load( pt_file, object_pairs_hook=collections.OrderedDict) subprocess.check_call(['git', 'push', cluster.git_push_url] + args.refspecs) arv = api_from_config('v1', model=OrderedJsonModel()) arv.pipeline_templates().update(uuid=cluster.template_uuid, body=pipeline_template).execute() config_dirty = False for key, value in cluster.config_changes(): pipeline_config.set(args.dest, key, value) config_dirty = True if config_dirty: pipeline_config.save()
def main(args, stdout, stderr, api_client=None): parser = arg_parser() job_order_object = None arvargs = parser.parse_args(args) if arvargs.create_template and not arvargs.job_order: job_order_object = ({}, "") try: if api_client is None: api_client = arvados.api('v1', model=OrderedJsonModel()) runner = ArvCwlRunner(api_client) except Exception as e: logger.error(e) return 1 return cwltool.main.main(args=arvargs, stdout=stdout, stderr=stderr, executor=runner.arvExecutor, makeTool=runner.arvMakeTool, versionfunc=versionstring, job_order_object=job_order_object)
def NewSummarizer(process_or_uuid, **kwargs): """Construct with the appropriate subclass for this uuid/object.""" if isinstance(process_or_uuid, dict): process = process_or_uuid uuid = process['uuid'] else: uuid = process_or_uuid process = None arv = arvados.api('v1', model=OrderedJsonModel()) if '-dz642-' in uuid: if process is None: # Get the associated CR. Doesn't matter which since they all have the same logs crs = arv.container_requests().list( filters=[['container_uuid', '=', uuid]], limit=1).execute()['items'] if len(crs) > 0: process = crs[0] klass = ContainerRequestTreeSummarizer elif '-xvhdp-' in uuid: if process is None: process = arv.container_requests().get(uuid=uuid).execute() klass = ContainerRequestTreeSummarizer elif '-8i9sb-' in uuid: if process is None: process = arv.jobs().get(uuid=uuid).execute() klass = JobTreeSummarizer elif '-d1hrv-' in uuid: if process is None: process = arv.pipeline_instances().get(uuid=uuid).execute() klass = PipelineSummarizer elif '-4zz18-' in uuid: return CollectionSummarizer(collection_id=uuid) else: raise ArgumentError("Unrecognized uuid %s", uuid) return klass(process, uuid=uuid, **kwargs)
def main(args, stdout, stderr, api_client=None): args.insert(0, "--leave-outputs") parser = cwltool.main.arg_parser() exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--enable-reuse", action="store_true", default=True, dest="enable_reuse", help="") exgroup.add_argument("--disable-reuse", action="store_false", default=True, dest="enable_reuse", help="") parser.add_argument("--project-uuid", type=str, help="Project that will own the workflow jobs") parser.add_argument( "--ignore-docker-for-reuse", action="store_true", help= "Ignore Docker image version when deciding whether to reuse past jobs.", default=False) exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--submit", action="store_true", help="Submit workflow to run on Arvados.", default=True, dest="submit") exgroup.add_argument( "--local", action="store_false", help="Run workflow on local host (submits jobs to Arvados).", default=True, dest="submit") exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument( "--wait", action="store_true", help="After submitting workflow runner job, wait for completion.", default=True, dest="wait") exgroup.add_argument("--no-wait", action="store_false", help="Submit workflow runner job and exit.", default=True, dest="wait") try: if api_client is None: api_client = arvados.api('v1', model=OrderedJsonModel()) runner = ArvCwlRunner(api_client) except Exception as e: logger.error(e) return 1 return cwltool.main.main(args, stdout=stdout, stderr=stderr, executor=runner.arvExecutor, makeTool=runner.arvMakeTool, parser=parser, versionfunc=versionstring)
def main(args, stdout, stderr, api_client=None, keep_client=None): parser = arg_parser() job_order_object = None arvargs = parser.parse_args(args) if arvargs.version: print versionstring() return if arvargs.update_workflow: if arvargs.update_workflow.find('-7fd4e-') == 5: want_api = 'containers' elif arvargs.update_workflow.find('-p5p6p-') == 5: want_api = 'jobs' else: want_api = None if want_api and arvargs.work_api and want_api != arvargs.work_api: logger.error( '--update-workflow arg {!r} uses {!r} API, but --api={!r} specified' .format(arvargs.update_workflow, want_api, arvargs.work_api)) return 1 arvargs.work_api = want_api if (arvargs.create_workflow or arvargs.update_workflow) and not arvargs.job_order: job_order_object = ({}, "") add_arv_hints() try: if api_client is None: api_client = arvados.api('v1', model=OrderedJsonModel()) if keep_client is None: keep_client = arvados.keep.KeepClient(api_client=api_client, num_retries=4) runner = ArvCwlRunner(api_client, work_api=arvargs.work_api, keep_client=keep_client, num_retries=4, output_name=arvargs.output_name, output_tags=arvargs.output_tags) except Exception as e: logger.error(e) return 1 if arvargs.debug: logger.setLevel(logging.DEBUG) logging.getLogger('arvados').setLevel(logging.DEBUG) if arvargs.quiet: logger.setLevel(logging.WARN) logging.getLogger('arvados').setLevel(logging.WARN) logging.getLogger('arvados.arv-run').setLevel(logging.WARN) if arvargs.metrics: metrics.setLevel(logging.DEBUG) logging.getLogger("cwltool.metrics").setLevel(logging.DEBUG) if arvargs.log_timestamps: arvados.log_handler.setFormatter( logging.Formatter( '%(asctime)s %(name)s %(levelname)s: %(message)s', '%Y-%m-%d %H:%M:%S')) else: arvados.log_handler.setFormatter( logging.Formatter('%(name)s %(levelname)s: %(message)s')) arvargs.conformance_test = None arvargs.use_container = True arvargs.relax_path_checks = True arvargs.validate = None return cwltool.main.main(args=arvargs, stdout=stdout, stderr=stderr, executor=runner.arv_executor, makeTool=runner.arv_make_tool, versionfunc=versionstring, job_order_object=job_order_object, make_fs_access=partial(CollectionFsAccess, api_client=api_client, keep_client=keep_client), fetcher_constructor=partial( CollectionFetcher, api_client=api_client, keep_client=keep_client, num_retries=runner.num_retries), resolver=partial(collectionResolver, api_client, num_retries=runner.num_retries), logger_handler=arvados.log_handler)
if not args.destination: if len(args.uuids) < 2: parser.error("missing destination project UUID argument") args.destination = args.uuids.pop() return args def setup_logging(args): log_handler = logging.StreamHandler() log_handler.setFormatter( logging.Formatter( '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s', '%Y-%m-%d %H:%M:%S')) logger.addHandler(log_handler) logger.setLevel(max(1, logging.WARNING - (10 * args.verbose))) def main(stdin, stdout, stderr, arglist, arv): args = parse_arguments(arglist) setup_logging(args) uuid_mapper = UUIDMapper(arv) dependencies = DependencyTracker(uuid_mapper, logger) for uuid in args.uuids: dependencies.add_object(uuid, arv) dependencies.move_to(args.destination, arv, args.request_handler) if __name__ == '__main__': main(sys.stdin, sys.stdout, sys.stderr, sys.argv[1:], arvados.api('v1', model=OrderedJsonModel()))
def main(args, stdout, stderr, api_client=None, keep_client=None, install_sig_handlers=True): parser = arg_parser() job_order_object = None arvargs = parser.parse_args(args) if len(arvargs.storage_classes.strip().split(',')) > 1: logger.error(str(u"Multiple storage classes are not supported currently.")) return 1 arvargs.use_container = True arvargs.relax_path_checks = True arvargs.print_supported_versions = False if install_sig_handlers: arv_cmd.install_signal_handlers() if arvargs.update_workflow: if arvargs.update_workflow.find('-7fd4e-') == 5: want_api = 'containers' else: want_api = None if want_api and arvargs.work_api and want_api != arvargs.work_api: logger.error(str(u'--update-workflow arg {!r} uses {!r} API, but --api={!r} specified').format( arvargs.update_workflow, want_api, arvargs.work_api)) return 1 arvargs.work_api = want_api if (arvargs.create_workflow or arvargs.update_workflow) and not arvargs.job_order: job_order_object = ({}, "") add_arv_hints() for key, val in viewitems(cwltool.argparser.get_default_args()): if not hasattr(arvargs, key): setattr(arvargs, key, val) try: if api_client is None: api_client = arvados.safeapi.ThreadSafeApiCache( api_params={"model": OrderedJsonModel(), "timeout": arvargs.http_timeout}, keep_params={"num_retries": 4}) keep_client = api_client.keep # Make an API object now so errors are reported early. api_client.users().current().execute() if keep_client is None: keep_client = arvados.keep.KeepClient(api_client=api_client, num_retries=4) executor = ArvCwlExecutor(api_client, arvargs, keep_client=keep_client, num_retries=4) except Exception: logger.exception("Error creating the Arvados CWL Executor") return 1 # Note that unless in debug mode, some stack traces related to user # workflow errors may be suppressed. if arvargs.debug: logger.setLevel(logging.DEBUG) logging.getLogger('arvados').setLevel(logging.DEBUG) if arvargs.quiet: logger.setLevel(logging.WARN) logging.getLogger('arvados').setLevel(logging.WARN) logging.getLogger('arvados.arv-run').setLevel(logging.WARN) if arvargs.metrics: metrics.setLevel(logging.DEBUG) logging.getLogger("cwltool.metrics").setLevel(logging.DEBUG) if arvargs.log_timestamps: arvados.log_handler.setFormatter(logging.Formatter( '%(asctime)s %(name)s %(levelname)s: %(message)s', '%Y-%m-%d %H:%M:%S')) else: arvados.log_handler.setFormatter(logging.Formatter('%(name)s %(levelname)s: %(message)s')) if stdout is sys.stdout: # cwltool.main has code to work around encoding issues with # sys.stdout and unix pipes (they default to ASCII encoding, # we want utf-8), so when stdout is sys.stdout set it to None # to take advantage of that. Don't override it for all cases # since we still want to be able to capture stdout for the # unit tests. stdout = None return cwltool.main.main(args=arvargs, stdout=stdout, stderr=stderr, executor=executor.arv_executor, versionfunc=versionstring, job_order_object=job_order_object, logger_handler=arvados.log_handler, custom_schema_callback=add_arv_hints, loadingContext=executor.loadingContext, runtimeContext=executor.runtimeContext, input_required=not (arvargs.create_workflow or arvargs.update_workflow))
def run(): # Timestamps are added by crunch-job, so don't print redundant timestamps. arvados.log_handler.setFormatter(logging.Formatter('%(name)s %(levelname)s: %(message)s')) # Print package versions logger.info(arvados_cwl.versionstring()) api = arvados.api("v1") arvados_cwl.add_arv_hints() runner = None try: job_order_object = arvados.current_job()['script_parameters'] toolpath = "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], job_order_object.pop("cwl:tool")) pdh_path = re.compile(r'^[0-9a-f]{32}\+\d+(/.+)?$') def keeppath(v): if pdh_path.match(v): return "keep:%s" % v else: return v def keeppathObj(v): if "location" in v: v["location"] = keeppath(v["location"]) for k,v in job_order_object.items(): if isinstance(v, basestring) and arvados.util.keep_locator_pattern.match(v): job_order_object[k] = { "class": "File", "location": "keep:%s" % v } adjustFileObjs(job_order_object, keeppathObj) adjustDirObjs(job_order_object, keeppathObj) normalizeFilesDirs(job_order_object) output_name = None output_tags = None enable_reuse = True on_error = "continue" debug = False if "arv:output_name" in job_order_object: output_name = job_order_object["arv:output_name"] del job_order_object["arv:output_name"] if "arv:output_tags" in job_order_object: output_tags = job_order_object["arv:output_tags"] del job_order_object["arv:output_tags"] if "arv:enable_reuse" in job_order_object: enable_reuse = job_order_object["arv:enable_reuse"] del job_order_object["arv:enable_reuse"] if "arv:on_error" in job_order_object: on_error = job_order_object["arv:on_error"] del job_order_object["arv:on_error"] if "arv:debug" in job_order_object: debug = job_order_object["arv:debug"] del job_order_object["arv:debug"] runner = arvados_cwl.ArvCwlRunner(api_client=arvados.safeapi.ThreadSafeApiCache( api_params={"model": OrderedJsonModel()}, keep_params={"num_retries": 4}), output_name=output_name, output_tags=output_tags) make_fs_access = functools.partial(CollectionFsAccess, collection_cache=runner.collection_cache) t = load_tool(toolpath, runner.arv_make_tool, fetcher_constructor=functools.partial(CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""), num_retries=runner.num_retries)) if debug: logger.setLevel(logging.DEBUG) logging.getLogger('arvados').setLevel(logging.DEBUG) logging.getLogger("cwltool").setLevel(logging.DEBUG) args = argparse.Namespace() args.project_uuid = arvados.current_job()["owner_uuid"] args.enable_reuse = enable_reuse args.on_error = on_error args.submit = False args.debug = debug args.quiet = False args.ignore_docker_for_reuse = False args.basedir = os.getcwd() args.name = None args.cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]} args.make_fs_access = make_fs_access args.trash_intermediate = False args.intermediate_output_ttl = 0 args.priority = arvados_cwl.DEFAULT_PRIORITY args.do_validate = True args.disable_js_validation = False runner.arv_executor(t, job_order_object, **vars(args)) except Exception as e: if isinstance(e, WorkflowException): logging.info("Workflow error %s", e) else: logging.exception("Unhandled exception") if runner and runner.final_output_collection: outputCollection = runner.final_output_collection.portable_data_hash() else: outputCollection = None api.job_tasks().update(uuid=arvados.current_task()['uuid'], body={ 'output': outputCollection, 'success': False, 'progress':1.0 }).execute()
def run(): # Print package versions logger.info(arvados_cwl.versionstring()) api = arvados.api("v1") arvados_cwl.add_arv_hints() runner = None try: job_order_object = arvados.current_job()['script_parameters'] pdh_path = re.compile(r'^[0-9a-f]{32}\+\d+(/.+)?$') def keeppath(v): if pdh_path.match(v): return "keep:%s" % v else: return v def keeppathObj(v): v["location"] = keeppath(v["location"]) job_order_object["cwl:tool"] = "file://%s/%s" % ( os.environ['TASK_KEEPMOUNT'], job_order_object["cwl:tool"]) for k, v in job_order_object.items(): if isinstance( v, basestring) and arvados.util.keep_locator_pattern.match(v): job_order_object[k] = { "class": "File", "location": "keep:%s" % v } adjustFileObjs(job_order_object, keeppathObj) adjustDirObjs(job_order_object, keeppathObj) normalizeFilesDirs(job_order_object) adjustDirObjs( job_order_object, functools.partial( getListing, arvados_cwl.fsaccess.CollectionFsAccess("", api_client=api))) output_name = None output_tags = None enable_reuse = True if "arv:output_name" in job_order_object: output_name = job_order_object["arv:output_name"] del job_order_object["arv:output_name"] if "arv:output_tags" in job_order_object: output_tags = job_order_object["arv:output_tags"] del job_order_object["arv:output_tags"] if "arv:enable_reuse" in job_order_object: enable_reuse = job_order_object["arv:enable_reuse"] del job_order_object["arv:enable_reuse"] runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api( 'v1', model=OrderedJsonModel()), output_name=output_name, output_tags=output_tags) t = load_tool(job_order_object, runner.arv_make_tool) args = argparse.Namespace() args.project_uuid = arvados.current_job()["owner_uuid"] args.enable_reuse = enable_reuse args.submit = False args.debug = True args.quiet = False args.ignore_docker_for_reuse = False args.basedir = os.getcwd() args.cwl_runner_job = { "uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"] } outputObj = runner.arv_executor(t, job_order_object, **vars(args)) except Exception as e: if isinstance(e, WorkflowException): logging.info("Workflow error %s", e) else: logging.exception("Unhandled exception") if runner and runner.final_output_collection: outputCollection = runner.final_output_collection.portable_data_hash( ) else: outputCollection = None api.job_tasks().update(uuid=arvados.current_task()['uuid'], body={ 'output': outputCollection, 'success': False, 'progress': 1.0 }).execute()
def main(args, stdout, stderr, api_client=None, keep_client=None, install_sig_handlers=True): parser = arg_parser() job_order_object = None arvargs = parser.parse_args(args) if len(arvargs.storage_classes.strip().split(',')) > 1: logger.error( str(u"Multiple storage classes are not supported currently.")) return 1 arvargs.use_container = True arvargs.relax_path_checks = True arvargs.print_supported_versions = False if install_sig_handlers: arv_cmd.install_signal_handlers() if arvargs.update_workflow: if arvargs.update_workflow.find('-7fd4e-') == 5: want_api = 'containers' elif arvargs.update_workflow.find('-p5p6p-') == 5: want_api = 'jobs' else: want_api = None if want_api and arvargs.work_api and want_api != arvargs.work_api: logger.error( str(u'--update-workflow arg {!r} uses {!r} API, but --api={!r} specified' ).format(arvargs.update_workflow, want_api, arvargs.work_api)) return 1 arvargs.work_api = want_api if (arvargs.create_workflow or arvargs.update_workflow) and not arvargs.job_order: job_order_object = ({}, "") add_arv_hints() for key, val in viewitems(cwltool.argparser.get_default_args()): if not hasattr(arvargs, key): setattr(arvargs, key, val) try: if api_client is None: api_client = arvados.safeapi.ThreadSafeApiCache( api_params={ "model": OrderedJsonModel(), "timeout": arvargs.http_timeout }, keep_params={"num_retries": 4}) keep_client = api_client.keep # Make an API object now so errors are reported early. api_client.users().current().execute() if keep_client is None: keep_client = arvados.keep.KeepClient(api_client=api_client, num_retries=4) executor = ArvCwlExecutor(api_client, arvargs, keep_client=keep_client, num_retries=4) except Exception as e: logger.error(e) return 1 if arvargs.debug: logger.setLevel(logging.DEBUG) logging.getLogger('arvados').setLevel(logging.DEBUG) if arvargs.quiet: logger.setLevel(logging.WARN) logging.getLogger('arvados').setLevel(logging.WARN) logging.getLogger('arvados.arv-run').setLevel(logging.WARN) if arvargs.metrics: metrics.setLevel(logging.DEBUG) logging.getLogger("cwltool.metrics").setLevel(logging.DEBUG) if arvargs.log_timestamps: arvados.log_handler.setFormatter( logging.Formatter( '%(asctime)s %(name)s %(levelname)s: %(message)s', '%Y-%m-%d %H:%M:%S')) else: arvados.log_handler.setFormatter( logging.Formatter('%(name)s %(levelname)s: %(message)s')) return cwltool.main.main(args=arvargs, stdout=stdout, stderr=stderr, executor=executor.arv_executor, versionfunc=versionstring, job_order_object=job_order_object, logger_handler=arvados.log_handler, custom_schema_callback=add_arv_hints, loadingContext=executor.loadingContext, runtimeContext=executor.runtimeContext)