def fix_type_error(exc_info, callable, varargs, kwargs): """ Given an exception, this will test if the exception was due to a signature error, and annotate the error with better information if so. Usage:: try: val = callable(*args, **kw) except TypeError: exc_info = fix_type_error(None, callable, args, kw) raise exc_info[0], exc_info[1], exc_info[2] """ if exc_info is None: exc_info = sys.exc_info() if (exc_info[0] != TypeError or str(exc_info[1]).find('argument') == -1 or getattr(exc_info[1], '_type_error_fixed', False)): return exc_info exc_info[1]._type_error_fixed = True argspec = inspect.formatargspec(*getfullargspec(callable)) args = ', '.join(map(_short_repr, varargs)) if kwargs and args: args += ', ' if kwargs: kwargs = sorted(kwargs.keys()) args += ', '.join(f'{n}=...' for n in kwargs) gotspec = f'({args})' msg = f'{exc_info[1]}; got {gotspec}, wanted {argspec}' exc_info[1].args = (msg,) return exc_info
def __extract_payload_from_request(trans, func, kwargs): content_type = trans.request.headers.get('content-type', '') if content_type.startswith( 'application/x-www-form-urlencoded') or content_type.startswith( 'multipart/form-data'): # If the content type is a standard type such as multipart/form-data, the wsgi framework parses the request body # and loads all field values into kwargs. However, kwargs also contains formal method parameters etc. which # are not a part of the request body. This is a problem because it's not possible to differentiate between values # which are a part of the request body, and therefore should be a part of the payload, and values which should not be # in the payload. Therefore, the decorated method's formal arguments are discovered through reflection and removed from # the payload dictionary. This helps to prevent duplicate argument conflicts in downstream methods. payload = kwargs.copy() named_args = getfullargspec(func).args for arg in named_args: payload.pop(arg, None) for k, v in payload.items(): if isinstance(v, str): try: # note: parse_non_hex_float only needed here for single string values where something like # 40000000000000e5 will be parsed as a scientific notation float. This is as opposed to hex strings # in larger JSON structures where quoting prevents this (further below) payload[k] = loads(v, parse_float=parse_non_hex_float) except Exception: # may not actually be json, just continue pass else: # Assume application/json content type and parse request body manually, since wsgi won't do it. However, the order of this check # should ideally be in reverse, with the if clause being a check for application/json and the else clause assuming a standard encoding # such as multipart/form-data. Leaving it as is for backward compatibility, just in case. payload = loads(unicodify(trans.request.body)) run_as = trans.request.headers.get('run-as') if run_as: payload['run_as'] = run_as return payload
def lint_tool_source_with(lint_context, tool_source, extra_modules=[]): import galaxy.tool_util.linters tool_xml = getattr(tool_source, "xml_tree", None) linter_modules = submodules.import_submodules(galaxy.tool_util.linters, ordered=True) linter_modules.extend(extra_modules) for module in linter_modules: tool_type = tool_source.parse_tool_type() or "default" lint_tool_types = getattr(module, "lint_tool_types", ["default"]) if not ("*" in lint_tool_types or tool_type in lint_tool_types): continue for (name, value) in inspect.getmembers(module): if callable(value) and name.startswith("lint_"): # Look at the first argument to the linter to decide # if we should lint the XML description or the abstract # tool parser object. first_arg = getfullargspec(value).args[0] if first_arg == "tool_xml": if tool_xml is None: # XML linter and non-XML tool, skip for now continue else: lint_context.lint(name, value, tool_xml) else: lint_context.lint(name, value, tool_source)
def __invoke_expand_function(self, expand_function, destination): function_arg_names = getfullargspec(expand_function).args app = self.job_wrapper.app possible_args = { "job_id": self.job_wrapper.job_id, "tool": self.job_wrapper.tool, "tool_id": self.job_wrapper.tool.id, "job_wrapper": self.job_wrapper, "rule_helper": RuleHelper(app), "app": app, "referrer": destination } actual_args = {} # Send through any job_conf.xml defined args to function for destination_param in destination.params.keys(): if destination_param in function_arg_names: actual_args[destination_param] = destination.params[destination_param] # Populate needed args for possible_arg_name in possible_args: if possible_arg_name in function_arg_names: actual_args[possible_arg_name] = possible_args[possible_arg_name] # Don't hit the DB to load the job object if not needed require_db = False for param in ["job", "user", "user_email", "resource_params", "workflow_invocation_uuid"]: if param in function_arg_names: require_db = True break if require_db: job = self.job_wrapper.get_job() user = job.user user_email = user and str(user.email) if "job" in function_arg_names: actual_args["job"] = job if "user" in function_arg_names: actual_args["user"] = user if "user_email" in function_arg_names: actual_args["user_email"] = user_email if "resource_params" in function_arg_names: actual_args["resource_params"] = self.job_wrapper.get_resource_parameters(job) if "workflow_invocation_uuid" in function_arg_names: param_values = job.raw_param_dict() workflow_invocation_uuid = param_values.get("__workflow_invocation_uuid__", None) actual_args["workflow_invocation_uuid"] = workflow_invocation_uuid if "workflow_resource_params" in function_arg_names: param_values = job.raw_param_dict() workflow_resource_params = param_values.get("__workflow_resource_params__", None) actual_args["workflow_resource_params"] = workflow_resource_params return expand_function(**actual_args)
def verify_assertion(data, assertion_description): tag = assertion_description["tag"] assert_function_name = "assert_" + tag assert_function = None for assertion_module in assertion_modules: if hasattr(assertion_module, assert_function_name): assert_function = getattr(assertion_module, assert_function_name) if assert_function is None: errmsg = "Unable to find test function associated with XML tag '%s'. Check your tool file syntax." % tag raise AssertionError(errmsg) assert_function_args = getfullargspec(assert_function).args args = {} for attribute, value in assertion_description["attributes"].items(): if attribute in assert_function_args: args[attribute] = value # Three special arguments automatically populated independently of # tool XML attributes. output is passed in as the contents of the # output file. verify_assertions_function is passed in as the # verify_assertions function defined above, this allows # recursively checking assertions on subsections of # output. children is the parsed version of the child elements of # the XML element describing this assertion. See # assert_element_text in test/base/asserts/xml.py as an example of # how to use verify_assertions_function and children in conjuction # to apply assertion checking to a subset of the input. The parsed # version of an elements child elements do not need to just define # assertions, developers of assertion functions can also use the # child elements in novel ways to define inputs the assertion # checking function (for instance consider the following fictional # assertion function for checking column titles of tabular output # - <has_column_titles><with_name name="sequence"><with_name # name="probability"></has_column_titles>.) if "output" in assert_function_args: # If the assert_function will have an attribute called "output" # the data passed from the test to the function will be unicodified. # This is because most of the assert functions are working on pure # text files. args["output"] = unicodify(data) if "output_bytes" in assert_function_args: # This will read in data as bytes and will not change it prior passing # it to the assert_function args["output_bytes"] = data if "verify_assertions_function" in assert_function_args: args["verify_assertions_function"] = verify_assertions if "children" in assert_function_args: args["children"] = assertion_description["children"] # TODO: Verify all needed function arguments are specified. assert_function(**args)
def do_merge(job_wrapper, task_wrappers): parallel_settings = job_wrapper.get_parallelism().attributes # Syntax: merge_outputs="export" pickone_outputs="genomesize" # Designates outputs to be merged, or selected from as a representative merge_outputs = parallel_settings.get("merge_outputs") if merge_outputs is None: merge_outputs = [] else: merge_outputs = [x.strip() for x in merge_outputs.split(",")] pickone_outputs = parallel_settings.get("pickone_outputs") if pickone_outputs is None: pickone_outputs = [] else: pickone_outputs = [x.strip() for x in pickone_outputs.split(",")] illegal_outputs = [x for x in merge_outputs if x in pickone_outputs] if len(illegal_outputs) > 0: return ('Tool file error', 'Outputs have conflicting parallelism attributes: %s' % str(illegal_outputs)) stdout = '' stderr = '' try: working_directory = job_wrapper.working_directory task_dirs = [ os.path.join(working_directory, x) for x in os.listdir(working_directory) if x.startswith('task_') ] assert task_dirs, "Should be at least one sub-task!" # TODO: Output datasets can be very complex. This doesn't handle metadata files outputs = job_wrapper.get_output_hdas_and_fnames() output_paths = job_wrapper.get_output_fnames() pickone_done = [] task_dirs = [ os.path.join(working_directory, x) for x in os.listdir(working_directory) if x.startswith('task_') ] task_dirs.sort(key=lambda x: int(x.split('task_')[-1])) for index, output in enumerate(outputs): output_file_name = str( output_paths[index]) # Use false_path if set, else real path. base_output_name = os.path.basename(output_file_name) if output in merge_outputs: output_dataset = outputs[output][0] output_type = output_dataset.datatype output_files = [ os.path.join(dir, base_output_name) for dir in task_dirs ] # Just include those files f in the output list for which the # file f exists; some files may not exist if a task fails. output_files = [f for f in output_files if os.path.exists(f)] if output_files: log.debug('files %s ' % output_files) if len(output_files) < len(task_dirs): log.debug( 'merging only %i out of expected %i files for %s' % (len(output_files), len(task_dirs), output_file_name)) # First two args to merge always output_files and path of dataset. More # complicated merge methods may require more parameters. Set those up here. extra_merge_arg_names = getfullargspec( output_type.merge).args[2:] extra_merge_args = {} if "output_dataset" in extra_merge_arg_names: extra_merge_args["output_dataset"] = output_dataset output_type.merge(output_files, output_file_name, **extra_merge_args) log.debug('merge finished: %s' % output_file_name) else: msg = 'nothing to merge for %s (expected %i files)' \ % (output_file_name, len(task_dirs)) log.debug(msg) stderr += msg + "\n" elif output in pickone_outputs: # just pick one of them if output not in pickone_done: task_file_name = os.path.join(task_dirs[0], base_output_name) shutil.move(task_file_name, output_file_name) pickone_done.append(output) else: log_error = "The output '%s' does not define a method for implementing parallelism" % output log.exception(log_error) raise Exception(log_error) except Exception as e: stdout = 'Error merging files' log.exception(stdout) stderr = util.unicodify(e) for tw in task_wrappers: # Prevent repetitive output, e.g. "Sequence File Aligned"x20 # Eventually do a reduce for jobs that output "N reads mapped", combining all N for tasks. out = tw.get_task().stdout.strip() err = tw.get_task().stderr.strip() if len(out) > 0: stdout += "\n" + tw.working_directory + ':\n' + out if len(err) > 0: stderr += "\n" + tw.working_directory + ':\n' + err return (stdout, stderr)