def _load_backend_decorator(inpt): _backend = known_backends.get(inpt, inpt) try: backend = util.load_obj_from_path( _backend, {'key': '%s_backend' % backend_type, backend_type: _backend}) except Exception as err: log.error( "Could not load %s backend. err: %s" % (backend_type, err), extra={'%s_backend' % backend_type: _backend, 'err_kls': type(err)}) raise return backend
def passes_filter(app_name, job_id): """Determine if this job matches certain criteria that state it is a valid job for this app_name. A partially out of scope for dag stuff, but important detail: Jobs that don't match the criteria should immediately be marked as completed """ # for now, if we can parse it, it's valid pjob_id = parse_job_id(app_name, job_id) # does this job matches criteria that makes it executable? if so, we can't # autocomplete it dg = cb.get_tasks_config() meta = dg[app_name] ld = dict(app_name=app_name, job_id=job_id) try: dct = dict(meta["valid_if_or"]) except (KeyError, TypeError): return True # everything is valid if "_func" in dct: import_path = dct.pop("_func") # safe because config is immutable try: func = load_obj_from_path(import_path, ld) except Exception as err: raise err.__class__("valid_if_or._func misconfigured: %s" % err.message) if func(app_name, **pjob_id): return True for k, v in dct.items(): try: kk = pjob_id[k] except KeyError: _log_raise( "valid_if_or contains a key that's not in the job_id", extra=dict(valid_if_or_key=k, **ld), exception_kls=DAGMisconfigured, ) vals = [get_NS().job_id_validations[k](x) for x in v] if kk in vals: return True return False
def passes_filter(app_name, job_id): """Determine if this job matches certain criteria that state it is a valid job for this app_name. A partially out of scope for dag stuff, but important detail: Jobs that don't match the criteria should immediately be marked as completed """ # for now, if we can parse it, it's valid pjob_id = parse_job_id(app_name, job_id) # does this job matches criteria that makes it executable? if so, we can't # autocomplete it dg = cb.get_tasks_config() meta = dg[app_name] ld = dict(app_name=app_name, job_id=job_id) try: dct = dict(meta['valid_if_or']) except (KeyError, TypeError): return True # everything is valid if '_func' in dct: import_path = dct.pop('_func') # safe because config is immutable try: func = load_obj_from_path(import_path, ld) except Exception as err: raise err.__class__("valid_if_or._func misconfigured: %s" % err.message) if func(app_name, **pjob_id): return True for k, v in dct.items(): try: kk = pjob_id[k] except KeyError: _log_raise("valid_if_or contains a key that's not in the job_id", extra=dict(valid_if_or_key=k, **ld), exception_kls=DAGMisconfigured) vals = [get_NS().job_id_validations[k](x) for x in v] if kk in vals: return True return False