def before_activity_control(context: Activity, **kwargs): """ Create a span, child of the method or rollback's span, before the activitiy is applied """ tracer = local.tracer name = context.get("name") parent_span = tracer.hypothesis_span or tracer.method_span or \ tracer.rollback_span or tracer.experiment_span span = tracer.start_span(name, child_of=parent_span) tracer.activity_span = span span.set_tag('type', 'activity') span.set_tag('activity', context.get("type")) # special treatment for HTTP activities # we inject the metadata of the HTTP request provider = context["provider"] span.log_kv(provider) if provider["type"] == "http": headers = provider.get("headers", {}) span.set_tag( 'http.method', provider.get("method", "GET").upper()) span.set_tag('http.url', provider["url"]) span.set_tag('span.kind', 'client') span.tracer.inject(span, 'http_headers', headers) provider["headers"] = headers if kwargs: span.log_kv(kwargs)
def before_activity_control(context: Activity, **kwargs): """ Prompt for Yes or No to executing an activity. """ logger.info("About to execute activity: " + context.get("name")) if click.confirm('Do you want to continue?'): logger.info("Continuing: " + context.get("name")) else: raise InterruptExecution("Experiment manually interrupted")
def run_python_activity(activity: Activity, configuration: Configuration, secrets: Secrets) -> Any: """ Run a Python activity. A python activity is a function from any importable module. The result of that function is returned as the activity's output. This should be considered as a private function. """ provider = activity["provider"] mod_path = provider["module"] func_name = provider["func"] mod = importlib.import_module(mod_path) func = getattr(mod, func_name) try: logger.debug("Activity '{}' loaded from '{}'".format( activity.get("name"), inspect.getfile(func))) except TypeError: pass arguments = provider.get("arguments", {}).copy() if configuration or secrets: arguments = substitute(arguments, configuration, secrets) sig = inspect.signature(func) if "secrets" in provider and "secrets" in sig.parameters: arguments["secrets"] = {} for s in provider["secrets"]: arguments["secrets"].update(secrets.get(s, {}).copy()) if "configuration" in sig.parameters: arguments["configuration"] = configuration.copy() try: return func(**arguments) except Exception as x: raise ActivityFailed( traceback.format_exception_only( type(x), x)[0].strip()).with_traceback(sys.exc_info()[2])
def ensure_activity_is_valid(activity: Activity): """ Goes through the activity and checks certain of its properties and raise :exc:`InvalidActivity` whenever one does not respect the expectations. An activity must at least take the following key: * `"type"` the kind of activity, one of `"python"`, `"process"` or `"http"` Depending on the type, an activity requires a variety of other keys. In all failing cases, raises :exc:`InvalidActivity`. """ if not activity: raise InvalidActivity("empty activity is no activity") # when the activity is just a ref, there is little to validate ref = activity.get("ref") if ref is not None: if not isinstance(ref, str) or ref == '': raise InvalidActivity( "reference to activity must be non-empty strings") return activity_type = activity.get("type") if not activity_type: raise InvalidActivity("an activity must have a type") if activity_type not in ("probe", "action"): raise InvalidActivity( "'{t}' is not a supported activity type".format(t=activity_type)) if not activity.get("name"): raise InvalidActivity("an activity must have a name") provider = activity.get("provider") if not provider: raise InvalidActivity("an activity requires a provider") provider_type = provider.get("type") if not provider_type: raise InvalidActivity("a provider must have a type") if provider_type not in ("python", "process", "http"): raise InvalidActivity( "unknown provider type '{type}'".format(type=provider_type)) if not activity.get("name"): raise InvalidActivity("activity must have a name (cannot be empty)") timeout = activity.get("timeout") if timeout is not None: if not isinstance(timeout, numbers.Number): raise InvalidActivity("activity timeout must be a number") pauses = activity.get("pauses") if pauses is not None: before = pauses.get("before") if before is not None and not isinstance(before, numbers.Number): raise InvalidActivity("activity before pause must be a number") after = pauses.get("after") if after is not None and not isinstance(after, numbers.Number): raise InvalidActivity("activity after pause must be a number") if "background" in activity: if not isinstance(activity["background"], bool): raise InvalidActivity("activity background must be a boolean") if provider_type == "python": validate_python_activity(activity) elif provider_type == "process": validate_process_activity(activity) elif provider_type == "http": validate_http_activity(activity)
def execute_activity(experiment: Experiment, activity: Activity, configuration: Configuration, secrets: Secrets, dry: bool = False) -> Run: """ Low-level wrapper around the actual activity provider call to collect some meta data (like duration, start/end time, exceptions...) during the run. """ ref = activity.get("ref") if ref: activity = lookup_activity(ref) if not activity: raise ActivityFailed( "could not find referenced activity '{r}'".format(r=ref)) with controls(level="activity", experiment=experiment, context=activity, configuration=configuration, secrets=secrets) as control: pauses = activity.get("pauses", {}) pause_before = pauses.get("before") if pause_before: logger.info("Pausing before next activity for {d}s...".format( d=pause_before)) # only pause when not in dry-mode if not dry: time.sleep(pause_before) if activity.get("background"): logger.info("{t}: {n} [in background]".format( t=activity["type"].title(), n=activity.get("name"))) else: logger.info("{t}: {n}".format( t=activity["type"].title(), n=activity.get("name"))) start = datetime.utcnow() run = { "activity": activity.copy(), "output": None } result = None interrupted = False try: # only run the activity itself when not in dry-mode if not dry: result = run_activity(activity, configuration, secrets) run["output"] = result run["status"] = "succeeded" if result is not None: logger.debug(" => succeeded with '{r}'".format(r=result)) else: logger.debug(" => succeeded without any result value") except ActivityFailed as x: error_msg = str(x) run["status"] = "failed" run["output"] = result run["exception"] = traceback.format_exception(type(x), x, None) logger.error(" => failed: {x}".format(x=error_msg)) finally: # capture the end time before we pause end = datetime.utcnow() run["start"] = start.isoformat() run["end"] = end.isoformat() run["duration"] = (end - start).total_seconds() pause_after = pauses.get("after") if pause_after and not interrupted: logger.info("Pausing after activity for {d}s...".format( d=pause_after)) # only pause when not in dry-mode if not dry: time.sleep(pause_after) control.with_state(run) return run
def execute_activity( experiment: Experiment, activity: Activity, configuration: Configuration, secrets: Secrets, dry: Dry, ) -> Run: """ Low-level wrapper around the actual activity provider call to collect some meta data (like duration, start/end time, exceptions...) during the run. """ ref = activity.get("ref") if ref: activity = lookup_activity(ref) if not activity: raise ActivityFailed(f"could not find referenced activity '{ref}'") with controls( level="activity", experiment=experiment, context=activity, configuration=configuration, secrets=secrets, ) as control: dry = activity.get("dry", dry) pauses = activity.get("pauses", {}) pauses = substitute(pauses, configuration, secrets) pause_before = pauses.get("before") is_dry = False activity_type = activity["type"] if dry == Dry.ACTIONS: is_dry = activity_type == "action" elif dry == Dry.PROBES: is_dry = activity_type == "probe" elif dry == Dry.ACTIVITIES: is_dry = True if pause_before: logger.info(f"Pausing before next activity for {pause_before}s...") # pause when one of the dry flags are set if dry != Dry.PAUSE and not is_dry: time.sleep(pause_before) if activity.get("background"): logger.info("{t}: {n} [in background]".format( t=activity["type"].title(), n=activity.get("name"))) else: logger.info("{t}: {n}".format(t=activity["type"].title(), n=activity.get("name"))) start = datetime.utcnow() run = {"activity": activity.copy(), "output": None} result = None interrupted = False try: # pause when one of the dry flags are set if not is_dry: result = run_activity(activity, configuration, secrets) run["output"] = result run["status"] = "succeeded" if result is not None: logger.debug(f" => succeeded with '{result}'") else: logger.debug(" => succeeded without any result value") except ActivityFailed as x: error_msg = str(x) run["status"] = "failed" run["output"] = result run["exception"] = traceback.format_exception(type(x), x, None) logger.error(f" => failed: {error_msg}") finally: # capture the end time before we pause end = datetime.utcnow() run["start"] = start.isoformat() run["end"] = end.isoformat() run["duration"] = (end - start).total_seconds() pause_after = pauses.get("after") if pause_after and not interrupted: logger.info(f"Pausing after activity for {pause_after}s...") # pause when one of the dry flags are set if dry != Dry.PAUSE and not is_dry: time.sleep(pause_after) control.with_state(run) return run
def before_activity_control(context: Activity, target_activity_name: str, **kwargs): if context.get("name") == target_activity_name: raise SystemExit("we are done here")
def before_activity_control(context: Activity, target_activity_name: str, **kwargs): if context.get("name") == target_activity_name: raise InterruptExecution("let's blow this up")