def superdot(state_dir, depfile, config_file):
from .scheduler import construct_dataflow, graph_to_dot
rules = read_rules(state_dir, depfile, config_file)
g = construct_dataflow(rules)
with open("dump.dot", "w") as fd:
fd.write(graph_to_dot(g))
def run_graph_to_dot(tmpdir, config):
state_dir = str(tmpdir) + "/state"
config_file = None # str(tmpdir.join("config"))
depfile = str(tmpdir) + "/t.conseq"
with open(depfile, "wt") as fd:
fd.write(config)
# with open(config_file, "wt") as fd:
# fd.write("let x='v'\n")
rules = read_rules(state_dir, depfile, config_file)
g = construct_dataflow(rules)
return graph_to_dot(g)
def print_rules(state_dir, depfile, config_file, mode, rule_name):
rules = read_rules(state_dir, depfile, config_file)
if mode == "all":
names = [rule.name for rule in rules]
elif mode == "up":
raise NotImplemented()
elif mode == "down":
raise NotImplemented()
else:
raise Exception(f"Expected {mode} to be all, up or down")
names.sort()
for name in names:
print(name)
def localize_cmd(state_dir, space, predicates, depfile, config_file):
rules = read_rules(state_dir, depfile, config_file)
resolver = xref.Resolver(state_dir, rules.vars)
j = dep.open_job_db(os.path.join(state_dir, "db.sqlite3"))
if space is None:
space = j.get_current_space()
subset = j.find_objs(space, dict(predicates))
for obj in subset:
for k, v in obj.props.items():
if isinstance(v, dict) and "$file_url" in v:
url = v["$file_url"]
r = resolver.resolve(url)
log.info("resolved %s to %s", url, r)
def debugrun(state_dir, depfile, target, override_vars, config_file):
db_path = os.path.join(state_dir, "db.sqlite3")
print("opening", db_path)
j = dep.open_job_db(db_path)
rules = read_rules(state_dir, depfile, config_file)
for var, value in override_vars.items():
rules.set_var(var, value)
rule = rules.get_rule(target)
queries, predicates = convert_input_spec_to_queries(
rules.jinja2_env, rule, rules.vars)
for q in queries:
t = dep.Template([q], [], rule.name)
applications = j.query_template(t)
log.info("{} matches for {}".format(len(applications), q))
applications = j.query_template(
dep.Template(queries, predicates, rule.name))
log.info("{} matches for entire rule".format(len(applications)))
def test_upstream(tmpdir):
from conseq.config import read_rules
state_dir = tmpdir.join("state")
state_dir.mkdir()
depfile = tmpdir.join("dep.conseq")
depfile.write("""
rule a:
outputs: {"type": "a"}
rule b:
inputs: in = {"type": "a"}
outputs: {"type": "b"}
rule c:
inputs: in = {"type": "b"}
outputs: {"type": "c"}
rule d:
inputs: in = {"type": "c"}
outputs: {"type": "d"}
rule e:
inputs: in = {"type": "d"}
outputs: {"type": "e"}
""")
configfile = tmpdir.join("config.conseq")
configfile.write("")
rules = read_rules(str(state_dir), str(depfile), str(configfile))
g = construct_dataflow(rules)
assert sorted(g.get_upstream_rules("c")) == ["a", "b"]
assert sorted(g.get_downstream_rules("c") ) == ["d", "e"]
def main(depfile: str,
state_dir: str,
forced_targets: List[Any],
override_vars: Dict[Any, Any],
max_concurrent_executions: int,
capture_output: bool,
req_confirm: bool,
config_file: str,
maxfail: int = 1,
maxstart: None = None,
force_no_targets: bool = False,
reattach_existing=None,
remove_unknown_artifacts=None,
properties_to_add=[],
rule_filter=None) -> int:
assert max_concurrent_executions > 0
if not os.path.exists(state_dir):
os.makedirs(state_dir)
db_path = os.path.join(state_dir, "db.sqlite3")
j = dep.open_job_db(db_path)
# handle case where we explicitly state some templates to execute. Make sure nothing else executes
if len(forced_targets) > 0 or force_no_targets:
forced_rule_names = force_execution_of_rules(j, forced_targets)
else:
forced_rule_names = []
if rule_filter:
assert len(forced_targets
) == 0, "Cannot specify allowed rules and forced rules"
# because force_execution_of_rules() call limitStartToTemplates
# and one will clobber the state of the other
j.limitStartToTemplates([rule_filter])
rules = read_rules(state_dir, depfile, config_file, initial_config={})
rule_specifications = rules.get_rule_specifications()
jinja2_env = rules.jinja2_env
if rules.get_client("default", must=False) is None:
rules.add_client("default", exec_client.LocalExecClient({}))
# override with max_concurrent_executions
rules.get_client("default").resources["slots"] = max_concurrent_executions
for var, value in override_vars.items():
rules.set_var(var, value)
# handle the "add-if-missing" objects and changes to rules
reconcile_db(j,
jinja2_env,
rule_specifications,
rules.objs,
rules.vars,
force=remove_unknown_artifacts)
# handle the remember-executed statements
with j.transaction():
for exec_ in rules.remember_executed:
j.remember_executed(exec_)
# finish initializing exec clients
for name, props in list(rules.exec_clients.items()):
if isinstance(props, dict):
config = rules.get_vars()
props = expand_dict(jinja2_env, props, config)
class VirtualDict():
def __getitem__(self, key):
value = rules.get_vars()[key]
return render_template(jinja2_env, value, config)
def get(self, key, default=None):
value = rules.get_vars().get(key, default)
if value is None:
return None
return render_template(jinja2_env, value, config)
client = exec_client.create_client(name, VirtualDict(), props)
rules.add_client(name, client)
# Reattach or cancel jobs from previous invocation
executing = []
pending_jobs = j.get_started_executions()
if len(pending_jobs) > 0:
log.warning(
"Reattaching jobs that were started in a previous invocation of conseq, but had not terminated before conseq exited: %s",
pending_jobs)
if reattach_existing is None:
reattach_existing = ui.user_wants_reattach()
if reattach_existing:
executing = reattach(j, rules, pending_jobs)
else:
pending_jobs = j.get_started_executions()
for e in pending_jobs:
log.warning(
"Canceling {} which was started from earlier execution".
format(e.id))
j.cancel_execution(e.id)
# any jobs killed or other failures need to be removed so we'll attempt to re-run them
j.cleanup_unsuccessful()
assert len(j.get_pending()) == 0
for dec in rules:
try:
j.add_template(to_template(jinja2_env, dec, rules.vars))
except MissingTemplateVar as ex:
log.error("Could not load rule {}: {}".format(
dec.name, ex.get_error()))
return -1
# now check the rules we requested exist
for rule_name in forced_rule_names:
if not (j.has_template(rule_name)):
raise Exception("No such rule: {}".format(rule_name))
def new_object_listener(obj):
timestamp = datetime.datetime.now().isoformat()
j.add_obj(timestamp, obj)
try:
ret = main_loop(jinja2_env,
j,
new_object_listener,
rules,
state_dir,
executing,
capture_output,
req_confirm,
maxfail,
maxstart,
properties_to_add=properties_to_add)
except FatalUserError as e:
print("Error: {}".format(e))
return -1
return ret
def alt_dot(state_dir, depfile, config_file):
rules = read_rules(state_dir, depfile, config_file)
print(_rules_to_dot(rules))
def export_cmd(state_dir, depfile, config_file, dest_s3_path):
out = StringIO()
rules = read_rules(state_dir, depfile, config_file)
j = dep.open_job_db(os.path.join(state_dir, "db.sqlite3"))
objs = j.find_objs(DEFAULT_SPACE, {})
print(len(objs))
vars = rules.vars
cas_remote = helper.Remote(
vars["S3_STAGING_URL"],
".",
helper.S3StorageConnection(vars["AWS_ACCESS_KEY_ID"],
vars["AWS_SECRET_ACCESS_KEY"]),
)
def process_value(value):
if isinstance(value, dict):
if "$filename" in value:
url = cas_remote.upload_to_cas(value["$filename"])
value = {"$file_url": url}
return value
def process_filenames(obj: Obj):
translated = {}
for key, value in obj.props.items():
if isinstance(value, list) or isinstance(value, tuple):
value = [process_value(x) for x in value]
else:
value = process_value(value)
translated[key] = value
if "$manually-added" not in translated:
translated["$manually-added"] = {"$value": "false"}
return translated
def reindent(s, ident):
indent_str = " " * ident
lines = s.split("\n")
return "\n".join([lines[0]] + [indent_str + x for x in lines[1:]])
for obj in objs:
try:
props = process_filenames(obj)
except Exception as e:
raise Exception(
"Could not process filenames in artifact: {}".format(
repr(obj))) from e
out.write("add-if-missing {}\n\n".format(reindent(
json.dumps(props), 3)))
def get_key_props(obj):
props = {}
for key, value in obj.props.items():
if isinstance(value, dict) and (("$filename" in value) or
("$file_url" in value) or
("$value" in value)):
continue
props[key] = value
return props
def value_as_json(value):
if isinstance(value, tuple):
return json.dumps([get_key_props(x) for x in value], indent=3)
else:
return json.dumps(get_key_props(value), indent=3)
executions = j.get_all_executions()
skipped = 0
for execution in executions:
if execution.status != "completed":
skipped += 1
continue
out.write('remember-executed transform : "{}"\n'.format(
execution.transform))
for input in execution.inputs:
out.write(' input "{}" : {}\n'.format(
input[0], reindent(value_as_json(input[1]), 3)))
for output in execution.outputs:
out.write(" output : {}\n".format(
reindent(value_as_json(output), 3)))
out.write("\n")
log.info(
"Skipping export of %d executions which did not complete successfully",
skipped)
if dest_s3_path.startswith("s3://"):
log.info("Uploading artifact metadata to %s", dest_s3_path)
cas_remote.upload_str(dest_s3_path, out.getvalue())
else:
log.info("Writing artifacts to %s", dest_s3_path)
with open(dest_s3_path, "wt") as fd:
fd.write(out.getvalue())