def _init_data_config(self, data_identifier=None, step=1, **kwargs):
data_dir_filepath = Path(abspath(self.io["fs"], node="data"))
train_fpath = dataset_filepath(
name=data_identifier,
kind="train",
step=step,
data_dir_filepath=data_dir_filepath,
extension="csv",
check=True,
)
test_fpath = dataset_filepath(
name=data_identifier,
kind="test",
step=step,
data_dir_filepath=data_dir_filepath,
extension="csv",
check=True,
)
data_config = dict(
data_identifier=data_identifier,
step=step,
train_fpath=train_fpath,
test_fpath=test_fpath,
)
return data_config
def get_dummy_flow(message="hi", content=dict(a=1, b=2), timeout_s=20):
# config
dummy_config = get_dummy_config(message=message, content=content)
dummy_fs = dummy_config.get("io").get("fs")
# flow-object
logs_directory_key = "out.flow.logs"
check_existence_of_directory(dummy_fs, logs_directory_key)
log_filepath = abspath(dummy_fs, logs_directory_key, "logfile" + message)
flows_directory_key = "out.flow.flows"
check_existence_of_directory(dummy_fs, flows_directory_key)
flow_filepath = abspath(dummy_fs, flows_directory_key,
"flowfile-{}.pkl".format(message))
f = Flow(
config=dummy_config,
imports=dummy_imports,
flow=dummy_flow,
timeout_s=timeout_s,
flow_filepath=flow_filepath,
log_filepath=log_filepath,
)
return f
def _init_qry_config(
self, data_identifier=None, qry_keyword="default", n_queries=None, **kwargs
):
qry_dir_filepath = Path(abspath(self.io["fs"], node="query"))
qry_filepath = query_filepath(
name=data_identifier,
keyword=qry_keyword,
query_dir_filepath=qry_dir_filepath,
extension="npy",
)
qry_config = dict(
filepath=qry_filepath, keyword=qry_keyword, n_queries=n_queries
)
return qry_config
def dummy_flow(config):
print("Hello world")
fs = config.get("io").get("fs")
content = config.get("content")
message = config.get("message")
results_directory_key = "out.flow.results"
check_existence_of_directory(fs, results_directory_key)
fn_results = abspath(fs,
results_directory_key,
filename="{}.json".format(message))
results = content
dump_object(results, fn_results)
# Some extra actions
sleep_a_few_s = 2
time.sleep(sleep_a_few_s)
print("{} secs passed".format(sleep_a_few_s))
print(message)
return content