def replace_env_variables(self, cmd) -> str:
xml_root_path = os.path.dirname(os.path.abspath(self.process.uri))
cmd_out = cmd.replace("$__tool_directory__", xml_root_path)
cmd_out = cmd_out.replace("$__fiji__",
ConfigAccess.instance().config['fiji'])
config = ConfigAccess.instance()
if config.is_key('env'):
for element in config.get('env'):
cmd_out = cmd_out.replace("${" + element["name"] + "}",
element["value"])
return cmd_out
def __init__(self, process: Process):
super().__init__()
self.process = process
config = ConfigAccess.instance().config['runner']
self.service = runnerServices.get(config['service'], **config)
self.metadataservice = metadataServices.get(
ConfigAccess.instance().config['metadata']['service'], **config)
self._inputs = (
[]
) # list of {"name": "i", "uri": "/my/directory/", "filter": "\.tif$""}
self._parameters = [] # [key1, value1, key2, value1, ...]
self._output = '' # output uri (/my/output/folder)
self._mode = ''
self.output_uris = [] # list of generated outputs
def __init__(self, md_uri=''):
Observable.__init__(self)
self.md_uri = md_uri
self.metadata = None
config = ConfigAccess.instance().config['metadata']
self.service = metadataServices.get(config["service"], **config)
self.read()
def format_output_tmp_value(self, process: Process, name: str):
"""create the path of the output files"""
tmp_dir = ConfigAccess.instance().config['unittesting']['tmp']
for output in process.metadata.outputs:
# output.display()
if output.name == name and output.io == 'output':
return os.path.join(tmp_dir, name + '.' + output.type)
return name
def setUp(self):
self.xml_file = os.path.abspath(
os.path.join('tests', 'test_processes_local', 'helloworld.xml'))
self.tst_out_file = os.path.abspath(
os.path.join('tests', 'test_processes_local', 'helloworld.txt'))
ConfigAccess(
os.path.abspath(
os.path.join('tests', 'test_config', 'config_local.json')))
def __init__(self, md_uri: str):
self.md_uri = md_uri
self.metadata = None # RawDataContainer()
config = ConfigAccess.instance().config['metadata']
self.service = metadataServices.get(config["service"], **config)
try:
self.read()
except MetadataServiceError:
self.metadata = RawDataContainer()
def get_docker_working_dir():
working_dir = ''
config = ConfigAccess.instance().config['runner']
if 'working_dir' in config:
working_dir = config['working_dir'].replace('\\\\',
'/').replace('\\', '/')
else:
raise RunnerExecError("The docker runner need a working_dir. "
"Please setup working_dir in your config file")
return working_dir
def _import_file_bioformat(self, raw_dataset_uri, file_path,
destination_dir, data_name, author, date):
fiji_exe = ConfigAccess.instance().get('fiji')
cmd = f'{fiji_exe} --headless -macro bioimageit_convert.ijm "file,{file_path},{destination_dir},{data_name},{author},{date}"'
print("import bioformat cmd:", cmd)
if platform.system() == 'Windows':
subprocess.run(cmd, check=True)
else:
subprocess.run(cmd, shell=True, executable='/bin/bash', check=True)
# add data to rawdataset
self._add_to_rawdataset_biobormat(destination_dir, raw_dataset_uri)
def exec(self, process: ProcessContainer, args):
"""Execute a process
Parameters
----------
process
Metadata of the process
args
list of arguments
"""
token = None
config = ConfigAccess.instance().config['runner']
if 'token' in config:
token = config['token']
client = ag.Client(token)
# exec the process
params = ' '.join(args[1:])
files = []
for input_ in process.inputs:
if input_.is_data:
filename = ntpath.basename(input_.value)
params = params.replace(input_.value, filename)
files.append(input_.value)
for output in process.outputs:
if output.is_data:
filename = ntpath.basename(output.value)
params = params.replace(output.value, filename)
# print('files:', files)
# print('params:', params)
try:
out_dict = client.run_job(process.id, files=files, params=params)
except ag.StatusError as e:
print('API status Error:', e.status_code)
print('API status Error:', e.msg)
# print(out_dict)
# get the outputs
job_id = out_dict['id']
for output in process.outputs:
output_filename = ntpath.basename(output.value)
output_dir = os.path.dirname(os.path.abspath(output.value))
url = out_dict[str(job_id)][output_filename]
filepath = client.download_file(file_url=url,
outdir=output_dir,
force=True)
def _import_dir_bioformat(self, raw_dataset_uri, dir_uri, filter_, author,
format_, date, directory_tag_key):
fiji_exe = ConfigAccess.instance().get('fiji')
rawdataset_uri = os.path.abspath(raw_dataset_uri)
data_dir_path = os.path.dirname(rawdataset_uri)
cmd = f'{fiji_exe} --headless -macro bioimageit_convert.ijm "folder,{dir_uri},{data_dir_path},false,{filter_},{author},{date},{directory_tag_key}"'
print("import bioformat cmd:", cmd)
if platform.system() == 'Windows':
subprocess.run(cmd, check=True)
else:
subprocess.run(cmd, shell=True, executable='/bin/bash', check=True)
self._add_to_rawdataset_biobormat(data_dir_path, raw_dataset_uri)
def run_merged(self):
"""Run the process that merge txt number inputs
This is the main function that run the process on the experiment data
Raises
------
RunnerExecError
"""
for observer in self._observers:
observer.notify({'progress': 0, 'message': 'start'})
# 1- Query all the input data and verify that the size
# are equal, if not raise an exception
input_data, data_count = self._query_inputs()
# 2- Create the ProcessedDataSet
processed_dataset = self.experiment.create_processed_dataset(
self._output_dataset
)
# 3- Create run
run = Run()
run.metadata.process_name = self.process.metadata.fullname()
run.metadata.process_uri = self.process.uri
for t in range(len(self._inputs_names)):
run.metadata.inputs.append(
RunInputContainer(
self._inputs_names[t],
self._inputs_datasets[t],
self._inputs_query[t],
self._inputs_origin_output_name[t],
)
)
for i in range(0, len(self._process_params), 2):
run.metadata.parameters.append(
RunParameterContainer(
self._process_params[i], self._process_params[i + 1]
)
)
processed_dataset.add_run(run)
# 4- merge Inputs
inputs_values = [0 for i in range(len(self._inputs_names))]
for n in range(len(self._inputs_names)):
inputs_values[n] = list()
for i in range(data_count):
data_info = RawData(input_data[n][i])
if (
data_info.metadata.format == "numbercsv"
):
with open(data_info.metadata.uri, 'r') as file:
value = file.read().replace('\n', '').replace(' ', '')
inputs_values[n].append(value)
else:
raise RunnerExecError(
'run merge can use only number datatype')
# 5- save data in tmp files files in the processed dataset dir
tmp_inputs_files = [0 for i in range(len(self._inputs_names))]
processed_data_dir = processed_dataset.md_uri.replace(
"processeddataset.md.json", ""
)
for n in range(len(self._inputs_names)):
tmp_inputs_files[n] = os.path.join(
processed_data_dir, self._inputs_names[n] + '.csv'
)
f = open(tmp_inputs_files[n], 'w')
for i in range(len(inputs_values[n])):
value = str(inputs_values[n][i])
if i < len(inputs_values[n]) - 1:
f.write(value + ",")
else:
f.write(value)
f.close()
# 6- create input metadata for output .md.json
inputs_metadata = []
for n in range(len(tmp_inputs_files)):
inp_metadata = ProcessedDataInputContainer()
inp_metadata.name = self._inputs_names[n]
inp_metadata.uri = tmp_inputs_files[n]
inp_metadata.type = 'txt'
inputs_metadata.append(inp_metadata)
# 7- run process on generated files
args = []
# 7.1- inputs
for n in range(len(self._inputs_names)):
args.append(self._inputs_names[n])
args.append(tmp_inputs_files[n])
# 7.2- params
for param in self._process_params:
args.append(param)
# 4.3- outputs
for output in self.process.metadata.outputs:
extension = '.' + FormatsAccess.instance().get(output.type).extension
# args
args.append(output.name)
output_file_name = output.name
args.append(os.path.join(processed_data_dir,
output_file_name + extension))
# output metadata
processed_data = ProcessedData()
processed_data.metadata.name = output.name
processed_data.metadata.author = \
ConfigAccess.instance().get('user')['name']
processed_data.metadata.date = format_date('now')
processed_data.metadata.format = output.type
processed_data.metadata.run_uri = run.md_uri
processed_data.metadata.inputs = inputs_metadata
processed_data.metadata.output = {
'name': output.name,
'label': output.description,
}
# save the metadata and create its md_uri and uri
processed_dataset.create_data(processed_data)
# 8- exec
runner = Runner(self.process)
runner.exec(*args)
# notify observers
for observer in self._observers:
observer.notify({'progress': 100, 'message': 'done'})
runner_name = sys.argv[2]
runner_name = runner_name.upper()
# create the config file from config_sample.json
bioimagepy_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)))
package_dir = Path(bioimagepy_dir).parent
config_json = read_json(os.path.join(bioimagepy_dir, 'config_sample.json'))
config_json["process"]["xml_dirs"] = [ os.path.join(package_dir,
"toolboxes", "tools")]
config_json["process"]["categories"] = os.path.join(package_dir,
"toolboxes",
"toolboxes.json")
config_json["process"]["tools"] = os.path.join(package_dir, "toolboxes",
"tools.json")
config_json["runner"]["service"] = runner_name
if runner_name == 'DOCKER':
config_json["runner"]["working_dir"] = os.path.join(package_dir, "userdata")
config_json["gui"] = {"tmp": os.path.join(package_dir, "userdata")}
config_json["user"]["name"] = user_name
config_json["formats"]["file"] = os.path.join(package_dir, "formats.json")
config_file = os.path.join(package_dir, 'config.json')
write_json(config_json, config_file)
# run toolboxes build
ConfigAccess(config_file)
builder = Toolboxes()
builder.build()
def __init__(self):
config = ConfigAccess.instance().config['process']
self.service = processServices.get(config['service'], **config)
def run_sequence(self):
"""Run the process in a sequence
This is the main function that run the process on the experiment data
Raises
------
RunnerExecError
"""
# 1- Query all the input data and verify that the size
# are equal, if not raise an exception
input_data, data_count = self._query_inputs()
# 2- Create the ProcessedDataSet
processed_dataset = self.experiment.create_processed_dataset(
self._output_dataset
)
# 3- Create run
run = Run()
run.metadata.process_name = self.process.metadata.fullname()
run.metadata.process_uri = self.process.uri
for t in range(len(self._inputs_names)):
run.metadata.inputs.append(
RunInputContainer(
self._inputs_names[t],
self._inputs_datasets[t],
self._inputs_query[t],
self._inputs_origin_output_name[t],
)
)
for i in range(0, len(self._process_params), 2):
run.metadata.parameters.append(
RunParameterContainer(
self._process_params[i], self._process_params[i + 1]
)
)
processed_dataset.add_run(run)
# 4- loop over the input data
for i in range(data_count):
data_info_zero = RawData(input_data[0][i].uri())
# 4.0- notify observers
for observer in self._observers:
notification = dict()
notification['progress'] = int(100 * i / data_count)
notification['message'] = "Process " + \
data_info_zero.metadata.name
observer.notify(notification)
# 4.1- Parse IO
args = []
# get the input arguments
inputs_metadata = []
for n in range(len(self._inputs_names)):
args.append(self._inputs_names[n])
data_info = RawData(input_data[n][i].uri())
# input data can be a processedData but
# we only read the common metadata
args.append(data_info.metadata.uri)
inp_metadata = ProcessedDataInputContainer()
inp_metadata.name = self._inputs_names[n]
inp_metadata.uri = input_data[n][i].uri()
inp_metadata.type = data_info.metadata.type
inputs_metadata.append(inp_metadata)
# get the params arguments
for param in self._process_params:
args.append(param)
# setup outputs
for output in self.process.metadata.outputs:
# output metadata
processed_data = ProcessedData()
processed_data.metadata.name = (
data_info_zero.metadata.name + "_" + output.name
)
processed_data.metadata.author = \
ConfigAccess.instance().get('user')['name']
processed_data.metadata.date = format_date('now')
processed_data.metadata.format = output.type
processed_data.metadata.run_uri = run.md_uri
processed_data.metadata.inputs = inputs_metadata
processed_data.metadata.output = {
'name': output.name,
'label': output.description,
}
# save the metadata and create its md_uri and uri
processed_dataset.create_data(processed_data)
# args
args.append(output.name)
args.append(processed_data.metadata.uri)
# 4.2- exec
runner = Runner(self.process)
# print("args = ", args)
runner.exec(*args)
# 4.0- notify observers
for observer in self._observers:
observer.notify({'progress': 100, 'message': 'done'})
def setUp(self):
self.xml_file = os.path.join('tests', 'test_processes_local', 'svdeconv', 'svdeconv2d.xml')
ConfigAccess(os.path.join('tests', 'test_config','config_local.json'))
def __init__(self):
super().__init__()
config = ConfigAccess.instance().config['process']
self.xml_dir = config['xml_dirs'][0]
self.tools_file = config['tools']
def test_access(self):
ConfigAccess(self.config_local_file)
content = ConfigAccess.instance().config
self.assertTrue(_check_content(content))
def __init__(self):
self.workspace_dir = ConfigAccess.instance().config['workspace']
config = ConfigAccess.instance().config['metadata']
self.service = metadataServices.get(config["service"], **config)
def __init__(self, md_uri: str = ''):
self.md_uri = md_uri
self.metadata = None # RunContainer()
config = ConfigAccess.instance().config['metadata']
self.service = metadataServices.get(config["service"], **config)
self.read()
def __init__(self):
super().__init__()
self.service_name = 'LocalRunnerService'
self.conda_dir = ConfigAccess.instance().get('runner')['conda_dir']
print(self.conda_dir)
def __init__(self, uri: str):
self.uri = uri
config = ConfigAccess.instance().config['process']
self.service = processServices.get(config['service'], **config)
self.metadata = self.service.read_process(self.uri)
