def _prepare_inputs(self, preview=False):
resource_merger = ResourceMerger(
[NodeResources.INPUT],
[input.name for input in self.node.inputs if input.is_array],
)
for input in self.node.inputs:
if preview:
for i, value in enumerate(range(input.min_count)):
filename = os.path.join(self.workdir,
'i_{}_{}'.format(i, input.name))
resource_merger.append(
self._resource_manager.kind_to_resource_class[
input.file_type].prepare_input(filename, preview),
input.name,
input.is_array,
)
else:
for i, value in enumerate(input.values):
filename = os.path.join(self.workdir,
'i_{}_{}'.format(i, input.name))
with open(filename, 'wb') as f:
f.write(get_file_stream(value).read())
resource_merger.append(
self._resource_manager.kind_to_resource_class[
input.file_type].prepare_input(filename, preview),
input.name,
input.is_array,
)
return resource_merger.get_dict()
def _prepare_inputs(inputs, preview=False, pythonize=False):
res = {}
for input in inputs:
filenames = []
if preview:
for i, value in enumerate(range(input.min_count)):
filename = os.path.join(
'/tmp', '{}_{}_{}'.format(
str(uuid.uuid1())[:8], i, input.name))
filenames.append(filename)
else:
for i, value in enumerate(input.values):
filename = os.path.join(
'/tmp', '{}_{}_{}'.format(str(uuid.uuid1()), i,
input.name))
with open(filename, 'wb') as f:
f.write(get_file_stream(value.resource_id).read())
if FileTypes.EXECUTABLE in input.file_types:
# `chmod +x` to the executable file
st = os.stat(filename)
os.chmod(filename, st.st_mode | stat.S_IEXEC)
filenames.append(filename)
if pythonize:
if input.min_count == 1 and input.max_count == 1:
res[input.name] = filenames[0]
else:
res[input.name] = filenames
else:
# TODO is ' ' standard separator?
res[input.name] = ' '.join(filenames)
return res
def _prepare_inputs(self, preview=False):
resource_merger = ResourceMerger()
for input in self.node.inputs:
is_list = not (input.min_count == 1 and input.max_count == 1)
if preview:
for i, value in enumerate(range(input.min_count)):
filename = os.path.join(self.workdir,
'i_{}_{}'.format(i, input.name))
resource_merger.append(
resource_manager[input.file_types[0]].prepare_input(
filename, preview),
input.name,
is_list,
)
else:
for i, value in enumerate(input.values):
filename = os.path.join(self.workdir,
'i_{}_{}'.format(i, input.name))
with open(filename, 'wb') as f:
f.write(get_file_stream(value.resource_id).read())
resource_merger.append(
resource_manager[input.file_types[0]].prepare_input(
filename, preview), input.name, is_list)
return resource_merger.get_dict()
def get_resource(resource_id):
preview = json.loads(request.args.get('preview', 'false'))
file_type = request.args.get('file_type', None)
if preview and not file_type:
return make_fail_response(
'In preview mode `file_type` must be specified'), 400
fp = get_file_stream(resource_id, preview=preview, file_type=file_type)
if preview:
preview_object = PreviewObject(
fp=fp,
resource_id=resource_id,
)
return resource_manager[file_type].preview(preview_object)
return send_file(fp, attachment_filename=resource_id)
def test_dag():
N = 100
executor = create_dag_executor(N)
validation_error = executor.validate()
assert validation_error is None, validation_error
executor.run()
output_ids = executor.node.get_parameter_by_name(
'_nodes').value.value[-1].get_output_by_name(SUM_OUTPUT).values
assert len(output_ids) == 1, 'Unexpected number of outputs: `{}`'.format(
len(output_ids))
res = int(file_handler.get_file_stream(output_ids[0]).read())
res_expected = ((1 + N) * N) // 2
assert res == res_expected, 'Expected `{}` but got `{}`'.format(
res_expected, res)
print('Test successfully passed with result `{}`'.format(res))
def get_resource(resource_id):
preview = json.loads(request.args.get('preview', 'false'))
return send_file(get_file_stream(resource_id, preview=preview),
attachment_filename=resource_id)