def evaluate(self, data):
"""Evaluate the code needed to compute a given Data object."""
try:
inputs = copy.deepcopy(data.input)
hydrate_input_references(inputs, data.process.input_schema)
hydrate_input_uploads(inputs, data.process.input_schema)
# Include special 'proc' variable in the context.
inputs["proc"] = {
"data_id": data.id,
"data_dir": self.manager.get_executor().resolve_data_path(),
}
# Include special 'requirements' variable in the context.
inputs["requirements"] = data.process.requirements
# Inject default values and change resources according to
# the current Django configuration.
inputs["requirements"]["resources"] = data.process.get_resource_limits()
script_template = data.process.run.get("program", "")
# Get the appropriate expression engine. If none is defined, do not evaluate
# any expressions.
expression_engine = data.process.requirements.get("expression-engine", None)
if not expression_engine:
return script_template
return self.get_expression_engine(expression_engine).evaluate_block(
script_template, inputs, escape=self._escape, safe_wrapper=SafeString,
)
except EvaluationError as error:
raise ExecutionError("{}".format(error))
def evaluate(self, data):
"""Evaluate the code needed to compute a given Data object."""
try:
inputs = copy.deepcopy(data.input)
hydrate_input_references(inputs, data.process.input_schema)
hydrate_input_uploads(inputs, data.process.input_schema)
# Include special 'proc' variable in the context.
inputs['proc'] = {
'data_id': data.id,
'data_dir': settings.FLOW_EXECUTOR['DATA_DIR'],
}
# Include special 'requirements' variable in the context.
inputs['requirements'] = data.process.requirements
script_template = data.process.run.get('program', '')
# Get the appropriate expression engine. If none is defined, do not evaluate
# any expressions.
expression_engine = data.process.requirements.get(
'expression-engine', None)
if not expression_engine:
return script_template
return self.get_expression_engine(
expression_engine).evaluate_block(
script_template,
inputs,
escape=self._escape,
safe_wrapper=SafeString,
)
except EvaluationError as error:
raise ExecutionError('{}'.format(error))
def prepare_runtime(self, runtime_dir, data):
"""Prepare runtime directory."""
# Copy over Python process runtime (resolwe.process).
import resolwe.process as runtime_package
src_dir = os.path.dirname(inspect.getsourcefile(runtime_package))
dest_package_dir = os.path.join(runtime_dir, PYTHON_RUNTIME_DIRNAME,
'resolwe', 'process')
shutil.copytree(src_dir, dest_package_dir)
os.chmod(dest_package_dir, 0o755)
# Write python source file.
source = data.process.run.get('program', '')
program_path = os.path.join(runtime_dir, PYTHON_PROGRAM_FILENAME)
with open(program_path, 'w') as file:
file.write(source)
os.chmod(program_path, 0o755)
# Write serialized inputs.
inputs = copy.deepcopy(data.input)
hydrate_input_references(inputs, data.process.input_schema)
hydrate_input_uploads(inputs, data.process.input_schema)
inputs_path = os.path.join(runtime_dir, PYTHON_INPUTS_FILENAME)
# XXX: Skip serialization of LazyStorageJSON. We should support
# LazyStorageJSON in Python processes on the new communication protocol
def default(obj):
"""Get default value."""
class_name = obj.__class__.__name__
if class_name == 'LazyStorageJSON':
return ''
raise TypeError(
f'Object of type {class_name} is not JSON serializable')
with open(inputs_path, 'w') as file:
json.dump(inputs, file, default=default)
# Write serialized requirements.
# Include special 'requirements' variable in the context.
requirements = copy.deepcopy(data.process.requirements)
# Inject default values and change resources according to
# the current Django configuration.
requirements['resources'] = data.process.get_resource_limits()
requirements_path = os.path.join(runtime_dir,
PYTHON_REQUIREMENTS_FILENAME)
with open(requirements_path, 'w') as file:
json.dump(requirements, file)
# Generate volume maps required to expose needed files.
volume_maps = {
PYTHON_RUNTIME_DIRNAME: PYTHON_RUNTIME_VOLUME,
PYTHON_PROGRAM_FILENAME: PYTHON_PROGRAM_VOLUME,
PYTHON_INPUTS_FILENAME: PYTHON_INPUTS_VOLUME,
PYTHON_REQUIREMENTS_FILENAME: PYTHON_REQUIREMENTS_VOLUME,
}
return volume_maps
def input_(data, field_path):
"""Return a hydrated value of the ``input`` field."""
data_obj = Data.objects.get(id=data["__id"])
inputs = copy.deepcopy(data_obj.input)
# XXX: Optimize by hydrating only the required field (major refactoring).
hydrate_input_references(inputs, data_obj.process.input_schema)
hydrate_input_uploads(inputs, data_obj.process.input_schema)
return dict_dot(inputs, field_path)