def get_tools_paths(self, from_applications=False):
"""Get tools' paths."""
if settings.DEBUG or is_testing() or from_applications:
return list(get_apps_tools().values())
else:
tools_root = storage_settings.FLOW_VOLUMES["tools"]["config"]["path"]
subdirs = next(os.walk(tools_root))[1]
return [os.path.join(tools_root, sdir) for sdir in subdirs]
def get_tools_paths(self):
"""Get tools' paths."""
if settings.DEBUG or is_testing():
return list(get_apps_tools().values())
else:
tools_root = settings.FLOW_TOOLS_ROOT
subdirs = next(os.walk(tools_root))[1]
return [os.path.join(tools_root, sdir) for sdir in subdirs]
def get_tools_paths(self):
"""Get tools' paths."""
if settings.DEBUG or is_testing():
return list(get_apps_tools().values())
else:
tools_root = settings.FLOW_TOOLS_ROOT
subdirs = next(os.walk(tools_root))[1]
return [os.path.join(tools_root, sdir) for sdir in subdirs]
def handle_resolve_data_path(self, message: Message[int],
manager: "Processor") -> Response[str]:
"""Return the base path that stores given data."""
data_pk = message.message_data
if data_pk not in self._hydrate_cache or is_testing():
mount_point = os.fspath(constants.INPUTS_VOLUME)
data_connectors = FileStorage.objects.get(
data__pk=data_pk).connectors
for connector in connectors.for_storage("data"):
if connector in data_connectors and connector.mountable:
mount_point = f"/data_{connector.name}"
break
self._hydrate_cache[data_pk] = mount_point
return message.respond_ok(self._hydrate_cache[data_pk])
def discover_indexes(self):
"""Save list of index builders into ``_index_builders``."""
self.indexes = []
for app_config in apps.get_app_configs():
indexes_path = "{}.elastic_indexes".format(app_config.name)
try:
indexes_module = import_module(indexes_path)
for attr_name in dir(indexes_module):
attr = getattr(indexes_module, attr_name)
if (
inspect.isclass(attr)
and issubclass(attr, BaseIndex)
and attr is not BaseIndex
):
# Make sure that parallel tests have different indices.
if is_testing():
index = attr.document_class._index._name
testing_postfix = "_test_{}_{}".format(
TESTING_UUID, os.getpid()
)
if not index.endswith(testing_postfix):
# Replace current postfix with the new one.
if attr.testing_postfix:
index = index[: -len(attr.testing_postfix)]
index = index + testing_postfix
attr.testing_postfix = testing_postfix
attr.document_class._index._name = index
index = attr()
# Apply any extensions defined for the given index. Currently index extensions are
# limited to extending "mappings".
for extension in composer.get_extensions(attr):
mapping = getattr(extension, "mapping", {})
index.mapping.update(mapping)
self.indexes.append(index)
except ImportError as ex:
if not re.match("No module named .*elastic_indexes.*", str(ex)):
raise
def discover_indexes(self):
"""Save list of index builders into ``_index_builders``."""
self.indexes = []
for app_config in apps.get_app_configs():
indexes_path = '{}.elastic_indexes'.format(app_config.name)
try:
indexes_module = import_module(indexes_path)
for attr_name in dir(indexes_module):
attr = getattr(indexes_module, attr_name)
if inspect.isclass(attr) and issubclass(attr, BaseIndex) and attr is not BaseIndex:
# Make sure that parallel tests have different indices.
if is_testing():
index = attr.document_class._index._name # pylint: disable=protected-access
testing_postfix = '_test_{}_{}'.format(TESTING_UUID, os.getpid())
if not index.endswith(testing_postfix):
# Replace current postfix with the new one.
if attr.testing_postfix:
index = index[:-len(attr.testing_postfix)]
index = index + testing_postfix
attr.testing_postfix = testing_postfix
attr.document_class._index._name = index # pylint: disable=protected-access
index = attr()
# Apply any extensions defined for the given index. Currently index extensions are
# limited to extending "mappings".
for extension in composer.get_extensions(attr):
mapping = getattr(extension, 'mapping', {})
index.mapping.update(mapping)
self.indexes.append(index)
except ImportError as ex:
if not re.match('No module named .*elastic_indexes.*', str(ex)):
raise
def handle_finish(self, obj):
"""Handle an incoming ``Data`` finished processing request.
:param obj: The Channels message object. Command object format:
.. code-block:: none
{
'command': 'finish',
'data_id': [id of the :class:`~resolwe.flow.models.Data` object
this command changes],
'process_rc': [exit status of the processing]
'spawn_processes': [optional; list of spawn dictionaries],
'exported_files_mapper': [if spawn_processes present]
}
"""
data_id = obj[ExecutorProtocol.DATA_ID]
logger.debug(__("Finishing Data with id {} (handle_finish).", data_id),
extra={
'data_id': data_id,
'packet': obj
})
with transaction.atomic():
# Spawn any new jobs in the request.
spawned = False
if ExecutorProtocol.FINISH_SPAWN_PROCESSES in obj:
if is_testing():
# NOTE: This is a work-around for Django issue #10827
# (https://code.djangoproject.com/ticket/10827), same as in
# TestCaseHelpers._pre_setup(). Because the listener is running
# independently, it must clear the cache on its own.
ContentType.objects.clear_cache()
spawned = True
exported_files_mapper = obj[
ExecutorProtocol.FINISH_EXPORTED_FILES]
logger.debug(__(
"Spawning new Data objects for Data with id {} (handle_finish).",
data_id),
extra={'data_id': data_id})
try:
# This transaction is needed because we're running
# asynchronously with respect to the main Django code
# here; the manager can get nudged from elsewhere.
with transaction.atomic():
parent_data = Data.objects.get(pk=data_id)
# Spawn processes.
for d in obj[ExecutorProtocol.FINISH_SPAWN_PROCESSES]:
d['contributor'] = parent_data.contributor
d['process'] = Process.objects.filter(
slug=d['process']).latest()
for field_schema, fields in iterate_fields(
d.get('input', {}),
d['process'].input_schema):
type_ = field_schema['type']
name = field_schema['name']
value = fields[name]
if type_ == 'basic:file:':
fields[name] = self.hydrate_spawned_files(
exported_files_mapper, value, data_id)
elif type_ == 'list:basic:file:':
fields[name] = [
self.hydrate_spawned_files(
exported_files_mapper, fn, data_id)
for fn in value
]
with transaction.atomic():
d = Data.objects.create(**d)
DataDependency.objects.create(
parent=parent_data,
child=d,
kind=DataDependency.KIND_SUBPROCESS,
)
# Copy permissions.
copy_permissions(parent_data, d)
# Entity is added to the collection only when it is
# created - when it only contains 1 Data object.
entities = Entity.objects.filter(
data=d).annotate(
num_data=Count('data')).filter(
num_data=1)
# Copy collections.
for collection in parent_data.collection_set.all(
):
collection.data.add(d)
# Add entities to which data belongs to the collection.
for entity in entities:
entity.collections.add(collection)
except Exception: # pylint: disable=broad-except
logger.error(__(
"Error while preparing spawned Data objects of process '{}' (handle_finish):\n\n{}",
parent_data.process.slug, traceback.format_exc()),
extra={'data_id': data_id})
# Data wrap up happens last, so that any triggered signals
# already see the spawned children. What the children themselves
# see is guaranteed by the transaction we're in.
if ExecutorProtocol.FINISH_PROCESS_RC in obj:
process_rc = obj[ExecutorProtocol.FINISH_PROCESS_RC]
try:
d = Data.objects.get(pk=data_id)
except Data.DoesNotExist:
logger.warning(
"Data object does not exist (handle_finish).",
extra={
'data_id': data_id,
})
async_to_sync(self._send_reply)(obj, {
ExecutorProtocol.RESULT:
ExecutorProtocol.RESULT_ERROR
})
return
if process_rc == 0 and not d.status == Data.STATUS_ERROR:
changeset = {
'status': Data.STATUS_DONE,
'process_progress': 100,
'finished': now()
}
else:
changeset = {
'status': Data.STATUS_ERROR,
'process_progress': 100,
'process_rc': process_rc,
'finished': now()
}
obj[ExecutorProtocol.UPDATE_CHANGESET] = changeset
self.handle_update(obj, internal_call=True)
if not getattr(settings, 'FLOW_MANAGER_KEEP_DATA', False):
try:
# Clean up after process
data_purge(data_ids=[data_id],
delete=True,
verbosity=self._verbosity)
except Exception: # pylint: disable=broad-except
logger.error(__("Purge error:\n\n{}",
traceback.format_exc()),
extra={'data_id': data_id})
# Notify the executor that we're done.
async_to_sync(self._send_reply)(
obj, {
ExecutorProtocol.RESULT: ExecutorProtocol.RESULT_OK
})
# Now nudge the main manager to perform final cleanup. This is
# needed even if there was no spawn baggage, since the manager
# may need to know when executors have finished, to keep count
# of them and manage synchronization.
async_to_sync(consumer.send_event)({
WorkerProtocol.COMMAND:
WorkerProtocol.FINISH,
WorkerProtocol.DATA_ID:
data_id,
WorkerProtocol.FINISH_SPAWNED:
spawned,
WorkerProtocol.FINISH_COMMUNICATE_EXTRA: {
'executor':
getattr(settings, 'FLOW_EXECUTOR',
{}).get('NAME', 'resolwe.flow.executors.local'),
},
})
def _data_scan(self,
data_id=None,
executor="resolwe.flow.executors.local",
**kwargs):
"""Scan for new Data objects and execute them.
:param data_id: Optional id of Data object which (+ its
children) should be scanned. If it is not given, all
resolving objects are processed.
:param executor: The fully qualified name of the executor to use
for all :class:`~resolwe.flow.models.Data` objects
discovered in this pass.
"""
def process_data_object(data):
"""Process a single data object."""
# Lock for update. Note that we want this transaction to be as short as possible in
# order to reduce contention and avoid deadlocks. This is why we do not lock all
# resolving objects for update, but instead only lock one object at a time. This
# allows managers running in parallel to process different objects.
data = Data.objects.select_for_update().get(pk=data.pk)
if data.status != Data.STATUS_RESOLVING:
# The object might have already been processed while waiting for the lock to be
# obtained. In this case, skip the object.
return
dep_status = dependency_status(data)
if dep_status == Data.STATUS_ERROR:
data.status = Data.STATUS_ERROR
data.process_error.append(
"One or more inputs have status ERROR")
data.process_rc = 1
data.save()
return
elif dep_status != Data.STATUS_DONE:
return
if data.process.run:
try:
execution_engine = data.process.run.get("language", None)
# Evaluation by the execution engine may spawn additional data objects and
# perform other queries on the database. Queries of all possible execution
# engines need to be audited for possibilities of deadlocks in case any
# additional locks are introduced. Currently, we only take an explicit lock on
# the currently processing object.
program = self.get_execution_engine(
execution_engine).evaluate(data)
except (ExecutionError, InvalidEngineError) as error:
data.status = Data.STATUS_ERROR
data.process_error.append(
"Error in process script: {}".format(error))
data.save()
return
# Set allocated resources:
resource_limits = data.process.get_resource_limits()
data.process_memory = resource_limits["memory"]
data.process_cores = resource_limits["cores"]
else:
# If there is no run section, then we should not try to run anything. But the
# program must not be set to None as then the process will be stuck in waiting
# state.
program = ""
if data.status != Data.STATUS_DONE:
# The data object may already be marked as done by the execution engine. In this
# case we must not revert the status to STATUS_WAITING.
data.status = Data.STATUS_WAITING
data.save(render_name=True)
# Actually run the object only if there was nothing with the transaction.
transaction.on_commit(
# Make sure the closure gets the right values here, since they're
# changed in the loop.
lambda d=data, p=program: self._data_execute(d, p, executor))
logger.debug(
__(
"Manager processing communicate command triggered by Data with id {}.",
data_id,
))
if is_testing():
# NOTE: This is a work-around for Django issue #10827
# (https://code.djangoproject.com/ticket/10827), same as in
# TestCaseHelpers._pre_setup(). Because the worker is running
# independently, it must clear the cache on its own.
ContentType.objects.clear_cache()
# Ensure settings overrides apply
self.discover_engines(executor=executor)
try:
queryset = Data.objects.filter(status=Data.STATUS_RESOLVING)
if data_id is not None:
# Scan only given data object and its children.
queryset = queryset.filter(Q(parents=data_id)
| Q(id=data_id)).distinct()
for data in queryset:
try:
with transaction.atomic():
process_data_object(data)
# All data objects created by the execution engine are commited after this
# point and may be processed by other managers running in parallel. At the
# same time, the lock for the current data object is released.
except Exception as error:
logger.exception(
__(
"Unhandled exception in _data_scan while processing data object {}.",
data.pk,
))
# Unhandled error while processing a data object. We must set its
# status to STATUS_ERROR to prevent the object from being retried
# on next _data_scan run. We must perform this operation without
# using the Django ORM as using the ORM may be the reason the error
# occurred in the first place.
error_msg = "Internal error: {}".format(error)
process_error_field = Data._meta.get_field("process_error")
max_length = process_error_field.base_field.max_length
if len(error_msg) > max_length:
error_msg = error_msg[:max_length - 3] + "..."
try:
with connection.cursor() as cursor:
cursor.execute(
"""
UPDATE {table}
SET
status = %(status)s,
process_error = process_error || (%(error)s)::varchar[]
WHERE id = %(id)s
""".format(table=Data._meta.db_table),
{
"status": Data.STATUS_ERROR,
"error": [error_msg],
"id": data.pk,
},
)
except Exception:
# If object's state cannot be changed due to some database-related
# issue, at least skip the object for this run.
logger.exception(
__(
"Unhandled exception in _data_scan while trying to emit error for {}.",
data.pk,
))
except IntegrityError as exp:
logger.error(__("IntegrityError in manager {}", exp))
return
def handle_bootstrap(self, message: Message[Tuple[int,
str]]) -> Response[Dict]:
"""Handle bootstrap request.
:raises RuntimeError: when settings name is not known.
"""
data_id, settings_name = message.message_data
data = Data.objects.get(pk=data_id)
if is_testing():
self._bootstrap_cache = defaultdict(dict)
self._bootstrap_prepare_static_cache()
response: Dict[str, Any] = dict()
self.bootstrap_prepare_process_cache(data)
if settings_name == "executor":
response[ExecutorFiles.EXECUTOR_SETTINGS] = {
"DATA_DIR": data.location.get_path()
}
response[ExecutorFiles.LOCATION_SUBPATH] = data.location.subpath
response[ExecutorFiles.DJANGO_SETTINGS] = self._bootstrap_cache[
"settings"].copy()
response[ExecutorFiles.
PROCESS_META] = self._bootstrap_cache["process_meta"]
response[ExecutorFiles.PROCESS] = self._bootstrap_cache["process"][
data.process.id]
elif settings_name == "init":
response[ExecutorFiles.DJANGO_SETTINGS] = {
"STORAGE_CONNECTORS":
self._bootstrap_cache["settings"]["STORAGE_CONNECTORS"],
"FLOW_STORAGE":
storage_settings.FLOW_STORAGE,
"FLOW_VOLUMES":
storage_settings.FLOW_VOLUMES,
}
if hasattr(settings, constants.INPUTS_VOLUME_NAME):
response[ExecutorFiles.DJANGO_SETTINGS][
constants.INPUTS_VOLUME_NAME] = self._bootstrap_cache[
"settings"][constants.INPUTS_VOLUME_NAME]
response[ExecutorFiles.
SECRETS_DIR] = self._bootstrap_cache["connector_secrets"]
try:
response[ExecutorFiles.SECRETS_DIR].update(
data.resolve_secrets())
except PermissionDenied as e:
data.process_error.append(str(e))
data.save(update_fields=["process_error"])
raise
response[ExecutorFiles.LOCATION_SUBPATH] = data.location.subpath
elif settings_name == "communication":
response[ExecutorFiles.DJANGO_SETTINGS] = {
"STORAGE_CONNECTORS":
self._bootstrap_cache["settings"]["STORAGE_CONNECTORS"],
"FLOW_STORAGE":
storage_settings.FLOW_STORAGE,
}
response[ExecutorFiles.LOCATION_SUBPATH] = data.location.subpath
else:
raise RuntimeError(
f"Settings {settings_name} sent by peer with id {data_id} unknown."
)
return message.respond_ok(response)
def handle_finish(self, obj):
"""Handle an incoming ``Data`` finished processing request.
:param obj: The Channels message object. Command object format:
.. code-block:: none
{
'command': 'finish',
'data_id': [id of the :class:`~resolwe.flow.models.Data` object
this command changes],
'process_rc': [exit status of the processing]
'spawn_processes': [optional; list of spawn dictionaries],
'exported_files_mapper': [if spawn_processes present]
}
"""
data_id = obj[ExecutorProtocol.DATA_ID]
logger.debug(
__("Finishing Data with id {} (handle_finish).", data_id),
extra={
"data_id": data_id,
"packet": obj
},
)
spawning_failed = False
with transaction.atomic():
# Spawn any new jobs in the request.
spawned = False
if ExecutorProtocol.FINISH_SPAWN_PROCESSES in obj:
if is_testing():
# NOTE: This is a work-around for Django issue #10827
# (https://code.djangoproject.com/ticket/10827), same as in
# TestCaseHelpers._pre_setup(). Because the listener is running
# independently, it must clear the cache on its own.
ContentType.objects.clear_cache()
spawned = True
exported_files_mapper = obj[
ExecutorProtocol.FINISH_EXPORTED_FILES]
logger.debug(
__(
"Spawning new Data objects for Data with id {} (handle_finish).",
data_id,
),
extra={"data_id": data_id},
)
try:
# This transaction is needed because we're running
# asynchronously with respect to the main Django code
# here; the manager can get nudged from elsewhere.
with transaction.atomic():
parent_data = Data.objects.get(pk=data_id)
# Spawn processes.
for d in obj[ExecutorProtocol.FINISH_SPAWN_PROCESSES]:
d["contributor"] = parent_data.contributor
d["process"] = Process.objects.filter(
slug=d["process"]).latest()
d["tags"] = parent_data.tags
d["collection"] = parent_data.collection
d["subprocess_parent"] = parent_data
for field_schema, fields in iterate_fields(
d.get("input", {}),
d["process"].input_schema):
type_ = field_schema["type"]
name = field_schema["name"]
value = fields[name]
if type_ == "basic:file:":
fields[name] = self.hydrate_spawned_files(
exported_files_mapper, value, data_id)
elif type_ == "list:basic:file:":
fields[name] = [
self.hydrate_spawned_files(
exported_files_mapper, fn, data_id)
for fn in value
]
d = Data.objects.create(**d)
except Exception:
logger.error(
__(
"Error while preparing spawned Data objects of process '{}' (handle_finish):\n\n{}",
parent_data.process.slug,
traceback.format_exc(),
),
extra={"data_id": data_id},
)
spawning_failed = True
# Data wrap up happens last, so that any triggered signals
# already see the spawned children. What the children themselves
# see is guaranteed by the transaction we're in.
if ExecutorProtocol.FINISH_PROCESS_RC in obj:
process_rc = obj[ExecutorProtocol.FINISH_PROCESS_RC]
try:
d = Data.objects.get(pk=data_id)
except Data.DoesNotExist:
logger.warning(
"Data object does not exist (handle_finish).",
extra={"data_id": data_id},
)
async_to_sync(self._send_reply)(obj, {
ExecutorProtocol.RESULT:
ExecutorProtocol.RESULT_ERROR
})
return
changeset = {
"process_progress": 100,
"finished": now(),
}
if spawning_failed:
changeset["status"] = Data.STATUS_ERROR
changeset["process_error"] = [
"Error while preparing spawned Data objects"
]
elif process_rc != 0:
changeset["status"] = Data.STATUS_ERROR
changeset["process_rc"] = process_rc
obj[ExecutorProtocol.UPDATE_CHANGESET] = changeset
self.handle_update(obj, internal_call=True)
# Unlock all inputs.
self.unlock_all_inputs(data_id)
# Notify the executor that we're done.
async_to_sync(self._send_reply)(
obj, {
ExecutorProtocol.RESULT: ExecutorProtocol.RESULT_OK
})
# Now nudge the main manager to perform final cleanup. This is
# needed even if there was no spawn baggage, since the manager
# may need to know when executors have finished, to keep count
# of them and manage synchronization.
async_to_sync(consumer.send_event)({
WorkerProtocol.COMMAND:
WorkerProtocol.FINISH,
WorkerProtocol.DATA_ID:
data_id,
WorkerProtocol.FINISH_SPAWNED:
spawned,
WorkerProtocol.FINISH_COMMUNICATE_EXTRA: {
"executor":
getattr(settings, "FLOW_EXECUTOR",
{}).get("NAME", "resolwe.flow.executors.local"),
},
})
def _data_scan(self, data_id=None, executor='resolwe.flow.executors.local', **kwargs):
"""Scan for new Data objects and execute them.
:param data_id: Optional id of Data object which (+ its
children) should be scanned. If it is not given, all
resolving objects are processed.
:param executor: The fully qualified name of the executor to use
for all :class:`~resolwe.flow.models.Data` objects
discovered in this pass.
"""
def process_data_object(data):
"""Process a single data object."""
# Lock for update. Note that we want this transaction to be as short as possible in
# order to reduce contention and avoid deadlocks. This is why we do not lock all
# resolving objects for update, but instead only lock one object at a time. This
# allows managers running in parallel to process different objects.
data = Data.objects.select_for_update().get(pk=data.pk)
if data.status != Data.STATUS_RESOLVING:
# The object might have already been processed while waiting for the lock to be
# obtained. In this case, skip the object.
return
dep_status = dependency_status(data)
if dep_status == Data.STATUS_ERROR:
data.status = Data.STATUS_ERROR
data.process_error.append("One or more inputs have status ERROR")
data.process_rc = 1
data.save()
return
elif dep_status != Data.STATUS_DONE:
return
if data.process.run:
try:
execution_engine = data.process.run.get('language', None)
# Evaluation by the execution engine may spawn additional data objects and
# perform other queries on the database. Queries of all possible execution
# engines need to be audited for possibilities of deadlocks in case any
# additional locks are introduced. Currently, we only take an explicit lock on
# the currently processing object.
program = self.get_execution_engine(execution_engine).evaluate(data)
except (ExecutionError, InvalidEngineError) as error:
data.status = Data.STATUS_ERROR
data.process_error.append("Error in process script: {}".format(error))
data.save()
return
# Set allocated resources:
resource_limits = data.process.get_resource_limits()
data.process_memory = resource_limits['memory']
data.process_cores = resource_limits['cores']
else:
# If there is no run section, then we should not try to run anything. But the
# program must not be set to None as then the process will be stuck in waiting
# state.
program = ''
if data.status != Data.STATUS_DONE:
# The data object may already be marked as done by the execution engine. In this
# case we must not revert the status to STATUS_WAITING.
data.status = Data.STATUS_WAITING
data.save(render_name=True)
# Actually run the object only if there was nothing with the transaction.
transaction.on_commit(
# Make sure the closure gets the right values here, since they're
# changed in the loop.
lambda d=data, p=program: self._data_execute(d, p, executor)
)
logger.debug(__("Manager processing communicate command triggered by Data with id {}.", data_id))
if is_testing():
# NOTE: This is a work-around for Django issue #10827
# (https://code.djangoproject.com/ticket/10827), same as in
# TestCaseHelpers._pre_setup(). Because the worker is running
# independently, it must clear the cache on its own.
ContentType.objects.clear_cache()
# Ensure settings overrides apply
self.discover_engines(executor=executor)
try:
queryset = Data.objects.filter(status=Data.STATUS_RESOLVING)
if data_id is not None:
# Scan only given data object and its children.
queryset = queryset.filter(Q(parents=data_id) | Q(id=data_id)).distinct()
for data in queryset:
try:
with transaction.atomic():
process_data_object(data)
# All data objects created by the execution engine are commited after this
# point and may be processed by other managers running in parallel. At the
# same time, the lock for the current data object is released.
except Exception as error: # pylint: disable=broad-except
logger.exception(__(
"Unhandled exception in _data_scan while processing data object {}.",
data.pk
))
# Unhandled error while processing a data object. We must set its
# status to STATUS_ERROR to prevent the object from being retried
# on next _data_scan run. We must perform this operation without
# using the Django ORM as using the ORM may be the reason the error
# occurred in the first place.
error_msg = "Internal error: {}".format(error)
process_error_field = Data._meta.get_field('process_error') # pylint: disable=protected-access
max_length = process_error_field.base_field.max_length
if len(error_msg) > max_length:
error_msg = error_msg[:max_length - 3] + '...'
try:
with connection.cursor() as cursor:
cursor.execute(
"""
UPDATE {table}
SET
status = %(status)s,
process_error = process_error || (%(error)s)::varchar[]
WHERE id = %(id)s
""".format(
table=Data._meta.db_table # pylint: disable=protected-access
),
{
'status': Data.STATUS_ERROR,
'error': [error_msg],
'id': data.pk
}
)
except Exception as error: # pylint: disable=broad-except
# If object's state cannot be changed due to some database-related
# issue, at least skip the object for this run.
logger.exception(__(
"Unhandled exception in _data_scan while trying to emit error for {}.",
data.pk
))
except IntegrityError as exp:
logger.error(__("IntegrityError in manager {}", exp))
return
def handle_finish(self, obj):
"""Handle an incoming ``Data`` finished processing request.
:param obj: The Channels message object. Command object format:
.. code-block:: none
{
'command': 'finish',
'data_id': [id of the :class:`~resolwe.flow.models.Data` object
this command changes],
'process_rc': [exit status of the processing]
'spawn_processes': [optional; list of spawn dictionaries],
'exported_files_mapper': [if spawn_processes present]
}
"""
data_id = obj[ExecutorProtocol.DATA_ID]
logger.debug(
__("Finishing Data with id {} (handle_finish).", data_id),
extra={
'data_id': data_id,
'packet': obj
}
)
spawning_failed = False
with transaction.atomic():
# Spawn any new jobs in the request.
spawned = False
if ExecutorProtocol.FINISH_SPAWN_PROCESSES in obj:
if is_testing():
# NOTE: This is a work-around for Django issue #10827
# (https://code.djangoproject.com/ticket/10827), same as in
# TestCaseHelpers._pre_setup(). Because the listener is running
# independently, it must clear the cache on its own.
ContentType.objects.clear_cache()
spawned = True
exported_files_mapper = obj[ExecutorProtocol.FINISH_EXPORTED_FILES]
logger.debug(
__("Spawning new Data objects for Data with id {} (handle_finish).", data_id),
extra={
'data_id': data_id
}
)
try:
# This transaction is needed because we're running
# asynchronously with respect to the main Django code
# here; the manager can get nudged from elsewhere.
with transaction.atomic():
parent_data = Data.objects.get(pk=data_id)
# Spawn processes.
for d in obj[ExecutorProtocol.FINISH_SPAWN_PROCESSES]:
d['contributor'] = parent_data.contributor
d['process'] = Process.objects.filter(slug=d['process']).latest()
d['tags'] = parent_data.tags
for field_schema, fields in iterate_fields(d.get('input', {}), d['process'].input_schema):
type_ = field_schema['type']
name = field_schema['name']
value = fields[name]
if type_ == 'basic:file:':
fields[name] = self.hydrate_spawned_files(
exported_files_mapper, value, data_id
)
elif type_ == 'list:basic:file:':
fields[name] = [self.hydrate_spawned_files(exported_files_mapper, fn, data_id)
for fn in value]
with transaction.atomic():
d = Data.objects.create(**d)
DataDependency.objects.create(
parent=parent_data,
child=d,
kind=DataDependency.KIND_SUBPROCESS,
)
# Copy permissions.
copy_permissions(parent_data, d)
# Entity is added to the collection only when it is
# created - when it only contains 1 Data object.
entities = Entity.objects.filter(data=d).annotate(num_data=Count('data')).filter(
num_data=1)
# Copy collections.
for collection in parent_data.collection_set.all():
collection.data.add(d)
# Add entities to which data belongs to the collection.
for entity in entities:
entity.collections.add(collection)
except Exception: # pylint: disable=broad-except
logger.error(
__(
"Error while preparing spawned Data objects of process '{}' (handle_finish):\n\n{}",
parent_data.process.slug,
traceback.format_exc()
),
extra={
'data_id': data_id
}
)
spawning_failed = True
# Data wrap up happens last, so that any triggered signals
# already see the spawned children. What the children themselves
# see is guaranteed by the transaction we're in.
if ExecutorProtocol.FINISH_PROCESS_RC in obj:
process_rc = obj[ExecutorProtocol.FINISH_PROCESS_RC]
try:
d = Data.objects.get(pk=data_id)
except Data.DoesNotExist:
logger.warning(
"Data object does not exist (handle_finish).",
extra={
'data_id': data_id,
}
)
async_to_sync(self._send_reply)(obj, {ExecutorProtocol.RESULT: ExecutorProtocol.RESULT_ERROR})
return
changeset = {
'process_progress': 100,
'finished': now(),
}
if spawning_failed:
changeset['status'] = Data.STATUS_ERROR
changeset['process_error'] = ["Error while preparing spawned Data objects"]
elif process_rc == 0 and not d.status == Data.STATUS_ERROR:
changeset['status'] = Data.STATUS_DONE
else:
changeset['status'] = Data.STATUS_ERROR
changeset['process_rc'] = process_rc
obj[ExecutorProtocol.UPDATE_CHANGESET] = changeset
self.handle_update(obj, internal_call=True)
if not getattr(settings, 'FLOW_MANAGER_KEEP_DATA', False):
# Purge worker is not running in test runner, so we should skip triggering it.
if not is_testing():
channel_layer = get_channel_layer()
try:
async_to_sync(channel_layer.send)(
CHANNEL_PURGE_WORKER,
{
'type': TYPE_PURGE_RUN,
'location_id': d.location.id,
'verbosity': self._verbosity,
}
)
except ChannelFull:
logger.warning(
"Cannot trigger purge because channel is full.",
extra={'data_id': data_id}
)
# Notify the executor that we're done.
async_to_sync(self._send_reply)(obj, {ExecutorProtocol.RESULT: ExecutorProtocol.RESULT_OK})
# Now nudge the main manager to perform final cleanup. This is
# needed even if there was no spawn baggage, since the manager
# may need to know when executors have finished, to keep count
# of them and manage synchronization.
async_to_sync(consumer.send_event)({
WorkerProtocol.COMMAND: WorkerProtocol.FINISH,
WorkerProtocol.DATA_ID: data_id,
WorkerProtocol.FINISH_SPAWNED: spawned,
WorkerProtocol.FINISH_COMMUNICATE_EXTRA: {
'executor': getattr(settings, 'FLOW_EXECUTOR', {}).get('NAME', 'resolwe.flow.executors.local'),
},
})
