def nix_code_from_minio(apps, schema_editor):
from server import minio
# Delete from S3
try:
minio.remove_recursive(minio.ExternalModulesBucket, 'extract-numbers/')
except minio.error.NoSuchBucket:
pass # we're unit-testing and the bucket doesn't exist
def delete_module_versions(apps, schema_editor):
from server import minio
# Delete from database
ModuleVersion = apps.get_model('server', 'ModuleVersion')
ModuleVersion.objects.filter(id_name='convert-text').delete()
# Delete from S3
try:
minio.remove_recursive(minio.ExternalModulesBucket, 'convert-text/')
except minio.error.NoSuchBucket:
pass # we're unit-testing and the bucket doesn't exist
def delete(self, *args, **kwargs):
if self.inprogress_file_upload_key:
try:
minio.abort_multipart_upload(minio.UserFilesBucket,
self.inprogress_file_upload_key,
self.inprogress_file_upload_id)
except minio.error.NoSuchUpload:
pass
minio.remove_recursive(minio.UserFilesBucket,
self.uploaded_file_prefix)
CachedRenderResult.clear_wf_module(self)
super().delete(*args, **kwargs)
def delete_parquet_files_for_wf_module(wf_module: 'WfModule') -> None:
"""
Ensures there are no Parquet files cached for `wf_module`.
Different deltas on the same module produce different Parquet
filenames. This function removes all of them.
This leaves `wf_module.cached_render_result` invalid: it will continue
to exist in the database, but callers who try to read from it will
see `FileNotFoundError.
"""
minio.remove_recursive(
minio.CachedRenderResultsBucket,
parquet_prefix(wf_module.workflow_id, wf_module.id))
def import_module_from_directory(version: str,
importdir: Path,
force_reload=False):
module_files = ModuleFiles.load_from_dirpath(importdir) # raise ValueError
spec = ModuleSpec.load_from_path(module_files.spec) # raise ValueError
validate_python_functions(module_files.code) # raise ValueError
if not force_reload:
# Don't allow importing the same version twice
try:
ModuleVersion.objects.get(id_name=spec.id_name,
source_version_hash=version)
raise ValueError(f'Version {version} of module {spec.id_name}'
' has already been imported')
except ModuleVersion.DoesNotExist:
# this is what we want
pass
if module_files.javascript:
js_module = module_files.javascript.read_text(encoding='utf-8')
else:
js_module = ''
# Copy whole directory to S3
prefix = '%s/%s/' % (spec.id_name, version)
try:
# If files already exist, delete them so we can overwrite them.
#
# This can race: a worker may be loading the code to execute it. But
# races are unlikely to affect anybody because:
#
# * If force_reload=True we're in dev or test, where we control
# everything.
# * Otherwise, we've already checked there's no ModuleVersion, so
# probably nothing is trying and load what we're deleting here.
minio.remove_recursive(minio.ExternalModulesBucket, prefix)
except FileNotFoundError:
pass # common case: we aren't overwriting code
minio.fput_directory_contents(minio.ExternalModulesBucket, prefix,
Path(importdir))
# If that succeeds, initialise module in our database
module_version = ModuleVersion.create_or_replace_from_spec(
spec, source_version_hash=version, js_module=js_module)
logger.info('Imported module %s' % spec.id_name)
return module_version
def clear_minio():
buckets = (
minio.UserFilesBucket,
minio.StoredObjectsBucket,
minio.ExternalModulesBucket,
minio.CachedRenderResultsBucket,
)
if not hasattr(clear_minio, '_initialized'):
# Ensure buckets exist -- only on first call
for bucket in buckets:
minio.ensure_bucket_exists(bucket)
clear_minio._initialized = True
for bucket in buckets:
minio.remove_recursive(bucket, '/', force=True)
def delete(self, *args, **kwargs):
# Clear delta history. Deltas can reference WfModules: if we don't
# clear the deltas, Django may decide to CASCADE to WfModule first and
# we'll raise a ProtectedError.
self.clear_deltas()
# Clear all minio data. We _should_ clear it in pre-delete hooks on
# StoredObject, UploadedFile, etc.; but [2019-06-03, adamhooper] the
# database is inconsistent and Django is hard to use so new bugs may
# crop up anyway.
#
# [2019-06-03, adamhooper] hooks never work in ORMs. Better would be
# to make `delete()` a controller method, not a funky mishmash of
# Django-ORM absurdities. TODO nix Django ORM.
#
# TL;DR we're double-deleting minio data, to be extra-safe. The user
# said "delete." We'll delete.
if self.id: # be extra-safe: use if-statement so we don't remove '/'
minio.remove_recursive(minio.StoredObjectsBucket, f'{self.id}/')
minio.remove_recursive(minio.UserFilesBucket, f'wf-{self.id}/')
super().delete(*args, **kwargs)
def _delete_from_s3_post_delete(sender, instance, **kwargs):
"""
Delete module _code_ from S3, now that ModuleVersion is gone.
"""
prefix = '%s/%s/' % (sender.id_name, sender.source_version_hash)
minio.remove_recursive(minio.ExternalModulesBucket, prefix)
def delete(self, *args, **kwargs):
for in_progress_upload in self.in_progress_uploads.all():
in_progress_upload.delete_s3_data()
minio.remove_recursive(minio.UserFilesBucket, self.uploaded_file_prefix)
CachedRenderResult.clear_wf_module(self)
super().delete(*args, **kwargs)
