def nix_code_from_minio(apps, schema_editor): from server import minio # Delete from S3 try: minio.remove_recursive(minio.ExternalModulesBucket, 'extract-numbers/') except minio.error.NoSuchBucket: pass # we're unit-testing and the bucket doesn't exist
def delete_module_versions(apps, schema_editor): from server import minio # Delete from database ModuleVersion = apps.get_model('server', 'ModuleVersion') ModuleVersion.objects.filter(id_name='convert-text').delete() # Delete from S3 try: minio.remove_recursive(minio.ExternalModulesBucket, 'convert-text/') except minio.error.NoSuchBucket: pass # we're unit-testing and the bucket doesn't exist
def delete(self, *args, **kwargs): if self.inprogress_file_upload_key: try: minio.abort_multipart_upload(minio.UserFilesBucket, self.inprogress_file_upload_key, self.inprogress_file_upload_id) except minio.error.NoSuchUpload: pass minio.remove_recursive(minio.UserFilesBucket, self.uploaded_file_prefix) CachedRenderResult.clear_wf_module(self) super().delete(*args, **kwargs)
def delete_parquet_files_for_wf_module(wf_module: 'WfModule') -> None: """ Ensures there are no Parquet files cached for `wf_module`. Different deltas on the same module produce different Parquet filenames. This function removes all of them. This leaves `wf_module.cached_render_result` invalid: it will continue to exist in the database, but callers who try to read from it will see `FileNotFoundError. """ minio.remove_recursive( minio.CachedRenderResultsBucket, parquet_prefix(wf_module.workflow_id, wf_module.id))
def import_module_from_directory(version: str, importdir: Path, force_reload=False): module_files = ModuleFiles.load_from_dirpath(importdir) # raise ValueError spec = ModuleSpec.load_from_path(module_files.spec) # raise ValueError validate_python_functions(module_files.code) # raise ValueError if not force_reload: # Don't allow importing the same version twice try: ModuleVersion.objects.get(id_name=spec.id_name, source_version_hash=version) raise ValueError(f'Version {version} of module {spec.id_name}' ' has already been imported') except ModuleVersion.DoesNotExist: # this is what we want pass if module_files.javascript: js_module = module_files.javascript.read_text(encoding='utf-8') else: js_module = '' # Copy whole directory to S3 prefix = '%s/%s/' % (spec.id_name, version) try: # If files already exist, delete them so we can overwrite them. # # This can race: a worker may be loading the code to execute it. But # races are unlikely to affect anybody because: # # * If force_reload=True we're in dev or test, where we control # everything. # * Otherwise, we've already checked there's no ModuleVersion, so # probably nothing is trying and load what we're deleting here. minio.remove_recursive(minio.ExternalModulesBucket, prefix) except FileNotFoundError: pass # common case: we aren't overwriting code minio.fput_directory_contents(minio.ExternalModulesBucket, prefix, Path(importdir)) # If that succeeds, initialise module in our database module_version = ModuleVersion.create_or_replace_from_spec( spec, source_version_hash=version, js_module=js_module) logger.info('Imported module %s' % spec.id_name) return module_version
def clear_minio(): buckets = ( minio.UserFilesBucket, minio.StoredObjectsBucket, minio.ExternalModulesBucket, minio.CachedRenderResultsBucket, ) if not hasattr(clear_minio, '_initialized'): # Ensure buckets exist -- only on first call for bucket in buckets: minio.ensure_bucket_exists(bucket) clear_minio._initialized = True for bucket in buckets: minio.remove_recursive(bucket, '/', force=True)
def delete(self, *args, **kwargs): # Clear delta history. Deltas can reference WfModules: if we don't # clear the deltas, Django may decide to CASCADE to WfModule first and # we'll raise a ProtectedError. self.clear_deltas() # Clear all minio data. We _should_ clear it in pre-delete hooks on # StoredObject, UploadedFile, etc.; but [2019-06-03, adamhooper] the # database is inconsistent and Django is hard to use so new bugs may # crop up anyway. # # [2019-06-03, adamhooper] hooks never work in ORMs. Better would be # to make `delete()` a controller method, not a funky mishmash of # Django-ORM absurdities. TODO nix Django ORM. # # TL;DR we're double-deleting minio data, to be extra-safe. The user # said "delete." We'll delete. if self.id: # be extra-safe: use if-statement so we don't remove '/' minio.remove_recursive(minio.StoredObjectsBucket, f'{self.id}/') minio.remove_recursive(minio.UserFilesBucket, f'wf-{self.id}/') super().delete(*args, **kwargs)
def _delete_from_s3_post_delete(sender, instance, **kwargs): """ Delete module _code_ from S3, now that ModuleVersion is gone. """ prefix = '%s/%s/' % (sender.id_name, sender.source_version_hash) minio.remove_recursive(minio.ExternalModulesBucket, prefix)
def delete(self, *args, **kwargs): for in_progress_upload in self.in_progress_uploads.all(): in_progress_upload.delete_s3_data() minio.remove_recursive(minio.UserFilesBucket, self.uploaded_file_prefix) CachedRenderResult.clear_wf_module(self) super().delete(*args, **kwargs)