def run_for_product(self, product, fn, use_enclosing_directory): tmp_root = self.get_tmp_root(product) product_path = self.product_path(product) with util.TemporaryDirectory(dir=tmp_root, prefix=".run_for_product-", suffix="-%s" % product.core.uuid.hex) as tmp_path: self.get(product, product_path, tmp_path, use_enclosing_directory) paths = [os.path.join(tmp_path, basename) for basename in os.listdir(tmp_path)] return fn(paths)
def put(self, paths, properties, use_enclosing_directory, use_symlinks=None, retrieve_files=None, run_for_product=None): if use_symlinks: raise Error("S3 storage backend does not support symlinks") anything_stored = False try: archive_path = properties.core.archive_path physical_name = properties.core.physical_name if not use_enclosing_directory and retrieve_files is None: assert(len(paths) == 1 and os.path.basename(paths[0]) == physical_name) tmp_root = self.get_tmp_root(properties) with util.TemporaryDirectory(dir=tmp_root, prefix=".put-", suffix="-%s" % properties.core.uuid.hex) as tmp_path: if retrieve_files: paths = retrieve_files(tmp_path) # Upload file(s) for path in paths: key = self._prefix + os.path.join(archive_path, physical_name) # Add enclosing dir if use_enclosing_directory: key = os.path.join(key, os.path.basename(path)) if os.path.isdir(path): self._create_dir(key) anything_stored = True for root, subdirs, files in os.walk(path): rel_root = os.path.relpath(root, path) for subdir in subdirs: dirkey = os.path.normpath(os.path.join(key, rel_root, subdir)) self._create_dir(dirkey) anything_stored = True for filename in files: filekey = os.path.normpath(os.path.join(key, rel_root, filename)) filepath = os.path.join(root, filename) self._upload_file(filekey, filepath) anything_stored = True else: self._upload_file(key, path) anything_stored = True if run_for_product is not None: run_for_product(paths) except Exception as e: raise StorageError(e, anything_stored)
def pull(self, archive, product): if getattr(product.core, "archive_path", None) is None: raise Error("cannot pull files that do not have archive_path set") # Determine the (absolute) path in the archive that will contain the product and create it if required. abs_archive_path = os.path.realpath( os.path.join(archive._root, product.core.archive_path)) abs_product_path = os.path.join(abs_archive_path, product.core.physical_name) # Create destination location for product try: util.make_path(abs_archive_path) except EnvironmentError as _error: raise Error("cannot create parent destination path '%s' [%s]" % (abs_archive_path, _error)) plugin = archive.product_type_plugin(product.core.product_type) # Create a temporary directory and download the product there, then move the product to its # destination within the archive. try: with util.TemporaryDirectory(prefix=".pull-", suffix="-%s" % product.core.uuid.hex, dir=abs_archive_path) as tmp_path: # Create enclosing directory if required. if plugin.use_enclosing_directory: tmp_path = os.path.join(tmp_path, product.core.physical_name) util.make_path(tmp_path) # Define a temp location and download the file tmp_file = os.path.join(tmp_path, product.core.physical_name) downloader = util.Downloader(product.core.remote_url, archive.auth_file()) downloader.save(tmp_file) # TODO: implement extraction of downloaded archives # for ftp and file check if url ends with 'core.physical_name + <archive ext>' # for http/https check the header for the line: # Content-Disposition: attachment; filename="**********" # end then use this ***** filename to match against core.physical_name + <archive ext> # Move the transferred product into its destination within the archive. if plugin.use_enclosing_directory: os.rename(tmp_path, abs_product_path) else: os.rename(tmp_file, abs_product_path) except EnvironmentError as _error: raise Error( "unable to transfer product to destination path '%s' [%s]" % (abs_product_path, _error))
def put(self, paths, properties, use_enclosing_directory, use_symlinks=None, move_files=False, retrieve_files=None): if use_symlinks: raise Error("S3 storage backend does not support symlinks") archive_path = properties.core.archive_path physical_name = properties.core.physical_name tmp_root = self.get_tmp_root(properties) with util.TemporaryDirectory(dir=tmp_root, prefix=".put-", suffix="-%s" % properties.core.uuid.hex) as tmp_path: if retrieve_files: paths = retrieve_files(tmp_path) # Upload file(s) for path in paths: key = self._prefix + os.path.join(archive_path, physical_name) # Add enclosing dir if use_enclosing_directory: key = os.path.join(key, os.path.basename(path)) if os.path.isdir(path): for root, subdirs, files in os.walk(path): rel_root = os.path.relpath(root, path) for filename in files: filekey = os.path.normpath( os.path.join(key, rel_root, filename)) filepath = os.path.join(root, filename) self._resource.Object( self.bucket, filekey).upload_file( filepath, ExtraArgs=self._upload_args, Config=self._transfer_config) else: self._resource.Object(self.bucket, key).upload_file( path, ExtraArgs=self._upload_args, Config=self._transfer_config)
def put(self, paths, properties, use_enclosing_directory, use_symlinks=None, move_files=False, retrieve_files=None): if use_symlinks: raise Error("Swift storage backend does not support symlinks") archive_path = properties.core.archive_path physical_name = properties.core.physical_name tmp_root = self.get_tmp_root(properties) with util.TemporaryDirectory(dir=tmp_root, prefix=".put-", suffix="-%s" % properties.core.uuid.hex) as tmp_path: if retrieve_files: paths = retrieve_files(tmp_path) # Upload file(s) for path in paths: key = os.path.join(archive_path, physical_name) # Add enclosing dir if use_enclosing_directory: key = os.path.join(key, os.path.basename(path)) if os.path.isdir(path): for root, subdirs, files in os.walk(path): rel_root = os.path.relpath(root, path) for filename in files: filekey = os.path.normpath( os.path.join(key, rel_root, filename)) filepath = os.path.join(root, filename) with open(filepath, 'rb') as f: self._conn.put_object(self.container, filekey, contents=f.read()) else: with open(path, 'rb') as f: self._conn.put_object(self.container, key, contents=f.read())
def delete(self, product_path, properties): if not os.path.lexists(product_path): # If the product does not exist, do not consider this an error. return try: tmp_root = self.get_tmp_root(properties) with util.TemporaryDirectory(prefix=".remove-", suffix="-%s" % properties.core.uuid.hex, dir=tmp_root) as tmp_path: # Move product into the temporary directory. When the temporary directory will be removed at the end of # this scope, the product will be removed along with it. assert properties.core.physical_name == os.path.basename( product_path) os.rename( product_path, os.path.join(tmp_path, os.path.basename(product_path))) except EnvironmentError as _error: raise Error( "unable to remove product '%s' (%s) [%s]" % (properties.core.product_name, properties.core.uuid, _error))
def put(self, paths, properties, use_enclosing_directory, use_symlinks=None, retrieve_files=None): if use_symlinks is None: use_symlinks = self._use_symlinks physical_name = properties.core.physical_name archive_path = properties.core.archive_path uuid = properties.core.uuid abs_archive_path = os.path.realpath( os.path.join(self._root, archive_path)) abs_product_path = os.path.join(abs_archive_path, physical_name) # TODO separate this out like 'current_archive_path' if paths is not None and util.is_sub_path(os.path.realpath(paths[0]), abs_product_path, allow_equal=True): # Product should already be in the target location for path in paths: if not os.path.exists(path): raise Error("product source path does not exist '%s'" % (path, )) if not util.is_sub_path(os.path.realpath(path), abs_product_path, allow_equal=True): raise Error( "cannot ingest product where only part of the files are already at the " "destination location") else: # Create destination location for product try: util.make_path(abs_archive_path) except EnvironmentError as _error: raise Error("cannot create parent destination path '%s' [%s]" % (abs_archive_path, _error)) # Create a temporary directory and transfer the product there, then move the product to its # destination within the archive. try: tmp_root = self.get_tmp_root(properties) with util.TemporaryDirectory(prefix=".put-", suffix="-%s" % uuid.hex, dir=tmp_root) as tmp_path: # Create enclosing directory if required. if use_enclosing_directory: tmp_path = os.path.join(tmp_path, physical_name) util.make_path(tmp_path) # Transfer the product (parts). if use_symlinks: if use_enclosing_directory: abs_path = abs_product_path else: abs_path = abs_archive_path # Create symbolic link(s) for the product (parts). for path in paths: if util.is_sub_path(path, self._root): # Create a relative symbolic link when the target is part of the archive # (i.e. when creating an intra-archive symbolic link). This ensures the # archive can be relocated without breaking intra-archive symbolic links. os.symlink( os.path.relpath(path, abs_path), os.path.join(tmp_path, os.path.basename(path))) else: os.symlink( path, os.path.join(tmp_path, os.path.basename(path))) else: # Copy/retrieve product (parts). if retrieve_files: paths = retrieve_files(tmp_path) else: for path in paths: util.copy_path(path, tmp_path, resolve_root=True) # Move the transferred product into its destination within the archive. if use_enclosing_directory: os.rename(tmp_path, abs_product_path) else: assert (len(paths) == 1 and os.path.basename(paths[0]) == physical_name) tmp_product_path = os.path.join( tmp_path, physical_name) os.rename(tmp_product_path, abs_product_path) except EnvironmentError as _error: raise Error( "unable to transfer product to destination path '%s' [%s]" % (abs_product_path, _error))
def pull(self, archive, product): from ecmwfapi import ECMWFDataServer, ECMWFService dataserver = ECMWFDataServer(log=logging.info) marsservice = ECMWFService("mars", log=logging.info) if getattr(product.core, "archive_path", None) is None: raise Error("cannot pull files that do not have archive_path set") # Determine the (absolute) path in the archive that will contain the product and create it if required. abs_archive_path = os.path.realpath( os.path.join(archive._root, product.core.archive_path)) abs_product_path = os.path.join(abs_archive_path, product.core.physical_name) # Create destination location for product try: util.make_path(abs_archive_path) except EnvironmentError as _error: raise Error("cannot create parent destination path '%s' [%s]" % (abs_archive_path, _error)) requests = [] for order in product.core.remote_url.split('?')[1].split( '&concatenate&'): request = {} for param in order.split('&'): key, value = param.split('=') request[key] = value requests.append(request) plugin = archive.product_type_plugin(product.core.product_type) # Create a temporary directory and download the product there, then move the product to its # destination within the archive. try: with util.TemporaryDirectory(prefix=".pull-", suffix="-%s" % product.core.uuid.hex, dir=abs_archive_path) as tmp_path: # Create enclosing directory if required. if plugin.use_enclosing_directory: tmp_path = os.path.join(tmp_path, product.core.physical_name) util.make_path(tmp_path) # Download product. tmp_file_combined = os.path.join(tmp_path, product.core.physical_name) tmp_file_download = os.path.join(tmp_path, "request.grib") combined_file = open(tmp_file_combined, "w") for request in requests: if 'dataset' in request: request['target'] = tmp_file_download dataserver.retrieve(request) else: marsservice.execute(request, tmp_file_download) result_file = open(tmp_file_download, "r") combined_file.write(result_file.read()) result_file.close() os.remove(tmp_file_download) combined_file.close() # Move the retrieved product into its destination within the archive. if plugin.use_enclosing_directory: os.rename(tmp_path, abs_product_path) else: os.rename(tmp_file_combined, abs_product_path) except EnvironmentError as _error: raise Error( "unable to transfer product to destination path '%s' [%s]" % (abs_product_path, _error))