def run_for_product(self, product, fn, use_enclosing_directory):
tmp_root = self.get_tmp_root(product)
product_path = self.product_path(product)
with util.TemporaryDirectory(dir=tmp_root, prefix=".run_for_product-",
suffix="-%s" % product.core.uuid.hex) as tmp_path:
self.get(product, product_path, tmp_path, use_enclosing_directory)
paths = [os.path.join(tmp_path, basename) for basename in os.listdir(tmp_path)]
return fn(paths)
def put(self, paths, properties, use_enclosing_directory, use_symlinks=None,
retrieve_files=None, run_for_product=None):
if use_symlinks:
raise Error("S3 storage backend does not support symlinks")
anything_stored = False
try:
archive_path = properties.core.archive_path
physical_name = properties.core.physical_name
if not use_enclosing_directory and retrieve_files is None:
assert(len(paths) == 1 and os.path.basename(paths[0]) == physical_name)
tmp_root = self.get_tmp_root(properties)
with util.TemporaryDirectory(dir=tmp_root, prefix=".put-",
suffix="-%s" % properties.core.uuid.hex) as tmp_path:
if retrieve_files:
paths = retrieve_files(tmp_path)
# Upload file(s)
for path in paths:
key = self._prefix + os.path.join(archive_path, physical_name)
# Add enclosing dir
if use_enclosing_directory:
key = os.path.join(key, os.path.basename(path))
if os.path.isdir(path):
self._create_dir(key)
anything_stored = True
for root, subdirs, files in os.walk(path):
rel_root = os.path.relpath(root, path)
for subdir in subdirs:
dirkey = os.path.normpath(os.path.join(key, rel_root, subdir))
self._create_dir(dirkey)
anything_stored = True
for filename in files:
filekey = os.path.normpath(os.path.join(key, rel_root, filename))
filepath = os.path.join(root, filename)
self._upload_file(filekey, filepath)
anything_stored = True
else:
self._upload_file(key, path)
anything_stored = True
if run_for_product is not None:
run_for_product(paths)
except Exception as e:
raise StorageError(e, anything_stored)
def pull(self, archive, product):
if getattr(product.core, "archive_path", None) is None:
raise Error("cannot pull files that do not have archive_path set")
# Determine the (absolute) path in the archive that will contain the product and create it if required.
abs_archive_path = os.path.realpath(
os.path.join(archive._root, product.core.archive_path))
abs_product_path = os.path.join(abs_archive_path,
product.core.physical_name)
# Create destination location for product
try:
util.make_path(abs_archive_path)
except EnvironmentError as _error:
raise Error("cannot create parent destination path '%s' [%s]" %
(abs_archive_path, _error))
plugin = archive.product_type_plugin(product.core.product_type)
# Create a temporary directory and download the product there, then move the product to its
# destination within the archive.
try:
with util.TemporaryDirectory(prefix=".pull-",
suffix="-%s" % product.core.uuid.hex,
dir=abs_archive_path) as tmp_path:
# Create enclosing directory if required.
if plugin.use_enclosing_directory:
tmp_path = os.path.join(tmp_path,
product.core.physical_name)
util.make_path(tmp_path)
# Define a temp location and download the file
tmp_file = os.path.join(tmp_path, product.core.physical_name)
downloader = util.Downloader(product.core.remote_url,
archive.auth_file())
downloader.save(tmp_file)
# TODO: implement extraction of downloaded archives
# for ftp and file check if url ends with 'core.physical_name + <archive ext>'
# for http/https check the header for the line:
# Content-Disposition: attachment; filename="**********"
# end then use this ***** filename to match against core.physical_name + <archive ext>
# Move the transferred product into its destination within the archive.
if plugin.use_enclosing_directory:
os.rename(tmp_path, abs_product_path)
else:
os.rename(tmp_file, abs_product_path)
except EnvironmentError as _error:
raise Error(
"unable to transfer product to destination path '%s' [%s]" %
(abs_product_path, _error))
def put(self,
paths,
properties,
use_enclosing_directory,
use_symlinks=None,
move_files=False,
retrieve_files=None):
if use_symlinks:
raise Error("S3 storage backend does not support symlinks")
archive_path = properties.core.archive_path
physical_name = properties.core.physical_name
tmp_root = self.get_tmp_root(properties)
with util.TemporaryDirectory(dir=tmp_root,
prefix=".put-",
suffix="-%s" %
properties.core.uuid.hex) as tmp_path:
if retrieve_files:
paths = retrieve_files(tmp_path)
# Upload file(s)
for path in paths:
key = self._prefix + os.path.join(archive_path, physical_name)
# Add enclosing dir
if use_enclosing_directory:
key = os.path.join(key, os.path.basename(path))
if os.path.isdir(path):
for root, subdirs, files in os.walk(path):
rel_root = os.path.relpath(root, path)
for filename in files:
filekey = os.path.normpath(
os.path.join(key, rel_root, filename))
filepath = os.path.join(root, filename)
self._resource.Object(
self.bucket, filekey).upload_file(
filepath,
ExtraArgs=self._upload_args,
Config=self._transfer_config)
else:
self._resource.Object(self.bucket, key).upload_file(
path,
ExtraArgs=self._upload_args,
Config=self._transfer_config)
def put(self,
paths,
properties,
use_enclosing_directory,
use_symlinks=None,
move_files=False,
retrieve_files=None):
if use_symlinks:
raise Error("Swift storage backend does not support symlinks")
archive_path = properties.core.archive_path
physical_name = properties.core.physical_name
tmp_root = self.get_tmp_root(properties)
with util.TemporaryDirectory(dir=tmp_root,
prefix=".put-",
suffix="-%s" %
properties.core.uuid.hex) as tmp_path:
if retrieve_files:
paths = retrieve_files(tmp_path)
# Upload file(s)
for path in paths:
key = os.path.join(archive_path, physical_name)
# Add enclosing dir
if use_enclosing_directory:
key = os.path.join(key, os.path.basename(path))
if os.path.isdir(path):
for root, subdirs, files in os.walk(path):
rel_root = os.path.relpath(root, path)
for filename in files:
filekey = os.path.normpath(
os.path.join(key, rel_root, filename))
filepath = os.path.join(root, filename)
with open(filepath, 'rb') as f:
self._conn.put_object(self.container,
filekey,
contents=f.read())
else:
with open(path, 'rb') as f:
self._conn.put_object(self.container,
key,
contents=f.read())
def delete(self, product_path, properties):
if not os.path.lexists(product_path):
# If the product does not exist, do not consider this an error.
return
try:
tmp_root = self.get_tmp_root(properties)
with util.TemporaryDirectory(prefix=".remove-",
suffix="-%s" %
properties.core.uuid.hex,
dir=tmp_root) as tmp_path:
# Move product into the temporary directory. When the temporary directory will be removed at the end of
# this scope, the product will be removed along with it.
assert properties.core.physical_name == os.path.basename(
product_path)
os.rename(
product_path,
os.path.join(tmp_path, os.path.basename(product_path)))
except EnvironmentError as _error:
raise Error(
"unable to remove product '%s' (%s) [%s]" %
(properties.core.product_name, properties.core.uuid, _error))
def put(self,
paths,
properties,
use_enclosing_directory,
use_symlinks=None,
retrieve_files=None):
if use_symlinks is None:
use_symlinks = self._use_symlinks
physical_name = properties.core.physical_name
archive_path = properties.core.archive_path
uuid = properties.core.uuid
abs_archive_path = os.path.realpath(
os.path.join(self._root, archive_path))
abs_product_path = os.path.join(abs_archive_path, physical_name)
# TODO separate this out like 'current_archive_path'
if paths is not None and util.is_sub_path(os.path.realpath(paths[0]),
abs_product_path,
allow_equal=True):
# Product should already be in the target location
for path in paths:
if not os.path.exists(path):
raise Error("product source path does not exist '%s'" %
(path, ))
if not util.is_sub_path(os.path.realpath(path),
abs_product_path,
allow_equal=True):
raise Error(
"cannot ingest product where only part of the files are already at the "
"destination location")
else:
# Create destination location for product
try:
util.make_path(abs_archive_path)
except EnvironmentError as _error:
raise Error("cannot create parent destination path '%s' [%s]" %
(abs_archive_path, _error))
# Create a temporary directory and transfer the product there, then move the product to its
# destination within the archive.
try:
tmp_root = self.get_tmp_root(properties)
with util.TemporaryDirectory(prefix=".put-",
suffix="-%s" % uuid.hex,
dir=tmp_root) as tmp_path:
# Create enclosing directory if required.
if use_enclosing_directory:
tmp_path = os.path.join(tmp_path, physical_name)
util.make_path(tmp_path)
# Transfer the product (parts).
if use_symlinks:
if use_enclosing_directory:
abs_path = abs_product_path
else:
abs_path = abs_archive_path
# Create symbolic link(s) for the product (parts).
for path in paths:
if util.is_sub_path(path, self._root):
# Create a relative symbolic link when the target is part of the archive
# (i.e. when creating an intra-archive symbolic link). This ensures the
# archive can be relocated without breaking intra-archive symbolic links.
os.symlink(
os.path.relpath(path, abs_path),
os.path.join(tmp_path,
os.path.basename(path)))
else:
os.symlink(
path,
os.path.join(tmp_path,
os.path.basename(path)))
else:
# Copy/retrieve product (parts).
if retrieve_files:
paths = retrieve_files(tmp_path)
else:
for path in paths:
util.copy_path(path,
tmp_path,
resolve_root=True)
# Move the transferred product into its destination within the archive.
if use_enclosing_directory:
os.rename(tmp_path, abs_product_path)
else:
assert (len(paths) == 1 and os.path.basename(paths[0])
== physical_name)
tmp_product_path = os.path.join(
tmp_path, physical_name)
os.rename(tmp_product_path, abs_product_path)
except EnvironmentError as _error:
raise Error(
"unable to transfer product to destination path '%s' [%s]"
% (abs_product_path, _error))
def pull(self, archive, product):
from ecmwfapi import ECMWFDataServer, ECMWFService
dataserver = ECMWFDataServer(log=logging.info)
marsservice = ECMWFService("mars", log=logging.info)
if getattr(product.core, "archive_path", None) is None:
raise Error("cannot pull files that do not have archive_path set")
# Determine the (absolute) path in the archive that will contain the product and create it if required.
abs_archive_path = os.path.realpath(
os.path.join(archive._root, product.core.archive_path))
abs_product_path = os.path.join(abs_archive_path,
product.core.physical_name)
# Create destination location for product
try:
util.make_path(abs_archive_path)
except EnvironmentError as _error:
raise Error("cannot create parent destination path '%s' [%s]" %
(abs_archive_path, _error))
requests = []
for order in product.core.remote_url.split('?')[1].split(
'&concatenate&'):
request = {}
for param in order.split('&'):
key, value = param.split('=')
request[key] = value
requests.append(request)
plugin = archive.product_type_plugin(product.core.product_type)
# Create a temporary directory and download the product there, then move the product to its
# destination within the archive.
try:
with util.TemporaryDirectory(prefix=".pull-",
suffix="-%s" % product.core.uuid.hex,
dir=abs_archive_path) as tmp_path:
# Create enclosing directory if required.
if plugin.use_enclosing_directory:
tmp_path = os.path.join(tmp_path,
product.core.physical_name)
util.make_path(tmp_path)
# Download product.
tmp_file_combined = os.path.join(tmp_path,
product.core.physical_name)
tmp_file_download = os.path.join(tmp_path, "request.grib")
combined_file = open(tmp_file_combined, "w")
for request in requests:
if 'dataset' in request:
request['target'] = tmp_file_download
dataserver.retrieve(request)
else:
marsservice.execute(request, tmp_file_download)
result_file = open(tmp_file_download, "r")
combined_file.write(result_file.read())
result_file.close()
os.remove(tmp_file_download)
combined_file.close()
# Move the retrieved product into its destination within the archive.
if plugin.use_enclosing_directory:
os.rename(tmp_path, abs_product_path)
else:
os.rename(tmp_file_combined, abs_product_path)
except EnvironmentError as _error:
raise Error(
"unable to transfer product to destination path '%s' [%s]" %
(abs_product_path, _error))
