def get(self,
product,
product_path,
target_path,
use_enclosing_directory,
use_symlinks=None):
if use_symlinks:
raise Error("Swift storage backend does not support symlinks")
archive_path = product.core.archive_path
keys = self._object_keys(product_path)
if not keys:
raise Error("no data for product '%s' (%s)" %
(product.core.product_name, product.core.uuid))
for key in keys:
rel_path = os.path.relpath(key, archive_path)
if use_enclosing_directory:
rel_path = '/'.join(rel_path.split('/')[1:])
target = os.path.normpath(os.path.join(target_path, rel_path))
if key.endswith('/'):
util.make_path(target)
else:
util.make_path(os.path.dirname(target))
binary = self._conn.get_object(self.container, key)[1]
with open(target, 'wb') as f:
f.write(binary)
def retrieve_function(archive, product, verify_hash_download):
if getattr(product.core, "archive_path", None) is None:
raise Error("cannot pull files that do not have archive_path set")
# determine the backend to use
backend = None
url = product.core.remote_url
for prot in archive.remote_backends():
_backend = archive.remote_backend(prot)
if _backend.indentify(url):
backend = _backend
if backend is None:
raise Error("The protocol of '%s' is not supported" % url)
def retrieve_files(target_dir):
paths = backend.pull(archive, product, target_dir)
# check that download matches stored hash
stored_hash = getattr(product.core, 'hash', None)
if verify_hash_download and stored_hash is not None:
hash_type = archive._extract_hash_type(stored_hash)
if hash_type is None:
stored_hash = 'sha1:' + stored_hash
hash_type = 'sha1'
calc_hash = util.product_hash(paths, hash_type=hash_type)
if calc_hash != stored_hash:
raise DownloadError("hash mismatch when pulling product '%s' (%s)" %
(product.core.product_name, product.core.uuid))
return paths
return retrieve_files
def pull(self, archive, product):
if getattr(product.core, "archive_path", None) is None:
raise Error("cannot pull files that do not have archive_path set")
# Determine the (absolute) path in the archive that will contain the product and create it if required.
abs_archive_path = os.path.realpath(
os.path.join(archive._root, product.core.archive_path))
abs_product_path = os.path.join(abs_archive_path,
product.core.physical_name)
# Create destination location for product
try:
util.make_path(abs_archive_path)
except EnvironmentError as _error:
raise Error("cannot create parent destination path '%s' [%s]" %
(abs_archive_path, _error))
plugin = archive.product_type_plugin(product.core.product_type)
# Create a temporary directory and download the product there, then move the product to its
# destination within the archive.
try:
with util.TemporaryDirectory(prefix=".pull-",
suffix="-%s" % product.core.uuid.hex,
dir=abs_archive_path) as tmp_path:
# Create enclosing directory if required.
if plugin.use_enclosing_directory:
tmp_path = os.path.join(tmp_path,
product.core.physical_name)
util.make_path(tmp_path)
# Define a temp location and download the file
tmp_file = os.path.join(tmp_path, product.core.physical_name)
downloader = util.Downloader(product.core.remote_url,
archive.auth_file())
downloader.save(tmp_file)
# TODO: implement extraction of downloaded archives
# for ftp and file check if url ends with 'core.physical_name + <archive ext>'
# for http/https check the header for the line:
# Content-Disposition: attachment; filename="**********"
# end then use this ***** filename to match against core.physical_name + <archive ext>
# Move the transferred product into its destination within the archive.
if plugin.use_enclosing_directory:
os.rename(tmp_path, abs_product_path)
else:
os.rename(tmp_file, abs_product_path)
except EnvironmentError as _error:
raise Error(
"unable to transfer product to destination path '%s' [%s]" %
(abs_product_path, _error))
def get(self,
product,
product_path,
target_path,
use_enclosing_directory,
use_symlinks=None):
if use_symlinks is None:
use_symlinks = self._use_symlinks
try:
if use_symlinks:
if use_enclosing_directory:
for basename in os.listdir(product_path):
os.symlink(os.path.join(product_path, basename),
os.path.join(target_path, basename))
else:
os.symlink(
product_path,
os.path.join(target_path,
os.path.basename(product_path)))
else:
if use_enclosing_directory:
for basename in os.listdir(product_path):
util.copy_path(os.path.join(product_path, basename),
target_path,
resolve_root=True)
else:
util.copy_path(product_path,
target_path,
resolve_root=True)
except EnvironmentError as _error:
raise Error("unable to retrieve product '%s' (%s) [%s]" %
(product.core.product_name, product.core.uuid, _error))
def prepare(self):
# Create the archive root path.
try:
util.make_path(self._root)
except EnvironmentError as _error:
raise Error("unable to create archive root path '%s' [%s]" %
(self._root, _error))
def destroy(self):
if self.exists():
try:
util.remove_path(self._root)
except EnvironmentError as _error:
raise Error("unable to remove archive root path '%s' [%s]" %
(self._root, _error))
def put(self, paths, properties, use_enclosing_directory, use_symlinks=None,
retrieve_files=None, run_for_product=None):
if use_symlinks:
raise Error("S3 storage backend does not support symlinks")
anything_stored = False
try:
archive_path = properties.core.archive_path
physical_name = properties.core.physical_name
if not use_enclosing_directory and retrieve_files is None:
assert(len(paths) == 1 and os.path.basename(paths[0]) == physical_name)
tmp_root = self.get_tmp_root(properties)
with util.TemporaryDirectory(dir=tmp_root, prefix=".put-",
suffix="-%s" % properties.core.uuid.hex) as tmp_path:
if retrieve_files:
paths = retrieve_files(tmp_path)
# Upload file(s)
for path in paths:
key = self._prefix + os.path.join(archive_path, physical_name)
# Add enclosing dir
if use_enclosing_directory:
key = os.path.join(key, os.path.basename(path))
if os.path.isdir(path):
self._create_dir(key)
anything_stored = True
for root, subdirs, files in os.walk(path):
rel_root = os.path.relpath(root, path)
for subdir in subdirs:
dirkey = os.path.normpath(os.path.join(key, rel_root, subdir))
self._create_dir(dirkey)
anything_stored = True
for filename in files:
filekey = os.path.normpath(os.path.join(key, rel_root, filename))
filepath = os.path.join(root, filename)
self._upload_file(filekey, filepath)
anything_stored = True
else:
self._upload_file(key, path)
anything_stored = True
if run_for_product is not None:
run_for_product(paths)
except Exception as e:
raise StorageError(e, anything_stored)
def pull(archive, product):
# determine the backend to use
backend = None
url = product.core.remote_url
for prot in archive.remote_backends():
_backend = archive.remote_backend(prot)
if _backend.indentify(url):
backend = _backend
if backend is None:
raise Error("The protocol of '%s' is not supported" % url)
backend.pull(archive, product)
def pull(archive, product, use_enclosing_directory):
if getattr(product.core, "archive_path", None) is None:
raise Error("cannot pull files that do not have archive_path set")
# determine the backend to use
backend = None
url = product.core.remote_url
for prot in archive.remote_backends():
_backend = archive.remote_backend(prot)
if _backend.indentify(url):
backend = _backend
if backend is None:
raise Error("The protocol of '%s' is not supported" % url)
def retrieve_files(target_dir):
return backend.pull(archive, product, target_dir)
archive._storage.put(None,
product,
use_enclosing_directory,
use_symlinks=False,
retrieve_files=retrieve_files)
def get_remote_url(filename, ecmwfmars, levtype_options):
"""
levtype_options should be a dict with for each 'levtype' field a dict containing:
- string 'param'
'/' separated list of parameters for the given level type
https://software.ecmwf.int/wiki/display/UDOC/Identification+keywords#Identificationkeywords-param
- string 'levelist' (optional)
'/' separated list of levels for the given level type
https://software.ecmwf.int/wiki/display/UDOC/Identification+keywords#Identificationkeywords-levelist
"""
if not levtype_options:
raise Error("no parameters to construct remote_url")
remote_url = "ecmwfapi:%s?" % (filename, )
request = {
'class': ecmwfmars.marsclass,
'stream': ecmwfmars.stream,
'expver': ecmwfmars.expver,
'type': ecmwfmars.type,
'date': ecmwfmars.date,
'time': ecmwfmars.time,
}
if 'dataset' in ecmwfmars:
request['dataset'] = ecmwfmars.dataset
if 'step' in ecmwfmars:
request['step'] = str(ecmwfmars.step)
if 'resol' in ecmwfmars:
request['resol'] = ecmwfmars.resol
if 'grid' in ecmwfmars:
request['grid'] = ecmwfmars.grid
if 'area' in ecmwfmars:
request['area'] = ecmwfmars.area
first = True
for levtype in levtype_options:
if first:
first = False
else:
# The '&concatenate&' is a muninn-specific way of combining multiple requests in one
remote_url += "&concatenate&"
request['levtype'] = levtype
request['param'] = levtype_options[levtype]['param']
if 'levelist' in levtype_options[levtype]:
request['levelist'] = levtype_options[levtype]['levelist']
elif 'levelist' in request:
del request['levelist']
remote_url += "&".join(
["%s=%s" % (key, str(request[key])) for key in request])
return remote_url
def get(self, product, product_path, target_path, use_enclosing_directory, use_symlinks=None):
if use_symlinks:
raise Error("S3 storage backend does not support symlinks")
archive_path = product.core.archive_path
prefix = self._prefix + product_path
objs = list(self._resource.Bucket(self.bucket).objects.filter(Prefix=prefix))
if not objs:
raise Error("no data for product '%s' (%s)" % (product.core.product_name, product.core.uuid))
for obj in objs:
rel_path = os.path.relpath(obj.key, self._prefix + archive_path)
if use_enclosing_directory:
rel_path = '/'.join(rel_path.split('/')[1:])
target = os.path.normpath(os.path.join(target_path, rel_path))
if obj.key.endswith('/'):
util.make_path(target)
else:
util.make_path(os.path.dirname(target))
self._resource.Object(self.bucket, obj.key).download_file(target, ExtraArgs=self._download_args,
Config=self._transfer_config)
def update(self, other):
'''update a struct, using the same semantics as dict.update()'''
for key in other:
other_item = other[key]
if isinstance(other_item, Struct):
if key not in self:
self[key] = Struct()
else:
if not isinstance(self[key], Struct):
raise Error('Incompatible structs: %s vs %s' %
(self, other))
self[key].update(other_item)
else:
self[key] = other_item
def current_archive_path(self, paths):
for path in paths:
if not util.is_sub_path(
os.path.realpath(path), self._root, allow_equal=True):
raise Error(
"cannot ingest a file in-place if it is not inside the muninn archive root"
)
abs_archive_path = os.path.dirname(os.path.realpath(paths[0]))
if len(paths) > 1:
# check whether all files have the right enclosing directory
for path in paths:
enclosing_directory = os.path.basename(
os.path.dirname(os.path.realpath(path)))
if enclosing_directory != properties.core.physical_name:
raise Error(
"multi-part product has invalid enclosing directory for in-place ingestion"
)
abs_archive_path = os.path.dirname(abs_archive_path)
# strip archive root
return os.path.relpath(abs_archive_path,
start=os.path.realpath(self._root))
def put(self,
paths,
properties,
use_enclosing_directory,
use_symlinks=None,
move_files=False,
retrieve_files=None):
if use_symlinks:
raise Error("S3 storage backend does not support symlinks")
archive_path = properties.core.archive_path
physical_name = properties.core.physical_name
tmp_root = self.get_tmp_root(properties)
with util.TemporaryDirectory(dir=tmp_root,
prefix=".put-",
suffix="-%s" %
properties.core.uuid.hex) as tmp_path:
if retrieve_files:
paths = retrieve_files(tmp_path)
# Upload file(s)
for path in paths:
key = self._prefix + os.path.join(archive_path, physical_name)
# Add enclosing dir
if use_enclosing_directory:
key = os.path.join(key, os.path.basename(path))
if os.path.isdir(path):
for root, subdirs, files in os.walk(path):
rel_root = os.path.relpath(root, path)
for filename in files:
filekey = os.path.normpath(
os.path.join(key, rel_root, filename))
filepath = os.path.join(root, filename)
self._resource.Object(
self.bucket, filekey).upload_file(
filepath,
ExtraArgs=self._upload_args,
Config=self._transfer_config)
else:
self._resource.Object(self.bucket, key).upload_file(
path,
ExtraArgs=self._upload_args,
Config=self._transfer_config)
def put(self,
paths,
properties,
use_enclosing_directory,
use_symlinks=None,
move_files=False,
retrieve_files=None):
if use_symlinks:
raise Error("Swift storage backend does not support symlinks")
archive_path = properties.core.archive_path
physical_name = properties.core.physical_name
tmp_root = self.get_tmp_root(properties)
with util.TemporaryDirectory(dir=tmp_root,
prefix=".put-",
suffix="-%s" %
properties.core.uuid.hex) as tmp_path:
if retrieve_files:
paths = retrieve_files(tmp_path)
# Upload file(s)
for path in paths:
key = os.path.join(archive_path, physical_name)
# Add enclosing dir
if use_enclosing_directory:
key = os.path.join(key, os.path.basename(path))
if os.path.isdir(path):
for root, subdirs, files in os.walk(path):
rel_root = os.path.relpath(root, path)
for filename in files:
filekey = os.path.normpath(
os.path.join(key, rel_root, filename))
filepath = os.path.join(root, filename)
with open(filepath, 'rb') as f:
self._conn.put_object(self.container,
filekey,
contents=f.read())
else:
with open(path, 'rb') as f:
self._conn.put_object(self.container,
key,
contents=f.read())
def move(self, product, archive_path, paths=None):
# Ignore if product already there
if product.core.archive_path == archive_path:
return paths
product_path = self._prefix + self.product_path(product)
new_product_path = self._prefix + os.path.join(archive_path, product.core.physical_name)
objs = list(self._resource.Bucket(self.bucket).objects.filter(Prefix=product_path))
if not objs:
raise Error("no data for product '%s' (%s)" % (product.core.product_name, product.core.uuid))
for obj in objs:
new_key = os.path.normpath(os.path.join(new_product_path, os.path.relpath(obj.key, product_path)))
self._resource.Object(self.bucket, new_key).copy(CopySource={'Bucket': self.bucket, 'Key': obj.key},
ExtraArgs=self._copy_args, Config=self._transfer_config)
self._resource.Object(self.bucket, obj.key).delete()
return paths
def move(self, product, archive_path, paths=None):
# Ignore if product already there
if product.core.archive_path == archive_path:
return paths
product_path = self.product_path(product)
new_product_path = os.path.join(archive_path,
product.core.physical_name)
keys = self._object_keys(product_path)
if not keys:
raise Error("no data for product '%s' (%s)" %
(product.core.product_name, product.core.uuid))
for key in keys:
new_key = os.path.normpath(
os.path.join(new_product_path,
os.path.relpath(key, product_path)))
self._conn.copy_object(self.container, key,
os.path.join(self.container, new_key))
self._conn.delete_object(self.container, key)
return paths
def delete(self, product_path, properties):
if not os.path.lexists(product_path):
# If the product does not exist, do not consider this an error.
return
try:
tmp_root = self.get_tmp_root(properties)
with util.TemporaryDirectory(prefix=".remove-",
suffix="-%s" %
properties.core.uuid.hex,
dir=tmp_root) as tmp_path:
# Move product into the temporary directory. When the temporary directory will be removed at the end of
# this scope, the product will be removed along with it.
assert properties.core.physical_name == os.path.basename(
product_path)
os.rename(
product_path,
os.path.join(tmp_path, os.path.basename(product_path)))
except EnvironmentError as _error:
raise Error(
"unable to remove product '%s' (%s) [%s]" %
(properties.core.product_name, properties.core.uuid, _error))
def current_archive_path(self, paths, properties):
raise Error("S3 storage backend does not support ingesting already archived products")
def __init__(self, geometries=[]):
super(LineString, self).__init__(geometries)
if len(self) == 1:
raise Error(
"line string should be empty or should contain >= 2 points")
def extract_grib_metadata(gribfile):
"""
this will return a tuple containing:
- ecmwfmars properties struct
- levtype_options struct (see set_remote_url())
"""
import coda
@contextlib.contextmanager
def coda_open(filename):
coda_handle = coda.open(filename)
try:
yield coda_handle
finally:
coda.close(coda_handle)
ecmwfmars = Struct()
levtype_options = {} # TODO: add extraction of levtype_options
with coda_open(gribfile) as coda_handle:
cursor = coda.Cursor()
coda.cursor_set_product(cursor, coda_handle)
num_messages = coda.cursor_get_num_elements(cursor)
coda.cursor_goto_first_array_element(cursor)
for i in range(num_messages):
index = coda.cursor_get_available_union_field_index(cursor)
coda.cursor_goto_record_field_by_index(cursor, index)
step = 0
if index == 0:
# grib1
centuryOfReferenceTimeOfData = coda.fetch(
cursor, "centuryOfReferenceTimeOfData")
yearOfCentury = coda.fetch(cursor, "yearOfCentury")
month = coda.fetch(cursor, "month")
day = coda.fetch(cursor, "day")
date = "%02d%02d-%02d-%02d" % (centuryOfReferenceTimeOfData -
1, yearOfCentury, month, day)
hour = coda.fetch(cursor, "hour")
minute = coda.fetch(cursor, "minute")
time = "%02d:%02d:00" % (hour, minute)
unitOfTimeRange = coda.fetch(cursor, "unitOfTimeRange")
if unitOfTimeRange != 0:
P1 = coda.fetch(cursor, "P1")
if unitOfTimeRange == 1:
step = P1
elif unitOfTimeRange == 2:
step = 24 * P1
elif unitOfTimeRange == 10:
step = 3 * P1
elif unitOfTimeRange == 11:
step = 6 * P1
elif unitOfTimeRange == 13:
step = 12 * P1
else:
raise Error("unsupported unitOfTimeRange: %d" %
(unitOfTimeRange, ))
local = coda.fetch(cursor, "local")
try:
local = local[1:9].tobytes()
except AttributeError:
# workaround for older numpy versions
local = local[1:9].tostring()
marsclass, marstype, stream, expver = struct.unpack(
'>BBH4s', local)
else:
# grib2
year = coda.fetch(cursor, "year")
month = coda.fetch(cursor, "month")
day = coda.fetch(cursor, "day")
date = "%04d-%02d-%02d" % (year, month, day)
hour = coda.fetch(cursor, "hour")
minute = coda.fetch(cursor, "minute")
second = coda.fetch(cursor, "second")
time = "%02d:%02d:%02d" % (hour, minute, second)
significanceOfReferenceTime = coda.fetch(
cursor, "significanceOfReferenceTime")
local = coda.fetch(cursor, "local[0]")
try:
local = local[2:12].tobytes()
except AttributeError:
# workaround for older numpy versions
local = local[2:12].tostring()
marsclass, marstype, stream, expver = struct.unpack(
'>HHH4s', local)
coda.cursor_goto_record_field_by_name(cursor, "data")
num_data = coda.cursor_get_num_elements(cursor)
coda.cursor_goto_first_array_element(cursor)
prev_step = None
for j in range(num_data):
forecastTime = coda.fetch(cursor, "forecastTime")
if forecastTime != 0:
indicatorOfUnitOfTimeRange = coda.fetch(
cursor, "indicatorOfUnitOfTimeRange")
if indicatorOfUnitOfTimeRange == 0:
# minutes
step = 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 1:
# hours
step = 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 2:
# days
step = 24 * 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 10:
# 3 hours
step = 3 * 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 11:
# 6 hours
step = 6 * 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 12:
# 12 hours
step = 12 * 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 13:
# seconds
step = forecastTime
step = int(step / 3600.) # convert seconds to hours
if prev_step is None:
prev_step = step
elif step != prev_step:
raise Error(
"not all data has the same 'step' time (%d) (%d)"
% (step, prev_step))
if j < num_data - 1:
coda.cursor_goto_next_array_element(cursor)
coda.cursor_goto_parent(cursor)
coda.cursor_goto_parent(cursor)
if marsclass not in MARSCLASSES:
raise Error("unsupported MARS class (%d)" % (marsclass, ))
marsclass = MARSCLASSES[marsclass]
if marstype not in MARSTYPES:
raise Error("unsupported MARS type (%d)" % (marstype, ))
marstype = MARSTYPES[marstype]
if stream not in MARSSTREAMS:
raise Error("unsupported MARS stream (%d)" % (stream, ))
stream = MARSSTREAMS[stream]
if 'date' in ecmwfmars:
if date != ecmwfmars.date:
raise Error("not all data is for the same date (%s) (%s)" %
(date, ecmwfmars.date))
if time != ecmwfmars.time:
raise Error("not all data is for the same time (%s) (%s)" %
(time, ecmwfmars.time))
if step != 0:
if 'step' in ecmwfmars:
if step != ecmwfmars.step:
raise Error(
"not all data has the same 'step' time (%d) (%d)"
% (step, ecmwfmars.step))
else:
raise Error("not all data has the same 'step' time")
else:
if 'step' in ecmwfmars and ecmwfmars.step != 0:
raise Error("not all data has the same 'step' time")
if marsclass != ecmwfmars.marsclass:
raise Error(
"not all data has the same MARS class (%s) (%s)" %
(marsclass, ecmwfmars.marsclass))
if marstype != ecmwfmars.type:
raise Error(
"not all data has the same MARS type (%s) (%s)" %
(marstype, ecmwfmars.type))
if stream != ecmwfmars.stream:
raise Error(
"not all data has the same MARS stream (%s) (%s)" %
(stream, ecmwfmars.stream))
if expver != ecmwfmars.expver:
raise Error(
"not all data has the same MARS experiment version (%s) (%s)"
% (expver, ecmwfmars.expver))
else:
ecmwfmars.date = date
ecmwfmars.time = time
if step != 0:
ecmwfmars.step = step
ecmwfmars.marsclass = marsclass
ecmwfmars.type = marstype
ecmwfmars.stream = stream
ecmwfmars.expver = expver
if (marsclass, stream, expver, marstype) in PUBLIC_DATASETS:
ecmwfmars.dataset = PUBLIC_DATASETS[(marsclass, stream,
expver, marstype)]
coda.cursor_goto_parent(cursor)
if i < num_messages - 1:
coda.cursor_goto_next_array_element(cursor)
return ecmwfmars, levtype_options
def put(self,
paths,
properties,
use_enclosing_directory,
use_symlinks=None,
retrieve_files=None):
if use_symlinks is None:
use_symlinks = self._use_symlinks
physical_name = properties.core.physical_name
archive_path = properties.core.archive_path
uuid = properties.core.uuid
abs_archive_path = os.path.realpath(
os.path.join(self._root, archive_path))
abs_product_path = os.path.join(abs_archive_path, physical_name)
# TODO separate this out like 'current_archive_path'
if paths is not None and util.is_sub_path(os.path.realpath(paths[0]),
abs_product_path,
allow_equal=True):
# Product should already be in the target location
for path in paths:
if not os.path.exists(path):
raise Error("product source path does not exist '%s'" %
(path, ))
if not util.is_sub_path(os.path.realpath(path),
abs_product_path,
allow_equal=True):
raise Error(
"cannot ingest product where only part of the files are already at the "
"destination location")
else:
# Create destination location for product
try:
util.make_path(abs_archive_path)
except EnvironmentError as _error:
raise Error("cannot create parent destination path '%s' [%s]" %
(abs_archive_path, _error))
# Create a temporary directory and transfer the product there, then move the product to its
# destination within the archive.
try:
tmp_root = self.get_tmp_root(properties)
with util.TemporaryDirectory(prefix=".put-",
suffix="-%s" % uuid.hex,
dir=tmp_root) as tmp_path:
# Create enclosing directory if required.
if use_enclosing_directory:
tmp_path = os.path.join(tmp_path, physical_name)
util.make_path(tmp_path)
# Transfer the product (parts).
if use_symlinks:
if use_enclosing_directory:
abs_path = abs_product_path
else:
abs_path = abs_archive_path
# Create symbolic link(s) for the product (parts).
for path in paths:
if util.is_sub_path(path, self._root):
# Create a relative symbolic link when the target is part of the archive
# (i.e. when creating an intra-archive symbolic link). This ensures the
# archive can be relocated without breaking intra-archive symbolic links.
os.symlink(
os.path.relpath(path, abs_path),
os.path.join(tmp_path,
os.path.basename(path)))
else:
os.symlink(
path,
os.path.join(tmp_path,
os.path.basename(path)))
else:
# Copy/retrieve product (parts).
if retrieve_files:
paths = retrieve_files(tmp_path)
else:
for path in paths:
util.copy_path(path,
tmp_path,
resolve_root=True)
# Move the transferred product into its destination within the archive.
if use_enclosing_directory:
os.rename(tmp_path, abs_product_path)
else:
assert (len(paths) == 1 and os.path.basename(paths[0])
== physical_name)
tmp_product_path = os.path.join(
tmp_path, physical_name)
os.rename(tmp_product_path, abs_product_path)
except EnvironmentError as _error:
raise Error(
"unable to transfer product to destination path '%s' [%s]"
% (abs_product_path, _error))
def pull(self, archive, product):
from ecmwfapi import ECMWFDataServer, ECMWFService
dataserver = ECMWFDataServer(log=logging.info)
marsservice = ECMWFService("mars", log=logging.info)
if getattr(product.core, "archive_path", None) is None:
raise Error("cannot pull files that do not have archive_path set")
# Determine the (absolute) path in the archive that will contain the product and create it if required.
abs_archive_path = os.path.realpath(
os.path.join(archive._root, product.core.archive_path))
abs_product_path = os.path.join(abs_archive_path,
product.core.physical_name)
# Create destination location for product
try:
util.make_path(abs_archive_path)
except EnvironmentError as _error:
raise Error("cannot create parent destination path '%s' [%s]" %
(abs_archive_path, _error))
requests = []
for order in product.core.remote_url.split('?')[1].split(
'&concatenate&'):
request = {}
for param in order.split('&'):
key, value = param.split('=')
request[key] = value
requests.append(request)
plugin = archive.product_type_plugin(product.core.product_type)
# Create a temporary directory and download the product there, then move the product to its
# destination within the archive.
try:
with util.TemporaryDirectory(prefix=".pull-",
suffix="-%s" % product.core.uuid.hex,
dir=abs_archive_path) as tmp_path:
# Create enclosing directory if required.
if plugin.use_enclosing_directory:
tmp_path = os.path.join(tmp_path,
product.core.physical_name)
util.make_path(tmp_path)
# Download product.
tmp_file_combined = os.path.join(tmp_path,
product.core.physical_name)
tmp_file_download = os.path.join(tmp_path, "request.grib")
combined_file = open(tmp_file_combined, "w")
for request in requests:
if 'dataset' in request:
request['target'] = tmp_file_download
dataserver.retrieve(request)
else:
marsservice.execute(request, tmp_file_download)
result_file = open(tmp_file_download, "r")
combined_file.write(result_file.read())
result_file.close()
os.remove(tmp_file_download)
combined_file.close()
# Move the retrieved product into its destination within the archive.
if plugin.use_enclosing_directory:
os.rename(tmp_path, abs_product_path)
else:
os.rename(tmp_file_combined, abs_product_path)
except EnvironmentError as _error:
raise Error(
"unable to transfer product to destination path '%s' [%s]" %
(abs_product_path, _error))