def entrypoint():
"""Entrypoint for running this as a module or from the binary.
It starts the Connexion Debug Server. It is not meant to be used for
production, but only during the development.
"""
logger.info("Starting the Connexion Debug Server")
app.run()
def download():
"""Download data files from ESA and store them in the configured storage
directory.
"""
# Load download configuration
storage = config('storage') or 'data'
date_begin = config('download', 'date', 'begin')\
or '2019-09-10T00:00:00.000Z'
date_end = config('download', 'date', 'end') or '2019-09-11T00:00:00.000Z'
countries = config('download', 'country') or ['DE']
# create storage folder if not existing
os.makedirs(storage, exist_ok=True)
for country in countries:
wkt = bounding_box_to_wkt(*country_bounding_boxes[country][1])
# Search for data matching the parameter
logger.info(f'Looking for products in country {country}')
result = sentinel5dl.search(wkt,
begin_ts=date_begin,
end_ts=date_end,
product=product,
processing_level=processing_level,
logger_fn=logger.info)
logger.info('Found {0} products'.format(len(result.get('products'))))
# Download data
sentinel5dl.download(result.get('products'),
output_dir=storage,
logger_fn=logger.info)
def entrypoint():
"""Entrypoint for running this as a module or from the binary.
Triggers the preprocessing of the data.
"""
# Iterate through all find nc files
for ncfile in list_ncfiles():
logger.info(f"Pre-process '{ncfile}'")
# Read data from nc file
data = read_file(ncfile)
# filter data
data = filter_data(data)
# Write the filtered data to the database
write_to_database(data)
pass
def list_ncfiles(session):
"""Generator yielding all nc files in download path.
"""
# Iterate through the files and directories in storage path
for f in os.listdir(storage):
# Check if file was already added
filename = session.query(emissionsapi.db.File).filter(
emissionsapi.db.File.filename == f).first()
if filename is not None:
logger.info(f"Skipping {f}")
continue
# Join directory and filename
filepath = os.path.join(storage, f)
# yield file ending with '.nc'
if os.path.isfile(filepath) and filepath.endswith('.nc'):
yield filepath
def download():
"""Download data files from ESA and store them in the configured storage
directory.
"""
wkt = bounding_box_to_wkt(*country_bounding_boxes['DE'][1])
# create storage folder if not existing
os.makedirs(storage, exist_ok=True)
# Search for data matching the parameter
result = sentinel5dl.search(wkt,
begin_ts=start_date,
end_ts=end_date,
product=product,
processing_level=processing_level,
logger_fn=logger.info)
logger.info('Found {0} products'.format(len(result.get('products'))))
# Download data
sentinel5dl.download(result.get('products'),
output_dir=storage,
logger_fn=logger.info)