def download():
"""Download data files from ESA and store them in the configured storage
directory.
"""
# Load download configuration
storage = config('storage') or 'data'
date_begin = config('download', 'date', 'begin')\
or '2019-09-10T00:00:00.000Z'
date_end = config('download', 'date', 'end') or '2019-09-11T00:00:00.000Z'
countries = config('download', 'country') or ['DE']
# create storage folder if not existing
os.makedirs(storage, exist_ok=True)
for country in countries:
wkt = bounding_box_to_wkt(*country_bounding_boxes[country][1])
# Search for data matching the parameter
logger.info(f'Looking for products in country {country}')
result = sentinel5dl.search(wkt,
begin_ts=date_begin,
end_ts=date_end,
product=product,
processing_level=processing_level,
logger_fn=logger.info)
logger.info('Found {0} products'.format(len(result.get('products'))))
# Download data
sentinel5dl.download(result.get('products'),
output_dir=storage,
logger_fn=logger.info)
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
import pandas
import psycopg2.extensions
import geoalchemy2
from emissionsapi.config import config
# Logger
logger = logging.getLogger(__name__)
# Database uri as described in
# https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls
# Retrieved as environment variable.
database = config('database') or 'postgresql://*****:*****@localhost/db'
# Products
products = config('products') or {
'carbonmonoxide': {
'storage': 'data',
'product': 'carbonmonoxide_total_column',
'product_key': 'L2__CO____'
}
}
# H3 Resolution
resolution = config('resolution') or 4
# Global session variable. Set on initialization.
__session__ = None
import itertools
import logging
import multiprocessing
import os
import tempfile
import sentinel5dl
from emissionsapi import db, preprocess
from emissionsapi.config import config
# Logger
logger = logging.getLogger(__name__)
# Number of workers
workers = config('workers') or 1
# Earliest data to process
earliest_data = datetime.datetime.fromisoformat(
config('earliest_data') or '2019-01-01')
def generate_intervals(start, end, days=5):
'''Generator for intervals between start and end in days intervals.
:param start: Starting datetime for the intervals.
:type start: datetime.datetime
:param end: End datetime for the intervals.
:type end: datetime.datetime
:param days: Days for each interval, defaults to 5
:type days: int, optional
import glob
import itertools
import logging
import multiprocessing
import os
import s5a
from emissionsapi.config import config
import emissionsapi.db
# Logger
logger = logging.getLogger(__name__)
# Number of workers to preprocess
workers = config('workers') or 1
@emissionsapi.db.with_session
def list_ncfiles(session, storage):
"""list all nc files in storage.
:param session: SQLAlchemy Session
:type session: sqlalchemy.orm.session.Session
:param storage: Path to the directory containing the `*.nc` files
:type storage: str
:return: Set of all unprocessed files
:rtype: set
"""
# Get all *.nc files as set
files = set(glob.glob(os.path.join(storage, "*.nc")))
"""Module to filter and download the data from the ESA and store it locally.
"""
import os
import sentinel5dl
from emissionsapi.config import config
from emissionsapi.country_bounding_boxes import country_bounding_boxes
import emissionsapi.logger
# Logger
logger = emissionsapi.logger.getLogger('emission-api.download')
storage = config('storage') or 'data'
start_date = '2019-09-10T00:00:00.000Z'
end_date = '2019-09-11T00:00:00.000Z'
product = 'L2__CO____'
processing_level = 'L2'
def bounding_box_to_wkt(lon1, lat1, lon2, lat2):
"""Convert a bounding box specified by its top-left and bottom-right
coordinates to a wkt string defining a polygon.
"""
return f'POLYGON(({lon1} {lat1},{lon1} {lat2},{lon2} {lat2},'\
f'{lon2} {lat1},{lon1} {lat1}))'
def download():
"""Download data files from ESA and store them in the configured storage
import itertools
import logging
import multiprocessing
import os
import tempfile
import sentinel5dl
from emissionsapi import db, preprocess
from emissionsapi.config import config
# Logger
logger = logging.getLogger(__name__)
# Number of workers
workers = config('workers') or 1
# Earliest data to process
earliest_data = datetime.datetime.fromisoformat(
config('earliest_data') or '2019-01-01')
# Latest data to process
latest_data = (datetime.datetime.fromisoformat(config('latest_data'))
if config('latest_data') else datetime.datetime.now())
def generate_intervals(start, end, days=5):
'''Generator for intervals between start and end in days intervals.
:param start: Starting datetime for the intervals.
:type start: datetime.datetime
:param end: End datetime for the intervals.
from sqlalchemy import create_engine, Column, DateTime, Integer, Float, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
import geoalchemy2
from emissionsapi.config import config
import emissionsapi.logger
# Logger
logger = emissionsapi.logger.getLogger('emission-api.db')
# Database uri as described in
# https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls
# Retrieved as environment variable.
database = config('database') or 'postgresql://*****:*****@localhost/db'
# Global session variable. Set on initialization.
__session__ = None
# Base Class of all ORM objects.
Base = declarative_base()
class File(Base):
"""ORM Object for the nc files.
"""
# Tablename
__tablename__ = 'file'
filename = Column(String, primary_key=True)