Beispiel #1
0
def entrypoint():
    """Entrypoint for running this as a module or from the binary.
    It starts the Connexion Debug Server. It is not meant to be used for
    production, but only during the development.
    """
    logger.info("Starting the Connexion Debug Server")
    app.run()
Beispiel #2
0
def download():
    """Download data files from ESA and store them in the configured storage
    directory.
    """

    # Load download configuration
    storage = config('storage') or 'data'
    date_begin = config('download', 'date', 'begin')\
        or '2019-09-10T00:00:00.000Z'
    date_end = config('download', 'date', 'end') or '2019-09-11T00:00:00.000Z'
    countries = config('download', 'country') or ['DE']

    # create storage folder if not existing
    os.makedirs(storage, exist_ok=True)

    for country in countries:
        wkt = bounding_box_to_wkt(*country_bounding_boxes[country][1])

        # Search for data matching the parameter
        logger.info(f'Looking for products in country {country}')
        result = sentinel5dl.search(wkt,
                                    begin_ts=date_begin,
                                    end_ts=date_end,
                                    product=product,
                                    processing_level=processing_level,
                                    logger_fn=logger.info)
        logger.info('Found {0} products'.format(len(result.get('products'))))

        # Download data
        sentinel5dl.download(result.get('products'),
                             output_dir=storage,
                             logger_fn=logger.info)
Beispiel #3
0
def entrypoint():
    """Entrypoint for running this as a module or from the binary.
    Triggers the preprocessing of the data.
    """
    # Iterate through all find nc files
    for ncfile in list_ncfiles():
        logger.info(f"Pre-process '{ncfile}'")
        # Read data from nc file
        data = read_file(ncfile)
        # filter data
        data = filter_data(data)
        # Write the filtered data to the database
        write_to_database(data)
    pass
Beispiel #4
0
def list_ncfiles(session):
    """Generator yielding all nc files in download path.
    """
    # Iterate through the files and directories in storage path
    for f in os.listdir(storage):
        # Check if file was already added
        filename = session.query(emissionsapi.db.File).filter(
            emissionsapi.db.File.filename == f).first()
        if filename is not None:
            logger.info(f"Skipping {f}")
            continue
        # Join directory and filename
        filepath = os.path.join(storage, f)
        # yield file ending with '.nc'
        if os.path.isfile(filepath) and filepath.endswith('.nc'):
            yield filepath
Beispiel #5
0
def download():
    """Download data files from ESA and store them in the configured storage
    directory.
    """
    wkt = bounding_box_to_wkt(*country_bounding_boxes['DE'][1])

    # create storage folder if not existing
    os.makedirs(storage, exist_ok=True)

    # Search for data matching the parameter
    result = sentinel5dl.search(wkt,
                                begin_ts=start_date,
                                end_ts=end_date,
                                product=product,
                                processing_level=processing_level,
                                logger_fn=logger.info)
    logger.info('Found {0} products'.format(len(result.get('products'))))

    # Download data
    sentinel5dl.download(result.get('products'),
                         output_dir=storage,
                         logger_fn=logger.info)