示例#1
0
    def __init__(self, release):
        self.BATCH_FILE = gs.get(f'{release}:lite:land:batches:rules'
                                 )  # -> '../data/land_cdmlite_batch_rules.txt'
        self.FILE_LIST = gs.get(f'{release}:lite:land:batches:input_files'
                                )  # -> '../data/cdmlite_input_files.txt'
        self.BASE_DIR = gs.get(
            f'{release}:lite:land:incoming:observations'
        )  # -> '/gws/nopw/j04/c3s311a_lot2/data/level2/land/cdm_lite/'

        self._load()
        self._input_files = None
示例#2
0
def initialise(release):
    if release not in gs.RELEASES:
        raise ValueError(
            f'Release {release} is not valid, must be one of: {gs.RELEASES.keys()}'
        )

    global COMMON_BASE
    global INPUT_FILES_FILE
    global LAND_BATCH_RULES

    COMMON_BASE = gs.get(f'{release}:lite:land:incoming:observations')
    INPUT_FILES_FILE = gs.get(f'{release}:lite:land:batches:input_files')
    LAND_BATCH_RULES = gs.get(f'{release}:lite:land:batches:rules')
示例#3
0
def initialise(release):
    if release not in gs.RELEASES:
        raise ValueError(f'Release {release} is not valid, must be one of: {gs.RELEASES.keys()}')

    # Initalise global variables based on the release
    global BASE_INPUT_DIR
    global BASE_OUTPUT_DIR
    global BASE_LOG_DIR

    BASE_INPUT_DIR = gs.get(f'{release}:lite:marine:incoming:header_table')
    BASE_OUTPUT_DIR = gs.get(f'{release}:lite:marine:outputs:workflow')
    BASE_LOG_DIR = gs.get(f'{release}:lite:marine:outputs:log')

    print(f'[INFO] Set variable: BASE_INPUT_DIR={BASE_INPUT_DIR}')
示例#4
0
def initialise(release):
    if release not in gs.RELEASES:
        raise ValueError(
            f'Release {release} is not valid, must be one of: {gs.RELEASES.keys()}'
        )

    # Initalise global variables based on the release
    global RELEASE
    global BASE_OUTPUT_DIR
    global BASE_LOG_DIR
    global YEARS_DICT

    RELEASE = release

    BASE_OUTPUT_DIR = gs.get(f'{release}:lite:land:outputs:workflow')
    BASE_LOG_DIR = gs.get(f'{release}:lite:land:outputs:log')

    years_dict_file = gs.get(f'{release}:lite:land:batches:years')
    YEARS_DICT = pdict.PickleDict(years_dict_file)
示例#5
0
def main():

    args = sys.argv[1:]
    if len(args) == 0 or args[0].lower() in ('-h', '--h', '-help', '--help', 'help'):
        return show_help()

    settings_string = args[0]
    
    try:
        print(gs.get(settings_string))
    except Exception:
        raise KeyError(f'Setting not found: {settings_string}')
示例#6
0
#!/usr/bin/env python

import os
import sys

import glamod.settings as gs

args = sys.argv[1:]
if not args:
    raise Exception('Must provide a release as the only argument!')

release = args[0]
years_dict_path = gs.get(f'{args[0]}:lite:land:batches:years')
lock_path = f'{years_dict_path}.lock'

if os.path.exists(lock_path):
    os.remove(lock_path)
    print(f'[INFO] Removed {lock_path}')
else:
    print(f'[INFO] No lock file found at {lock_path}')
示例#7
0
def initialise(release):
    global BASE_UPDATE_DIR = gs.get(f'{release}:lite:land:incoming:daily_updates')
    global INCOMING_UPDATE_DIR = os.path.join(BASE_UPDATE_DIR, 'incoming')
    global PROCESSING_UPDATE_DIR = os.path.join(BASE_UPDATE_DIR, 'processing')
    global FAILED_UPDATE_DIR = os.path.join(BASE_UPDATE_DIR, 'failed')
    global COMPLETE_UPDATE_DIR = os.path.join(BASE_UPDATE_DIR, 'complete')
示例#8
0
def test_settings():
    assert(get('r2.0:full:land:incoming:source_configuration') == \
           '/gws/nopw/j04/c3s311a_lot2/data/level2/land/r202005/source_configuration')
    assert(get('r2.0:full:land:incoming:source_configuration') == \
           get(':full:land::source_configuration'))
示例#9
0
 - CHECK: there are no NULL values for "report_id" 
 - write to `output_file`
 - IF FAILURE: write `failure_file`
 - IF SUCCESS: write `success_file` 

"""

import os, re, glob, sys, time

import pandas as pd
import click

import glamod.settings as gs
import glamod.prepare.utils as prep_utils

BASE_INPUT_DIR = gs.get('r2.0:lite:marine:incoming:header_table')
# _EG_SUB_DIR = '001-110'

#BASE_OUTPUT_DIR = '/gws/nopw/j04/c3s311a_lot2/data/cdmlite/r201910/marine'
BASE_OUTPUT_DIR = gs.get('r2.0:lite:marine:outputs:workflow')
###'/work/scratch-nompiio/astephen/glamod/r202001/cdmlite/marine'

# .../gws/nopw/j04/c3s311a_lot2/data/marine/r092019_cdm_lite/ICOADS_R3.0.0T/level1a'

BASE_LOG_DIR = gs.get('r2.0:lite:marine:outputs:log')
### '/gws/smf/j04/c3s311a_lot2/cdmlite/log/r202001/prep/marine'

#FILE_PATTN = re.compile('^(observations|header)-?(\w+)?-(?P<year>\d{4})-(\d{2})-(?P<revision>r\d{1,})-(?P<other>\d{1,})\.psv')

# Note: header field "report_quality" is kept for filtering values of 0 only.
#    The out_fields will exclude it later.