def __init__(self, release):
self.BATCH_FILE = gs.get(f'{release}:lite:land:batches:rules'
) # -> '../data/land_cdmlite_batch_rules.txt'
self.FILE_LIST = gs.get(f'{release}:lite:land:batches:input_files'
) # -> '../data/cdmlite_input_files.txt'
self.BASE_DIR = gs.get(
f'{release}:lite:land:incoming:observations'
) # -> '/gws/nopw/j04/c3s311a_lot2/data/level2/land/cdm_lite/'
self._load()
self._input_files = None
def initialise(release):
if release not in gs.RELEASES:
raise ValueError(
f'Release {release} is not valid, must be one of: {gs.RELEASES.keys()}'
)
global COMMON_BASE
global INPUT_FILES_FILE
global LAND_BATCH_RULES
COMMON_BASE = gs.get(f'{release}:lite:land:incoming:observations')
INPUT_FILES_FILE = gs.get(f'{release}:lite:land:batches:input_files')
LAND_BATCH_RULES = gs.get(f'{release}:lite:land:batches:rules')
def initialise(release):
if release not in gs.RELEASES:
raise ValueError(f'Release {release} is not valid, must be one of: {gs.RELEASES.keys()}')
# Initalise global variables based on the release
global BASE_INPUT_DIR
global BASE_OUTPUT_DIR
global BASE_LOG_DIR
BASE_INPUT_DIR = gs.get(f'{release}:lite:marine:incoming:header_table')
BASE_OUTPUT_DIR = gs.get(f'{release}:lite:marine:outputs:workflow')
BASE_LOG_DIR = gs.get(f'{release}:lite:marine:outputs:log')
print(f'[INFO] Set variable: BASE_INPUT_DIR={BASE_INPUT_DIR}')
def initialise(release):
if release not in gs.RELEASES:
raise ValueError(
f'Release {release} is not valid, must be one of: {gs.RELEASES.keys()}'
)
# Initalise global variables based on the release
global RELEASE
global BASE_OUTPUT_DIR
global BASE_LOG_DIR
global YEARS_DICT
RELEASE = release
BASE_OUTPUT_DIR = gs.get(f'{release}:lite:land:outputs:workflow')
BASE_LOG_DIR = gs.get(f'{release}:lite:land:outputs:log')
years_dict_file = gs.get(f'{release}:lite:land:batches:years')
YEARS_DICT = pdict.PickleDict(years_dict_file)
def main():
args = sys.argv[1:]
if len(args) == 0 or args[0].lower() in ('-h', '--h', '-help', '--help', 'help'):
return show_help()
settings_string = args[0]
try:
print(gs.get(settings_string))
except Exception:
raise KeyError(f'Setting not found: {settings_string}')
#!/usr/bin/env python
import os
import sys
import glamod.settings as gs
args = sys.argv[1:]
if not args:
raise Exception('Must provide a release as the only argument!')
release = args[0]
years_dict_path = gs.get(f'{args[0]}:lite:land:batches:years')
lock_path = f'{years_dict_path}.lock'
if os.path.exists(lock_path):
os.remove(lock_path)
print(f'[INFO] Removed {lock_path}')
else:
print(f'[INFO] No lock file found at {lock_path}')
def initialise(release):
global BASE_UPDATE_DIR = gs.get(f'{release}:lite:land:incoming:daily_updates')
global INCOMING_UPDATE_DIR = os.path.join(BASE_UPDATE_DIR, 'incoming')
global PROCESSING_UPDATE_DIR = os.path.join(BASE_UPDATE_DIR, 'processing')
global FAILED_UPDATE_DIR = os.path.join(BASE_UPDATE_DIR, 'failed')
global COMPLETE_UPDATE_DIR = os.path.join(BASE_UPDATE_DIR, 'complete')
def test_settings():
assert(get('r2.0:full:land:incoming:source_configuration') == \
'/gws/nopw/j04/c3s311a_lot2/data/level2/land/r202005/source_configuration')
assert(get('r2.0:full:land:incoming:source_configuration') == \
get(':full:land::source_configuration'))
- CHECK: there are no NULL values for "report_id"
- write to `output_file`
- IF FAILURE: write `failure_file`
- IF SUCCESS: write `success_file`
"""
import os, re, glob, sys, time
import pandas as pd
import click
import glamod.settings as gs
import glamod.prepare.utils as prep_utils
BASE_INPUT_DIR = gs.get('r2.0:lite:marine:incoming:header_table')
# _EG_SUB_DIR = '001-110'
#BASE_OUTPUT_DIR = '/gws/nopw/j04/c3s311a_lot2/data/cdmlite/r201910/marine'
BASE_OUTPUT_DIR = gs.get('r2.0:lite:marine:outputs:workflow')
###'/work/scratch-nompiio/astephen/glamod/r202001/cdmlite/marine'
# .../gws/nopw/j04/c3s311a_lot2/data/marine/r092019_cdm_lite/ICOADS_R3.0.0T/level1a'
BASE_LOG_DIR = gs.get('r2.0:lite:marine:outputs:log')
### '/gws/smf/j04/c3s311a_lot2/cdmlite/log/r202001/prep/marine'
#FILE_PATTN = re.compile('^(observations|header)-?(\w+)?-(?P<year>\d{4})-(\d{2})-(?P<revision>r\d{1,})-(?P<other>\d{1,})\.psv')
# Note: header field "report_quality" is kept for filtering values of 0 only.
# The out_fields will exclude it later.