def plot(rundate: "datedoy", tech: "pipeline", stage: "option",
writers: "option"):
log.init(log_level="info")
# Get options
writer_names = writers.replace(",", " ").split()
dataset_id = util.read_option_value("--dset_id", default="last")
dataset_id = "last" if dataset_id == "last" else int(dataset_id)
identifier = util.read_option_value("--id", default="")
identifier = f"-{identifier}" if identifier else ""
session = util.read_option_value("--session", default="")
dset = data.Dataset(rundate=rundate,
tech=tech,
stage=stage,
id=identifier,
dataset_name=session,
dataset_id=dataset_id)
path_hdf5 = files.path("dataset_hdf5", file_vars=dset.vars)
path_json = files.path("dataset_json", file_vars=dset.vars)
log.info(f"Read Where dataset files {path_hdf5} and {path_json}.")
if dset.num_obs == 0:
log.fatal(f"No data to read for date {rundate}.")
# Loop over writers
for writer in writer_names:
write(writer, dset=dset)
def main():
"""Parse command line options and set up an Where analysis
Do simple parsing of command line arguments. Set up config-files and show the configuration.
"""
util.check_help_and_version(doc_module=__name__)
# Start logging
log.init()
# Read command line options
pipeline = pipelines.get_from_options()
config.read_pipeline(pipeline)
if util.check_options("--doy"):
rundate = util.parse_args("doy", doc_module=__name__)
else:
rundate = util.parse_args("date", doc_module=__name__)
session = pipelines.get_session(rundate, pipeline)
# Set up the configuration for the analysis
setup_config(rundate, pipeline, session)
# Show current configuration
show_config(rundate, pipeline, session)
# Store configuration in library
store_config_to_library(rundate, pipeline, session)
def write(rundate: "datedoy", pipeline: "pipeline", stage: "option",
writers: "option"):
log.init(log_level="info")
# Get options
label = util.read_option_value("--label", default="None")
# TODO: label = "last" if label == "last" else label
id_ = util.read_option_value("--id", default="")
session = util.read_option_value("--session", default="")
station = util.read_option_value("--station", default="")
writers = writers.replace(",", " ").split()
dset_vars = dict(pipeline=pipeline,
stage=stage,
session=session,
station=station,
label=label,
id=id_)
try:
dset = dataset.Dataset.read(**dict(dset_vars, rundate=rundate))
path = config.files.path("dataset",
file_vars={
**dset.vars,
**dset.analysis
})
log.info(f"Read Where dataset files {path}.")
except OSError as err:
log.fatal(f"Unable to read data for {rundate}: {err}")
# Loop over writers
for writer in writers:
log.info(f"Apply writer '{writer}'.")
write_(writer, dset=dset)
def concatenate(
from_date: "datedoy", to_date: "datedoy", tech: "pipeline", stage: "option", writers: "option", id: "option"
):
log.init(log_level="info")
# Get options
writer_names = writers.replace(",", " ").split()
dataset_id = util.read_option_value("--dset_id", default="last")
dataset_id = "last" if dataset_id == "last" else int(dataset_id)
dataset_name = util.read_option_value("--dset_name", default="")
session = util.read_option_value("--session", default="")
dset_vars = dict(
tech=tech,
stage=stage,
session=session,
dataset_name=session,
dataset_id=dataset_id,
session_name=id + "_concatenated",
)
dset = concatenate_datasets(from_date, to_date, dset_vars)
if dset.num_obs == 0:
log.fatal(f"No data to read period from {from_date} to {to_date}.")
dset.write()
# Loop over writers
for writer in writer_names:
write(writer, dset=dset)
def concatenate(from_date: "datedoy", to_date: "datedoy", pipeline: "pipeline",
stage: "option"):
log.init(log_level="info")
# Get options
label = util.read_option_value("--label", default="None")
# TODO: label = "last" if label == "last" else label
id_ = util.read_option_value("--id", default="")
only_for_rundate = True if util.check_options(
"--only_for_rundate") else False
session = util.read_option_value("--session", default="")
station = util.read_option_value("--station", default="")
writers = util.read_option_value("--writers",
default="").replace(",", " ").split()
# Update configuration of Where analysis
config.where.update_from_options(_clean_sys_argv(pipeline))
dset_vars = dict(pipeline=pipeline,
stage=stage,
session=session,
station=station,
label=label,
id=id_)
dset_vars = config.create_file_vars(rundate=from_date, **dset_vars)
dset = _concatenate_datasets(from_date, to_date, dset_vars,
only_for_rundate)
if dset.num_obs == 0:
log.fatal(f"No data to read period from {from_date} to {to_date}.")
dset.write()
# Loop over writers
for writer in writers:
write(writer, dset=dset)
def main():
"""Parse command line options and run the Where analysis
Do simple parsing of command line arguments. Set up config-files and start the analysis. See the help docstring at
the top of the file for more information about the workflow.
"""
# Start logging
log.init()
# Read command line options
if util.check_options("--doy"):
rundate = util.parse_args("doy", doc_module=__name__)
else:
rundate = util.parse_args("date", doc_module=__name__)
pipeline = pipelines.get_from_options()
session = pipelines.get_session(rundate, pipeline)
# Pretend to empty mailbox
pretend_to_empty_mailbox()
# Start an interactive session
if util.check_options("-I", "--interactive"):
from where.tools import interactive
interactive.interactive(rundate, pipeline, session)
return
# Set up the configuration for a new analysis or update an existing one
setup.setup_config(rundate, pipeline, session)
# Run the analysis
setup.add_timestamp(rundate, pipeline, session, "last run")
with timer(f"Finish pipeline {pipeline.upper()} in"):
pipelines.run(rundate, pipeline, session)
def main():
"""Invoke where_tools
To add a new tool, simply add a new .py-file with a registered plugin that
will be called when the tool is called.
"""
# Start logging
log.init(log_level="info")
# First read tool from command line, don't use util.parse_args() as that overrides -h for each tool
try:
tool = [a for a in sys.argv[1:] if not a.startswith("-")][0]
sys.argv.remove(tool)
except IndexError:
util._print_help_from_doc(__name__)
raise SystemExit
# Check that tool is available and figure out signature
try:
sig = plugins.signature(__name__, tool)
tool_module = plugins.get(__name__, tool).function.__module__
except exceptions.UnknownPluginError as err:
util._print_help_from_doc(__name__)
err.args = (
f"{err.args[0]}\n Available tools are {', '.join(plugins.names(__name__))}",
)
raise
# Parse parameters
util.check_help_and_version(doc_module=tool_module)
tool_args = dict()
for key, param in sig.parameters.items():
if param.annotation is None:
raise SystemExit(f"{param} in {tool} tool is not annotated")
if param.annotation == "datedoy":
if util.check_options("--doy"):
date = util.parse_args("doy", doc_module=__name__)
else:
date = util.parse_args("date", doc_module=__name__)
tool_args[key] = date
elif param.annotation == "pipeline":
tool_args[key] = pipelines.get_from_options()
config.read_pipeline(tool_args[key])
elif param.annotation == "option":
tool_args[key] = util.read_option_value(f"--{key}")
else:
tool_args[key] = util.parse_args(param.annotation,
doc_module=tool_module)
# Call tool
plugins.call(__name__, tool, **tool_args)
def main(date: "datedoy", pipeline: "pipeline", ids: "option"):
log.init(log_level="info")
# Additional required options
identifiers = [id_.strip() for id_ in ids.split(",")]
difference_by = util.read_option_value("--difference_by").replace(
",", " ").split()
stage = util.read_option_value("--stage")
# Get optional options
dataset_id = util.read_option_value("--dset_id", default="last")
dataset_id = "last" if dataset_id == "last" else int(dataset_id)
dataset_name = util.read_option_value("--dset_name", default="")
writer_names = util.read_option_value("--writers",
default="").replace(",",
" ").split()
session = util.read_option_value("--session", default="")
station = util.read_option_value("--station", default="")
# Get datasets
dset = data.Dataset(
rundate=date,
pipeline=pipeline,
stage=stage,
session=session,
station=station,
label=label,
id="-" + identifiers[0],
)
dset_other = data.Dataset(
rundate=date,
pipeline=pipeline,
stage=stage,
session=session,
station=station,
label=label,
id="-" + identifiers[1],
)
if dset.num_obs == 0:
log.warn(
f"Nothing to differentiate. Dataset '{identifiers[0]}' is empty.")
return 1
if dset_other.num_obs == 0:
log.warn(
f"Nothing to differentiate. Dataset '{identifiers[1]}' is empty.")
return 1
# Differentiate dataset
dset_diff = dset.difference(dset_other, index_by=','.join(difference_by))
dset_diff.write_as(stage="difference")
# Loop over writers
for writer in writer_names:
write(writer, dset_diff)
def main():
"""Parse command line options and run the Where analysis
Do simple parsing of command line arguments. Set up config-files and start the analysis. See the help docstring at
the top of the file for more information about the workflow.
"""
util.check_help_and_version(doc_module=__name__)
# Start logging
log.init(config.where.log.default_level.str)
log.debug(
f"Use {util.get_python_version()} on process {util.get_pid_and_server()}"
)
# Read command line options
pipeline = pipelines.get_from_options()
config.read_pipeline(pipeline)
if util.check_options("--doy"):
rundate = util.parse_args("doy", doc_module=__name__)
else:
rundate = util.parse_args("date", doc_module=__name__)
args, kwargs = util.options2args(sys.argv[1:])
# Start an interactive session
if util.check_options("-I", "--interactive"):
from where.tools import interactive # Local import because interactive imports many external packages
interactive.interactive(rundate, pipeline, **kwargs)
return
# Set up the configuration for a new analysis or update an existing one
unused_options = setup.setup_config(rundate, pipeline, *args, **kwargs)
pipeline_args, pipeline_kwargs = util.options2args(unused_options)
# Run the analysis
setup.add_timestamp(rundate, pipeline, "last run", **kwargs)
with Timer(f"Finish pipeline {pipeline.upper()} in"):
pipelines.run(rundate, pipeline, *pipeline_args, **pipeline_kwargs)
def concatenate(from_date: "datedoy", to_date: "datedoy", tech: "pipeline",
stage: "option"):
log.init(log_level="info")
# Get options
dataset_id = util.read_option_value("--dset_id", default="last")
dataset_id = "last" if dataset_id == "last" else int(dataset_id)
dataset_name = util.read_option_value("--dset_name", default="")
id = util.read_option_value("--id", default="")
only_for_rundate = True if util.check_options(
"--only_for_rundate") else False
session = util.read_option_value("--session", default="")
writer_names = util.read_option_value("--writers",
default="").replace(",",
" ").split()
# Update configuration of Where analysis
config.where.update_from_options(_clean_sys_argv(tech))
dset_vars = dict(
tech=tech,
stage=stage,
session=session,
dataset_name=session,
dataset_id=dataset_id,
session_name=id + "_concatenated",
)
dset = _concatenate_datasets(from_date, to_date, dset_vars,
only_for_rundate)
if dset.num_obs == 0:
log.fatal(f"No data to read period from {from_date} to {to_date}.")
dset.write()
# Loop over writers
for writer in writer_names:
write(writer, dset=dset)
def run(rundate, pipeline, *args, **kwargs):
"""Run a Where pipeline for a given date and session
Args:
rundate: Rundate of analysis.
pipeline: Pipeline used for analysis.
session: Session in analysis.
"""
if not setup.has_config(rundate, pipeline, *args, **kwargs):
log.fatal(
f"No configuration found for {pipeline.upper()} {rundate.strftime(config.FMT_date)}"
)
# Set up config
config.init(rundate, pipeline, **kwargs)
# Register filekey suffix
filekey_suffix = config.tech.filekey_suffix.list
if filekey_suffix:
config.files.profiles = filekey_suffix
# Validate input arguments
try:
prefix = plugins.call(package_name=__name__,
plugin_name=pipeline,
part="validate_args",
rundate=rundate,
**kwargs)
except mg_exceptions.UnknownPluginError:
log.warn(
f"Pipeline {pipeline} has not defined function 'validate_args'")
except exceptions.InvalidArgsError as err:
from where.tools import delete
# Clean up {placeholder} directories created by config
delete.delete_analysis(rundate, pipeline, **kwargs)
log.fatal(err)
# Set up console logger and start file logger
try:
prefix = plugins.call(package_name=__name__,
plugin_name=pipeline,
part="log_prefix",
rundate=rundate,
**kwargs)
except mg_exceptions.UnknownPluginError:
log.warn(f"Pipeline {pipeline} has not defined function 'log_prefix'")
prefix = ""
log_cfg = config.where.log
log.init(log_level=log_cfg.default_level.str, prefix=prefix)
if log_cfg.log_to_file.bool:
log.file_init(
file_path=config.files.path("log"),
log_level=log_cfg.default_level.str,
prefix=prefix,
rotation=log_cfg.number_of_log_backups.int,
)
# Update analysis config and file variables
config.set_analysis(rundate, pipeline=pipeline, **kwargs)
config.set_file_vars(file_vars())
log.blank() # Empty line for visual clarity
# Read which stages that should be executed once for each iterable
skip_stages = config.tech.skip_stages.list
stage_iterate = config.tech.stage_iterate.list
dset_list = []
dset = None
if stage_iterate:
# Read which list should be iterated over and the placeholder name of each entry
iterate_over, _, var_name = config.tech.stage_iterate_over.str.partition(
":")
var_name = var_name.strip()
# Iterate
for item in config.tech[iterate_over].list:
kwargs[var_name] = item
log.blank()
log.info(f"***** Running {item} *****")
for prev_stage, stage in zip([None] + stage_iterate,
stage_iterate):
if stage not in skip_stages:
dset = run_stage(rundate, pipeline, dset, stage,
prev_stage, **kwargs)
if dset is not None:
dset_list.append(dset)
dset = None
kwargs[var_name] = "combined"
if dset_list:
dset_list[0].merge_with(*dset_list[1:], sort_by="time")
dset = dset_list[0]
if len(dset_list) > 1:
log.info(f"Combining dataset for {len(dset_list)} {iterate_over}")
dset.write_as(stage=stage_iterate[-1], label=2, **kwargs)
# Read which stages that should be executed once
stage_once = config.tech.stage_once.list
# Find which stages we will run analysis for
if not stage_once and not stage_iterate:
stage_list = [s for s in stages(pipeline)]
prev_stage_start = None
else:
stage_list = [s for s in stage_once]
prev_stage_start = stage_iterate[-1] if stage_iterate else None
for prev_stage, stage in zip([prev_stage_start] + stage_list, stage_list):
if stage not in skip_stages:
dset = run_stage(rundate, pipeline, dset, stage, prev_stage,
**kwargs)
log.blank()
if dset is not None and dset.num_obs == 0:
log.warn(f"No observations in dataset after {stage} stage.")
break
# Store configuration to library
setup.store_config_to_library(rundate, pipeline, **kwargs)
# Write requirements to file for reproducibility
util.write_requirements()
def main(date: "datedoy", pipeline: "pipeline", items: "option",
specifier: "option"):
log.init(log_level="info")
dsets = dict()
# Additional options
stage = util.read_option_value("--stage")
writer_names = util.read_option_value("--writers").replace(",",
" ").split()
items_ = [s.strip() for s in items.split(",")]
# Get optional options
label = util.read_option_value("--label", default="None")
# TODO label = "last" if label == "last" else label
station = util.read_option_value("--station", default="")
id_ = util.read_option_value("--id", default="")
# Get dataset variables
dset_vars = dict(pipeline=pipeline,
stage=stage,
station=station,
label=label,
id=id_)
dset_vars = config.create_file_vars(rundate=date, **dset_vars)
# Read datasets for given specifier
if specifier == "id":
for id_ in items_:
dset = dataset.Dataset().read(rundate=date,
pipeline=pipeline,
stage=stage,
label=label,
id=id_,
station=station)
if dset.num_obs == 0:
log.warn(f"Dataset '{id_}' is empty.")
continue
dset_vars[
"id"] = id_ #TODO: Better solution for handling of dataset variables?
dset.vars.update(
dset_vars
) # Necessary for example for getting correct file path in used writers.
dsets.update({id_: dset})
elif specifier == "station":
for station in items_:
dset = dataset.Dataset().read(rundate=date,
pipeline=pipeline,
stage=stage,
label=label,
id=id_,
station=station)
if dset.num_obs == 0:
log.warn(f"Dataset '{station}' is empty.")
continue
dset_vars[
"station"] = station #TODO: Better solution for handling of dataset variables?
dset.vars.update(
dset_vars
) # Necessary for example for getting correct file path in used writers.
dsets.update({station: dset})
elif specifier == "stage":
for stage in items_:
dset = dataset.Dataset().read(rundate=date,
pipeline=pipeline,
stage=stage,
label=label,
id=id_,
station=station)
if dset.num_obs == 0:
log.warn(f"Dataset '{stage}' is empty.")
continue
dset_vars[
"stage"] = stage #TODO: Better solution for handling of dataset variables?
dset.vars.update(
dset_vars
) # Necessary for example for getting correct file path in used writers.
dsets.update({stage: dset})
else:
log.fatal(
f"Specifier {specifier} is not defined. It should be either 'id', 'station' or 'stage'."
)
if len(dsets) == 0:
log.fatal(f"All given datasets are empty [{', '.join(dsets.keys())}].")
elif len(dsets) == 1:
log.warn(
f"Nothing to compare. Only dataset '{list(dsets.keys())[0]}' is available."
)
# Loop over writers
for writer in writer_names:
write(writer, dset=dsets)
def run(rundate, pipeline, session=""):
"""Run a Where pipeline for a given date and session
Args:
rundate: Rundate of analysis.
pipeline: Pipeline used for analysis.
session: Session in analysis.
"""
if not setup.has_config(rundate, pipeline, session):
log.fatal(
f"No configuration found for {pipeline.upper()} {session} {rundate.strftime(config.FMT_date)}"
)
# Set up session config
config.init(rundate=rundate, tech_name=pipeline, session=session)
# Set up prefix for console logger and start file logger
log_cfg = config.where.log
prefix = f"{pipeline.upper()} {session} {rundate:%Y-%m-%d}"
log.init(log_level=log_cfg.default_level.str, prefix=prefix)
if log_cfg.log_to_file.bool:
log.file_init(
file_path=files.path("log"),
log_level=log_cfg.default_level.str,
prefix=prefix,
rotation=log_cfg.number_of_log_backups.int,
)
# Read which stages to skip from technique configuration file.
skip_stages = config.tech.get("skip_stages", default="").list
# Register filekey suffix
filekey_suffix = config.tech.filekey_suffix.list
if filekey_suffix:
config.files.profiles = filekey_suffix
# Find which stages we will run analysis for
# TODO: Specify stage_list in config
stage_list = [s for s in stages(pipeline) if s not in skip_stages]
# Start file logging and reporting
reports.report.init(sessions=[session])
reports.report.start_session(session)
reports.report.text("header", session.replace("_", " ").title())
# Update analysis config and file variables
config.set_analysis(rundate=rundate,
tech=pipeline,
analysis=pipeline,
session=session)
config.set_file_vars(file_vars())
# Log the name of the session
log.blank() # Empty line for visual clarity
log.info(f"Start session {session}")
session_timer = timer(f"Finish session {session} in")
session_timer.start()
# Run stages, keep track of previous stage
dset = None
dep_fast = config.where.files.dependencies_fast.bool
for prev_stage, stage in zip([None] + stage_list, stage_list):
# Skip stages where no dependencies have changed
dep_path = files.path("depends", file_vars=dict(stage=stage))
if not (dependencies.changed(dep_path, fast_check=dep_fast)
or util.check_options("-F", "--force")):
log.info(
f"Not necessary to run {stage} for {pipeline.upper()} {rundate.strftime(config.FMT_date)}"
)
continue
elif dset is None:
# Create or read dataset
empty = stage == stage_list[0]
dset = dataset.Dataset(rundate,
tech=pipeline,
stage=prev_stage,
dataset_name=session,
dataset_id="last",
empty=empty)
# Report on the stage
reports.report.start_section(stage)
reports.report.text("header", stage.replace("_", " ").title())
if prev_stage:
log.blank() # Empty line for visual clarity
# Set up dependencies. Add dependencies to previous stage and config file
dependencies.init(dep_path, fast_check=dep_fast)
dependencies.add(files.path("depends",
file_vars=dict(stage=prev_stage)),
label="depends")
dependencies.add(*config.tech.sources, label="config")
# Delete old datasets for this stage
dset.delete_from_file(stage=stage, dataset_id="all")
# Call the current stage. Skip rest of stages if current stage returns False (compare with is since by
# default stages return None)
plugins.call(package_name=__name__,
plugin_name=pipeline,
part=stage,
stage=stage,
dset=dset,
plugin_logger=log.info)
dependencies.write()
if dset.num_obs == 0:
log.warn(
f"No observations in dataset after {stage} stage. Exiting pipeline"
)
break
else: # Only done if loop does not break (all stages finish normally)
# Publish files for session
files.publish_files()
session_timer.end()
# Store configuration to library
setup.store_config_to_library(rundate, pipeline, session)
# Write reports specified in config
reports.write(rundate, pipeline)
# Write requirements to file for reproducibility
util.write_requirements()
def main():
"""Parse command line options and loop over the Where analysis
Do simple parsing of command line arguments. Set up config-files and potentially start the analysis. See the help
docstring at the top of the file for more information about the workflow.
"""
util.check_help_and_version(doc_module=__name__)
log.init(log_level=config.where.log.default_level.str, prefix="Runner")
# Initialize
pipeline = pipelines.get_from_options()
config.read_pipeline(pipeline)
if util.check_options("--doy"):
from_date = util.parse_args("doy", doc_module=__name__)
to_date = util.parse_args("doy", doc_module=__name__)
sys.argv.remove("--doy")
else:
from_date = util.parse_args("date", doc_module=__name__)
to_date = util.parse_args("date", doc_module=__name__)
# Handle list of sessions
session_list = set(
util.read_option_value("--session", default="").replace(",",
" ").split())
sys.argv = [o for o in sys.argv if not o.startswith("--session=")]
# Start logging
file_vars = dict(**util.get_user_info())
log.file_init(
file_path=files.path("log_runner", file_vars=file_vars),
log_level=config.where.log.default_level.str,
prefix="Runner",
rotation=config.where.log.number_of_log_backups.int,
)
atexit.register(log_statistics)
# Should where_runner crash if Where crashes?
stop_on_error_opts = None
if util.check_options("--stop-on-error"):
stop_on_error_opts = True
elif util.check_options("--continue-on-error"):
stop_on_error_opts = False
stop_on_error = config.where.get("stop_on_error",
section="runner",
value=stop_on_error_opts).bool
error_logger = log.fatal if stop_on_error else log.error
# Loop over dates
rundate = from_date
while rundate <= to_date:
available_sessions = set(pipelines.list_sessions(rundate, pipeline))
sessions = available_sessions & session_list if session_list else available_sessions
where_args = remove_runner_args(sys.argv[1:])
for session in sorted(sessions):
cmd = f"{where.__executable__} {rundate:%Y %m %d} --session={session}".split(
) + where_args
log.info(f"Running '{' '.join(cmd)}'")
count("Number of analyses")
try:
subprocess.run(cmd,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except subprocess.CalledProcessError as err:
count("Failed analyses")
error_msg = err.stderr.decode().strip().split("\n")[-1]
error_logger(f"Command '{' '.join(cmd)}' failed: {error_msg}")
else:
count("Successful analyses")
copy_log_from_where(rundate, pipeline, session)
rundate += timedelta(days=1)
def compare(date: "datedoy", pipeline: "pipeline", items: "option",
specifier: "option"):
log.init(log_level="info")
dsets = dict()
# Additional options
stage = util.read_option_value("--stage")
writer_names = util.read_option_value("--writers").replace(",",
" ").split()
items_ = [s.strip() for s in items.split(",")]
# Get optional options
label = util.read_option_value("--label", default="None")
# TODO label = "last" if label == "last" else label
station = util.read_option_value("--station", default="")
id_ = util.read_option_value("--id", default="")
# Update configuration of Where analysis
config.where.update_from_options(_clean_sys_argv(pipeline))
# Get dataset variables
dset_vars = config.create_file_vars(rundate=date, pipeline=pipeline)
# Read datasets for given specifier
if specifier == "id":
for id_ in items_:
try:
dset = dataset.Dataset().read(rundate=date,
pipeline=pipeline,
stage=stage,
label=label,
id=id_,
station=station)
except OSError:
log.warn(f"No data to read for Dataset id '{id_}'.")
continue
dset.vars.update(dset_vars)
dset.vars["id"] = id_
dsets.update({id_: dset})
elif specifier == "station":
for station in items_:
try:
dset = dataset.Dataset().read(rundate=date,
pipeline=pipeline,
stage=stage,
label=label,
id=id_,
station=station)
except OSError:
log.warn(f"No data to read for Dataset station '{station}'.")
continue
dset.vars.update(dset_vars)
dset.vars["station"] = station
dsets.update({station: dset})
elif specifier == "stage":
for stage in items_:
try:
dset = dataset.Dataset().read(rundate=date,
pipeline=pipeline,
stage=stage,
label=label,
id=id_,
station=station)
except OSError:
log.warn(f"No data to read for Dataset stage '{stage}'.")
continue
dset.vars.update(dset_vars)
dset.vars["stage"] = stage
dsets.update({stage: dset})
else:
log.fatal(
f"Specifier {specifier} is not defined. It should be either 'id', 'station' or 'stage'."
)
if len(dsets) == 0:
log.fatal(f"All given datasets are empty [{', '.join(dsets.keys())}].")
elif len(dsets) == 1:
log.warn(
f"Nothing to compare. Only dataset '{list(dsets.keys())[0]}' is available."
)
# Loop over writers
for writer in writer_names:
write(writer, dset=dsets)
def main(date: "datedoy", tech: "pipeline", items: "option",
specifier: "option"):
log.init(log_level="info")
dsets = dict()
# Additional options
stage = util.read_option_value("--stage")
writer_names = util.read_option_value("--writers").replace(",",
" ").split()
items_ = [s.strip() for s in items.split(",")]
# Get optional options
dataset_id = util.read_option_value("--dset_id", default="last")
dataset_id = "last" if dataset_id == "last" else int(dataset_id)
dataset_name = util.read_option_value("--dset_name", default="")
session = util.read_option_value("--session", default="")
id_ = "-" + util.read_option_value(
"--id", default="") if util.read_option_value("--id",
default="") else ""
# Read datasets for given specifier
if specifier == "id":
for id_ in items_:
dset = data.Dataset(rundate=date,
tech=tech,
stage=stage,
dataset_name=dataset_name,
dataset_id=dataset_id,
id="-" + id_)
if dset.num_obs == 0:
log.warn(f"Dataset '{id_}' is empty.")
continue
dsets.update({id_: dset})
elif specifier == "session":
for session in items_:
dset = data.Dataset(rundate=date,
tech=tech,
stage=stage,
dataset_name=session,
dataset_id=dataset_id,
id=id_)
if dset.num_obs == 0:
log.warn(f"Dataset '{session}' is empty.")
continue
dsets.update({session: dset})
elif specifier == "stage":
for stage in items_:
dset = data.Dataset(rundate=date,
tech=tech,
stage=stage,
dataset_name=dataset_name,
dataset_id=dataset_id,
id=id_)
if dset.num_obs == 0:
log.warn(f"Dataset '{stage}' is empty.")
continue
dsets.update({stage: dset})
else:
log.fatal(
f"Specifier {specifier} is not defined. It should be either 'id', 'session' or 'stage'."
)
if len(dsets) == 0:
log.fatal(f"All given datasets are empty [{', '.join(dsets.keys())}].")
elif len(dsets) == 1:
log.warn(
f"Nothing to compare. Only dataset '{list(dsets.keys())[0]}' is available."
)
# Loop over writers
for writer in writer_names:
write(writer, dset=dsets)
def setUp(self):
"""
The first test set up is based on the bc_velo.c program, which is published in :cite:`remondi2004` and
following RINEX navigation file sample:
/* Sample Broadcast Message in unit of radians, seconds, meters.
20 01 7 23 2 0 0.0 -.857324339449D-04 -.272848410532D-11 .000000000000D+00
.200000000000D+02 .886875000000D+02 .465376527657D-08 .105827953357D+01
.457651913166D-05 .223578442819D-02 .177137553692D-05 .515379589081D+04
.936000000000D+05 .651925802231D-07 .164046615454D+01 -.856816768646D-07
.961685061380D+00 .344968750000D+03 .206374037770D+01 -.856928551657D-08
.342514267094D-09 .000000000000D+00 .112400000000D+04 .000000000000D+00
.200000000000D+01 .000000000000D+00 -.651925802231D-08 .276000000000D+03
.865800000000D+05 .000000000000D+00 .000000000000D+00 .000000000000D+00
*/
The second test set up compares results from Where against gLAB solution for satellite G20 and epoch
2016-03-01 00:00:00.0.
/* Sample Broadcast Message in unit of radians, seconds, meters for satellite G20 and
/ epoch 2016-03-01 00:00:00.0
20 16 3 1 0 0 0.0 0.396233052015D-03 0.261479726760D-11 0.000000000000D+00
0.100000000000D+02-0.231562500000D+02 0.530236372187D-08 0.253477496869D+00
-0.111199915409D-05 0.483385741245D-02 0.810064375401D-05 0.515369705963D+04
0.172800000000D+06-0.141561031342D-06 0.304306271006D+00 0.372529029846D-08
0.926615731710D+00 0.207250000000D+03 0.133849764271D+01-0.843427989304D-08
-0.164292557730D-09 0.100000000000D+01 0.188600000000D+04 0.000000000000D+00
0.200000000000D+01 0.000000000000D+00-0.838190317154D-08 0.100000000000D+02
0.172770000000D+06 0.400000000000D+01 0.000000000000D+00 0.000000000000D+00
The third test set up compares results from Where against CNES solution for satellite E01 and epoch
2016-03-01 01:30:00.0.
/* Sample Broadcast Message in unit of radians, seconds, meters for satellite E01 and
/ epoch 2019-07-01 00:00:00.0
E01 2019 07 01 00 00 00-6.374700460583D-04-8.085976332950D-12 0.000000000000D+00
1.600000000000D+01 2.106250000000D+02 2.413671967697D-09 5.641607228729D-01
9.929761290550D-06 1.870252890512D-04 7.383525371552D-06 5.440612319946D+03
8.640000000000D+04-3.725290298462D-09 2.424721177655D-01 1.657754182816D-07
9.878562635157D-01 1.958125000000D+02 3.073143357419D+00-5.291648989849D-09
2.003654888840D-10 2.580000000000D+02 2.060000000000D+03 0.000000000000D+00
3.120000000000D+00 0.000000000000D+00-1.862645149231D-09 0.000000000000D+00
8.714000000000D+04 0.000000000000D+00 0.000000000000D+00 0.000000000000D+00
"""
# Initialize logging
log.init(log_level="debug")
# Get GNSS ephemeris data for testing
if TEST == "test_1":
file_name = "test2040.01n"
year = 2001
month = 7
day = 23
hour = 2
minute = 0
second = 0
satellite = "G20"
self.system = "G" # GNSS identifier
# Satellite transmission time
self.t_sat_gpsweek = 1124.0
self.t_sat_gpssec = 86400.00
elif TEST == "test_2":
file_name = "test0610.16n"
year = 2016
month = 3
day = 1
hour = 0
minute = 0
second = 0
satellite = "G20"
self.system = "G" # GNSS identifier
# Satellite transmission time
self.t_sat_gpsweek = 1886.0
self.t_sat_gpssec = 172799.92312317
elif TEST == "test_3":
file_name = "TEST00CNS_R_20191820000_01D_EN.rnx"
year = 2019
month = 7
day = 1
hour = 1
minute = 30
second = 0
satellite = "E01"
self.system = "E" # GNSS identifier
# Satellite transmission time
self.t_sat_gpsweek = 2060.0
self.t_sat_gpssec = 91800.0
rundate = datetime(year, month, day, hour, minute, second)
# Initialize configuration
config.init(rundate=rundate, pipeline="gnss")
# Generate observation datast
self.dset = dataset.Dataset(num_obs=1, rundate=rundate)
self.dset.add_time(name="time",
val=rundate,
scale="gps",
fmt="datetime")
#self.dset.add_time(
# name="time", val=Time(val=[self.t_sat_gpsweek], val2=[self.t_sat_gpssec], fmt="gps_ws", scale="gps")
#)
self.dset.add_text(name="satellite", val=[satellite])
# Get broadcast ephemeris
self.brdc = apriori.get(
"orbit",
rundate=rundate,
system=tuple({self.system}),
station="test",
apriori_orbit="broadcast",
file_path=pathlib.Path(__file__).parent / "files" / file_name,
)
self.idx = 0 # Broadcast ephemeris index
