def plot(rundate: "datedoy", tech: "pipeline", stage: "option", writers: "option"): log.init(log_level="info") # Get options writer_names = writers.replace(",", " ").split() dataset_id = util.read_option_value("--dset_id", default="last") dataset_id = "last" if dataset_id == "last" else int(dataset_id) identifier = util.read_option_value("--id", default="") identifier = f"-{identifier}" if identifier else "" session = util.read_option_value("--session", default="") dset = data.Dataset(rundate=rundate, tech=tech, stage=stage, id=identifier, dataset_name=session, dataset_id=dataset_id) path_hdf5 = files.path("dataset_hdf5", file_vars=dset.vars) path_json = files.path("dataset_json", file_vars=dset.vars) log.info(f"Read Where dataset files {path_hdf5} and {path_json}.") if dset.num_obs == 0: log.fatal(f"No data to read for date {rundate}.") # Loop over writers for writer in writer_names: write(writer, dset=dset)
def main(): """Parse command line options and set up an Where analysis Do simple parsing of command line arguments. Set up config-files and show the configuration. """ util.check_help_and_version(doc_module=__name__) # Start logging log.init() # Read command line options pipeline = pipelines.get_from_options() config.read_pipeline(pipeline) if util.check_options("--doy"): rundate = util.parse_args("doy", doc_module=__name__) else: rundate = util.parse_args("date", doc_module=__name__) session = pipelines.get_session(rundate, pipeline) # Set up the configuration for the analysis setup_config(rundate, pipeline, session) # Show current configuration show_config(rundate, pipeline, session) # Store configuration in library store_config_to_library(rundate, pipeline, session)
def write(rundate: "datedoy", pipeline: "pipeline", stage: "option", writers: "option"): log.init(log_level="info") # Get options label = util.read_option_value("--label", default="None") # TODO: label = "last" if label == "last" else label id_ = util.read_option_value("--id", default="") session = util.read_option_value("--session", default="") station = util.read_option_value("--station", default="") writers = writers.replace(",", " ").split() dset_vars = dict(pipeline=pipeline, stage=stage, session=session, station=station, label=label, id=id_) try: dset = dataset.Dataset.read(**dict(dset_vars, rundate=rundate)) path = config.files.path("dataset", file_vars={ **dset.vars, **dset.analysis }) log.info(f"Read Where dataset files {path}.") except OSError as err: log.fatal(f"Unable to read data for {rundate}: {err}") # Loop over writers for writer in writers: log.info(f"Apply writer '{writer}'.") write_(writer, dset=dset)
def concatenate( from_date: "datedoy", to_date: "datedoy", tech: "pipeline", stage: "option", writers: "option", id: "option" ): log.init(log_level="info") # Get options writer_names = writers.replace(",", " ").split() dataset_id = util.read_option_value("--dset_id", default="last") dataset_id = "last" if dataset_id == "last" else int(dataset_id) dataset_name = util.read_option_value("--dset_name", default="") session = util.read_option_value("--session", default="") dset_vars = dict( tech=tech, stage=stage, session=session, dataset_name=session, dataset_id=dataset_id, session_name=id + "_concatenated", ) dset = concatenate_datasets(from_date, to_date, dset_vars) if dset.num_obs == 0: log.fatal(f"No data to read period from {from_date} to {to_date}.") dset.write() # Loop over writers for writer in writer_names: write(writer, dset=dset)
def concatenate(from_date: "datedoy", to_date: "datedoy", pipeline: "pipeline", stage: "option"): log.init(log_level="info") # Get options label = util.read_option_value("--label", default="None") # TODO: label = "last" if label == "last" else label id_ = util.read_option_value("--id", default="") only_for_rundate = True if util.check_options( "--only_for_rundate") else False session = util.read_option_value("--session", default="") station = util.read_option_value("--station", default="") writers = util.read_option_value("--writers", default="").replace(",", " ").split() # Update configuration of Where analysis config.where.update_from_options(_clean_sys_argv(pipeline)) dset_vars = dict(pipeline=pipeline, stage=stage, session=session, station=station, label=label, id=id_) dset_vars = config.create_file_vars(rundate=from_date, **dset_vars) dset = _concatenate_datasets(from_date, to_date, dset_vars, only_for_rundate) if dset.num_obs == 0: log.fatal(f"No data to read period from {from_date} to {to_date}.") dset.write() # Loop over writers for writer in writers: write(writer, dset=dset)
def main(): """Parse command line options and run the Where analysis Do simple parsing of command line arguments. Set up config-files and start the analysis. See the help docstring at the top of the file for more information about the workflow. """ # Start logging log.init() # Read command line options if util.check_options("--doy"): rundate = util.parse_args("doy", doc_module=__name__) else: rundate = util.parse_args("date", doc_module=__name__) pipeline = pipelines.get_from_options() session = pipelines.get_session(rundate, pipeline) # Pretend to empty mailbox pretend_to_empty_mailbox() # Start an interactive session if util.check_options("-I", "--interactive"): from where.tools import interactive interactive.interactive(rundate, pipeline, session) return # Set up the configuration for a new analysis or update an existing one setup.setup_config(rundate, pipeline, session) # Run the analysis setup.add_timestamp(rundate, pipeline, session, "last run") with timer(f"Finish pipeline {pipeline.upper()} in"): pipelines.run(rundate, pipeline, session)
def main(): """Invoke where_tools To add a new tool, simply add a new .py-file with a registered plugin that will be called when the tool is called. """ # Start logging log.init(log_level="info") # First read tool from command line, don't use util.parse_args() as that overrides -h for each tool try: tool = [a for a in sys.argv[1:] if not a.startswith("-")][0] sys.argv.remove(tool) except IndexError: util._print_help_from_doc(__name__) raise SystemExit # Check that tool is available and figure out signature try: sig = plugins.signature(__name__, tool) tool_module = plugins.get(__name__, tool).function.__module__ except exceptions.UnknownPluginError as err: util._print_help_from_doc(__name__) err.args = ( f"{err.args[0]}\n Available tools are {', '.join(plugins.names(__name__))}", ) raise # Parse parameters util.check_help_and_version(doc_module=tool_module) tool_args = dict() for key, param in sig.parameters.items(): if param.annotation is None: raise SystemExit(f"{param} in {tool} tool is not annotated") if param.annotation == "datedoy": if util.check_options("--doy"): date = util.parse_args("doy", doc_module=__name__) else: date = util.parse_args("date", doc_module=__name__) tool_args[key] = date elif param.annotation == "pipeline": tool_args[key] = pipelines.get_from_options() config.read_pipeline(tool_args[key]) elif param.annotation == "option": tool_args[key] = util.read_option_value(f"--{key}") else: tool_args[key] = util.parse_args(param.annotation, doc_module=tool_module) # Call tool plugins.call(__name__, tool, **tool_args)
def main(date: "datedoy", pipeline: "pipeline", ids: "option"): log.init(log_level="info") # Additional required options identifiers = [id_.strip() for id_ in ids.split(",")] difference_by = util.read_option_value("--difference_by").replace( ",", " ").split() stage = util.read_option_value("--stage") # Get optional options dataset_id = util.read_option_value("--dset_id", default="last") dataset_id = "last" if dataset_id == "last" else int(dataset_id) dataset_name = util.read_option_value("--dset_name", default="") writer_names = util.read_option_value("--writers", default="").replace(",", " ").split() session = util.read_option_value("--session", default="") station = util.read_option_value("--station", default="") # Get datasets dset = data.Dataset( rundate=date, pipeline=pipeline, stage=stage, session=session, station=station, label=label, id="-" + identifiers[0], ) dset_other = data.Dataset( rundate=date, pipeline=pipeline, stage=stage, session=session, station=station, label=label, id="-" + identifiers[1], ) if dset.num_obs == 0: log.warn( f"Nothing to differentiate. Dataset '{identifiers[0]}' is empty.") return 1 if dset_other.num_obs == 0: log.warn( f"Nothing to differentiate. Dataset '{identifiers[1]}' is empty.") return 1 # Differentiate dataset dset_diff = dset.difference(dset_other, index_by=','.join(difference_by)) dset_diff.write_as(stage="difference") # Loop over writers for writer in writer_names: write(writer, dset_diff)
def main(): """Parse command line options and run the Where analysis Do simple parsing of command line arguments. Set up config-files and start the analysis. See the help docstring at the top of the file for more information about the workflow. """ util.check_help_and_version(doc_module=__name__) # Start logging log.init(config.where.log.default_level.str) log.debug( f"Use {util.get_python_version()} on process {util.get_pid_and_server()}" ) # Read command line options pipeline = pipelines.get_from_options() config.read_pipeline(pipeline) if util.check_options("--doy"): rundate = util.parse_args("doy", doc_module=__name__) else: rundate = util.parse_args("date", doc_module=__name__) args, kwargs = util.options2args(sys.argv[1:]) # Start an interactive session if util.check_options("-I", "--interactive"): from where.tools import interactive # Local import because interactive imports many external packages interactive.interactive(rundate, pipeline, **kwargs) return # Set up the configuration for a new analysis or update an existing one unused_options = setup.setup_config(rundate, pipeline, *args, **kwargs) pipeline_args, pipeline_kwargs = util.options2args(unused_options) # Run the analysis setup.add_timestamp(rundate, pipeline, "last run", **kwargs) with Timer(f"Finish pipeline {pipeline.upper()} in"): pipelines.run(rundate, pipeline, *pipeline_args, **pipeline_kwargs)
def concatenate(from_date: "datedoy", to_date: "datedoy", tech: "pipeline", stage: "option"): log.init(log_level="info") # Get options dataset_id = util.read_option_value("--dset_id", default="last") dataset_id = "last" if dataset_id == "last" else int(dataset_id) dataset_name = util.read_option_value("--dset_name", default="") id = util.read_option_value("--id", default="") only_for_rundate = True if util.check_options( "--only_for_rundate") else False session = util.read_option_value("--session", default="") writer_names = util.read_option_value("--writers", default="").replace(",", " ").split() # Update configuration of Where analysis config.where.update_from_options(_clean_sys_argv(tech)) dset_vars = dict( tech=tech, stage=stage, session=session, dataset_name=session, dataset_id=dataset_id, session_name=id + "_concatenated", ) dset = _concatenate_datasets(from_date, to_date, dset_vars, only_for_rundate) if dset.num_obs == 0: log.fatal(f"No data to read period from {from_date} to {to_date}.") dset.write() # Loop over writers for writer in writer_names: write(writer, dset=dset)
def run(rundate, pipeline, *args, **kwargs): """Run a Where pipeline for a given date and session Args: rundate: Rundate of analysis. pipeline: Pipeline used for analysis. session: Session in analysis. """ if not setup.has_config(rundate, pipeline, *args, **kwargs): log.fatal( f"No configuration found for {pipeline.upper()} {rundate.strftime(config.FMT_date)}" ) # Set up config config.init(rundate, pipeline, **kwargs) # Register filekey suffix filekey_suffix = config.tech.filekey_suffix.list if filekey_suffix: config.files.profiles = filekey_suffix # Validate input arguments try: prefix = plugins.call(package_name=__name__, plugin_name=pipeline, part="validate_args", rundate=rundate, **kwargs) except mg_exceptions.UnknownPluginError: log.warn( f"Pipeline {pipeline} has not defined function 'validate_args'") except exceptions.InvalidArgsError as err: from where.tools import delete # Clean up {placeholder} directories created by config delete.delete_analysis(rundate, pipeline, **kwargs) log.fatal(err) # Set up console logger and start file logger try: prefix = plugins.call(package_name=__name__, plugin_name=pipeline, part="log_prefix", rundate=rundate, **kwargs) except mg_exceptions.UnknownPluginError: log.warn(f"Pipeline {pipeline} has not defined function 'log_prefix'") prefix = "" log_cfg = config.where.log log.init(log_level=log_cfg.default_level.str, prefix=prefix) if log_cfg.log_to_file.bool: log.file_init( file_path=config.files.path("log"), log_level=log_cfg.default_level.str, prefix=prefix, rotation=log_cfg.number_of_log_backups.int, ) # Update analysis config and file variables config.set_analysis(rundate, pipeline=pipeline, **kwargs) config.set_file_vars(file_vars()) log.blank() # Empty line for visual clarity # Read which stages that should be executed once for each iterable skip_stages = config.tech.skip_stages.list stage_iterate = config.tech.stage_iterate.list dset_list = [] dset = None if stage_iterate: # Read which list should be iterated over and the placeholder name of each entry iterate_over, _, var_name = config.tech.stage_iterate_over.str.partition( ":") var_name = var_name.strip() # Iterate for item in config.tech[iterate_over].list: kwargs[var_name] = item log.blank() log.info(f"***** Running {item} *****") for prev_stage, stage in zip([None] + stage_iterate, stage_iterate): if stage not in skip_stages: dset = run_stage(rundate, pipeline, dset, stage, prev_stage, **kwargs) if dset is not None: dset_list.append(dset) dset = None kwargs[var_name] = "combined" if dset_list: dset_list[0].merge_with(*dset_list[1:], sort_by="time") dset = dset_list[0] if len(dset_list) > 1: log.info(f"Combining dataset for {len(dset_list)} {iterate_over}") dset.write_as(stage=stage_iterate[-1], label=2, **kwargs) # Read which stages that should be executed once stage_once = config.tech.stage_once.list # Find which stages we will run analysis for if not stage_once and not stage_iterate: stage_list = [s for s in stages(pipeline)] prev_stage_start = None else: stage_list = [s for s in stage_once] prev_stage_start = stage_iterate[-1] if stage_iterate else None for prev_stage, stage in zip([prev_stage_start] + stage_list, stage_list): if stage not in skip_stages: dset = run_stage(rundate, pipeline, dset, stage, prev_stage, **kwargs) log.blank() if dset is not None and dset.num_obs == 0: log.warn(f"No observations in dataset after {stage} stage.") break # Store configuration to library setup.store_config_to_library(rundate, pipeline, **kwargs) # Write requirements to file for reproducibility util.write_requirements()
def main(date: "datedoy", pipeline: "pipeline", items: "option", specifier: "option"): log.init(log_level="info") dsets = dict() # Additional options stage = util.read_option_value("--stage") writer_names = util.read_option_value("--writers").replace(",", " ").split() items_ = [s.strip() for s in items.split(",")] # Get optional options label = util.read_option_value("--label", default="None") # TODO label = "last" if label == "last" else label station = util.read_option_value("--station", default="") id_ = util.read_option_value("--id", default="") # Get dataset variables dset_vars = dict(pipeline=pipeline, stage=stage, station=station, label=label, id=id_) dset_vars = config.create_file_vars(rundate=date, **dset_vars) # Read datasets for given specifier if specifier == "id": for id_ in items_: dset = dataset.Dataset().read(rundate=date, pipeline=pipeline, stage=stage, label=label, id=id_, station=station) if dset.num_obs == 0: log.warn(f"Dataset '{id_}' is empty.") continue dset_vars[ "id"] = id_ #TODO: Better solution for handling of dataset variables? dset.vars.update( dset_vars ) # Necessary for example for getting correct file path in used writers. dsets.update({id_: dset}) elif specifier == "station": for station in items_: dset = dataset.Dataset().read(rundate=date, pipeline=pipeline, stage=stage, label=label, id=id_, station=station) if dset.num_obs == 0: log.warn(f"Dataset '{station}' is empty.") continue dset_vars[ "station"] = station #TODO: Better solution for handling of dataset variables? dset.vars.update( dset_vars ) # Necessary for example for getting correct file path in used writers. dsets.update({station: dset}) elif specifier == "stage": for stage in items_: dset = dataset.Dataset().read(rundate=date, pipeline=pipeline, stage=stage, label=label, id=id_, station=station) if dset.num_obs == 0: log.warn(f"Dataset '{stage}' is empty.") continue dset_vars[ "stage"] = stage #TODO: Better solution for handling of dataset variables? dset.vars.update( dset_vars ) # Necessary for example for getting correct file path in used writers. dsets.update({stage: dset}) else: log.fatal( f"Specifier {specifier} is not defined. It should be either 'id', 'station' or 'stage'." ) if len(dsets) == 0: log.fatal(f"All given datasets are empty [{', '.join(dsets.keys())}].") elif len(dsets) == 1: log.warn( f"Nothing to compare. Only dataset '{list(dsets.keys())[0]}' is available." ) # Loop over writers for writer in writer_names: write(writer, dset=dsets)
def run(rundate, pipeline, session=""): """Run a Where pipeline for a given date and session Args: rundate: Rundate of analysis. pipeline: Pipeline used for analysis. session: Session in analysis. """ if not setup.has_config(rundate, pipeline, session): log.fatal( f"No configuration found for {pipeline.upper()} {session} {rundate.strftime(config.FMT_date)}" ) # Set up session config config.init(rundate=rundate, tech_name=pipeline, session=session) # Set up prefix for console logger and start file logger log_cfg = config.where.log prefix = f"{pipeline.upper()} {session} {rundate:%Y-%m-%d}" log.init(log_level=log_cfg.default_level.str, prefix=prefix) if log_cfg.log_to_file.bool: log.file_init( file_path=files.path("log"), log_level=log_cfg.default_level.str, prefix=prefix, rotation=log_cfg.number_of_log_backups.int, ) # Read which stages to skip from technique configuration file. skip_stages = config.tech.get("skip_stages", default="").list # Register filekey suffix filekey_suffix = config.tech.filekey_suffix.list if filekey_suffix: config.files.profiles = filekey_suffix # Find which stages we will run analysis for # TODO: Specify stage_list in config stage_list = [s for s in stages(pipeline) if s not in skip_stages] # Start file logging and reporting reports.report.init(sessions=[session]) reports.report.start_session(session) reports.report.text("header", session.replace("_", " ").title()) # Update analysis config and file variables config.set_analysis(rundate=rundate, tech=pipeline, analysis=pipeline, session=session) config.set_file_vars(file_vars()) # Log the name of the session log.blank() # Empty line for visual clarity log.info(f"Start session {session}") session_timer = timer(f"Finish session {session} in") session_timer.start() # Run stages, keep track of previous stage dset = None dep_fast = config.where.files.dependencies_fast.bool for prev_stage, stage in zip([None] + stage_list, stage_list): # Skip stages where no dependencies have changed dep_path = files.path("depends", file_vars=dict(stage=stage)) if not (dependencies.changed(dep_path, fast_check=dep_fast) or util.check_options("-F", "--force")): log.info( f"Not necessary to run {stage} for {pipeline.upper()} {rundate.strftime(config.FMT_date)}" ) continue elif dset is None: # Create or read dataset empty = stage == stage_list[0] dset = dataset.Dataset(rundate, tech=pipeline, stage=prev_stage, dataset_name=session, dataset_id="last", empty=empty) # Report on the stage reports.report.start_section(stage) reports.report.text("header", stage.replace("_", " ").title()) if prev_stage: log.blank() # Empty line for visual clarity # Set up dependencies. Add dependencies to previous stage and config file dependencies.init(dep_path, fast_check=dep_fast) dependencies.add(files.path("depends", file_vars=dict(stage=prev_stage)), label="depends") dependencies.add(*config.tech.sources, label="config") # Delete old datasets for this stage dset.delete_from_file(stage=stage, dataset_id="all") # Call the current stage. Skip rest of stages if current stage returns False (compare with is since by # default stages return None) plugins.call(package_name=__name__, plugin_name=pipeline, part=stage, stage=stage, dset=dset, plugin_logger=log.info) dependencies.write() if dset.num_obs == 0: log.warn( f"No observations in dataset after {stage} stage. Exiting pipeline" ) break else: # Only done if loop does not break (all stages finish normally) # Publish files for session files.publish_files() session_timer.end() # Store configuration to library setup.store_config_to_library(rundate, pipeline, session) # Write reports specified in config reports.write(rundate, pipeline) # Write requirements to file for reproducibility util.write_requirements()
def main(): """Parse command line options and loop over the Where analysis Do simple parsing of command line arguments. Set up config-files and potentially start the analysis. See the help docstring at the top of the file for more information about the workflow. """ util.check_help_and_version(doc_module=__name__) log.init(log_level=config.where.log.default_level.str, prefix="Runner") # Initialize pipeline = pipelines.get_from_options() config.read_pipeline(pipeline) if util.check_options("--doy"): from_date = util.parse_args("doy", doc_module=__name__) to_date = util.parse_args("doy", doc_module=__name__) sys.argv.remove("--doy") else: from_date = util.parse_args("date", doc_module=__name__) to_date = util.parse_args("date", doc_module=__name__) # Handle list of sessions session_list = set( util.read_option_value("--session", default="").replace(",", " ").split()) sys.argv = [o for o in sys.argv if not o.startswith("--session=")] # Start logging file_vars = dict(**util.get_user_info()) log.file_init( file_path=files.path("log_runner", file_vars=file_vars), log_level=config.where.log.default_level.str, prefix="Runner", rotation=config.where.log.number_of_log_backups.int, ) atexit.register(log_statistics) # Should where_runner crash if Where crashes? stop_on_error_opts = None if util.check_options("--stop-on-error"): stop_on_error_opts = True elif util.check_options("--continue-on-error"): stop_on_error_opts = False stop_on_error = config.where.get("stop_on_error", section="runner", value=stop_on_error_opts).bool error_logger = log.fatal if stop_on_error else log.error # Loop over dates rundate = from_date while rundate <= to_date: available_sessions = set(pipelines.list_sessions(rundate, pipeline)) sessions = available_sessions & session_list if session_list else available_sessions where_args = remove_runner_args(sys.argv[1:]) for session in sorted(sessions): cmd = f"{where.__executable__} {rundate:%Y %m %d} --session={session}".split( ) + where_args log.info(f"Running '{' '.join(cmd)}'") count("Number of analyses") try: subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) except subprocess.CalledProcessError as err: count("Failed analyses") error_msg = err.stderr.decode().strip().split("\n")[-1] error_logger(f"Command '{' '.join(cmd)}' failed: {error_msg}") else: count("Successful analyses") copy_log_from_where(rundate, pipeline, session) rundate += timedelta(days=1)
def compare(date: "datedoy", pipeline: "pipeline", items: "option", specifier: "option"): log.init(log_level="info") dsets = dict() # Additional options stage = util.read_option_value("--stage") writer_names = util.read_option_value("--writers").replace(",", " ").split() items_ = [s.strip() for s in items.split(",")] # Get optional options label = util.read_option_value("--label", default="None") # TODO label = "last" if label == "last" else label station = util.read_option_value("--station", default="") id_ = util.read_option_value("--id", default="") # Update configuration of Where analysis config.where.update_from_options(_clean_sys_argv(pipeline)) # Get dataset variables dset_vars = config.create_file_vars(rundate=date, pipeline=pipeline) # Read datasets for given specifier if specifier == "id": for id_ in items_: try: dset = dataset.Dataset().read(rundate=date, pipeline=pipeline, stage=stage, label=label, id=id_, station=station) except OSError: log.warn(f"No data to read for Dataset id '{id_}'.") continue dset.vars.update(dset_vars) dset.vars["id"] = id_ dsets.update({id_: dset}) elif specifier == "station": for station in items_: try: dset = dataset.Dataset().read(rundate=date, pipeline=pipeline, stage=stage, label=label, id=id_, station=station) except OSError: log.warn(f"No data to read for Dataset station '{station}'.") continue dset.vars.update(dset_vars) dset.vars["station"] = station dsets.update({station: dset}) elif specifier == "stage": for stage in items_: try: dset = dataset.Dataset().read(rundate=date, pipeline=pipeline, stage=stage, label=label, id=id_, station=station) except OSError: log.warn(f"No data to read for Dataset stage '{stage}'.") continue dset.vars.update(dset_vars) dset.vars["stage"] = stage dsets.update({stage: dset}) else: log.fatal( f"Specifier {specifier} is not defined. It should be either 'id', 'station' or 'stage'." ) if len(dsets) == 0: log.fatal(f"All given datasets are empty [{', '.join(dsets.keys())}].") elif len(dsets) == 1: log.warn( f"Nothing to compare. Only dataset '{list(dsets.keys())[0]}' is available." ) # Loop over writers for writer in writer_names: write(writer, dset=dsets)
def main(date: "datedoy", tech: "pipeline", items: "option", specifier: "option"): log.init(log_level="info") dsets = dict() # Additional options stage = util.read_option_value("--stage") writer_names = util.read_option_value("--writers").replace(",", " ").split() items_ = [s.strip() for s in items.split(",")] # Get optional options dataset_id = util.read_option_value("--dset_id", default="last") dataset_id = "last" if dataset_id == "last" else int(dataset_id) dataset_name = util.read_option_value("--dset_name", default="") session = util.read_option_value("--session", default="") id_ = "-" + util.read_option_value( "--id", default="") if util.read_option_value("--id", default="") else "" # Read datasets for given specifier if specifier == "id": for id_ in items_: dset = data.Dataset(rundate=date, tech=tech, stage=stage, dataset_name=dataset_name, dataset_id=dataset_id, id="-" + id_) if dset.num_obs == 0: log.warn(f"Dataset '{id_}' is empty.") continue dsets.update({id_: dset}) elif specifier == "session": for session in items_: dset = data.Dataset(rundate=date, tech=tech, stage=stage, dataset_name=session, dataset_id=dataset_id, id=id_) if dset.num_obs == 0: log.warn(f"Dataset '{session}' is empty.") continue dsets.update({session: dset}) elif specifier == "stage": for stage in items_: dset = data.Dataset(rundate=date, tech=tech, stage=stage, dataset_name=dataset_name, dataset_id=dataset_id, id=id_) if dset.num_obs == 0: log.warn(f"Dataset '{stage}' is empty.") continue dsets.update({stage: dset}) else: log.fatal( f"Specifier {specifier} is not defined. It should be either 'id', 'session' or 'stage'." ) if len(dsets) == 0: log.fatal(f"All given datasets are empty [{', '.join(dsets.keys())}].") elif len(dsets) == 1: log.warn( f"Nothing to compare. Only dataset '{list(dsets.keys())[0]}' is available." ) # Loop over writers for writer in writer_names: write(writer, dset=dsets)
def setUp(self): """ The first test set up is based on the bc_velo.c program, which is published in :cite:`remondi2004` and following RINEX navigation file sample: /* Sample Broadcast Message in unit of radians, seconds, meters. 20 01 7 23 2 0 0.0 -.857324339449D-04 -.272848410532D-11 .000000000000D+00 .200000000000D+02 .886875000000D+02 .465376527657D-08 .105827953357D+01 .457651913166D-05 .223578442819D-02 .177137553692D-05 .515379589081D+04 .936000000000D+05 .651925802231D-07 .164046615454D+01 -.856816768646D-07 .961685061380D+00 .344968750000D+03 .206374037770D+01 -.856928551657D-08 .342514267094D-09 .000000000000D+00 .112400000000D+04 .000000000000D+00 .200000000000D+01 .000000000000D+00 -.651925802231D-08 .276000000000D+03 .865800000000D+05 .000000000000D+00 .000000000000D+00 .000000000000D+00 */ The second test set up compares results from Where against gLAB solution for satellite G20 and epoch 2016-03-01 00:00:00.0. /* Sample Broadcast Message in unit of radians, seconds, meters for satellite G20 and / epoch 2016-03-01 00:00:00.0 20 16 3 1 0 0 0.0 0.396233052015D-03 0.261479726760D-11 0.000000000000D+00 0.100000000000D+02-0.231562500000D+02 0.530236372187D-08 0.253477496869D+00 -0.111199915409D-05 0.483385741245D-02 0.810064375401D-05 0.515369705963D+04 0.172800000000D+06-0.141561031342D-06 0.304306271006D+00 0.372529029846D-08 0.926615731710D+00 0.207250000000D+03 0.133849764271D+01-0.843427989304D-08 -0.164292557730D-09 0.100000000000D+01 0.188600000000D+04 0.000000000000D+00 0.200000000000D+01 0.000000000000D+00-0.838190317154D-08 0.100000000000D+02 0.172770000000D+06 0.400000000000D+01 0.000000000000D+00 0.000000000000D+00 The third test set up compares results from Where against CNES solution for satellite E01 and epoch 2016-03-01 01:30:00.0. /* Sample Broadcast Message in unit of radians, seconds, meters for satellite E01 and / epoch 2019-07-01 00:00:00.0 E01 2019 07 01 00 00 00-6.374700460583D-04-8.085976332950D-12 0.000000000000D+00 1.600000000000D+01 2.106250000000D+02 2.413671967697D-09 5.641607228729D-01 9.929761290550D-06 1.870252890512D-04 7.383525371552D-06 5.440612319946D+03 8.640000000000D+04-3.725290298462D-09 2.424721177655D-01 1.657754182816D-07 9.878562635157D-01 1.958125000000D+02 3.073143357419D+00-5.291648989849D-09 2.003654888840D-10 2.580000000000D+02 2.060000000000D+03 0.000000000000D+00 3.120000000000D+00 0.000000000000D+00-1.862645149231D-09 0.000000000000D+00 8.714000000000D+04 0.000000000000D+00 0.000000000000D+00 0.000000000000D+00 """ # Initialize logging log.init(log_level="debug") # Get GNSS ephemeris data for testing if TEST == "test_1": file_name = "test2040.01n" year = 2001 month = 7 day = 23 hour = 2 minute = 0 second = 0 satellite = "G20" self.system = "G" # GNSS identifier # Satellite transmission time self.t_sat_gpsweek = 1124.0 self.t_sat_gpssec = 86400.00 elif TEST == "test_2": file_name = "test0610.16n" year = 2016 month = 3 day = 1 hour = 0 minute = 0 second = 0 satellite = "G20" self.system = "G" # GNSS identifier # Satellite transmission time self.t_sat_gpsweek = 1886.0 self.t_sat_gpssec = 172799.92312317 elif TEST == "test_3": file_name = "TEST00CNS_R_20191820000_01D_EN.rnx" year = 2019 month = 7 day = 1 hour = 1 minute = 30 second = 0 satellite = "E01" self.system = "E" # GNSS identifier # Satellite transmission time self.t_sat_gpsweek = 2060.0 self.t_sat_gpssec = 91800.0 rundate = datetime(year, month, day, hour, minute, second) # Initialize configuration config.init(rundate=rundate, pipeline="gnss") # Generate observation datast self.dset = dataset.Dataset(num_obs=1, rundate=rundate) self.dset.add_time(name="time", val=rundate, scale="gps", fmt="datetime") #self.dset.add_time( # name="time", val=Time(val=[self.t_sat_gpsweek], val2=[self.t_sat_gpssec], fmt="gps_ws", scale="gps") #) self.dset.add_text(name="satellite", val=[satellite]) # Get broadcast ephemeris self.brdc = apriori.get( "orbit", rundate=rundate, system=tuple({self.system}), station="test", apriori_orbit="broadcast", file_path=pathlib.Path(__file__).parent / "files" / file_name, ) self.idx = 0 # Broadcast ephemeris index