def main(): parser = ArgumentParser() parser = process_args(parser) parser = logging_args(parser) args = parser.parse_args() setup_logging(args.log_conf, args.log_filename, args.error_email, args.log_level, "crmprtd") utc = pytz.utc args.start_date = utc.localize( verify_date(args.start_date, datetime.min, "start date")) args.end_date = utc.localize( verify_date(args.end_date, datetime.max, "end date")) process( args.connection_string, args.sample_size, args.network, args.start_date, args.end_date, args.diag, args.infer, )
def test_verify_date_exception(datestring): default = datetime.datetime.now() warning = ( f"Parameter '{datestring}' is undefined or unparseable. Using the " f"default '{default:%Y-%m-%d %H:%M:%S}'") with pytest.warns(UserWarning, match=warning): assert verify_date(datestring, default, "") == default
def main(partner): """Main download function to use for download scripts for the EC_SWOB provincial partners (e.g. bc-env-snow, bc-env-aq, bc-forestry and bc-tran). Args: partner (str): The partner abbreviation found in the SWOB URL (e.g. bc-tran) Returns: No return value. Produces side-effect of sending downloaded XML files to STDOUT """ desc = globals()["__doc__"] parser = ArgumentParser(description=desc) parser = logging_args(parser) parser.add_argument( "-d", "--date", help=("Alternate date to use for downloading " "(interpreted with dateutil.parser.parse)." "Defaults to now."), ) args = parser.parse_args() setup_logging( args.log_conf, args.log_filename, args.error_email, args.log_level, "crmprtd.{}".format(partner), ) if not args.date: dl_date = datetime.datetime.now() else: dl_date = verify_date(args.date, datetime.datetime.now(), "date") download( "https://dd.weather.gc.ca/observations/swob-ml/partners/{}/".format( partner), dl_date, )
def download(time, frequency, province, language, baseurl): log.info("Starting EC rtd") try: # Determine time parameter deltat = timedelta(1 / 24.0) if frequency == "hourly" else timedelta(1) if time: time = verify_date(time, datetime.utcnow() - deltat, "time") log.info("Starting manual run " f"using timestamp {time}") else: # go back a day time = datetime.utcnow() - deltat log.info("Starting automatic run " f"using timestamp {time}") # Construct and download the xml url = makeurl(frequency, province, language, time, baseurl) scheme, _ = url.split(":", 1) https_download(url, scheme, log) except IOError: log.exception("Unable to download or open xml data") sys.exit(1)
def test_verify_date(datestring, default): assert verify_date(datestring, default, "") == datetime.datetime(2020, 1, 1)
def main(): # pragma: no cover desc = globals()["__doc__"] parser = ArgumentParser(description=desc) parser = logging_args(parser) parser.add_argument("-u", "--username", default="pcic", help=("Username for the ftp server ")) parser.add_argument( "-f", "--ftp_server", default="sftp2.bchydro.com", help=("Full uri to BC Hydro's ftp " "server"), ) parser.add_argument( "-F", "--ftp_dir", default=("pcic"), help=("FTP Directory containing BC hydro's " "data files"), ) parser.add_argument( "-S", "--ssh_private_key", required=True, help=("Path to file with SSH private key"), ) end = datetime.now() start = end - relativedelta.relativedelta(months=1) parser.add_argument( "-s", "--start_date", help=("Optional start time to use for downloading " "(interpreted with dateutil.parser.parse). " "Defaults to one month prior to now."), ) parser.add_argument( "-e", "--end_date", help=("Optional end time to use for downloading " "(interpreted with dateutil.parser.parse). " "Defaults to now."), ) args = parser.parse_args() setup_logging( args.log_conf, args.log_filename, args.error_email, args.log_level, "crmprtd.bc_hydro", ) args.start_date = verify_date(args.start_date, start, "start date") args.end_date = verify_date(args.end_date, end, "end date") download( args.username, args.ssh_private_key, args.ftp_server, args.ftp_dir, args.start_date, args.end_date, )