def main():
parser = ArgumentParser()
parser.add_argument('-p', '--province', required=True,
help='2 letter province code')
parser.add_argument('-g', '--language', default='e',
choices=['e', 'f'],
help="'e' (english) | 'f' (french)")
parser.add_argument('-F', '--frequency', required=True,
choices=['daily', 'hourly'],
help='daily|hourly')
parser.add_argument('-t', '--time',
help=("Alternate *UTC* time to use for downloading "
"(interpreted using "
"format=YYYY/MM/DD HH:MM:SS)"))
parser.add_argument('-T', '--threshold', default=1000,
help=('Distance threshold (in meters) to use when '
'matching stations. Stations are considered a '
'match if they have the same id, name, and are '
'within this threshold'))
parser = logging_args(parser)
args = parser.parse_args()
setup_logging(args.log_conf, args.log_filename, args.error_email,
args.log_level, 'crmprtd.ec')
download(args.time, args.frequency, args.province, args.language)
def main():
parser = ArgumentParser()
parser = process_args(parser)
parser = logging_args(parser)
args = parser.parse_args()
setup_logging(args.log_conf, args.log_filename, args.error_email,
args.log_level, "crmprtd")
utc = pytz.utc
args.start_date = utc.localize(
verify_date(args.start_date, datetime.min, "start date"))
args.end_date = utc.localize(
verify_date(args.end_date, datetime.max, "end date"))
process(
args.connection_string,
args.sample_size,
args.network,
args.start_date,
args.end_date,
args.diag,
args.infer,
)
def main():
desc = globals()["__doc__"]
parser = ArgumentParser(description=desc)
parser.add_argument(
"-f",
"--ftp_server",
default="ftp.env.gov.bc.ca",
help=("Full hostname of Water and Air Monitoring and "
"Reporting's ftp server"),
)
parser.add_argument(
"-F",
"--ftp_dir",
default=("pub/outgoing/AIR/Hourly_Raw_Air_Data/"
"Meteorological/"),
help="FTP Directory containing WAMR's data files",
)
parser = logging_args(parser)
args = parser.parse_args()
setup_logging(
args.log_conf,
args.log_filename,
args.error_email,
args.log_level,
"crmprtd.wamr",
)
download(args.ftp_server, args.ftp_dir)
def main():
desc = globals()["__doc__"]
parser = ArgumentParser(description=desc)
parser.add_argument(
"-p", "--province", required=True, help="2 letter province code"
)
parser.add_argument(
"-g",
"--language",
default="e",
choices=["e", "f"],
help="'e' (english) | 'f' (french)",
)
parser.add_argument(
"-F",
"--frequency",
required=True,
choices=["daily", "hourly"],
help="daily|hourly",
)
parser.add_argument(
"-t",
"--time",
help=(
"Alternate *UTC* time to use for downloading "
"(interpreted using dateutil.parser.parse)."
"Defaults to the previous hour/day (depending on"
" --frequency)."
),
)
parser.add_argument(
"-T",
"--threshold",
default=1000,
help=(
"Distance threshold (in meters) to use when "
"matching stations. Stations are considered a "
"match if they have the same id, name, and are "
"within this threshold"
),
)
parser.add_argument(
"-b",
"--baseurl",
default="https://dd.weather.gc.ca",
help=(
"Base URL (scheme and hostname components) for"
"the meteorological observations service"
),
)
parser = logging_args(parser)
args = parser.parse_args()
setup_logging(
args.log_conf, args.log_filename, args.error_email, args.log_level, "crmprtd.ec"
)
download(args.time, args.frequency, args.province, args.language, args.baseurl)
def main():
parser = ArgumentParser()
parser = process_args(parser)
parser = logging_args(parser)
args = parser.parse_args()
setup_logging(args.log_conf, args.log_filename, args.error_email,
args.log_level, 'crmprtd')
process(args.connection_string, args.sample_size, args.network)
def main(): # pragma: no cover
desc = globals()["__doc__"]
parser = ArgumentParser(description=desc)
parser = logging_args(parser)
parser = common_auth_arguments(parser)
parser.add_argument(
"-S",
"--start_time",
help=("Alternate time to use for downloading "
"(interpreted with dateutil.parser.parse)."
"Defaults to one hour prior to now"),
)
parser.add_argument(
"-E",
"--end_time",
help=("Alternate time to use for downloading "
"(interpreted with dateutil.parser.parse)."
"Defaults to now."),
)
parser.add_argument("-s",
"--station_id",
default=None,
help="Station ID for which to download data")
parser.add_argument(
"-u",
"--base_url",
default="https://prdoas5.apps.th.gov.bc.ca/saw-data/sawr7110",
help="Base URL for the MoTI SAW service",
)
args = parser.parse_args()
setup_logging(
args.log_conf,
args.log_filename,
args.error_email,
args.log_level,
"crmprtd.moti",
)
download(
args.username,
args.password,
args.auth_fname,
args.auth_key,
args.start_time,
args.end_time,
args.station_id,
args.base_url,
)
def main():
parser = ArgumentParser()
parser.add_argument('-f', '--ftp_server',
default='ftp.env.gov.bc.ca',
help=('Full hostname of Water and Air Monitoring and '
'Reporting\'s ftp server'))
parser.add_argument('-F', '--ftp_dir',
default=('pub/outgoing/AIR/Hourly_Raw_Air_Data/'
'Meteorological/'),
help='FTP Directory containing WAMR\'s data files')
parser = logging_args(parser)
args = parser.parse_args()
setup_logging(args.log_conf, args.log_filename, args.error_email,
args.log_level, 'crmprtd.wamr')
download(args.ftp_server, args.ftp_dir)
def main(partner):
"""Main download function to use for download scripts for the EC_SWOB
provincial partners (e.g. bc-env-snow, bc-env-aq, bc-forestry and
bc-tran).
Args:
partner (str): The partner abbreviation found in the SWOB URL
(e.g. bc-tran)
Returns:
No return value. Produces side-effect of sending downloaded
XML files to STDOUT
"""
desc = globals()["__doc__"]
parser = ArgumentParser(description=desc)
parser = logging_args(parser)
parser.add_argument(
"-d",
"--date",
help=("Alternate date to use for downloading "
"(interpreted with dateutil.parser.parse)."
"Defaults to now."),
)
args = parser.parse_args()
setup_logging(
args.log_conf,
args.log_filename,
args.error_email,
args.log_level,
"crmprtd.{}".format(partner),
)
if not args.date:
dl_date = datetime.datetime.now()
else:
dl_date = verify_date(args.date, datetime.datetime.now(), "date")
download(
"https://dd.weather.gc.ca/observations/swob-ml/partners/{}/".format(
partner),
dl_date,
)
def main():
parser = ArgumentParser()
parser = logging_args(parser)
parser = common_auth_arguments(parser)
parser.add_argument('-f', '--ftp_server',
default='BCFireweatherFTPp1.nrs.gov.bc.ca',
help=('Full uri to Wildfire Management Branch\'s ftp '
'server'))
parser.add_argument('-F', '--ftp_file',
default='HourlyWeatherAllFields_WA.txt',
help=('Filename to open on the Wildfire Management '
'Branch\'s ftp site'))
args = parser.parse_args()
setup_logging(args.log_conf, args.log_filename, args.error_email,
args.log_level, 'crmprtd.wmb')
download(args.username, args.password, args.auth_fname, args.auth_key,
args.ftp_server, args.ftp_file)
def main():
parser = ArgumentParser()
parser = logging_args(parser)
parser = common_auth_arguments(parser)
parser.add_argument('-S', '--start_time',
help=("Alternate time to use for downloading "
"(interpreted with "
"strptime(format='Y/m/d H:M:S')"))
parser.add_argument('-E', '--end_time',
help=("Alternate time to use for downloading "
"(interpreted with "
"strptime(format='Y/m/d H:M:S')"))
parser.add_argument('-s', '--station_id',
default=None,
help="Station ID for which to download data")
args = parser.parse_args()
setup_logging(args.log_conf, args.log_filename, args.error_email,
args.log_level, 'crmprtd.moti')
download(args.username, args.password, args.auth_fname, args.auth_key,
args.start_time, args.end_time, args.station_id)
def main(): # pragma: no cover
desc = globals()["__doc__"]
parser = ArgumentParser(description=desc)
parser = logging_args(parser)
parser = common_auth_arguments(parser)
parser.add_argument(
"-S",
"--start_time",
type=dateutil.parser.parse,
help=("Optional start time to use for downloading "
"(interpreted with dateutil.parser.parse)."
"Defaults to one day prior to now"),
)
parser.add_argument(
"-E",
"--end_time",
type=dateutil.parser.parse,
help=("Optional end time to use for downloading "
"(interpreted with dateutil.parser.parse)."
"Defaults to now."),
)
args = parser.parse_args()
setup_logging(
args.log_conf,
args.log_filename,
args.error_email,
args.log_level,
"crmprtd.crd",
)
verify_dates(args.start_time, args.end_time)
auth_yaml = open(args.auth_fname, "r").read() if args.auth_fname else None
auth = crmprtd.download.extract_auth(args.username, None, auth_yaml,
args.auth_key)
download(auth["u"], args.start_time, args.end_time)
def main():
desc = globals()["__doc__"]
parser = ArgumentParser(description=desc)
parser = logging_args(parser)
parser = common_auth_arguments(parser)
parser.add_argument(
"-f",
"--ftp_server",
default="BCFireweatherFTPp1.nrs.gov.bc.ca",
help=("Full uri to Wildfire Management Branch's ftp "
"server"),
)
parser.add_argument(
"-F",
"--ftp_file",
default="HourlyWeatherAllFields_WA.txt",
help=("Filename to open on the Wildfire Management "
"Branch's ftp site"),
)
args = parser.parse_args()
setup_logging(
args.log_conf,
args.log_filename,
args.error_email,
args.log_level,
"crmprtd.wmb",
)
download(
args.username,
args.password,
args.auth_fname,
args.auth_key,
args.ftp_server,
args.ftp_file,
)
def main():
desc = globals()["__doc__"]
parser = ArgumentParser(description=desc)
parser.add_argument(
"-S",
"--start_time",
help=(
"Alternate time to use for downloading "
"(interpreted with "
"strptime(format='Y/m/d H:M:S')."
),
)
parser.add_argument(
"-E",
"--end_time",
help=(
"Alternate time to use for downloading "
"(interpreted with "
"strptime(format='Y/m/d H:M:S')."
),
)
parser.add_argument(
"-a",
"--auth_fname",
help="Yaml file with plaintext usernames/passwords. "
"For simplicity the auth keys are *not* "
"configurable (unlike the download scripts. The "
"supplied file must contain keys with names "
"'moti', 'moti2' and 'wmb' if you want to infill "
"those networks.",
)
parser.add_argument(
"-c", "--connection_string", help="PostgreSQL connection string", required=True
)
parser.add_argument(
"-N",
"--networks",
nargs="*",
default="crd ec moti wamr wmb ec_swob",
help="Set of networks for which to infill",
)
parser = logging_args(parser)
args = parser.parse_args()
log_args = [
args.log_conf,
args.log_filename,
args.error_email,
args.log_level,
"infill_all",
]
setup_logging(*log_args)
fmt = "%Y/%m/%d %H:%M:%S"
s = datetime.datetime.strptime(args.start_time, fmt).astimezone(tzlocal())
e = datetime.datetime.strptime(args.end_time, fmt).astimezone(tzlocal())
log_args = [(x, y) for (x, y) in zip(["-L", "-l", "-m", "-o"], log_args) if y]
log_args = list(itertools.chain(*log_args))
infill(args.networks, s, e, args.auth_fname, args.connection_string, log_args)
def test_setup_logging():
setup_logging(None, "mof.log", "*****@*****.**", "CRITICAL", "test")
log = logging.getLogger("test")
assert log.name == "test"
assert log.level == 0
assert log.parent.level == 50
def main(): # pragma: no cover
desc = globals()["__doc__"]
parser = ArgumentParser(description=desc)
parser = logging_args(parser)
parser.add_argument("-u",
"--username",
default="pcic",
help=("Username for the ftp server "))
parser.add_argument(
"-f",
"--ftp_server",
default="sftp2.bchydro.com",
help=("Full uri to BC Hydro's ftp "
"server"),
)
parser.add_argument(
"-F",
"--ftp_dir",
default=("pcic"),
help=("FTP Directory containing BC hydro's "
"data files"),
)
parser.add_argument(
"-S",
"--ssh_private_key",
required=True,
help=("Path to file with SSH private key"),
)
end = datetime.now()
start = end - relativedelta.relativedelta(months=1)
parser.add_argument(
"-s",
"--start_date",
help=("Optional start time to use for downloading "
"(interpreted with dateutil.parser.parse). "
"Defaults to one month prior to now."),
)
parser.add_argument(
"-e",
"--end_date",
help=("Optional end time to use for downloading "
"(interpreted with dateutil.parser.parse). "
"Defaults to now."),
)
args = parser.parse_args()
setup_logging(
args.log_conf,
args.log_filename,
args.error_email,
args.log_level,
"crmprtd.bc_hydro",
)
args.start_date = verify_date(args.start_date, start, "start date")
args.end_date = verify_date(args.end_date, end, "end date")
download(
args.username,
args.ssh_private_key,
args.ftp_server,
args.ftp_dir,
args.start_date,
args.end_date,
)