def test_raddo_complete_download():
START_DATE = datetime.datetime.strftime(
datetime.datetime.today().date() - datetime.timedelta(days=4),
"%Y-%m-%d")
END_DATE = _date_str(datetime.datetime.today().date() -
datetime.timedelta(days=2)) # Yesterday
with tempfile.TemporaryDirectory() as tmpdirname:
os.chdir(tmpdirname)
RAD_DIR = tmpdirname
rd = Raddo()
tiff_dir = rd.try_create_directory(os.path.join(RAD_DIR, "tiff"))
successfull_down = rd.radolan_down(rad_dir_dwd=RAD_DIR_DWD,
rad_dir_dwd_hist=RAD_DIR_DWD_HIST,
rad_dir=RAD_DIR,
errors_allowed=ERRORS_ALLOWED,
start_date=START_DATE,
end_date=END_DATE,
force=True,
force_down=True)
new_paths = sort_tars.sort_tars(files=successfull_down)
untarred_dirs = untar.untar(files=new_paths)
tiff_dir = rd.try_create_directory(os.path.join(RAD_DIR, "tiff"))
asc_files = rd.get_asc_files(untarred_dirs)
gtiff_files = rd.create_geotiffs(asc_files, tiff_dir)
rd.create_netcdf(gtiff_files, RAD_DIR)
tempfile.TemporaryDirectory().cleanup()
def main():
rd = Raddo()
class MyParser(argparse.ArgumentParser):
def error(self, message):
sys.stderr.write('error: %s\n' % message)
self.print_help()
sys.exit(2)
parser = MyParser(
description=(' raddo - utility to download and preprocess RADOLAN RW\n'
' weather radar data from DWD servers.'),
prog="raddo",
# usage='run "%(prog)s -h" for all cli options.'
)
parser.add_argument('-s',
'--start',
required=False,
default=rd.START_DATE,
action='store',
dest='start',
help=(f'Start date as parsable string '
f'(e.g. "2018-05-20").'
f'\nDefault: {rd.START_DATE} '
f'(14 days ago).'))
parser.add_argument('-e',
'--end',
required=False,
default=rd.END_DATE,
action='store',
dest='end',
help=(f'End date as parsable string '
f'(e.g. "2020-05-20").'
f'\nDefault: {rd.END_DATE_STR} (yesterday)'))
parser.add_argument(
'-d',
'--directory',
required=False,
default=f"{os.getcwd()}",
action='store',
dest='directory',
help=(f'Absolute path to local directory where RADOLAN'
f' data should be (and may already be) saved. Ch'
f'ecks for existing files only if this flag is se'
f't.\nDefault: {os.getcwd()} (current directory)'))
parser.add_argument('-C',
'--complete',
required=False,
default=False,
action='store_true',
dest='complete',
help=(f'Run all subcommands. Same as using flags '
f'-fxgn.'))
parser.add_argument('-f',
'--sort-in-folders',
required=False,
default=False,
action='store_true',
dest='sort',
help=(f'Should the data be sorted in folders?'))
parser.add_argument('-x',
'--extract',
required=False,
default=False,
action='store_true',
dest='extract',
help=(f'Should the data be extracted?'))
parser.add_argument('-g',
'--geotiff',
required=False,
default=False,
action='store_true',
dest='geotiff',
help=(f'Set if GeoTiffs in EPSG:4326 should be '
f'created for newly downloaded files.'))
parser.add_argument('-n',
'--netcdf',
required=False,
default=False,
action='store_true',
dest='netcdf',
help=(f'Create a NetCDF from GeoTiffs?'))
parser.add_argument('-N',
'--netcdf-file',
required=False,
default=None,
action='store',
dest='outfile',
help=(f'Name of the output NetCDF file.'))
parser.add_argument('-m',
'--mask',
required=False,
default=False,
action='store',
dest='mask',
help=(f'Use mask when creating NetCDF.'))
parser.add_argument('-p',
'--point',
required=False,
default=False,
action='store',
dest='point',
help=(f'Extract precipitation for specific '
f'point coordinates.'))
parser.add_argument('-b',
'--buffer',
required=False,
default=1400,
action='store',
dest='buffersize',
help=(f'Buffer in meter around mask shapefile'
' (Default 1400m).'))
# parser.add_argument('-q', '--quiet',
# required=False,
# default=False,
# action='store_true', dest='quiet',
# help=(f'Be quiet.'))
parser.add_argument('-F',
'--force',
required=False,
default=False,
action='store_true',
dest='force',
help=(f'Forces local file search. Omits faster check '
'of ".raddo_local_files.txt".'))
parser.add_argument('-D',
'--force-download',
required=False,
default=False,
action='store_true',
dest='force_down',
help=(f'Forces download of all files.'))
parser.add_argument('-y',
'--yes',
required=False,
default=False,
action='store_true',
dest='yes',
help=(f'Skip user input. Just accept to download to '
'current directory if not specified otherwise.'))
parser.add_argument('-v',
'--version',
required=False,
default=False,
action='store_true',
dest='version',
help=(f'Print information on software version.'))
parser.add_argument('-u',
'--radolan_server_url',
required=False,
default=rd.RAD_DIR_DWD,
action='store',
dest='url',
help=(f'Path to recent .asc RADOLAN data on '
f'DWD servers.\nDefault: {rd.RAD_DIR_DWD}'))
parser.add_argument('-r',
'--errors-allowed',
required=False,
default=rd.ERRORS_ALLOWED,
action='store',
dest='errors',
help=(f'Errors allowed when contacting DWD Server.'
f'\nDefault: {rd.ERRORS_ALLOWED}'))
parser.add_argument('-t',
'--no-time-correction',
required=False,
default=False,
action='store_true',
dest='tcorr',
help=(f'Omit time adjustment to previous hour in '
f'netCDF file creation and just use RADOLANs '
f'sum up time HH:50 (Default: false).'))
args = parser.parse_args()
if args.complete:
args.netcdf = True
args.geotiff = True
args.extract = True
args.sort = True
if (args.geotiff or args.netcdf or args.point):
args.extract = True
args.sort = True
# print version
if args.version:
sys.stdout.write(f"raddo {__version__}\n")
sys.exit()
# if not args.quiet:
sys.stdout.write(pcol.HEADER + 59 * '=' + '\n' + parser.description +
"\n" + 59 * "=" + pcol.ENDC + "\n\n")
# if no -d flag:
if args.directory == os.getcwd():
# if (args.start == rd.START_DATE) & (args.end == rd.END_DATE):
# sys.stdout.write(f"{parser.print_help()}\n\n")
if not args.yes:
if not user_check(f"Do you really want to store RADOLAN data in "
f"\"{os.getcwd()}\"?"):
sys.stderr.write(f"User Interruption.\n")
sys.exit()
args.directory = os.path.abspath(args.directory)
if args.start == rd.START_DATE:
if not args.yes:
if not user_check(
f"Do you really want to download RADOLAN data from "
f"{rd.START_DATE} on?"):
sys.stderr.write(f"User Interruption.\n")
sys.exit()
assert args.errors < 21, \
"Error value too high. Please be respectful with the data provider."
successfull_down = rd.radolan_down(rad_dir_dwd=args.url,
rad_dir=args.directory,
errors_allowed=int(args.errors),
start_date=args.start,
end_date=args.end,
no_time_correction=args.tcorr,
force=args.force,
force_down=args.force_down,
yes=args.yes,
buffer=args.buffersize)
if len(successfull_down) > 0:
new_paths = []
untarred_dirs = []
if args.sort:
new_paths = sort_tars.sort_tars(files=successfull_down)
if args.extract:
untarred_dirs = untar.untar(files=new_paths, hist=rd.hist_files)
if (args.geotiff or args.netcdf or args.point):
if len(untarred_dirs) > 0:
asc_files = rd.get_asc_files(untarred_dirs)
if args.mask:
rd.read_mask(args.mask)
if args.point:
try:
assert args.mask is False, \
"Only specify mask or point coordinates!"
except AssertionError as e:
sys.stderr.write(f"{e}")
sys.exit()
rd.read_coords(args.point)
# create tiff directory
if args.geotiff:
tiff_dir = rd.try_create_directory(
os.path.join(os.path.abspath(args.directory), "tiff"))
if not args.yes:
if len(asc_files) > 7 * 24:
if not user_check("Do you really want to create "
f"{len(asc_files)} geotiffs?\n[These"
" files are only created if not "
"already available.]"):
sys.exit("\nExiting.")
# create geotiffs
gtiff_files = rd.create_geotiffs(asc_files, tiff_dir)
# create temporary directory if geotiffs are not wanted:
else:
args.yes = True
# TODO change to current dir (avoid /tmp overflow?)
tmpd = tempfile.TemporaryDirectory()
tiff_dir = tmpd.name
# create temporary geotiffs
gtiff_files = rd.create_geotiffs(asc_files, tiff_dir)
# create netcdf file
if args.netcdf:
rd.create_netcdf(gtiff_files, args.directory, args.outfile,
args.tcorr)
if args.point:
if not args.netcdf:
if args.outfile is None:
netcdf_outf = tempfile.NamedTemporaryFile(
prefix="RADOLAN_TEMP_").name
else:
netcdf_outf = args.outfile
rd.create_netcdf(gtiff_files, args.directory, netcdf_outf,
args.tcorr)
rd.create_point_from_netcdf()
else:
print("Cannot create GeoTiffs - no newly extracted *.asc files.")
try:
tmpd.cleanup()
except NameError:
pass