def download_raw_file(unix_time_sec,
secondary_source,
top_local_directory_name,
protocol,
user_name=None,
password=None,
raise_error_if_fails=True):
"""Downloads raw file from either FTP or HTTP server.
:param unix_time_sec: Valid time.
:param secondary_source: String ID for secondary data source.
:param top_local_directory_name: Name of top-level directory with raw MADIS
files on local machine.
:param protocol: Protocol (either "http" or "ftp").
:param user_name: User name on FTP or HTTP server. To login anonymously,
leave this as None.
:param password: Password on FTP or HTTP server. To login anonymously,
leave this as None.
:param raise_error_if_fails: Boolean flag. If True and download fails, this
method will raise an error.
:return: local_gzip_file_name: Local path to file that was just downloaded.
If download failed but raise_error_if_fails = False, this will be None.
:raises: ValueError: if protocol is neither "ftp" nor "http".
"""
error_checking.assert_is_string(protocol)
if protocol not in ['ftp', 'http']:
error_string = (
'Protocol should be either "ftp" or "http", not "{0:s}"'.format(
protocol))
raise ValueError(error_string)
raw_wind_io.check_data_sources(raw_wind_io.MADIS_DATA_SOURCE,
secondary_source)
online_file_name = _get_online_file_name(unix_time_sec=unix_time_sec,
secondary_source=secondary_source,
protocol=protocol)
local_gzip_file_name = find_local_raw_file(
unix_time_sec=unix_time_sec,
secondary_source=secondary_source,
top_directory_name=top_local_directory_name,
raise_error_if_missing=False)
if protocol == 'ftp':
return downloads.download_file_via_ftp(
server_name=FTP_SERVER_NAME,
user_name=user_name,
password=password,
ftp_file_name=online_file_name,
local_file_name=local_gzip_file_name,
raise_error_if_fails=raise_error_if_fails)
return downloads.download_files_via_http(
online_file_names=[online_file_name],
local_file_names=[local_gzip_file_name],
user_name=user_name,
password=password,
host_name=HTTP_HOST_NAME,
raise_error_if_fails=raise_error_if_fails)[0]
def download_file(top_local_dir_name,
field_name,
month_string,
is_surface=False,
raise_error_if_fails=False):
"""Downloads NetCDF file to the local machine.
:param top_local_dir_name: Name of top-level target directory (local
directory for NetCDF files with NARR data).
:param field_name: See doc for `_get_pathless_file_name`.
:param month_string: Same.
:param is_surface: Same.
:param raise_error_if_fails: Boolean flag. If download fails and
`raise_error_if_fails = True`, this method will error out.
:return: local_file_name: Path to downloaded file on local machine. If
download failed and `raise_error_if_fails = False`, this is None.
"""
local_file_name = find_file(top_directory_name=top_local_dir_name,
field_name=field_name,
month_string=month_string,
is_surface=is_surface,
raise_error_if_missing=False)
pathless_file_name = _get_pathless_file_name(field_name=field_name,
month_string=month_string,
is_surface=is_surface)
if is_surface:
online_dir_name = ONLINE_SURFACE_DIR_NAME + ''
else:
online_dir_name = ONLINE_PRESSURE_LEVEL_DIR_NAME + ''
online_file_name = '{0:s}/{1:s}'.format(online_dir_name,
pathless_file_name)
return downloads.download_files_via_http(
online_file_names=[online_file_name],
local_file_names=[local_file_name],
raise_error_if_fails=raise_error_if_fails)
def download_grib_file(init_time_unix_sec,
lead_time_hours=None,
model_name=None,
grid_id=None,
top_local_directory_name=None,
raise_error_if_fails=True):
"""Downloads grib file to local machine.
:param init_time_unix_sec: Model-initialization time (Unix format).
:param lead_time_hours: Lead time (valid time minus init time). If model is
a reanalysis, you can leave this as None (always zero).
:param model_name: Name of model.
:param grid_id: String ID for model grid.
:param top_local_directory_name: Name of top local directory with grib files
for the given model/grib combo. File will be saved here.
:param raise_error_if_fails: Boolean flag. If True and download fails, will
raise an error.
:return: local_file_name: Path to grib file on local machine. If download
failed but raise_error_if_fails = False, this will be None.
"""
error_checking.assert_is_boolean(raise_error_if_fails)
pathless_file_names = _get_pathless_grib_file_names(
init_time_unix_sec,
lead_time_hours=lead_time_hours,
model_name=model_name,
grid_id=grid_id)
top_online_directory_names = nwp_model_utils.get_top_online_directories(
model_name, grid_id)
desired_local_file_name = find_grib_file(
init_time_unix_sec,
lead_time_hours=lead_time_hours,
model_name=model_name,
grid_id=grid_id,
top_directory_name=top_local_directory_name,
raise_error_if_missing=False)
for i in range(len(pathless_file_names)):
for j in range(len(top_online_directory_names)):
this_online_file_name = '{0:s}/{1:s}/{2:s}/{3:s}'.format(
top_online_directory_names[j],
time_conversion.unix_sec_to_string(init_time_unix_sec,
TIME_FORMAT_MONTH),
time_conversion.unix_sec_to_string(init_time_unix_sec,
TIME_FORMAT_DATE),
pathless_file_names[i])
this_raise_error_flag = (raise_error_if_fails
and i == len(pathless_file_names) - 1 and
j == len(top_online_directory_names) - 1)
local_file_name = downloads.download_files_via_http(
online_file_names=[this_online_file_name],
local_file_names=[desired_local_file_name],
raise_error_if_fails=this_raise_error_flag)[0]
if local_file_name is not None:
extensionless_local_file_name, local_file_extension = (
os.path.splitext(local_file_name))
if this_online_file_name.endswith(local_file_extension):
break
if local_file_extension == grib_io.GRIB1_FILE_EXTENSION:
new_file_extension = grib_io.GRIB2_FILE_EXTENSION
else:
new_file_extension = grib_io.GRIB1_FILE_EXTENSION
os.rename(local_file_name,
extensionless_local_file_name + new_file_extension)
local_file_name = (extensionless_local_file_name +
new_file_extension)
break
if local_file_name is not None:
break
return local_file_name
field_matrix = _read_single_field_from_file(
single_field_file_name,
num_grid_rows=num_grid_rows,
num_grid_columns=num_grid_columns,
sentinel_value=sentinel_value,
raise_error_if_fails=raise_error_if_fails)
if delete_single_field_file and os.path.isfile(single_field_file_name):
os.remove(single_field_file_name)
return field_matrix
if __name__ == '__main__':
downloads.download_files_via_http(
online_file_names=[NARR_FILE_NAME_ONLINE],
local_file_names=[NARR_FILE_NAME_LOCAL])
downloads.download_files_via_http(online_file_names=[RAP_FILE_NAME_ONLINE],
local_file_names=[RAP_FILE_NAME_LOCAL])
NARR_H500_MATRIX_METRES = read_field_from_grib_file(
NARR_FILE_NAME_LOCAL,
grib1_field_name=H500_NAME_GRIB,
single_field_file_name=NARR_H500_FILE_NAME,
wgrib_exe_name=WGRIB_EXE_NAME_DEFAULT,
wgrib2_exe_name=WGRIB2_EXE_NAME_DEFAULT,
num_grid_rows=NUM_ROWS_IN_NARR,
num_grid_columns=NUM_COLUMNS_IN_NARR,
sentinel_value=NARR_SENTINEL_VALUE)
print NARR_H500_MATRIX_METRES
print numpy.nanmin(NARR_H500_MATRIX_METRES)
def download_grib_file(top_local_directory_name,
init_time_unix_sec,
model_name,
grid_id=None,
lead_time_hours=None,
raise_error_if_fails=None):
"""Downloads grib file to local machine.
:param top_local_directory_name: Name of top-level directory for grib files
on local machine.
:param init_time_unix_sec: Model-initialization time.
:param model_name: See doc for `nwp_model_utils.check_grid_id`.
:param grid_id: Same.
:param lead_time_hours: Lead time.
:param raise_error_if_fails: Boolean flag. If download fails and
raise_error_if_fails = True, this method will error out.
:return: local_file_name: Path to grib file on local machine. If download
fails and raise_error_if_fails = False, this will be None.
"""
error_checking.assert_is_boolean(raise_error_if_fails)
pathless_file_names = _get_pathless_grib_file_names(
init_time_unix_sec=init_time_unix_sec,
model_name=model_name,
grid_id=grid_id,
lead_time_hours=lead_time_hours)
top_online_dir_names = nwp_model_utils.get_top_online_directories(
model_name=model_name, grid_id=grid_id)
desired_local_file_name = find_grib_file(
top_directory_name=top_local_directory_name,
init_time_unix_sec=init_time_unix_sec,
model_name=model_name,
grid_id=grid_id,
lead_time_hours=lead_time_hours,
raise_error_if_missing=False)
for i in range(len(pathless_file_names)):
for j in range(len(top_online_dir_names)):
this_online_file_name = '{0:s}/{1:s}/{2:s}/{3:s}'.format(
top_online_dir_names[j],
time_conversion.unix_sec_to_string(init_time_unix_sec,
TIME_FORMAT_MONTH),
time_conversion.unix_sec_to_string(init_time_unix_sec,
TIME_FORMAT_DATE),
pathless_file_names[i])
raise_error_now = (raise_error_if_fails
and i == len(pathless_file_names) - 1
and j == len(top_online_dir_names) - 1)
local_file_name = downloads.download_files_via_http(
online_file_names=[this_online_file_name],
local_file_names=[desired_local_file_name],
raise_error_if_fails=raise_error_now)[0]
if local_file_name is None:
continue
extensionless_local_file_name, local_file_extension = (
os.path.splitext(local_file_name))
if this_online_file_name.endswith(local_file_extension):
break
if local_file_extension == grib_io.GRIB1_FILE_EXTENSION:
new_local_file_name = '{0:s}{1:s}'.format(
extensionless_local_file_name,
grib_io.GRIB2_FILE_EXTENSION)
else:
new_local_file_name = '{0:s}{1:s}'.format(
extensionless_local_file_name,
grib_io.GRIB1_FILE_EXTENSION)
os.rename(local_file_name, new_local_file_name)
local_file_name = copy.deepcopy(new_local_file_name)
break
if local_file_name is not None:
break
return local_file_name