def get_match_files(weather_index):
"""!
Generate URL containing data
@param weather_index WeatherIndex to get files for
"""
# Get full url for the file that has the data we're asking for
diff = for_date - for_run
real_hour = int((diff.days * 24) + (diff.seconds / 60 / 60))
file = self.mask.format('{}', for_run.hour, real_hour)
var = r'&{}=on&var_{}=on'.format(weather_index.layer, weather_index.name)
subregion_mask = r'&subregion=&leftlon={}&rightlon={}&toplat={}&bottomlat={}'
subregion = subregion_mask.format(common.BOUNDS['longitude']['min'],
common.BOUNDS['longitude']['max'],
common.BOUNDS['latitude']['max'],
common.BOUNDS['latitude']['min'])
date = for_run.strftime(r'%Y%m%d')
time = for_run.strftime(r'%H')
dir = self.dir.format(date, time)
partial_url = r'{}{}{}{}{}{}'.format(self.host, self.script, file, var, subregion, dir)
def get_local_name(partial_url):
"""!
Return file name that will be saved locally from partial_url
@param partial_url Partial url to use for determining name
@return Path to save to locally when using URL to retrieve
"""
save_as = '{}_{}{}_{}_{:03d}'.format(self.name, date, time, weather_index.name, real_hour)
return os.path.join(self.DIR_DATA, save_as)
def save_file(partial_url):
"""!
Save the given url
@param partial_url Partial url to use for determining name
@return Path saved to when using URL to retrieve
"""
out_file = get_local_name(partial_url)
if os.path.isfile(out_file):
# HACK: no timestamp so don't download if exists
return out_file
logging.debug("Downloading {}".format(out_file))
def generate_member(member):
member_name = r'ge{}{:02d}'.format('c' if member == 0 else 'p', member)
u = partial_url.format(member_name)
assert('{}' not in u)
return u
urls = list(map(generate_member, range(0, self.num_members)))
results = common.download_many(urls, fct=download_and_wait)
assert(self.num_members == len(results))
try:
# logging.debug("Writing file")
with open(out_file, "wb") as f:
for result in results:
f.write(result)
except:
# get rid of file that's there if there was an error
common.try_remove(out_file)
raise
return out_file
if self.no_download:
# return file that would have been saved without actually trying to save it
return get_local_name(partial_url)
return common.try_save(save_file, partial_url)
def get_Dataset(year, colname):
"""!
Open netCDF4 for desired data
@param year Year to download data for
@param colname Name of field to read data for
@return netCDF4.Dataset of data read
"""
print year, colname
file = file_masks[colname].format(year)
filename = None
if 1981 == year and 'TMP' == colname:
logging.debug("Using workaround for 1981 temperature download issue")
# HACK to use file from alternate source since other won't download
other_url = r'http://database.rish.kyoto-u.ac.jp/arch/ncep/data/ncep.reanalysis/surface_gauss/air.2m.gauss.1981.nc'
def save_other(url):
return common.save_http(DIR_DATA, url)
filename = common.try_save(save_other, other_url)
else:
filename = common.try_save(save_file, file)
#~ filename = os.path.join(DIR_DATA, os.path.basename(file))
return netCDF4.Dataset(filename)
def get_Dataset(year, colname):
"""!
Open netCDF4 for desired data
@param year Year to download data for
@param colname Name of field to read data for
@return netCDF4.Dataset of data read
"""
print(year, colname)
file = file_masks[colname].format(year)
filename = common.try_save(save_file, file)
#~ filename = os.path.join(DIR_DATA, os.path.basename(file))
return netCDF4.Dataset(filename)
def do_save(args):
"""!
Generate URL containing data
@param weather_index WeatherIndex to get files for
"""
host, dir, mask, save_dir, date, time, real_hour, save_as, weather_index = args
# Get full url for the file that has the data we're asking for
partial_url = r'{}{}{}'.format(
host, dir.format(date=date, time=time, real_hour=real_hour),
mask.format(date=date,
time=time,
real_hour=real_hour,
index='{}',
layer='{}'))
partial_url = partial_url.format(weather_index.name, weather_index.layer)
def save_file(partial_url):
"""!
Save the given url
@param partial_url Partial url to use for determining name
@return Path saved to when using URL to retrieve
"""
out_file = os.path.join(save_dir, save_as.format(weather_index.name))
if os.path.isfile(out_file):
# HACK: no timestamp so don't download if exists
# logging.debug("Have {}".format(out_file))
return out_file
# logging.debug("Downloading {}".format(out_file))
try:
common.save_http(save_dir, partial_url, save_as=out_file)
except:
# get rid of file that's there if there was an error
common.try_remove(out_file)
raise
return out_file
return common.try_save(save_file, partial_url)