def get_match_files(weather_index): """! Generate URL containing data @param weather_index WeatherIndex to get files for """ # Get full url for the file that has the data we're asking for diff = for_date - for_run real_hour = int((diff.days * 24) + (diff.seconds / 60 / 60)) file = self.mask.format('{}', for_run.hour, real_hour) var = r'&{}=on&var_{}=on'.format(weather_index.layer, weather_index.name) subregion_mask = r'&subregion=&leftlon={}&rightlon={}&toplat={}&bottomlat={}' subregion = subregion_mask.format(common.BOUNDS['longitude']['min'], common.BOUNDS['longitude']['max'], common.BOUNDS['latitude']['max'], common.BOUNDS['latitude']['min']) date = for_run.strftime(r'%Y%m%d') time = for_run.strftime(r'%H') dir = self.dir.format(date, time) partial_url = r'{}{}{}{}{}{}'.format(self.host, self.script, file, var, subregion, dir) def get_local_name(partial_url): """! Return file name that will be saved locally from partial_url @param partial_url Partial url to use for determining name @return Path to save to locally when using URL to retrieve """ save_as = '{}_{}{}_{}_{:03d}'.format(self.name, date, time, weather_index.name, real_hour) return os.path.join(self.DIR_DATA, save_as) def save_file(partial_url): """! Save the given url @param partial_url Partial url to use for determining name @return Path saved to when using URL to retrieve """ out_file = get_local_name(partial_url) if os.path.isfile(out_file): # HACK: no timestamp so don't download if exists return out_file logging.debug("Downloading {}".format(out_file)) def generate_member(member): member_name = r'ge{}{:02d}'.format('c' if member == 0 else 'p', member) u = partial_url.format(member_name) assert('{}' not in u) return u urls = list(map(generate_member, range(0, self.num_members))) results = common.download_many(urls, fct=download_and_wait) assert(self.num_members == len(results)) try: # logging.debug("Writing file") with open(out_file, "wb") as f: for result in results: f.write(result) except: # get rid of file that's there if there was an error common.try_remove(out_file) raise return out_file if self.no_download: # return file that would have been saved without actually trying to save it return get_local_name(partial_url) return common.try_save(save_file, partial_url)
def get_Dataset(year, colname): """! Open netCDF4 for desired data @param year Year to download data for @param colname Name of field to read data for @return netCDF4.Dataset of data read """ print year, colname file = file_masks[colname].format(year) filename = None if 1981 == year and 'TMP' == colname: logging.debug("Using workaround for 1981 temperature download issue") # HACK to use file from alternate source since other won't download other_url = r'http://database.rish.kyoto-u.ac.jp/arch/ncep/data/ncep.reanalysis/surface_gauss/air.2m.gauss.1981.nc' def save_other(url): return common.save_http(DIR_DATA, url) filename = common.try_save(save_other, other_url) else: filename = common.try_save(save_file, file) #~ filename = os.path.join(DIR_DATA, os.path.basename(file)) return netCDF4.Dataset(filename)
def get_Dataset(year, colname): """! Open netCDF4 for desired data @param year Year to download data for @param colname Name of field to read data for @return netCDF4.Dataset of data read """ print(year, colname) file = file_masks[colname].format(year) filename = common.try_save(save_file, file) #~ filename = os.path.join(DIR_DATA, os.path.basename(file)) return netCDF4.Dataset(filename)
def do_save(args): """! Generate URL containing data @param weather_index WeatherIndex to get files for """ host, dir, mask, save_dir, date, time, real_hour, save_as, weather_index = args # Get full url for the file that has the data we're asking for partial_url = r'{}{}{}'.format( host, dir.format(date=date, time=time, real_hour=real_hour), mask.format(date=date, time=time, real_hour=real_hour, index='{}', layer='{}')) partial_url = partial_url.format(weather_index.name, weather_index.layer) def save_file(partial_url): """! Save the given url @param partial_url Partial url to use for determining name @return Path saved to when using URL to retrieve """ out_file = os.path.join(save_dir, save_as.format(weather_index.name)) if os.path.isfile(out_file): # HACK: no timestamp so don't download if exists # logging.debug("Have {}".format(out_file)) return out_file # logging.debug("Downloading {}".format(out_file)) try: common.save_http(save_dir, partial_url, save_as=out_file) except: # get rid of file that's there if there was an error common.try_remove(out_file) raise return out_file return common.try_save(save_file, partial_url)