def __init__(self,variable,search_pattern=None,period=None):
self.variable = variable
self.db_open_with = db_open_with()
if not search_pattern:
search_pattern = variable+"*.nc"
self.matcher = FileMatcher(search_pattern)
try:
self._all_files = sorted(self.matcher.locate())
except:
from awrams.utils.io.messages import MSG_CANT_MATCH_VARIABLE_WITH_PATTERN
logger.critical(MSG_CANT_MATCH_VARIABLE_WITH_PATTERN,variable,search_pattern)
raise
if not len(self._all_files):
raise BaseException("No matching files for variable %s using search pattern %s"%(variable,search_pattern))
if period is not None:
years = dt.years_for_period(period)
f_files = []
for in_file in self._all_files:
file_year = re.match('.*([0-9]{4})',in_file).groups()[0]
if int(file_year) in years:
f_files.append(in_file)
self.files = f_files
else:
self.files = self._all_files
if len(self.files) == 0:
# +++ quick fix for solar if whole period is less than 1990
self.files = self._all_files[:1]
ref_file = self._open_data(self.files[0])
end_ref_file = self._open_data(self.files[-1]) if len(self.files)>1 else ref_file
self.epoch_offset = int(ref_file.variables['time'][0])
end_offset = int(end_ref_file.variables['time'][-1])
self.epoch = epoch_from_nc(ref_file)
self.start_date = dt.datetime.fromordinal(self.epoch.toordinal()+self.epoch_offset)
self.end_date = dt.datetime.fromordinal(self.epoch.toordinal()+end_offset)
self.geospatial_reference = geospatial_reference_from_nc(ref_file)
self.latitudes = aquantize(np.float64(ref_file.variables['latitude'][:]),0.05)
self.longitudes = aquantize(np.float64(ref_file.variables['longitude'][:]),0.05)
if 'var_name' in ref_file.ncattrs():
self.variable = ref_file.getncattr('var_name')
try:
self.units = ref_file.variables[self.variable].units
except AttributeError:
pass
ref_file.close()
if len(self.files)>1:
end_ref_file.close()
def __init__(self,
variable,
search_pattern,
period=None,
day_exist_chn_name=None):
self.variable = variable
self._reader = InputReader(variable)
self._pattern = search_pattern
self.db_open_with = db_open_with()
if not search_pattern:
search_pattern = variable + "*.nc"
self.matcher = FileMatcher(search_pattern)
self._all_files = sorted(self.matcher.locate())
if not len(self._all_files):
raise NoMatchingFilesException(
"No matching files for variable %s using search pattern %s" %
(variable, search_pattern))
if period is not None:
years = dt.years_for_period(period)
f_files = []
for in_file in self._all_files:
file_year = re.match('.*([0-9]{4})', in_file).groups()[0]
if int(file_year) in years:
f_files.append(in_file)
self.files = f_files
else:
self.files = self._all_files
if len(self.files) == 0:
# +++ quick fix for solar if whole period is less than 1990
self.files = self._all_files[:1]
self.open_files = [self._open_data(fn) for fn in self.files]
ref_file = self.open_files[0]
end_ref_file = self.open_files[-1]
self.epoch_offset = int(ref_file.variables['time'][0])
self.epoch = epoch_from_nc(ref_file)
self.start_date = start_date(ref_file)
self.end_date = end_date(end_ref_file, day_exist_chn_name)
self.file_map = {}
self._map_files()
self.length = sum(
[ds.variables[self.variable].shape[0] for ds in self.open_files])
self.shape = np.ma.concatenate(
[[self.length], ref_file.variables[self.variable].shape[1:]])
self.geospatial_reference = geospatial_reference_from_nc(ref_file)
self.latitudes = ref_file.variables['latitude'][:]
self.longitudes = ref_file.variables['longitude'][:]
from awrams.utils.messaging.buffers import shm_as_ndarray
# +++ Tidy namespace
from awrams.utils.messaging.general import *
from awrams.utils.io.netcdf_wrapper import NCVariableParameters
#from awrams.utils.settings import DB_OPEN_WITH, MAX_FILES_PER_SFM #pylint: disable=no-name-in-module
from awrams.utils.awrams_log import get_module_logger as _get_module_logger
logger = _get_module_logger('data_mapping')
import glob
from awrams.utils.io import db_open_with
db_opener = db_open_with()
import awrams.utils.io.db_helper as dbh
set_h5nc_attr = dbh.set_h5nc_attr
from awrams.utils.config_manager import get_system_profile
sys_settings = get_system_profile().get_settings()
DB_OPEN_WITH = sys_settings['IO_SETTINGS']['DB_OPEN_WITH']
MAX_FILES_PER_SFM = sys_settings['IO_SETTINGS']['MAX_FILES_PER_SFM']
DEFAULT_CHUNKSIZE = sys_settings['IO_SETTINGS']['DEFAULT_CHUNKSIZE']
def find_time_coord(coordinates):
'''
Locate the time dimension in a set of coordinates