def retrieve_variables(self,output,zlib=False):
#open and record:
try:
self.open_with_error()
for var_name in self.Dataset.variables.keys():
netcdf_utils.replicate_and_copy_variable(output,self.Dataset,var_name,zlib=True)
#netcdf_utils.replicate_netcdf_var(output,self.Dataset,var_name)
#output.variables[var_name][:]=self.Dataset.variables[var_name][:]
self.close()
except dodsError as e:
e_mod=" This is an uncommon error. It is likely to be FATAL."
self.close()
print e.value+e_mod
return
def retrieve_variables(self, output, zlib=False):
#open and record:
try:
self.open_with_error()
for var_name in self.Dataset.variables.keys():
netcdf_utils.replicate_and_copy_variable(output,
self.Dataset,
var_name,
zlib=True)
#netcdf_utils.replicate_netcdf_var(output,self.Dataset,var_name)
#output.variables[var_name][:]=self.Dataset.variables[var_name][:]
self.close()
except dodsError as e:
e_mod = " This is an uncommon error. It is likely to be FATAL."
self.close()
print e.value + e_mod
return
def replicate(self, output, hdf5=None, check_empty=False):
#replicate attributes
netcdf_utils.replicate_netcdf_file(output, self.data_root)
#replicate and copy variables:
for var_name in self.data_root.variables.keys():
netcdf_utils.replicate_and_copy_variable(output,
self.data_root,
var_name,
hdf5=hdf5,
check_empty=check_empty)
if 'soft_links' in self.data_root.groups.keys():
output_grp = netcdf_utils.replicate_group(output, self.data_root,
'soft_links')
netcdf_utils.replicate_netcdf_file(
output_grp, self.data_root.groups['soft_links'])
for var_name in self.data_root.groups['soft_links'].variables.keys(
):
if hdf5 != None:
netcdf_utils.replicate_and_copy_variable(
output_grp,
self.data_root.groups['soft_links'],
var_name,
hdf5=hdf5['soft_links'],
check_empty=check_empty)
else:
netcdf_utils.replicate_and_copy_variable(
output_grp,
self.data_root.groups['soft_links'],
var_name,
check_empty=check_empty)
return
def record_indices(self,output,data,var):
if data!=None:
#Create descriptive vars:
for other_var in data.variables.keys():
if ( (not 'time' in data.variables[other_var].dimensions) and
(not other_var in output.variables.keys()) ):
netcdf_utils.replicate_and_copy_variable(output,data,other_var)
#CREATE LOOK-UP TABLE:
output_grp=output.groups['soft_links']
output_grp.createDimension('indices',2)
indices=output_grp.createVariable('indices',np.str,('indices',))
indices[0]='path'
indices[1]='time'
#Create main variable:
if data!=None:
netcdf_utils.replicate_netcdf_var(output,data,var,chunksize=-1,zlib=True)
else:
output.createVariable(var,np.float32,('time',),zlib=True)
var_out = output_grp.createVariable(var,np.int32,('time','indices'),zlib=False,fill_value=np.iinfo(np.int32).max)
#Create soft links:
for time_id, time in enumerate(self.time_axis_unique):
var_out[time_id,0]=np.min(self.paths_indices[time==self.time_axis])
var_out[time_id,1]=self.table['indices'][np.logical_and(self.paths_indices==var_out[time_id,0],time==self.time_axis)][0]
if data!=None:
#Create support variables:
for other_var in data.variables.keys():
if ( ('time' in data.variables[other_var].dimensions) and (other_var!=var) and
(not other_var in output.variables.keys()) ):
netcdf_utils.replicate_netcdf_var(output,data,other_var,chunksize=-1,zlib=True)
var_out = output_grp.createVariable(other_var,np.int32,('time','indices'),zlib=False,fill_value=np.iinfo(np.int32).max)
#Create soft links:
for time_id, time in enumerate(self.time_axis_unique):
var_out[time_id,0]=np.min(self.paths_indices[time==self.time_axis])
var_out[time_id,1]=self.table['indices'][np.logical_and(self.paths_indices==var_out[time_id,0],time==self.time_axis)][0]
return
def retrieve(self,
output,
retrieval_function,
options,
semaphores=[],
username=None,
user_pass=None):
#year=None,month=None,day=None, min_year=None,previous=0,next=0,source_dir=None,username=None,user_pass=None):
self.initialize_retrieval()
if 'source_dir' in dir(options) and options.source_dir != None:
#Check if the file has already been retrieved:
self.paths_list, self.file_type_list = retrieval_utils.find_local_file(
options.source_dir, self.data_root.groups['soft_links'])
#Define tree:
self.tree = self.data_root.path.split('/')[1:]
if 'time' in self.data_root.variables.keys():
#Then find time axis, time restriction and which variables to retrieve:
time_axis, time_restriction = self.retrieve_time_axis(options)
#years=year,months=month,days=day,min_year=min_year,previous=previous,next=next)
#Record to output if output is a netCDF4 Dataset:
if (isinstance(output, netCDF4.Dataset)
or isinstance(output, netCDF4.Group)):
if not 'time' in output.dimensions.keys():
netcdf_utils.create_time_axis(output, self.data_root,
time_axis[time_restriction])
#Replicate all the other variables:
for var in set(self.data_root.variables.keys()).difference(
self.retrievable_vars):
if not var in output.variables.keys():
output = netcdf_utils.replicate_and_copy_variable(
output, self.data_root, var)
#output=netcdf_utils.replicate_netcdf_var(output,self.data_root,var)
#output.variables[var][:]=self.data_root.variables[var][:]
#print var_to_retrieve
#import time
#time.sleep(1000)
for var_to_retrieve in self.retrievable_vars:
self.retrieve_variables(retrieval_function,
var_to_retrieve,
time_restriction,
output,
semaphores=semaphores,
username=username,
user_pass=user_pass)
#paths_list,file_type_list,paths_id_list,checksums_list,version_list,
else:
if (isinstance(output, netCDF4.Dataset)
or isinstance(output, netCDF4.Group)):
#for var in set(self.data_root.variables.keys()).difference(self.retrievable_vars):
# if not var in output.variables.keys():
# output=netcdf_utils.replicate_and_copy_variable(output,self.data_root,var)
#Fixed variables. Do not retrieve, just copy:
for var in self.retrievable_vars:
output = netcdf_utils.replicate_and_copy_variable(
output, self.data_root, var)
output.sync()
else:
#Downloading before a complete validate has been performed:
self.retrieve_without_time(retrieval_function,
output,
username=username,
user_pass=user_pass,
semaphores=semaphores)
return