示例#1
0
 def retrieve_variables(self,output,zlib=False):
     #open and record:
     try:
         self.open_with_error()
         for var_name in self.Dataset.variables.keys():
             netcdf_utils.replicate_and_copy_variable(output,self.Dataset,var_name,zlib=True)
             #netcdf_utils.replicate_netcdf_var(output,self.Dataset,var_name)
             #output.variables[var_name][:]=self.Dataset.variables[var_name][:]
         self.close()
     except dodsError as e:
         e_mod=" This is an uncommon error. It is likely to be FATAL."
         self.close()
         print e.value+e_mod
     return
示例#2
0
 def retrieve_variables(self, output, zlib=False):
     #open and record:
     try:
         self.open_with_error()
         for var_name in self.Dataset.variables.keys():
             netcdf_utils.replicate_and_copy_variable(output,
                                                      self.Dataset,
                                                      var_name,
                                                      zlib=True)
             #netcdf_utils.replicate_netcdf_var(output,self.Dataset,var_name)
             #output.variables[var_name][:]=self.Dataset.variables[var_name][:]
         self.close()
     except dodsError as e:
         e_mod = " This is an uncommon error. It is likely to be FATAL."
         self.close()
         print e.value + e_mod
     return
示例#3
0
 def replicate(self, output, hdf5=None, check_empty=False):
     #replicate attributes
     netcdf_utils.replicate_netcdf_file(output, self.data_root)
     #replicate and copy variables:
     for var_name in self.data_root.variables.keys():
         netcdf_utils.replicate_and_copy_variable(output,
                                                  self.data_root,
                                                  var_name,
                                                  hdf5=hdf5,
                                                  check_empty=check_empty)
     if 'soft_links' in self.data_root.groups.keys():
         output_grp = netcdf_utils.replicate_group(output, self.data_root,
                                                   'soft_links')
         netcdf_utils.replicate_netcdf_file(
             output_grp, self.data_root.groups['soft_links'])
         for var_name in self.data_root.groups['soft_links'].variables.keys(
         ):
             if hdf5 != None:
                 netcdf_utils.replicate_and_copy_variable(
                     output_grp,
                     self.data_root.groups['soft_links'],
                     var_name,
                     hdf5=hdf5['soft_links'],
                     check_empty=check_empty)
             else:
                 netcdf_utils.replicate_and_copy_variable(
                     output_grp,
                     self.data_root.groups['soft_links'],
                     var_name,
                     check_empty=check_empty)
     return
示例#4
0
    def record_indices(self,output,data,var):
        if data!=None:
            #Create descriptive vars:
            for other_var in data.variables.keys():
                if ( (not 'time' in data.variables[other_var].dimensions) and 
                     (not other_var in output.variables.keys()) ):
                    netcdf_utils.replicate_and_copy_variable(output,data,other_var)

        #CREATE LOOK-UP TABLE:
        output_grp=output.groups['soft_links']
        output_grp.createDimension('indices',2)
        indices=output_grp.createVariable('indices',np.str,('indices',))
        indices[0]='path'
        indices[1]='time'

        #Create main variable:
        if data!=None:
            netcdf_utils.replicate_netcdf_var(output,data,var,chunksize=-1,zlib=True)
        else:
            output.createVariable(var,np.float32,('time',),zlib=True)

        var_out = output_grp.createVariable(var,np.int32,('time','indices'),zlib=False,fill_value=np.iinfo(np.int32).max)
        #Create soft links:
        for time_id, time in enumerate(self.time_axis_unique):
            var_out[time_id,0]=np.min(self.paths_indices[time==self.time_axis])
            var_out[time_id,1]=self.table['indices'][np.logical_and(self.paths_indices==var_out[time_id,0],time==self.time_axis)][0]

        if data!=None:
            #Create support variables:
            for other_var in data.variables.keys():
                if ( ('time' in data.variables[other_var].dimensions) and (other_var!=var) and
                     (not other_var in output.variables.keys()) ):
                    netcdf_utils.replicate_netcdf_var(output,data,other_var,chunksize=-1,zlib=True)
                    var_out = output_grp.createVariable(other_var,np.int32,('time','indices'),zlib=False,fill_value=np.iinfo(np.int32).max)
                    #Create soft links:
                    for time_id, time in enumerate(self.time_axis_unique):
                        var_out[time_id,0]=np.min(self.paths_indices[time==self.time_axis])
                        var_out[time_id,1]=self.table['indices'][np.logical_and(self.paths_indices==var_out[time_id,0],time==self.time_axis)][0]
        return
示例#5
0
    def retrieve(self,
                 output,
                 retrieval_function,
                 options,
                 semaphores=[],
                 username=None,
                 user_pass=None):
        #year=None,month=None,day=None, min_year=None,previous=0,next=0,source_dir=None,username=None,user_pass=None):

        self.initialize_retrieval()
        if 'source_dir' in dir(options) and options.source_dir != None:
            #Check if the file has already been retrieved:
            self.paths_list, self.file_type_list = retrieval_utils.find_local_file(
                options.source_dir, self.data_root.groups['soft_links'])

        #Define tree:
        self.tree = self.data_root.path.split('/')[1:]

        if 'time' in self.data_root.variables.keys():
            #Then find time axis, time restriction and which variables to retrieve:
            time_axis, time_restriction = self.retrieve_time_axis(options)
            #years=year,months=month,days=day,min_year=min_year,previous=previous,next=next)

            #Record to output if output is a netCDF4 Dataset:
            if (isinstance(output, netCDF4.Dataset)
                    or isinstance(output, netCDF4.Group)):

                if not 'time' in output.dimensions.keys():
                    netcdf_utils.create_time_axis(output, self.data_root,
                                                  time_axis[time_restriction])

                #Replicate all the other variables:
                for var in set(self.data_root.variables.keys()).difference(
                        self.retrievable_vars):
                    if not var in output.variables.keys():
                        output = netcdf_utils.replicate_and_copy_variable(
                            output, self.data_root, var)
                        #output=netcdf_utils.replicate_netcdf_var(output,self.data_root,var)
                        #output.variables[var][:]=self.data_root.variables[var][:]

            #print var_to_retrieve
            #import time
            #time.sleep(1000)
            for var_to_retrieve in self.retrievable_vars:
                self.retrieve_variables(retrieval_function,
                                        var_to_retrieve,
                                        time_restriction,
                                        output,
                                        semaphores=semaphores,
                                        username=username,
                                        user_pass=user_pass)
                #paths_list,file_type_list,paths_id_list,checksums_list,version_list,
        else:
            if (isinstance(output, netCDF4.Dataset)
                    or isinstance(output, netCDF4.Group)):
                #for var in set(self.data_root.variables.keys()).difference(self.retrievable_vars):
                #    if not var in output.variables.keys():
                #        output=netcdf_utils.replicate_and_copy_variable(output,self.data_root,var)
                #Fixed variables. Do not retrieve, just copy:
                for var in self.retrievable_vars:
                    output = netcdf_utils.replicate_and_copy_variable(
                        output, self.data_root, var)
                output.sync()
            else:
                #Downloading before a complete validate has been performed:
                self.retrieve_without_time(retrieval_function,
                                           output,
                                           username=username,
                                           user_pass=user_pass,
                                           semaphores=semaphores)
        return