예제 #1
0
def read_input_file(self, filename, name, info=[]):
    '''

        Read state data from ASCII filename

        Returns:

        thisdata,is_error

        where:

        is_error = (error,error_msg)

        Load data from a file (e.g. BRDF data or parameters data)

        The file format is flat ASCII with a header, and needs to
        be one of the formats appearing in self.headers

        '''
    from eoldas_Lib import set_default_limits,\
        check_limits_valid,quantize_location, sortopt
    try:
        f = open(filename, 'r')
    except:
        return 0,(True,'Failed to open load file %s with call to %s' % \
                  (filename,str('read_input_file')))
    try:
        if f.errors != None:
            error_msg = str(f.errors)
            return 0, (True, error_msg)
    except:
        pass
    # try to read a PARAMETERS file
    find_col = lambda name: np.where(np.array(params) == name)

    # read the first line
    header = f.readline().replace('#', '').split()
    MAGIC = header[0]
    found = False
    nl = 0
    for (k, v) in self.headers.iteritems():
        if MAGIC == v:
            found = True
            nl = 1
            basic = header[1:]
            if k in self.headers_2:
                header2 = f.readline().replace('#', '').split()
                if header2[0] != self.headers_2[k]:
                    found = False
                else:
                    nl = 2
                    extras = header2[1:]
        if found:
            fmt = k
            break

    if nl == 0:
        f.close()
        return 0,(True,'File %s not recognised by %s'\
                  % (filename,str('read_input_file')))
    if 'logger' in self or 'logger' in self.dict():
        self.logger.info("Interpreted format of %s as %s" % (filename, k))
    f.close()
    f = open(filename, 'r')
    [f.readline() for i in xrange(nl)]

    # the limits info is used to only read observations
    # within these limits
    # The size depends on location and should have 3 numbers
    # for each location entry
    try:
        location = self.Name.location
    except:
        try:
            location = self.name.location
        except:
            if fmt == 'BRDF':
                location = ['time']
            else:
                location = 'time row col'.split()
    location = np.array(
        [i.replace('[', '').replace(']', '') for i in location])
    try:
        limits = self.name.qlocation
    except:
        limits = set_default_limits(location)
    try:
        names = np.array(self._state.name.state)
    except:
        try:
            names = np.array(self.name.state)
        except:
            names = ['default']

    limits = np.array(check_limits_valid(limits))

    sd_params = []
    names = np.atleast_1d(names)
    try:
        for i in xrange(len(names)):
            sd_params.append("sd-%s" % names[i])
    except:
        pass
    sd_params = np.array(sd_params)
    if (fmt == 'BRDF' or fmt == 'BRDF-UCL'):
        # unpack the header
        nbands = int(basic[1])
        bands = basic[2:nbands + 2]
        try:
            if self.name.datatype == 'y':
                names = bands
        except:
            names = bands
        sd_params = []
        for i in xrange(len(np.atleast_1d(names))):
            sd_params.append("sd-%s" % names[i])
        sd_params = np.array(sd_params)
        sd = np.zeros(sd_params.shape[0])
        for i in xrange(len(np.atleast_1d(names))):
            this = np.where(np.array(bands) == names[i])[0]
            if this.size:
                sd[i] = float(basic[2 + nbands + this[0]])
        #sd = np.array([float(i) for i in basic[2+nbands:]])
        if fmt == 'BRDF-UCL':
            params = extras
        #location = extras
        else:
            params = ['time']
        nlocation = len(np.atleast_1d(params))
        params.extend("mask vza vaa sza saa".split())
        params.extend(bands)
        if fmt == 'BRDF-UCL':
            params.extend(sd_params)
        params = np.array(params)
        #names = bands
    else:
        params = basic
        sd = np.zeros_like(names).astype(float)

    # check to see if any location information given
    # loop over self._state.name.location and see which
    # columns appear in params
    loccols = []
    for i in xrange(len(np.atleast_1d(location))):
        ccc = find_col(location[i])
        if len(np.atleast_1d(ccc)):
            loccols.append(ccc[0])
        else:
            loccols.append(0)
    # now do the same for control
    controlcols = []
    try:
        control = self.name.control
    except:
        try:
            control = self.Name.control
        except:
            control = 'mask vza vaa sza saa'.split()
    try:
        if len(np.atleast_1d(control)) == 0:
            control = np.array("mask".split())
    except:
        if control.size == 0:
            control = np.array("mask".split())
    control = control.reshape(control.size)
    #strip out superflous brackets
    control = np.array([i.replace('[', '').replace(']', '') for i in control])
    for i in xrange(control.size):
        ccc = find_col(control[i])
        if len(np.atleast_1d(ccc)):
            controlcols.append(ccc[0])
    # if the datatype is y, then we get the names from the file
    # which we suppose by default to be anything
    # other than options & control
    # but first we see if we can find anything defined in names
    # now for the bands
    wnames = [find_col(i) for i in names]
    # and sd
    wsdnames = [find_col(i) for i in sd_params]
    have_names = False
    # check to see if any names data found
    nnames = np.array([np.array(i).size for i in wnames]).sum()
    if nnames ==0 and (self.datatype == None or \
                                 self.datatype[0] == 'y'):
        # we found no names so check datatype is None or y & guess the
        # names from the params fields that arent used as control or location
        names = []
        sd_params = []
        p_orig = params
        wnames = []
        wsdnames = []
        for i in xrange(len(np.atleast_1d(p_orig))):
            taken = False
            params = control
            taken = taken or \
                    bool(np.array(find_col(p_orig[i])).flatten().shape[0])
            params = location
            taken = taken or \
                    bool(np.array(find_col(p_orig[i])).flatten().shape[0])
            params = names
            taken = taken or \
                    bool(np.array(find_col(p_orig[i])).flatten().shape[0])
            params = sd_params
            taken = taken or \
                    bool(np.array(find_col(p_orig[i])).flatten().shape[0])
            if not taken:
                names.append(p_orig[i])
                sd_params.append("sd-%s" % p_orig[i])
                params = p_orig
                wnames.append(find_col(names[-1]))
                wsdnames.append(find_col(sd_params[-1]))
        params = p_orig
    data = f.readlines()
    f.close()
    # check to see if there is a mask column
    is_mask = 'mask' in params
    want_mask = True or 'mask' in control

    # so we need a grid to stroe the data
    # [p,t,r,c ...] or similar
    # the total datasize will be len(data) * (len(names) + len(location))
    # set to nan ... but we'll return a mask later
    grid = []
    locations = []
    qlocations = []
    controls = []
    sd2 = []
    maxi = [(limits[i, 1] - limits[i, 0] * 1.) / limits[i, 2]
            for i in xrange(len(np.atleast_1d(limits)))]
    for i in xrange(len(np.atleast_1d(data))):
        ok = True
        liner = data[i].split()
        get_col = lambda index, liner: float(
            len(np.atleast_1d(index)) and liner[index])
        ldata = []
        for c in xrange(len(np.atleast_1d(location))):
            ldata.append(get_col(loccols[c], liner))
        qldata = quantize_location(ldata, limits)
        if (np.array(qldata) < 0).all() or (maxi - np.array(qldata) < 0).all():
            ok = False
        cdata = []
        for c in xrange(len(np.atleast_1d(controlcols))):
            if want_mask and not is_mask and \
   (control[c] == 'mask' or control[c] == '[mask]'):
                cdata.append(1)
            else:
                cdata.append(get_col(controlcols[c], liner))

# check the mask value
        try:
            if not (want_mask and not is_mask):
                c = np.where(control == 'mask')[0]
                if c.size == 0: c = np.where(control == '[mask]')[0]
                if c.size == 0:
                    ok = True
                elif int(cdata[c]) != 1:
                    ok = False
        except:
            ok = True
            cdata.append(1)
        if ok:
            this = np.zeros(len(np.atleast_1d(names)))
            this[:] = None
            # this will set unread fields to nan
            for (j, k) in enumerate(wnames):
                if np.array(k).size > 0:
                    this[j] = float(liner[k[0]])
            that = np.zeros(len(np.atleast_1d(names)))
            that[:] = None
            # this will set unread fields to nan
            for (j, k) in enumerate(wsdnames):
                if np.array(k[0]).shape[0] > 0:
                    that[j] = float(liner[k[0]])
            locations.extend(ldata)
            controls.append(cdata)
            qlocations.append(qldata)
            grid.append(this)
            sd2.append(that)
    # check to see if the sd data are any good
    sd2a = np.array(sd2)
    if sd2a.flatten().sum() > 0:
        sd = sd2a
    n_samples = len(np.atleast_1d(data))
    data = {}
    name = {}
    data['state'] = np.array(grid)
    nsamples = data['state'].shape[0]
    if not 'datatype' in self.name.dict() or self.name.datatype == None or \
                                            self.name.datatype[0] == 'y':
        # its a y or its a bit broken
        name['state'] = np.array(names)
    # note, the state list can be updated
    # by what it finds, but only for y states
    name['fmt'] = fmt
    name['location'] = np.array(location)
    nlocations = name['location'].shape[0]
    data['location'] = np.array(locations).reshape(nsamples, nlocations)
    name['location'] = np.array(location)
    name['qlocation'] = np.array(limits)
    #orig = np.repeat(np.array(name['qlocation'][:,0]),nsamples).reshape(nlocations,nsamples).T
    data['qlocation'] = np.array(qlocations).reshape(nsamples,
                                                     nlocations)  #+ orig
    name['qlocation'] = np.array(limits)
    name['control'] = np.array(control)
    ncontrol = np.max((1, name['control'].shape[0]))
    if name['control'].shape[0]:
        data['control'] = np.array(controls).reshape(nsamples, ncontrol)
    else:
        data['control'] = np.array(controls)
    # only return sd if its > 0
    if sd.size != data['state'].size:
        try:
            sd = np.tile(np.array(sd), nsamples).reshape(data['state'].shape)
        except:
            self.logger.info("can't tile sd data: %s" % str(sd))
        sd = np.array([0.])
    if sd.flatten().sum() > 0:
        name['sd'] = np.array(names)
        data['sd'] = sd
    datasets = {'data': data, 'name': name}
    return datasets,(False,'Data read from %s with %s fmt %s'% \
                     (filename,str('read_input_file'),fmt))
예제 #2
0
def read_numpy(self, filename, name, info=[]):
    '''
        Try to read the file as as a NpzFile file
        '''
    from eoldas_Lib import set_default_limits,check_limits_valid,\
        quantize_location,dequantize_location

    # none of these ciritical to functioning
    try:
        info = self._state.info
    except:
        info = []
    try:
        names = self.name.state
    except:
        try:
            names = self.Name.state
        except:
            names = None
    try:
        control = self.Name.control
    except:
        try:
            control = self.name.control
        except:
            control = None
    try:
        location = self.name.location
    except:
        try:
            location = self.Name.location
        except:
            location = ['time', 'row', 'col']
    try:
        limits = self.name.qlocation
    except:
        try:
            limits = self.Name.qlocation
        except:
            limits = set_default_limits(location)
    # refl_check=False,names=None,\
    # control=['mask','vza','vaa','sza','saa'],\
    # location=['time','row','col'],limits=None

    # location specifies the dimesions and names of the
    # problem, e.g., & typically [time,row,col]
    limits = np.array(check_limits_valid(limits))

    try:
        f = np.load(filename)
        if not type(f).__name__ == 'NpzFile':
            f.close()
            self.error_msg = "%s is not a NpzFile" % filename
            self.error = True
            if 'logger' in self or 'logger' in self.dict():
                self.logger.info(self.error_msg)
            return 0, (self.error, self.error_msg)
    except:
        self.error_msg = "a problem opening %s as a NpzFile" % filename
        self.error = True
        if 'logger' in self or 'logger' in self.dict():
            self.logger.info(self.error_msg)
        return 0, (self.error, self.error_msg)
    # ok so far then
    # lets have a look inside

    ncontents = np.array(f.files)
    contents = np.array(f.files)
    # translation table for default names
    def_names = 'b1 b2 b3 b4 b5 b6 b7'.split()
    if names == None:
        # assume MODIS
        names = def_names
    def_alt_names = \
        '645.5 856.5 465.6 553.6 1241.6 1629.1 2114.1'.split()
    # look for any of names in contents
    datasets = []
    alt_datasets = []
    alt_names = names
    for i in xrange(len(np.atleast_1d(contents))):
        if contents[i] in names:
            datasets.append(i)

    if not len(np.atleast_1d(datasets)):
        if 'logger' in self or 'logger' in self.dict():
            self.logger.error(\
                          "None of requested datasets %s found in %s ..." \
                          %(str(names),filename) + \
                          " trying default MODIS names: only %s"\
                          %(str(contents)))
        names = def_names
        alt_names = def_alt_names
        for i in xrange(len(np.atleast_1d(contents))):
            if contents[i] in names:
                datasets.append(i)
        if not len(np.atleast_1d(datasets)):
            self.error_msg = "None of requested datasets %s found in %s"\
                %(str(names),filename) + ' ' + \
                "... trying default MODIS names: only %s"\
                %(str(contents))
            self.error = True
            if 'logger' in self or 'logger' in self.dict():
                self.logger.error(self.error_msg)
            return 0, (self.error, self.error_msg)
    trans_names = {}
    for (i, j) in enumerate(alt_names):
        trans_names[names[i]] = j


#trans_names = {names[i]:j for (i,j) in enumerate(alt_names)}
    alt_name = []
    this_name = []
    for i in datasets:
        this_name.append(contents[i])
        alt_name.append(trans_names[contents[i]])

    # Translate  some old stylies...
    trans = {'raa': 'vaa', 'doys': 'time'}
    for i in trans:
        if i in contents:
            ncontents[np.where(contents == i)[0]] = trans[i]
    # as a minimum, there needs to be some definition of one of
    # the terms in location

    # check how many dimensions this has
    # now find a dataset
    try:
        # This could be more general, but this will do for now as its useful
        # for spatial datasets
        QA_OK = np.array(\
                         [8, 72, 136, 200, 1032, 1288, 2056,2120, 2184, 2248])
        doy = f['doys'] - 2004000
        qa = f['qa']
        vza = f['vza']
        sza = f['sza']
        raa = f['raa']
        y = []
        for i in this_name:
            y.append(f[i])
        #mask = np.logical_or.reduce([qa==x for x in QA_OK ])
        if 'logger' in self or 'logger' in self.dict():
            self.logger.info(\
                         "sucessfully interpreted NpzFile dataset from %s"\
                             %filename)
            self.logger.info("sub-setting ...")
        controls = []
        locations = []
        grid = []
        qlocations = []
        thisshape = vza.shape
        starter = {'time': np.min(doy), 'row': 0, 'col': 0}
        delta = {'time': 1, 'row': 1, 'col': 1}
        if len(np.atleast_1d(limits)) < 3:
            from eoldas_Lib import set_default_limits
            old_loc = location
            location = np.array(['time', 'row', 'col'])
            lim2 = set_default_limits(location)
            for i in xrange(len(np.atleast_1d(limits))):
                ww = np.where(old_loc[i] == location)[0]
                lim2[ww] = list(limits[i])
            limits = lim2
        for i in xrange(len(np.atleast_1d(limits))):
            if limits[i][0] == None:
                limits[i][0] = starter[location[i]]
            if limits[i][1] == None:
                limits[i][1] = (thisshape[i] - 1) + starter[location[i]]
            if limits[i][2] == None:
                limits[i][2] = delta[location[i]]
        limits = np.array(limits)
        start_doy = limits[0][0]
        end_doy = limits[0][1]
        step_doy = limits[0][2]
        start_row = limits[1][0]
        end_row = limits[1][1]
        step_row = limits[1][2]
        start_col = limits[2][0]
        end_col = limits[2][1]
        step_col = limits[2][2]
        gooddays = np.logical_and.reduce(np.concatenate(\
                            ([doy >= start_doy],[doy  <=end_doy])))
        qa = qa[gooddays, start_row:end_row + 1, start_col:end_col + 1]
        vza = vza[gooddays, start_row:end_row + 1,
                  start_col:end_col + 1] * 0.01
        sza = sza[gooddays, start_row:end_row + 1,
                  start_col:end_col + 1] * 0.01
        raa = raa[gooddays, start_row:end_row + 1,
                  start_col:end_col + 1] * 0.01
        yy = []
        for i in xrange(len(np.atleast_1d(this_name))):
            this = y[i]
            yy.append(this[gooddays,start_row:end_row+1,\
                           start_col:end_col+1]*0.0001)
        doy = doy[gooddays]
        # now do QA
        mask = np.zeros_like(qa).astype(bool)
        # loop over qa
        for j in xrange(len(np.atleast_1d(QA_OK))):
            ww = np.where(qa == QA_OK[j])
            mask[ww] = True
        # better look over data to check valid
        for j in xrange(len(np.atleast_1d(yy))):
            ww = np.where(yy[j] < 0)
            mask[ww] = False
        ww = np.where(mask)
        if 'logger' in self or 'logger' in self.dict():
            self.logger.debug('parsing dataset: %d samples look ok'\
                          %np.array(ww).shape[1])
        vza = vza[ww]
        sza = sza[ww]
        raa = raa[ww]
        doy = doy[ww[0]]
        row = ww[1] + start_row
        col = ww[2] + start_col
        locations = np.array([doy, row, col])
        nnn = len(np.atleast_1d(locations[0]))
        orig = np.repeat(np.array([start_doy, start_row, start_col]),
                         locations.shape[1]).reshape(locations.shape).T
        div = np.repeat(np.array([step_doy, step_row, step_col]),
                        locations.shape[1]).reshape(locations.shape).T
        qlocations = ((locations.T - orig) / div.astype(float)).astype(int).T
        controls = np.array([np.ones_like(doy).astype(bool),\
                             vza,raa,sza,0*doy])
        y = []
        for i in xrange(len(np.atleast_1d(this_name))):
            this = yy[i]
            y.append(this[ww])
        grid = np.array(y)
        fmt = 'BRDF-UCL'
        control = ['mask', 'vza', 'vaa', 'sza', 'saa']
        bands = alt_name
        if not np.array(grid).size:
            if 'logger' in self or 'logger' in self.dict():
                self.logger.error(\
                              "Warning: returning a zero-sized dataset ... "+\
                              " I wouldn;t try to do anything with it")
        # in case we dont have data for all bands
        mask =  np.logical_or.reduce([[this_name[i]==x for x in names] \
                                      for i in xrange(len(np.atleast_1d(this_name)))])
        sd = np.array('0.004 0.015 0.003 0.004 0.013 0.01 0.006'\
                      .split())[mask]
        sd = np.array([float(i) for i in sd.flatten()])\
            .reshape(sd.shape)
        nsamps = grid.shape[1]
        sd = sd.repeat(nsamps).reshape(grid.shape).T
        datasets = ParamStorage()
        datasets.data = ParamStorage()
        datasets.name = ParamStorage()
        datasets.name.fmt = fmt
        grid = grid.T
        datasets.data[name] = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\
                                                        .astype(object)
        datasets.data[name][:, :] = None
        for i in xrange(len(np.atleast_1d(this_name))):
            ww = np.where(names == this_name[i])[0][0]
            datasets.data[name][:, ww] = grid[:, i]
        datasets.data.location = np.array(locations).T
        datasets.data.control = np.array(controls).T
        datasets.data.qlocation = np.array(qlocations).T
        datasets.name[name] = np.array(names)
        datasets.name.location = np.array(['time', 'row', 'col'])
        datasets.name.control = np.array(control)
        datasets.name.qlocation = limits
        datasets.name.bands = np.array(bands)
        datasets.data.sd = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\
                                                        .astype(object)
        # for i in xrange(grid.shape[0]):
        # datasets.data.sd[i,:] = self.options.sd
        datasets.data.sd[:, :] = None
        for i in xrange(len(np.atleast_1d(this_name))):
            ww = np.where(names == this_name[i])[0][0]
            datasets.data.sd[:, ww] = sd[:, i]
        datasets.name.sd = np.array(names)
        if 'logger' in self or 'logger' in self.dict():
            self.logger.debug('finished parsing dataset')
    except:
        self.error_msg=\
            "a problem processing information from  %s as a NpzFile"\
                %filename
        self.error = True
        if 'logger' in self or 'logger' in self.dict():
            self.logger.info(self.error_msg)
        return 0, (self.error, self.error_msg)
    f.close()
    if 'logger' in self or 'logger' in self.dict():
        self.logger.info('... done')
    self.error = False
    self.error_msg = ""
    return datasets, (self.error, self.error_msg)
예제 #3
0
    def reinit(self,options,names=None,datatype=None,limits=None,\
            bounds=None,control=None,location=None,env=None,header=None,\
               logdir=None,writers={},grid=False,logger=None,\
            datadir=None,logfile=None,name=None,info=[],readers=[],debug=None):
        '''
        Method to re-initialise the class instance
            
        The setup is on the whole controlled by the datatype which
            contains e.g. 'x'. This is used to set up the members
            self.x and self.y as SpecialVariables (see SpecialVariable
            in eoldas_SpecialVariable.py). There are some special attributes
            for datatypes starting with 'y'. These are assumed to be 
            observational data, which means that when they are read, the 
            data names associated with them are not limited to those in 
            self.names but rather set to whatever is read in in the data. This 
            is because the data names for observational data may be terms such 
            as waveband names etc that need special interpretation. Also,
            the default output format for observational data is 
            different to that of other data.  
            
        The elements self.state is a SpecialVariables which means 
            that they can be assigned various data types (see SpecialVariables)
            and loaded accordingly (e.g. if a filename is specified, this is 
            read in to the data structure. The SpecialVariables contain 'hidden'
            datasets, which here are mainly the 'control' and 'location'
            information. A SpecialVariable has two internal structures: `data`
            and `name`. The former is used to store data values (e.g. the 
            state values) and the latter to store associated metadata.
            For example, `control` is passed here e.g. as [`mask`,`vza`] and this
            gives the metadata that are stored in `name`. The actual values
            of the control data are stored in the `data` section. For
            location, we might be passed [`time`,`row`,`col`], so this is set
            in names.location, and the data.location contains the values
            of the location at each of these elements. For the actual state
            dataset, this is stored according to its name, so for `x` the values
            are stored in data.x and the associated data names in name.x.
        
        State datasets must represent at least the mean and standard deviation
            of a state for them to be of value in EOLDAS. TThe mean is accessed
            as e.g. self.state for the state dataset. 
            The sd is accessed can be accessed as self._state.sd if it 
            has been set. This reference can also be used to directly set 
            data associated with a SpecialVariable, e.g.
            self.Data.control = np.zeros([2,3])
            
            to represent 2 samples with 3 control variables. You can access name
            information similarly with
            
            print self.Name.control
            
            but this will generate a KeyError if the term has not been set. You
            can check it exists with:
            
            key = 'control'
            if key in self.Name:
                this = (self.Data[key],self.Name[key])
            
            To get e.g. a dictionary representation of a SpecialVariable
            you can use eg:
            
                self.Name.to_dict()
            
            to get the name dictionary, or
            
                thisdict = self._state.to_dict()
            
            to get the full representation, which then contains 'data' and
            'name' as well as some other information stored in the 
            SpecialVariable.
            
            You can similarly load them using e.g. 
            
                self.Data.update(
                        ParamStorage().from_dict(thisdict['data'])
                        combine=True)
        
        '''

        # set up a fakes dictionary from the data types
        self.set('datatype',datatype)
        self.set('fakes', {'state':'_state'})
        
        # first check that options is sensible
        self.__check_type(options,ParamStorage,fatal=True)
        self.options = options
       
        from eoldas_Lib import set_default_limits,\
                    check_limits_valid,quantize_location, sortopt
   
        nSpecial = 1
        if name == None:
           import time
           thistime = str(time.time())
           name = type(self).__name__
           name =  "%s.%s" % (name,thistime)
        self.thisname = name
        self.options.thisname = str(name).replace(' ','_')
        

        log_terms = {\
            'logfile':logfile or sortopt(self.options,'logfile',None),\
            'logdir':logdir or sortopt(self.options,'logdir',None),\
            'debug' : debug or sortopt(self.options,'debug',True)}
        
        self.datadir = datadir or sortopt(self.options,'datadir',["."])
        self.header = header or "EOLDAS pickle V1.0 - plewis"
        env = env or sortopt(self.options,'env',None)
        names = names or sortopt(self.options,'names',None)
        location = location or sortopt(self.options,'location',['time'])
        control = control or sortopt(self.options,'control',[])
        limits = limits or sortopt(self.options,'limits',\
                        set_default_limits(np.array(location)))
        limits = limits or self.options.limits
        limits = np.array(check_limits_valid(limits))
        bounds = bounds or sortopt(self.options,'bounds',\
                    [[None,None]] * xlen(names))
        self.options.bounds = bounds
       
        self.headers = {'PARAMETERS-V2':"PARAMETERS-V2", \
            'PARAMETERS':"PARAMETERS", \
            'BRDF-UCL':'BRDF-UCL',\
            'BRDF': 'BRDF'}
        self.headers_2 = {'BRDF-UCL':'location'}

        #  The ones pre-loaded are
        # self.read_functions = [self.read_pickle,self.read_numpy_fromfile]
        self._state = SpecialVariable(info=info,name=self.thisname,\
                                  readers=readers,datadir=self.datadir,\
                                  env=env,writers=writers,\
                                  header=self.header,\
                                  logger=logger,log_terms=log_terms,\
                                  simple=False)
        # self._state is where data are read into
        # but self.Data and self.Name are where we access them from
        self.grid=grid
        # this is so we can access this object from
        # inside a SpecialVariable
        self.state = np.array([0.])
        # a default data fmt output     
        if datatype[0] == 'y':
            self.Name.fmt = 'BRDF'
            self.Name.state = np.array(['dummy'])
        else:
            self.Name.fmt = 'PARAMETERS'
            n_params = xlen(names)
            if not n_params:
                error_msg = \
                    "The field 'names' must be defined in options or"+ \
                    "passed directly to this method if you have the data type x"
                raise Exception(error_msg)
        self.Name.state = np.array(names)   
        self.Name.location = np.array(location)
        self.Name.control = np.array(control)
        self.Name.header = self.header
        self.Name.bounds = np.array(bounds)
        self.Name.qlocation = np.array(limits)
        self.Name.datadir = datadir
        #
        # sort this object's name 
        # sort logging
        self.logger = sortlog(self,log_terms['logfile'],logger,name=self.thisname,
			logdir=log_terms['logdir'],debug=log_terms['debug'])
        self.logger.info('Initialising %s' % type(self).__name__)
예제 #4
0
def read_input_file(self,filename,name,info=[]):
    '''

        Read state data from ASCII filename

        Returns:

        thisdata,is_error

        where:

        is_error = (error,error_msg)

        Load data from a file (e.g. BRDF data or parameters data)

        The file format is flat ASCII with a header, and needs to
        be one of the formats appearing in self.headers

        '''
    from eoldas_Lib import set_default_limits,\
        check_limits_valid,quantize_location, sortopt
    try:
        f = open(filename,'r')
    except:
        return 0,(True,'Failed to open load file %s with call to %s' % \
                  (filename,str('read_input_file')))
    try:
        if f.errors != None:
            error_msg = str(f.errors)
            return 0,(True,error_msg)
    except:
        pass
    # try to read a PARAMETERS file
    find_col = lambda name :np.where(np.array(params) == name)

    # read the first line
    header = f.readline().replace('#','').split()
    MAGIC = header[0]
    found = False
    nl = 0
    for (k,v) in self.headers.iteritems():
        if MAGIC == v:
            found = True
            nl = 1
            basic = header[1:]
            if k in self.headers_2:
                header2 = f.readline().replace('#','').split()
                if header2[0] != self.headers_2[k]:
                    found = False
                else:
                    nl = 2
                    extras = header2[1:]
        if found:
            fmt = k
            break

    if nl == 0:
        f.close()
        return 0,(True,'File %s not recognised by %s'\
                  % (filename,str('read_input_file')))
    if 'logger' in self or 'logger' in self.dict():
        self.logger.info("Interpreted format of %s as %s"%(filename,k))
    f.close()
    f = open(filename,'r')
    [f.readline() for i in xrange(nl)]


    # the limits info is used to only read observations
    # within these limits
    # The size depends on location and should have 3 numbers
    # for each location entry
    try:
        location = self.Name.location
    except:
        try:
	    location = self.name.location
        except:
            if fmt == 'BRDF':
                location = ['time']
            else:
                location = 'time row col'.split()
    location = np.array([i.replace('[','').replace(']','') for i in location])
    try:
        limits = self.name.qlocation
    except:
        limits = set_default_limits(location)
    try:
        names = np.array(self._state.name.state)
    except:
	try:
	    names = np.array(self.name.state)
	except:
	    names = ['default']

    limits = np.array(check_limits_valid(limits))

    sd_params = []
    names = np.atleast_1d(names)
    try:
        for i in xrange(len(names)):
            sd_params.append("sd-%s"%names[i])
    except:
        pass
    sd_params = np.array(sd_params)
    if (fmt == 'BRDF' or fmt == 'BRDF-UCL'):
        # unpack the header
        nbands = int(basic[1])
        bands = basic[2:nbands+2]
        try:
            if self.name.datatype == 'y':
                names = bands
        except:
            names = bands
        sd_params = []
        for i in xrange(len(np.atleast_1d(names))):
            sd_params.append("sd-%s"%names[i])
        sd_params = np.array(sd_params)
        sd = np.zeros(sd_params.shape[0])
        for i in xrange(len(np.atleast_1d(names))):
            this = np.where(np.array(bands) == names[i])[0]
            if this.size:
                sd[i] = float(basic[2+nbands+this[0]])
        #sd = np.array([float(i) for i in basic[2+nbands:]])
        if fmt == 'BRDF-UCL':
            params = extras
        #location = extras
        else:
            params = ['time']
        nlocation = len(np.atleast_1d(params))
        params.extend("mask vza vaa sza saa".split())
        params.extend(bands)
        if fmt == 'BRDF-UCL':
            params.extend(sd_params)
        params = np.array(params)
            #names = bands
    else:
        params = basic
        sd = np.zeros_like(names).astype(float)

    # check to see if any location information given
    # loop over self._state.name.location and see which
    # columns appear in params
    loccols = []
    for i in xrange(len(np.atleast_1d(location))):
        ccc = find_col(location[i])
        if len(np.atleast_1d(ccc)):
            loccols.append(ccc[0])
        else:
            loccols.append(0)
    # now do the same for control
    controlcols = []
    try:
        control=self.name.control
    except:
        try:
            control=self.Name.control
        except:
            control = 'mask vza vaa sza saa'.split()
    try:
        if len(np.atleast_1d(control)) == 0:
            control = np.array("mask".split())
    except:
        if control.size == 0:
            control = np.array("mask".split())
    control = control.reshape(control.size)
    #strip out superflous brackets
    control = np.array([i.replace('[','').replace(']','') for i in control])
    for i in xrange(control.size):
        ccc = find_col(control[i])
        if len(np.atleast_1d(ccc)):
            controlcols.append(ccc[0])
    # if the datatype is y, then we get the names from the file
    # which we suppose by default to be anything
    # other than options & control
    # but first we see if we can find anything defined in names
    # now for the bands
    wnames = [find_col(i) for i in names]
    # and sd
    wsdnames = [find_col(i) for i in sd_params]
    have_names = False
    # check to see if any names data found
    nnames = np.array([np.array(i).size for i in wnames]).sum()
    if nnames ==0 and (self.datatype == None or \
                                 self.datatype[0] == 'y'):
        # we found no names so check datatype is None or y & guess the 
        # names from the params fields that arent used as control or location
        names = []
        sd_params = []
        p_orig = params
        wnames = []
        wsdnames = []
        for i in xrange(len(np.atleast_1d(p_orig))):
            taken = False
            params = control 
            taken = taken or \
                    bool(np.array(find_col(p_orig[i])).flatten().shape[0])
            params = location
            taken = taken or \
                    bool(np.array(find_col(p_orig[i])).flatten().shape[0])
            params = names
            taken = taken or \
                    bool(np.array(find_col(p_orig[i])).flatten().shape[0])
            params = sd_params
            taken = taken or \
                    bool(np.array(find_col(p_orig[i])).flatten().shape[0])
            if not taken:
                names.append(p_orig[i])
                sd_params.append("sd-%s"%p_orig[i])
                params = p_orig
                wnames.append(find_col(names[-1]))
                wsdnames.append(find_col(sd_params[-1]))
        params = p_orig
    data = f.readlines()
    f.close()
    # check to see if there is a mask column
    is_mask = 'mask' in params
    want_mask = True or  'mask' in control

    # so we need a grid to stroe the data
    # [p,t,r,c ...] or similar
    # the total datasize will be len(data) * (len(names) + len(location))
    # set to nan ... but we'll return a mask later
    grid = []
    locations = []
    qlocations = []
    controls = []
    sd2 = []
    maxi = [(limits[i,1]-limits[i,0]*1.)/limits[i,2] for i in xrange(len(np.atleast_1d(limits)))]
    for i in xrange(len(np.atleast_1d(data))):
        ok = True
        liner = data[i].split()
        get_col = lambda index,liner : float(len(np.atleast_1d(index)) and liner[index])
        ldata = []
        for c in xrange(len(np.atleast_1d(location))):
            ldata.append(get_col(loccols[c],liner))
        qldata = quantize_location(ldata,limits)
        if (np.array(qldata) < 0).all() or (maxi - np.array(qldata) < 0).all():
	    ok = False
        cdata = []
        for c in xrange(len(np.atleast_1d(controlcols))):
            if want_mask and not is_mask and \
			(control[c] == 'mask' or control[c] == '[mask]'):
                cdata.append(1)
            else:
                cdata.append(get_col(controlcols[c],liner))
	# check the mask value
        try:
	    if not (want_mask and not is_mask):
	        c = np.where(control=='mask')[0]
	        if c.size == 0: c = np.where(control=='[mask]')[0]
                if c.size == 0:
	 	    ok = True
	        elif int(cdata[c]) != 1:
	            ok = False  
        except:
	    ok = True
            cdata.append(1)  
        if ok:
 	    this = np.zeros(len(np.atleast_1d(names)))
            this[:] = None
            # this will set unread fields to nan
            for (j,k) in enumerate(wnames):
                if np.array(k).size >0:
                    this[j] = float(liner[k[0]])
            that = np.zeros(len(np.atleast_1d(names)))
            that[:] = None
            # this will set unread fields to nan
            for (j,k) in enumerate(wsdnames):
                if np.array(k[0]).shape[0] > 0:
                    that[j] = float(liner[k[0]])
            locations.extend(ldata)
            controls.append(cdata)
            qlocations.append(qldata)
            grid.append(this)
            sd2.append(that)
    # check to see if the sd data are any good
    sd2a = np.array(sd2)
    if sd2a.flatten().sum() > 0:
        sd = sd2a
    n_samples = len(np.atleast_1d(data))
    data = {}
    name = {}
    data['state'] = np.array(grid)
    nsamples = data['state'].shape[0]
    if not 'datatype' in self.name.dict() or self.name.datatype == None or \
                                            self.name.datatype[0] == 'y':
        # its a y or its a bit broken
        name['state'] = np.array(names)
    # note, the state list can be updated
    # by what it finds, but only for y states
    name['fmt'] = fmt
    name['location'] = np.array(location)
    nlocations = name['location'].shape[0]
    data['location'] = np.array(locations).reshape(nsamples,nlocations)
    name['location'] = np.array(location)
    name['qlocation'] = np.array(limits)
    #orig = np.repeat(np.array(name['qlocation'][:,0]),nsamples).reshape(nlocations,nsamples).T
    data['qlocation'] = np.array(qlocations).reshape(nsamples,nlocations) #+ orig
    name['qlocation'] = np.array(limits)
    name['control'] =  np.array(control)
    ncontrol = np.max((1,name['control'].shape[0]))
    if name['control'].shape[0]:
        data['control'] =  np.array(controls).reshape(nsamples,ncontrol)
    else:
        data['control'] =  np.array(controls)
    # only return sd if its > 0
    if sd.size != data['state'].size:
	try:
            sd = np.tile(np.array(sd),nsamples).reshape(data['state'].shape)
	except:
	    self.logger.info("can't tile sd data: %s"%str(sd))
	sd = np.array([0.])
    if sd.flatten().sum() > 0:
        name['sd'] = np.array(names)
        data['sd'] = sd
    datasets = {'data':data,'name':name}
    return datasets,(False,'Data read from %s with %s fmt %s'% \
                     (filename,str('read_input_file'),fmt))
예제 #5
0
def read_numpy(self,filename,name,info=[]):
    '''
        Try to read the file as as a NpzFile file
        '''
    from eoldas_Lib import set_default_limits,check_limits_valid,\
        quantize_location,dequantize_location
    
    # none of these ciritical to functioning
    try:
        info = self._state.info
    except:
        info = []
    try:
        names = self.name.state
    except:
        try:
            names = self.Name.state
        except:
            names = None
    try:
        control = self.Name.control
    except:
        try:
            control = self.name.control
        except:
            control = None
    try:
        location = self.name.location
    except:
        try:
            location = self.Name.location
        except:
            location = ['time','row','col']
    try:
        limits = self.name.qlocation
    except:
        try:
            limits = self.Name.qlocation
        except:
            limits = set_default_limits(location)
    # refl_check=False,names=None,\
    # control=['mask','vza','vaa','sza','saa'],\
    # location=['time','row','col'],limits=None
    
    # location specifies the dimesions and names of the
    # problem, e.g., & typically [time,row,col]
    limits = np.array(check_limits_valid(limits))
    
    try:
        f = np.load(filename)
        if not type(f).__name__ == 'NpzFile':
            f.close()
            self.error_msg="%s is not a NpzFile"%filename
            self.error=True
            if 'logger' in self or 'logger' in self.dict():
                self.logger.info(self.error_msg)
            return 0,(self.error,self.error_msg)
    except:
        self.error_msg="a problem opening %s as a NpzFile"%filename
        self.error=True
        if 'logger' in self or 'logger' in self.dict():
            self.logger.info(self.error_msg)
        return 0,(self.error,self.error_msg)
    # ok so far then
    # lets have a look inside
    
    ncontents = np.array(f.files)  
    contents = np.array(f.files)
    # translation table for default names
    def_names = 'b1 b2 b3 b4 b5 b6 b7'.split()
    if names == None:
        # assume MODIS
        names = def_names
    def_alt_names = \
        '645.5 856.5 465.6 553.6 1241.6 1629.1 2114.1'.split()
    # look for any of names in contents
    datasets = []
    alt_datasets = []
    alt_names = names
    for i in xrange(len(np.atleast_1d(contents))):
        if contents[i] in names:
            datasets.append(i)
    
    if not len(np.atleast_1d(datasets)):
        if 'logger' in self or 'logger' in self.dict():
            self.logger.error(\
                          "None of requested datasets %s found in %s ..." \
                          %(str(names),filename) + \
                          " trying default MODIS names: only %s"\
                          %(str(contents)))
        names = def_names
        alt_names = def_alt_names
        for i in xrange(len(np.atleast_1d(contents))):
            if contents[i] in names:
                datasets.append(i)
        if not len(np.atleast_1d(datasets)):
            self.error_msg = "None of requested datasets %s found in %s"\
                %(str(names),filename) + ' ' + \
                "... trying default MODIS names: only %s"\
                %(str(contents))
            self.error = True
            if 'logger' in self or 'logger' in self.dict():
                self.logger.error(self.error_msg)
            return 0,(self.error,self.error_msg)
    trans_names = {}
    for (i,j) in enumerate(alt_names):
        trans_names[names[i]] = j
#trans_names = {names[i]:j for (i,j) in enumerate(alt_names)}
    alt_name = []
    this_name = []
    for i in datasets:
        this_name.append(contents[i])
        alt_name.append(trans_names[contents[i]])
    
    # Translate  some old stylies...
    trans = {'raa':'vaa','doys':'time'}
    for i in trans:
        if i in contents:
            ncontents[np.where(contents==i)[0]]=trans[i]
    # as a minimum, there needs to be some definition of one of
    # the terms in location

    # check how many dimensions this has
    # now find a dataset
    try:
        # This could be more general, but this will do for now as its useful
        # for spatial datasets
        QA_OK = np.array(\
                         [8, 72, 136, 200, 1032, 1288, 2056,2120, 2184, 2248])
        doy = f['doys'] - 2004000
        qa = f['qa']
        vza = f['vza']
        sza = f['sza']
        raa = f['raa']
        y = []
        for i in this_name:
            y.append(f[i])
        #mask = np.logical_or.reduce([qa==x for x in QA_OK ])
        if 'logger' in self or 'logger' in self.dict():
            self.logger.info(\
                         "sucessfully interpreted NpzFile dataset from %s"\
                             %filename)
            self.logger.info("sub-setting ...")
        controls = []
        locations = []
        grid = []
        qlocations = []
        thisshape = vza.shape
        starter = {'time':np.min(doy),'row':0,'col':0}
        delta = {'time':1,'row':1,'col':1}
        if len(np.atleast_1d(limits)) <3:
            from eoldas_Lib import set_default_limits
            old_loc = location
            location = np.array(['time','row','col'])
            lim2 = set_default_limits(location)
            for i in xrange(len(np.atleast_1d(limits))):
                ww = np.where(old_loc[i] == location)[0]
                lim2[ww] = list(limits[i])
            limits = lim2
        for i in xrange(len(np.atleast_1d(limits))):
            if limits[i][0] == None:
                limits[i][0] = starter[location[i]]
            if limits[i][1] == None:
                limits[i][1] = (thisshape[i]-1) + starter[location[i]]
            if limits[i][2] == None:
                limits[i][2]= delta[location[i]]
        limits = np.array(limits)
        start_doy = limits[0][0]
        end_doy =   limits[0][1]
        step_doy =  limits[0][2]
        start_row = limits[1][0]
        end_row =   limits[1][1]
        step_row =  limits[1][2]
        start_col = limits[2][0]
        end_col =   limits[2][1]
        step_col =  limits[2][2]
        gooddays = np.logical_and.reduce(np.concatenate(\
                            ([doy >= start_doy],[doy  <=end_doy])))
        qa = qa[gooddays,start_row:end_row+1,start_col:end_col+1]
        vza = vza[gooddays,start_row:end_row+1,start_col:end_col+1]*0.01
        sza = sza[gooddays,start_row:end_row+1,start_col:end_col+1]*0.01
        raa = raa[gooddays,start_row:end_row+1,start_col:end_col+1]*0.01
        yy = []
        for i in xrange(len(np.atleast_1d(this_name))):
            this = y[i]
            yy.append(this[gooddays,start_row:end_row+1,\
                           start_col:end_col+1]*0.0001)
        doy = doy[gooddays]
        # now do QA
        mask = np.zeros_like(qa).astype(bool)
        # loop over qa
        for j in xrange(len(np.atleast_1d(QA_OK))):
            ww = np.where(qa==QA_OK[j])
            mask[ww] = True
        # better look over data to check valid
        for j in xrange(len(np.atleast_1d(yy))):
            ww = np.where(yy[j] < 0)
            mask[ww] = False
        ww = np.where(mask)
        if 'logger' in self or 'logger' in self.dict():
            self.logger.debug('parsing dataset: %d samples look ok'\
                          %np.array(ww).shape[1]) 
        vza = vza[ww]
        sza = sza[ww]
        raa = raa[ww]
        doy= doy[ww[0]]
        row = ww[1]+start_row
        col = ww[2]+start_col
	locations  = np.array([doy,row,col])
	nnn = len(np.atleast_1d(locations[0]))
        orig = np.repeat(np.array([start_doy,start_row,start_col]),locations.shape[1]).reshape(locations.shape).T
        div = np.repeat(np.array([step_doy,step_row,step_col]),locations.shape[1]).reshape(locations.shape).T
        qlocations = ((locations.T - orig)/div.astype(float)).astype(int).T
        controls = np.array([np.ones_like(doy).astype(bool),\
                             vza,raa,sza,0*doy])
        y = []
        for i in xrange(len(np.atleast_1d(this_name))):
            this = yy[i]
            y.append(this[ww])
        grid = np.array(y)   
        fmt = 'BRDF-UCL'
        control = ['mask','vza','vaa','sza','saa']
        bands = alt_name
        if not np.array(grid).size:
            if 'logger' in self or 'logger' in self.dict():
                self.logger.error(\
                              "Warning: returning a zero-sized dataset ... "+\
                              " I wouldn;t try to do anything with it")
        # in case we dont have data for all bands
        mask =  np.logical_or.reduce([[this_name[i]==x for x in names] \
                                      for i in xrange(len(np.atleast_1d(this_name)))])
        sd = np.array('0.004 0.015 0.003 0.004 0.013 0.01 0.006'\
                      .split())[mask]
        sd = np.array([float(i) for i in sd.flatten()])\
            .reshape(sd.shape)
        nsamps = grid.shape[1]
        sd = sd.repeat(nsamps).reshape(grid.shape).T
        datasets = ParamStorage()
        datasets.data  = ParamStorage()
        datasets.name  = ParamStorage()
        datasets.name.fmt = fmt
        grid = grid.T
        datasets.data[name] = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\
                                                        .astype(object)
        datasets.data[name][:,:] = None
        for i in xrange(len(np.atleast_1d(this_name))):
            ww = np.where(names == this_name[i])[0][0]
            datasets.data[name][:,ww] = grid[:,i]
        datasets.data.location = np.array(locations).T
        datasets.data.control = np.array(controls).T
        datasets.data.qlocation = np.array(qlocations).T
        datasets.name[name] = np.array(names)
        datasets.name.location = np.array(['time','row','col'])
        datasets.name.control = np.array(control)
        datasets.name.qlocation = limits
        datasets.name.bands = np.array(bands)
        datasets.data.sd = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\
                                                        .astype(object)
        # for i in xrange(grid.shape[0]):
        # datasets.data.sd[i,:] = self.options.sd
        datasets.data.sd[:,:] = None
        for i in xrange(len(np.atleast_1d(this_name))):
            ww = np.where(names == this_name[i])[0][0]
            datasets.data.sd[:,ww] = sd[:,i]
        datasets.name.sd = np.array(names)
        if 'logger' in self or 'logger' in self.dict():
            self.logger.debug('finished parsing dataset')
    except:
        self.error_msg=\
            "a problem processing information from  %s as a NpzFile"\
                %filename
        self.error=True
        if 'logger' in self or 'logger' in self.dict():
            self.logger.info(self.error_msg)
        return 0,(self.error,self.error_msg)
    f.close()
    if 'logger' in self or 'logger' in self.dict():
        self.logger.info('... done')
    self.error=False
    self.error_msg=""
    return datasets,(self.error,self.error_msg)
예제 #6
0
    def reinit(self,options,names=None,datatype=None,limits=None,\
            bounds=None,control=None,location=None,env=None,header=None,\
               logdir=None,writers={},grid=False,logger=None,\
            datadir=None,logfile=None,name=None,info=[],readers=[],debug=None):
        '''
        Method to re-initialise the class instance
            
        The setup is on the whole controlled by the datatype which
            contains e.g. 'x'. This is used to set up the members
            self.x and self.y as SpecialVariables (see SpecialVariable
            in eoldas_SpecialVariable.py). There are some special attributes
            for datatypes starting with 'y'. These are assumed to be 
            observational data, which means that when they are read, the 
            data names associated with them are not limited to those in 
            self.names but rather set to whatever is read in in the data. This 
            is because the data names for observational data may be terms such 
            as waveband names etc that need special interpretation. Also,
            the default output format for observational data is 
            different to that of other data.  
            
        The elements self.state is a SpecialVariables which means 
            that they can be assigned various data types (see SpecialVariables)
            and loaded accordingly (e.g. if a filename is specified, this is 
            read in to the data structure. The SpecialVariables contain 'hidden'
            datasets, which here are mainly the 'control' and 'location'
            information. A SpecialVariable has two internal structures: `data`
            and `name`. The former is used to store data values (e.g. the 
            state values) and the latter to store associated metadata.
            For example, `control` is passed here e.g. as [`mask`,`vza`] and this
            gives the metadata that are stored in `name`. The actual values
            of the control data are stored in the `data` section. For
            location, we might be passed [`time`,`row`,`col`], so this is set
            in names.location, and the data.location contains the values
            of the location at each of these elements. For the actual state
            dataset, this is stored according to its name, so for `x` the values
            are stored in data.x and the associated data names in name.x.
        
        State datasets must represent at least the mean and standard deviation
            of a state for them to be of value in EOLDAS. TThe mean is accessed
            as e.g. self.state for the state dataset. 
            The sd is accessed can be accessed as self._state.sd if it 
            has been set. This reference can also be used to directly set 
            data associated with a SpecialVariable, e.g.
            self.Data.control = np.zeros([2,3])
            
            to represent 2 samples with 3 control variables. You can access name
            information similarly with
            
            print self.Name.control
            
            but this will generate a KeyError if the term has not been set. You
            can check it exists with:
            
            key = 'control'
            if key in self.Name:
                this = (self.Data[key],self.Name[key])
            
            To get e.g. a dictionary representation of a SpecialVariable
            you can use eg:
            
                self.Name.to_dict()
            
            to get the name dictionary, or
            
                thisdict = self._state.to_dict()
            
            to get the full representation, which then contains 'data' and
            'name' as well as some other information stored in the 
            SpecialVariable.
            
            You can similarly load them using e.g. 
            
                self.Data.update(
                        ParamStorage().from_dict(thisdict['data'])
                        combine=True)
        
        '''

        # set up a fakes dictionary from the data types
        self.set('datatype', datatype)
        self.set('fakes', {'state': '_state'})

        # first check that options is sensible
        self.__check_type(options, ParamStorage, fatal=True)
        self.options = options

        from eoldas_Lib import set_default_limits,\
                    check_limits_valid,quantize_location, sortopt

        nSpecial = 1
        if name == None:
            import time
            thistime = str(time.time())
            name = type(self).__name__
            name = "%s.%s" % (name, thistime)
        self.thisname = name
        self.options.thisname = str(name).replace(' ', '_')


        log_terms = {\
            'logfile':logfile or sortopt(self.options,'logfile',None),\
            'logdir':logdir or sortopt(self.options,'logdir',None),\
            'debug' : debug or sortopt(self.options,'debug',True)}

        self.datadir = datadir or sortopt(self.options, 'datadir', ["."])
        self.header = header or "EOLDAS pickle V1.0 - plewis"
        env = env or sortopt(self.options, 'env', None)
        names = names or sortopt(self.options, 'names', None)
        location = location or sortopt(self.options, 'location', ['time'])
        control = control or sortopt(self.options, 'control', [])
        limits = limits or sortopt(self.options,'limits',\
                        set_default_limits(np.array(location)))
        limits = limits or self.options.limits
        limits = np.array(check_limits_valid(limits))
        bounds = bounds or sortopt(self.options,'bounds',\
                    [[None,None]] * xlen(names))
        self.options.bounds = bounds

        self.headers = {'PARAMETERS-V2':"PARAMETERS-V2", \
            'PARAMETERS':"PARAMETERS", \
            'BRDF-UCL':'BRDF-UCL',\
            'BRDF': 'BRDF'}
        self.headers_2 = {'BRDF-UCL': 'location'}

        #  The ones pre-loaded are
        # self.read_functions = [self.read_pickle,self.read_numpy_fromfile]
        self._state = SpecialVariable(info=info,name=self.thisname,\
                                  readers=readers,datadir=self.datadir,\
                                  env=env,writers=writers,\
                                  header=self.header,\
                                  logger=logger,log_terms=log_terms,\
                                  simple=False)
        # self._state is where data are read into
        # but self.Data and self.Name are where we access them from
        self.grid = grid
        # this is so we can access this object from
        # inside a SpecialVariable
        self.state = np.array([0.])
        # a default data fmt output
        if datatype[0] == 'y':
            self.Name.fmt = 'BRDF'
            self.Name.state = np.array(['dummy'])
        else:
            self.Name.fmt = 'PARAMETERS'
            n_params = xlen(names)
            if not n_params:
                error_msg = \
                    "The field 'names' must be defined in options or"+ \
                    "passed directly to this method if you have the data type x"
                raise Exception(error_msg)
        self.Name.state = np.array(names)
        self.Name.location = np.array(location)
        self.Name.control = np.array(control)
        self.Name.header = self.header
        self.Name.bounds = np.array(bounds)
        self.Name.qlocation = np.array(limits)
        self.Name.datadir = datadir
        #
        # sort this object's name
        # sort logging
        self.logger = sortlog(self,
                              log_terms['logfile'],
                              logger,
                              name=self.thisname,
                              logdir=log_terms['logdir'],
                              debug=log_terms['debug'])
        self.logger.info('Initialising %s' % type(self).__name__)