def __init__(self, head=None): """ Constructor, either just a default () or with a header as a dictionary (preferably an ordered dictionary). This is tricky to construct properly, and it is probably easier to use several lines of 'add_entry' statements. Whatever you supply will be passed to add_entry within a loop iterating over the elements of head. Args: head : a dictionary of key, value pairs, with keys having a hierarchical structure with '.' separators and values each a tuple of (value,type,comment). See add_entry for more. """ Odict.__init__(self)
def add_entry(self, *args): """ Adds a new Uhead item, checking the various arguments to reduce the chances of problems. This can have either 2 or 4 arguments. The 4 argument case is as follows: Args key : hierarchical string of the form 'User.Filter' where 'User' is a directory or folder of grouped entries. It cannot have blanks and any implied directories must already exists. Thus to set a key 'User.Filter.Wheel', 'User.Filter' would need to exist and be a directory. The existence of the implied 'User' would not be checked in this case, on the assumption that it was checked when 'User.Filter' was created. value : value to associate (will be ignored in the case of directories, but see the 2 argument case below). The nature of the value varies with the itype; see next. itype : one of a range of possible data types. This rather 'unpythonic' argument is to address the need to match up with data files and the C++ ULTRACAM pipeline when it comes to writing to disk. Look for integers called 'ITYPE_*' to see the set of potential types. The meaning of most data types is obvious. e.g. ITYPE_DOUBLE or ITYPE_FLOAT expect floating point numbers. In this case both will be stored as a Python float in memory, but will be saved to disk with different numbers of bytes. Less obvious ones are: ITYPE_TIME : the corresponding value should be a two-element tuple or list with first an integer for the number of days and then a float for the number of hours passed. comment : comment string with details of the variable. If just 2 arguments are given, they will be interpreted as just a key and comment for a directory. """ # elementary checks if len(args) == 2: key, comment = args itype = ITYPE_DIR value = None elif len(args) == 4: key, value, itype, comment = args else: raise UltracamError("Uhead.add_entry: takes either 2 or 4 arguments") if not isinstance(key, basestring): raise UltracamError('Uhead.add_entry: argument "key" must be a string.') if not isinstance(comment, basestring): raise UltracamError("Uhead.add_entry: key = " + key + ': "comment" must be a string.') # now look at the key: must have no blanks if key.find(" ") > -1: raise UltracamError('Uhead.add_entry: key = "' + key + '" contains at least one blank.') # if key has a '.' then the part before last dot must already exist # and must be a directory. Search in reverse order, as all being well, it # should usually be fastest. ldot = key.rfind(".") if ldot > -1: dir = key[:ldot] for kold in self.keys()[::-1]: if dir == kold and self[kold][1] == ITYPE_DIR: break else: raise UltracamError("Uhead.add_entry: key = " + key + ": could not locate directory = " + key[:ldot]) # determine position of key within Odict. Must add onto # whichever directory it belongs to. for index, kold in enumerate(self.keys()): if kold.startswith(dir): lind = index # the next implicitly check the value: if they can't be converted to # the right type, something is wrong. if itype == ITYPE_DOUBLE or itype == ITYPE_FLOAT: value = float(value) elif itype == ITYPE_INT or itype == ITYPE_UINT or itype == ITYPE_UCHAR or itype == ITYPE_USINT: value = int(value) elif itype == ITYPE_STRING: value = str(value) elif itype == ITYPE_BOOL: value = bool(value) elif itype == ITYPE_DIR: pass elif itype == ITYPE_TIME: if len(value) != 2: raise UltracamError( "Uhead.add_entry: key = " + key + ": require a 2-element tuple or list (int,float) for ITYPE_TIME)" ) value = (int(value[0]), float(value[1])) elif itype == ITYPE_DVECTOR: if not isinstance(value, np.ndarray) or len(value.shape) != 1: raise UltracamError("Uhead.add_entry: key = " + key + ": require a 1D numpy.ndarray for ITYPE_DVECTOR)") value = value.astype(float64) elif itype == ITYPE_IVECTOR: if not isinstance(value, np.ndarray) or len(value.shape) != 1: raise UltracamError("Uhead.add_entry: key = " + key + ": require a 1D numpy.ndarray for ITYPE_IVECTOR)") value = value.astype(int) elif itype == ITYPE_FVECTOR: if not isinstance(value, np.ndarray) or len(value.shape) != 1: raise UltracamError("Uhead.add_entry: key = " + key + ": require a 1D numpy.ndarray for ITYPE_FVECTOR)") value = value.astype(float32) else: raise UltracamError("Uhead.add_entry: key = " + key + ": itype = " + str(itype) + " not recognised.") # checks passed, finally set item if ldot > -1: self.insert(key, (value, itype, comment), lind + 1) else: Odict.__setitem__(self, key, (value, itype, comment))
def __repr__(self): rep = "Uhead(" + Odict.__repr__(self)[6:] return rep
def rucm(cls, fname, flt=True): """ Factory method to produce an MCCD from a ucm file. fname -- ucm file name. '.ucm' will be appended if not supplied. flt -- convert to 4-byte floats whatever the input data, or not. ucm files can either contain 4-bytes floats or for reduced disk footprint, unsigned 2-byte integers. If flt=True, either type will end up as float32 internally. If flt=False, the disk type will be retained. The latter is unsafe when arithematic is involved hence the default is to convert to 4-byte floats. Exceptions are thrown if the file cannot be found, or an error during the read occurs. """ # Assume it is a file object, if that fails, assume it is # the name of a file. if not fname.endswith('.ucm'): fname += '.ucm' uf = open(fname, 'rb') start_format = check_ucm(uf) # read the header lmap = struct.unpack(start_format + 'i', uf.read(4))[0] head = Uhead() for i in range(lmap): name = read_string(uf, start_format) itype = struct.unpack(start_format + 'i', uf.read(4))[0] comment = read_string(uf, start_format) if itype == ITYPE_DOUBLE: value = struct.unpack(start_format + 'd', uf.read(8))[0] elif itype == ITYPE_INT: value = struct.unpack(start_format + 'i', uf.read(4))[0] elif itype == ITYPE_UINT: value = struct.unpack(start_format + 'I', uf.read(4))[0] elif itype == ITYPE_FLOAT: value = struct.unpack(start_format + 'f', uf.read(4))[0] elif itype == ITYPE_STRING: value = read_string(uf, start_format) elif itype == ITYPE_BOOL: value = struct.unpack(start_format + 'B', uf.read(1))[0] elif itype == ITYPE_DIR: value = None elif itype == ITYPE_TIME: value = struct.unpack(start_format + 'id', uf.read(12)) elif itype == ITYPE_DVECTOR: nvec = struct.unpack(start_format + 'i', uf.read(4))[0] value = struct.unpack(start_format + str(nvec) + 'd', uf.read(8*nvec)) elif itype == ITYPE_UCHAR: value = struct.unpack(start_format + 'c', uf.read(1))[0] elif itype == ITYPE_USINT: value = struct.unpack(start_format + 'H', uf.read(2))[0] elif itype == ITYPE_IVECTOR: nvec = struct.unpack(start_format + 'i', uf.read(4))[0] value = struct.unpack(start_format + str(nvec) + 'i', uf.read(4*nvec)) elif itype == ITYPE_FVECTOR: nvec = struct.unpack(start_format + 'i', uf.read(4))[0] value = struct.unpack(start_format + str(nvec) + 'f', uf.read(4*nvec)) else: raise UltracamError('ultracam.MCCD.rucm: do not recognize itype = ' + str(itype)) # store header information, fast method Odict.__setitem__(head, name, (value, itype, comment)) # now for the data data = [] # read number of CCDs nccd = struct.unpack(start_format + 'i', uf.read(4))[0] for nc in range(nccd): # read number of wndows nwin = struct.unpack(start_format + 'i', uf.read(4))[0] wins = [] for nw in range(nwin): llx,lly,nx,ny,xbin,ybin,nxmax,nymax,iout = struct.unpack(start_format + '9i', uf.read(36)) if iout == 0: win = np.fromfile(file=uf, dtype=np.float32, count=nx*ny) elif iout == 1: if flt: win = np.fromfile(file=uf, dtype=np.uint16, count=nx*ny).astype(np.float32) else: win = np.fromfile(file=uf, dtype=np.uint16, count=nx*ny) else: raise UltracamError('ultracam.MCCD.rucm: iout = ' + str(iout) + ' not recognised') win = win.reshape((ny,nx)) wins.append(Window(win,llx,lly,xbin,ybin)) data.append(CCD(wins,None,nxmax,nymax,True,None)) uf.close() return cls(data, head)
def __repr__(self): rep = 'Uhead(' + Odict.__repr__(self)[6:] return rep
def rucm(cls, fname, flt=True): """ Factory method to produce an MCCD from a ucm file. fname -- ucm file name. '.ucm' will be appended if not supplied. flt -- convert to 4-byte floats whatever the input data, or not. ucm files can either contain 4-bytes floats or for reduced disk footprint, unsigned 2-byte integers. If flt=True, either type will end up as float32 internally. If flt=False, the disk type will be retained. The latter is unsafe when arithematic is involved hence the default is to convert to 4-byte floats. Exceptions are thrown if the file cannot be found, or an error during the read occurs. """ # Assume it is a file object, if that fails, assume it is # the name of a file. if not fname.endswith('.ucm'): fname += '.ucm' uf = open(fname, 'rb') start_format = check_ucm(uf) # read the header lmap = struct.unpack(start_format + 'i', uf.read(4))[0] head = Uhead() for i in range(lmap): name = read_string(uf, start_format) itype = struct.unpack(start_format + 'i', uf.read(4))[0] comment = read_string(uf, start_format) if itype == ITYPE_DOUBLE: value = struct.unpack(start_format + 'd', uf.read(8))[0] elif itype == ITYPE_INT: value = struct.unpack(start_format + 'i', uf.read(4))[0] elif itype == ITYPE_UINT: value = struct.unpack(start_format + 'I', uf.read(4))[0] elif itype == ITYPE_FLOAT: value = struct.unpack(start_format + 'f', uf.read(4))[0] elif itype == ITYPE_STRING: value = read_string(uf, start_format) elif itype == ITYPE_BOOL: value = struct.unpack(start_format + 'B', uf.read(1))[0] elif itype == ITYPE_DIR: value = None elif itype == ITYPE_TIME: value = struct.unpack(start_format + 'id', uf.read(12)) elif itype == ITYPE_DVECTOR: nvec = struct.unpack(start_format + 'i', uf.read(4))[0] value = struct.unpack(start_format + str(nvec) + 'd', uf.read(8 * nvec)) elif itype == ITYPE_UCHAR: value = struct.unpack(start_format + 'c', uf.read(1))[0] elif itype == ITYPE_USINT: value = struct.unpack(start_format + 'H', uf.read(2))[0] elif itype == ITYPE_IVECTOR: nvec = struct.unpack(start_format + 'i', uf.read(4))[0] value = struct.unpack(start_format + str(nvec) + 'i', uf.read(4 * nvec)) elif itype == ITYPE_FVECTOR: nvec = struct.unpack(start_format + 'i', uf.read(4))[0] value = struct.unpack(start_format + str(nvec) + 'f', uf.read(4 * nvec)) else: raise UltracamError( 'ultracam.MCCD.rucm: do not recognize itype = ' + str(itype)) # store header information, fast method Odict.__setitem__(head, name, (value, itype, comment)) # now for the data data = [] # read number of CCDs nccd = struct.unpack(start_format + 'i', uf.read(4))[0] for nc in range(nccd): # read number of wndows nwin = struct.unpack(start_format + 'i', uf.read(4))[0] wins = [] for nw in range(nwin): llx, lly, nx, ny, xbin, ybin, nxmax, nymax, iout = struct.unpack( start_format + '9i', uf.read(36)) if iout == 0: win = np.fromfile(file=uf, dtype=np.float32, count=nx * ny) elif iout == 1: if flt: win = np.fromfile(file=uf, dtype=np.uint16, count=nx * ny).astype(np.float32) else: win = np.fromfile(file=uf, dtype=np.uint16, count=nx * ny) else: raise UltracamError('ultracam.MCCD.rucm: iout = ' + str(iout) + ' not recognised') win = win.reshape((ny, nx)) wins.append(Window(win, llx, lly, xbin, ybin)) data.append(CCD(wins, None, nxmax, nymax, True, None)) uf.close() return cls(data, head)
def add_entry(self, *args): """ Adds a new Uhead item, checking the various arguments to reduce the chances of problems. This can have either 2 or 4 arguments. The 4 argument case is as follows: Args key : hierarchical string of the form 'User.Filter' where 'User' is a directory or folder of grouped entries. It cannot have blanks and any implied directories must already exists. Thus to set a key 'User.Filter.Wheel', 'User.Filter' would need to exist and be a directory. The existence of the implied 'User' would not be checked in this case, on the assumption that it was checked when 'User.Filter' was created. value : value to associate (will be ignored in the case of directories, but see the 2 argument case below). The nature of the value varies with the itype; see next. itype : one of a range of possible data types. This rather 'unpythonic' argument is to address the need to match up with data files and the C++ ULTRACAM pipeline when it comes to writing to disk. Look for integers called 'ITYPE_*' to see the set of potential types. The meaning of most data types is obvious. e.g. ITYPE_DOUBLE or ITYPE_FLOAT expect floating point numbers. In this case both will be stored as a Python float in memory, but will be saved to disk with different numbers of bytes. Less obvious ones are: ITYPE_TIME : the corresponding value should be a two-element tuple or list with first an integer for the number of days and then a float for the number of hours passed. comment : comment string with details of the variable. If just 2 arguments are given, they will be interpreted as just a key and comment for a directory. """ # elementary checks if len(args) == 2: key, comment = args itype = ITYPE_DIR value = None elif len(args) == 4: key, value, itype, comment = args else: raise UltracamError( 'Uhead.add_entry: takes either 2 or 4 arguments') if not isinstance(key, six.string_types): raise UltracamError( 'Uhead.add_entry: argument "key" must be a string.') if not isinstance(comment, six.string_types): raise UltracamError('Uhead.add_entry: key = ' + key + ': "comment" must be a string.') # now look at the key: must have no blanks if key.find(' ') > -1: raise UltracamError('Uhead.add_entry: key = "' + key + '" contains at least one blank.') # if key has a '.' then the part before last dot must already exist # and must be a directory. Search in reverse order, as all being well, it # should usually be fastest. ldot = key.rfind('.') if ldot > -1: dir = key[:ldot] for kold in list(self.keys())[::-1]: if dir == kold and self[kold][1] == ITYPE_DIR: break else: raise UltracamError('Uhead.add_entry: key = ' + key + ': could not locate directory = ' + key[:ldot]) # determine position of key within Odict. Must add onto # whichever directory it belongs to. for index, kold in enumerate(self.keys()): if kold.startswith(dir): lind = index # the next implicitly check the value: if they can't be converted to # the right type, something is wrong. if itype == ITYPE_DOUBLE or itype == ITYPE_FLOAT: value = float(value) elif itype == ITYPE_INT or itype == ITYPE_UINT or \ itype == ITYPE_UCHAR or itype == ITYPE_USINT: value = int(value) elif itype == ITYPE_STRING: value = str(value) elif itype == ITYPE_BOOL: value = bool(value) elif itype == ITYPE_DIR: pass elif itype == ITYPE_TIME: if len(value) != 2: raise UltracamError( 'Uhead.add_entry: key = ' + key + ': require a 2-element tuple or list (int,float) for ITYPE_TIME)' ) value = (int(value[0]), float(value[1])) elif itype == ITYPE_DVECTOR: if not isinstance(value, np.ndarray) or len(value.shape) != 1: raise UltracamError( 'Uhead.add_entry: key = ' + key + ': require a 1D numpy.ndarray for ITYPE_DVECTOR)') value = value.astype(float64) elif itype == ITYPE_IVECTOR: if not isinstance(value, np.ndarray) or len(value.shape) != 1: raise UltracamError( 'Uhead.add_entry: key = ' + key + ': require a 1D numpy.ndarray for ITYPE_IVECTOR)') value = value.astype(int) elif itype == ITYPE_FVECTOR: if not isinstance(value, np.ndarray) or len(value.shape) != 1: raise UltracamError( 'Uhead.add_entry: key = ' + key + ': require a 1D numpy.ndarray for ITYPE_FVECTOR)') value = value.astype(float32) else: raise UltracamError('Uhead.add_entry: key = ' + key + ': itype = ' + str(itype) + ' not recognised.') # checks passed, finally set item if ldot > -1: self.insert(key, (value, itype, comment), lind + 1) else: Odict.__setitem__(self, key, (value, itype, comment))