def set_table_type(self, target_dtype): '''Convert NumPy dtype to PyTable descriptor (adapted from blaze.pytables). E.g.: -------- >>> dt = np.dtype([('name', 'S7'), ('amount', 'i4'), ('time', 'M8[us]')]) >>> this.set_table_type(dt) # doctest: +SKIP {'amount': Int32Col(shape=(), dflt=0, pos=1), 'name': StringCol(itemsize=7, shape=(), dflt='', pos=0)} ''' # This lets us deal with things like python list based specifications target_dtype = np.dtype(target_dtype) tb_desc = {} for pos, name in enumerate(target_dtype.names): dt, _ = target_dtype.fields[name] # We may eventually want to deal with datetime64 columns, but for # now, we don't need to # if issubclass(dt.type, np.datetime64): # desc = tb.Description({name: tb.Time64Col(pos=pos)}) # else: desc, _ = tb.descr_from_dtype(np.dtype([(name, dt)])) getattr(desc, name)._v_pos = pos tb_desc.update(desc._v_colobjects) self.tb_desc = tb_desc
class DataHelper(object): default_double = np.nan default_int = -10 default_str = '' unknown_double = -10101010. unknown_int = -20 unknown_str = '?' dtype = np.dtype([('DEFAULT_DOUBLE', '<f8'), ('DEFAULT_INT', '<i8'), ('DEFAULT_STR', 'S8'), ('UNKNOWN_DOUBLE', '<f8'), ('UNKNOWN_INT', '<i8'), ('UNKNOWN_STR', 'S8')]) Format = tables.descr_from_dtype(dtype)[0] def save(self): data = np.empty(1, dtype=self.dtype) data['DEFAULT_DOUBLE'][0] = self.default_double data['DEFAULT_INT'][0] = self.default_int data['DEFAULT_STR'][0] = self.default_str data['UNKNOWN_DOUBLE'][0] = self.unknown_double data['UNKNOWN_INT'][0] = self.unknown_int data['UNKNOWN_STR'][0] = self.unknown_str return data def load(self, data): self.default_double = data['DEFAULT_DOUBLE'][0] self.default_int = data['DEFAULT_INT'][0] self.default_str = data['DEFAULT_STR'][0].decode() self.unknown_double = data['UNKNOWN_DOUBLE'][0] self.unknown_int = data['UNKNOWN_INT'][0] self.unknown_str = data['UNKNOWN_STR'][0].decode()
def pytables_desc(self): """ Convert NumPy dtype to PyTable descriptor (lifted from blaze.pytables). Examples -------- >>> from tables import Int32Col, StringCol, Time64Col >>> dt = np.dtype([('name', 'S7'), ('amount', 'i4'), ('time', 'M8[us]')]) >>> dtype_to_pytables(dt) # doctest: +SKIP {'amount': Int32Col(shape=(), dflt=0, pos=1), 'name': StringCol(itemsize=7, shape=(), dflt='', pos=0), 'time': Time64Col(shape=(), dflt=0.0, pos=2)} """ dtype = np.dtype(self.target_dtype) d = {} for pos, name in enumerate(dtype.names): dt, _ = dtype.fields[name] if issubclass(dt.type, np.datetime64): tdtype = tb.defscription({name: tb.Time64Col(pos = pos)}), else: tdtype = tb.descr_from_dtype(np.dtype([(name, dt)])) el = tdtype[0] # removed dependency on toolz -DJC getattr(el, name)._v_pos = pos d.update(el._v_colobjects) return d
def pytables_desc(self): """ Convert NumPy dtype to PyTable descriptor (lifted from blaze.pytables). Examples -------- >>> from tables import Int32Col, StringCol, Time64Col >>> dt = np.dtype([('name', 'S7'), ('amount', 'i4'), ('time', 'M8[us]')]) >>> dtype_to_pytables(dt) # doctest: +SKIP {'amount': Int32Col(shape=(), dflt=0, pos=1), 'name': StringCol(itemsize=7, shape=(), dflt='', pos=0), 'time': Time64Col(shape=(), dflt=0.0, pos=2)} """ dtype = np.dtype(self.target_dtype) d = {} for pos, name in enumerate(dtype.names): dt, _ = dtype.fields[name] if issubclass(dt.type, np.datetime64): tdtype = tb.defscription({name: tb.Time64Col(pos=pos)}), else: tdtype = tb.descr_from_dtype(np.dtype([(name, dt)])) el = tdtype[0] # removed dependency on toolz -DJC getattr(el, name)._v_pos = pos d.update(el._v_colobjects) return d
def _write_info(self): import pyNastran info = 'h5Nastran version %s\nPowered by pyNastran version %s' % ( self.h5n_version_str, pyNastran.__version__) self.h5f.create_array(self.table_paths.about_path, self.table_paths.about_table, obj=info.encode(), title='h5Nastran Info', createparents=True) versioning_dtype = np.dtype([('H5NASTRAN_VERSION_STR', 'S8'), ('H5NASTRAN_VERSION', '<i8', (3, )), ('NASTRAN_TYPE', 'S8'), ('NASTRAN_VERSION', '<i8', (3, ))]) format = tables.descr_from_dtype(versioning_dtype)[0] self.h5f.create_table(self.table_paths.versioning_path, self.table_paths.versioning_table, format, 'VERSIONING', expectedrows=1, createparents=True) table = self.h5f.get_node(self.table_paths.versioning) data = np.zeros(1, dtype=versioning_dtype) data['H5NASTRAN_VERSION_STR'][0] = self.h5n_version_str data['H5NASTRAN_VERSION'][0] = self.h5n_version nastran_type = self.nastran_type if nastran_type is None: nastran_type = '' data['NASTRAN_TYPE'][0] = nastran_type data['NASTRAN_VERSION'][0] = self.nastran_version table.append(data) self.h5f.flush()
def _save_gui_stats(t, table_file=None): if table_file is None: return if not os.path.exists(table_file): cols = (GUIDE_COLS['obs'] + GUIDE_COLS['cat'] + GUIDE_COLS['stat'] + GUIDE_COLS['agasc'] + GUIDE_COLS['temp'] + GUIDE_COLS['bad']) desc, byteorder = tables.descr_from_dtype(np.dtype(cols)) filters = tables.Filters(complevel=5, complib='zlib') h5 = tables.open_file(table_file, 'a') tbl = h5.create_table('/', 'data', desc, filters=filters, expectedrows=1e6) tbl.cols.obsid.create_index() tbl.cols.kalman_tstart.create_csindex() tbl.cols.agasc_id.create_index() h5.close() del h5 h5 = tables.open_file(table_file, 'a') tbl = h5.get_node('/', 'data') have_obsid_coord = tbl.get_where_list('(obsid == {}) & (obi == {})'.format( t[0]['obsid'], t[0]['obi']), sort=True) if len(have_obsid_coord): obsid_rec = tbl.read_coordinates(have_obsid_coord) if len(obsid_rec) != len(t): raise ValueError( "Could not update {}; different number of slots".format( t[0]['obsid'])) # preserve any 'known_bad' status for row in obsid_rec: slot = row['slot'] t['known_bad'][t['slot'] == slot] = row['known_bad'] t['bad_comment'][t['slot'] == slot] = row['bad_comment'] tbl.modify_coordinates(have_obsid_coord, t.as_array()) else: tbl.append(t.as_array()) logger.info("saving stats to h5 table") tbl.flush() h5.flush() h5.close()
def dtype_to_pytables(dtype): """ Convert NumPy dtype to PyTable descriptor Examples -------- >>> from tables import Int32Col, StringCol, Time64Col >>> dt = np.dtype([('name', 'S7'), ('amount', 'i4'), ('time', 'M8[us]')]) >>> dtype_to_pytables(dt) # doctest: +SKIP {'amount': Int32Col(shape=(), dflt=0, pos=1), 'name': StringCol(itemsize=7, shape=(), dflt='', pos=0), 'time': Time64Col(shape=(), dflt=0.0, pos=2)} """ d = {} for pos, name in enumerate(dtype.names): dt, _ = dtype.fields[name] if issubclass(dt.type, np.datetime64): tdtype = tb.Description({name: tb.Time64Col(pos=pos)}), else: tdtype = tb.descr_from_dtype(np.dtype([(name, dt)])) el = first(tdtype) getattr(el, name)._v_pos = pos d.update(el._v_colobjects) return d
def _save_gui_stats(t): if not os.path.exists(TABLE_FILE): cols = (GUIDE_COLS['obs'] + GUIDE_COLS['cat'] + GUIDE_COLS['stat'] + GUIDE_COLS['agasc'] + GUIDE_COLS['temp'] + GUIDE_COLS['bad']) desc, byteorder = tables.descr_from_dtype(np.dtype(cols)) filters = tables.Filters(complevel=5, complib='zlib') h5 = tables.open_file(TABLE_FILE, 'a') tbl = h5.create_table('/', 'data', desc, filters=filters, expectedrows=1e6) tbl.cols.obsid.create_index() tbl.cols.kalman_tstart.create_csindex() tbl.cols.agasc_id.create_index() h5.close() del h5 h5 = tables.open_file(TABLE_FILE, 'a') tbl = h5.get_node('/', 'data') have_obsid_coord = tbl.get_where_list( '(obsid == {}) & (obi == {})'.format( t[0]['obsid'], t[0]['obi']), sort=True) if len(have_obsid_coord): obsid_rec = tbl.read_coordinates(have_obsid_coord) if len(obsid_rec) != len(t): raise ValueError( "Could not update {}; different number of slots".format( t[0]['obsid'])) # preserve any 'known_bad' status for row in obsid_rec: slot = row['slot'] t['known_bad'][t['slot'] == slot] = row['known_bad'] t['bad_comment'][t['slot'] == slot] = row['bad_comment'] tbl.modify_coordinates(have_obsid_coord, t.as_array()) else: tbl.append(t.as_array()) logger.info("saving stats to h5 table") tbl.flush() h5.flush() h5.close()
def dtype_to_table(dtype): """ Convert a NumPy dtype to a PyTables Table description Essentially just :ref:`tables.descr_from_dtype` but it works on :ref:`np.datetime64` Args: dtype (np.dtype): NumPy data type Returns: dict: PyTables description """ desc = {} for idx, name in enumerate(dtype.names): _dt, _ = dtype.fields[name] if issubclass(_dt.type, np.datetime64): tb_dtype = tb.Description({name: tb.Time64Col(pos=idx)}) else: tb_dtype, byteorder = tb.descr_from_dtype(np.dtype([(name, _dt)])) _tb_dtype = tb_dtype._v_colobjects _tb_dtype[name]._v_pos = idx desc.update(_tb_dtype) return desc
def _punch_finalize(self): dtype = np.dtype([('SUBCASE_ID', '<i8'), ('LOAD_FACTOR', '<f8'), ('DOMAIN_ID', '<i8')]) format = tables.descr_from_dtype(dtype)[0] self.h5f.create_table(self.table_paths.subcase_path, self.table_paths.subcase_table, format, 'SUBCASES', expectedrows=len(self._punch_subcase_ids), createparents=True) table = self.h5f.get_node(self.table_paths.subcase) data = np.zeros(len(self._punch_subcase_ids), dtype=dtype) subcase_id = data['SUBCASE_ID'] load_factor = data['LOAD_FACTOR'] domain_id = data['DOMAIN_ID'] for key, domain_id_ in iteritems(self._punch_subcase_ids): index = domain_id_ - 1 subcase_id_, load_factor_ = key subcase_id[index] = subcase_id_ load_factor[index] = load_factor_ domain_id[index] = domain_id_ table.append(data) self.h5f.flush()
def process(obsid, version='last'): """ For requested obsid/version, run V&V, make plots, save plots and JSON, save info to shelve file, and update RMS HDF5. :param obsid: obsid :param version: 'last', 'default' or revision number of ASP1 products :returns: mica.vv.Obi V&V object """ obi = get_arch_vv(obsid, version) if not os.path.exists(FILES['temp_root']): os.makedirs(FILES['temp_root']) if obi is None: return None obi.tempdir = tempfile.mkdtemp(dir=FILES['temp_root']) obi.save_plots_and_resid() _file_vv(obi) if not os.path.exists(FILES['h5_file']): vv_desc, byteorder = tables.descr_from_dtype(VV_DTYPE) filters = tables.Filters(complevel=5, complib='zlib') h5f = tables.open_file(FILES['h5_file'], 'a') tbl = h5f.create_table('/', 'vv', vv_desc, filters=filters, expectedrows=1e6) h5f.close() h5 = tables.open_file(FILES['h5_file'], 'a') tbl = h5.get_node('/', 'vv') obi.set_tbl(tbl) obi.slots_to_table() tbl.flush() h5.flush() h5.close() return obi
def __init__(self, table_id, group, results_type, index_id, dtype, indices, validator=None, len_id=None, pos_id=None, subtables=None, rename=None, is_subtable=False): self.implemented = True self.table_id = table_id self.group = group self.dtype = np.dtype(dtype) if subtables is None: subtables = [] assert len( subtables ) == 0, 'Cannot have subtables defined for this result table type.' self.subtables = subtables self.attrs = list(self.dtype.names) try: self.attrs.remove('DOMAIN_ID') except ValueError: pass def _zero(): return 0 self._pos = defaultdict(_zero) for subtable in self.subtables: try: self.attrs.remove(subtable.pos_id) self.attrs.remove(subtable.len_id) except ValueError: print(self.path(), self.attrs, subtable.pos_id, subtable.len_id) raise self.table = None self.h5f = None self.h5n = None if len_id is None: len_id = '%s_LEN' % self.table_id.replace('_', '') if pos_id is None: pos_id = '%s_POS' % self.table_id.replace('_', '') if rename is None: rename = {} self.len_id = rename.get(len_id, len_id) self.pos_id = rename.get(pos_id, pos_id) cols = set(self.dtype.names) for subtable in self.subtables: assert subtable.len_id in cols assert subtable.pos_id in cols ################################################################################################################ self.is_subtable = is_subtable self.index_id = index_id try: self.results_type = results_type[0] except IndexError: self.results_type = results_type if validator is None: validator = _validator self.validator = validator try: self.Format = tables.descr_from_dtype(self.dtype)[0] except NotImplementedError: dtype, indices = _convert_dtype(dtype, indices) self.dtype = np.dtype(dtype) self.Format = tables.descr_from_dtype(self.dtype)[0] self.indices = deepcopy(indices) assert isinstance(self.indices, DataGetter) assert len(self.indices) == len(self.dtype.names) - 1, str( (len(self.indices), len(self.dtype.names))) self.domain_count = 0 self.IndexFormat = IndexFormat self.PrivateIndexFormat = PrivateIndexFormat self.PrivateIndexDataFormat = PrivateIndexDataFormat self._index_table = None self._private_index_table = None self._index_data = [] self._subcase_index = [] self._index_offset = 0 self._subcase_ids = set() self._index_options = {} self.result_table = None
# Licensed under a 3-clause BSD style license - see LICENSE.rst import tables import numpy as np VV_DTYPE = np.dtype( [('obsid', '<i4'), ('revision', '<i4'), ('isdefault', '<i4'), ('aspect_1_id', '<i4'), ('used', '<i4'), ('vv_version', '<i4'), ('ap_date', '|S21'), ('tstart', '<f8'), ('tstop', '<f8'), ('sim_z', '<f8'), ('sim_z_offset', '<f8'), ('instrument', '|S10'), ('ra_pnt', '<f8'), ('dec_pnt', '<f8'), ('roll_pnt', '<f8'), ('slot', '<i4'), ('type', '|S10'), ('n_pts', '<i4'), ('rad_off', '<f8'), ('frac_dy_big', '<f8'), ('frac_dz_big', '<f8'), ('frac_mag_big', '<f8'), ('mean_y', '<f8'), ('mean_z', '<f8'), ('dy_mean', '<f8'), ('dy_med', '<f8'), ('dy_rms', '<f8'), ('dz_mean', '<f8'), ('dz_med', '<f8'), ('dz_rms', '<f8'), ('dr_mean', '<f8'), ('dr_med', '<f8'), ('dr_rms', '<f8'), ('mag_mean', '<f8'), ('mag_med', '<f8'), ('mag_rms', '<f8'), ('mean_aacccdpt', '<f8')]) vv_desc, byteorder = tables.descr_from_dtype(VV_DTYPE) filters = tables.Filters(complevel=5, complib='zlib') h5f = tables.openFile('vv.h5', 'a') tbl = h5f.createTable('/', 'vv', vv_desc, filters=filters, expectedrows=1e6) h5f.close()
import numpy as np import Ska.Table import tables import os from glob import glob import Ska.Numpy example_file = "/data/agasc1p6/agasc/n0000/0001.fit" dtype = Ska.Table.read_fits_table(example_file).dtype table_desc, bo = tables.descr_from_dtype(dtype) # filters = tables.Filters(complevel=5, complib='zlib') h5 = tables.openFile("agasc1p6.h5", mode="w") tbl = h5.createTable("/", "data", table_desc, title="AGASC 1.6") tbl.flush() h5.flush() h5.close() AGASC_DIR = "/data/agasc1p6/agasc" h5 = tables.openFile("agasc1p6.h5", mode="a") tbl = h5.getNode("/", "data") for chunk in glob(os.path.join(AGASC_DIR, "?????")): for file in glob(os.path.join(chunk, "????.fit")): print "processing %s" % file stars = Ska.Table.read_table(file) tbl.append(stars) tbl.flush() h5.flush() h5.flush() h5.close()
def __init__(self, table_id, group, dtype, defaults=None, len_id=None, pos_id=None, subtables=None, rename=None): self.implemented = True self.table_id = table_id self.group = group self.dtype = np.dtype(dtype) try: self.Format = tables.descr_from_dtype(self.dtype)[0] except NotImplementedError: dtype = _convert_dtype(dtype) self.dtype = np.dtype(dtype) self.Format = tables.descr_from_dtype(self.dtype)[0] self.subtables = subtables self.attrs = list(self.dtype.names) try: self.attrs.remove('DOMAIN_ID') except ValueError: pass def _zero(): return 0 self._pos = defaultdict(_zero) _dtypes = {_[0]: (_[1], _[2]) for _ in dtype} for subtable in self.subtables: try: self.attrs.remove(subtable.pos_id) self.attrs.remove(subtable.len_id) except ValueError: print(self.path(), self.attrs, subtable.pos_id, subtable.len_id) raise self.table = None self.h5f = None if defaults is not None: self.defaults = defaults else: self.defaults = {} for i in range(len(self.attrs)): name = self.attrs[i] _type = _dtypes[name][0] if _type.startswith('S'): self.defaults[name] = DataHelper.default_str else: self.defaults[name] = _defaults[_type] if len_id is None: len_id = '%s_LEN' % self.table_id.replace('_', '') if pos_id is None: pos_id = '%s_POS' % self.table_id.replace('_', '') if rename is None: rename = {} self.len_id = rename.get(len_id, len_id) self.pos_id = rename.get(pos_id, pos_id) cols = set(self.dtype.names) for subtable in self.subtables: assert subtable.len_id in cols assert subtable.pos_id in cols
from __future__ import print_function, absolute_import import numpy as np import tables from numpy import cross from pyNastran.op2.data_in_material_coord import is_mcid, angle2vec, check_theta, calc_imat shell_element_info_dtype = np.dtype([('EID', '<i8', ()), ('CENTER', '<f8', (3, )), ('THETA_RAD', '<f8', ())]) shell_element_info_format = tables.descr_from_dtype( shell_element_info_dtype)[0] class QuadShell(object): def get_shell_info(self, bdf, cards): elms = cards.get(self.__class__.__name__, []) elm_count = len(elms) result = np.empty(elm_count, dtype=shell_element_info_dtype) if len(result) == 0: return result _eids = result['EID'] _centers = result['CENTER'] _theta_rad = result['THETA_RAD'] eids = np.array(sorted(elms.keys())) elems = np.array([elms[eid] for eid in eids])
class Defaults(object): default_double = np.nan default_int = -10 default_str = '' unknown_double = -10101010. unknown_int = -20 unknown_str = '?' dtype = np.dtype([ ('DEFAULT_DOUBLE', '<f8'), ('DEFAULT_INT', '<i8'), ('DEFAULT_STR', 'S8'), ('UNKNOWN_DOUBLE', '<f8'), ('UNKNOWN_INT', '<i8'), ('UNKNOWN_STR', 'S8') ]) Format = tables.descr_from_dtype(dtype)[0] def get_value_int(self, val): if val is None: return self.default_int else: return val def get_value_double(self, val): if val is None: return self.default_double else: return val def get_value_str(self, val): if val is None: return self.default_str else: return val def get_list_double(self, vals): if vals[0] is None: return [self.default_double] * len(vals) else: return vals def get_list_int(self, vals): if vals[0] is None: return [self.default_int] * len(vals) else: return vals def get_list_str(self, vals): if vals[0] is None: return [self.default_str] * len(vals) else: return vals def to_value_int(self, val): if val == self.default_int: return None else: return val def to_value_double(self, val): if val == self.default_double: return None else: return val def to_value_str(self, val): if val == self.default_str: return None else: return val def to_list_double(self, vals): if vals[0] == self.default_double: return None else: return vals def to_list_int(self, vals): if vals[0] == self.default_int: return None else: return vals def to_list_str(self, vals): if vals[0] == self.default_str: return None else: return vals def save(self, h5n): h5f = h5n.h5f h5f.create_table(h5n.table_paths.defaults_path, h5n.table_paths.defaults_table, self.Format, 'DATA DEFAULTS', expectedrows=1, createparents=True) table = h5f.get_node(h5n.table_paths.defaults) table.append(self._save()) def load(self, h5n): self._load(h5n.h5f.get_node(h5n.table_paths.defaults).read()) def _load(self, data): self.default_double = data['DEFAULT_DOUBLE'][0] self.default_int = data['DEFAULT_INT'][0] self.default_str = data['DEFAULT_STR'][0].decode() self.unknown_double = data['UNKNOWN_DOUBLE'][0] self.unknown_int = data['UNKNOWN_INT'][0] self.unknown_str = data['UNKNOWN_STR'][0].decode() def _save(self): data = np.empty(1, dtype=self.dtype) data['DEFAULT_DOUBLE'][0] = self.default_double data['DEFAULT_INT'][0] = self.default_int data['DEFAULT_STR'][0] = self.default_str data['UNKNOWN_DOUBLE'][0] = self.unknown_double data['UNKNOWN_INT'][0] = self.unknown_int data['UNKNOWN_STR'][0] = self.unknown_str return data
names = [name for name in stars.dtype.names if name not in excludes] print('Dtype before excluding:\n', stars.dtype) stars = Table([stars[name] for name in names], names=names, copy=False) stars = stars.as_array() print('Dtype after excluding:\n', stars.dtype) print('Sorting on Dec and re-ordering') idx = np.argsort(stars['DEC']) stars = stars.take(idx) print('Creating miniagasc.h5 file') rootname = 'proseco_agasc' if args.proseco else 'miniagasc' filename = '{}_{}.h5'.format(rootname, args.version) table_desc, bo = tables.descr_from_dtype(stars.dtype) minih5 = tables.open_file(filename, mode='w') minitbl = minih5.create_table('/', 'data', table_desc, title='AGASC {}'.format(num_version)) print('Appending stars to {} file'.format(filename)) minitbl.append(stars) minitbl.flush() print('Creating indexes in miniagasc.h5 file') if not args.proseco: minitbl.cols.RA.create_csindex() minitbl.cols.DEC.create_csindex() minitbl.cols.AGASC_ID.create_csindex() print('Flush and close miniagasc.h5 file') minitbl.flush()