class AssociationLandscapeTable(tables.IsDescription):
id = UInt64Col(pos=0)
start_locus_id = UInt64Col(pos=1)
stop_locus_id = UInt64Col(pos=2)
no_of_loci = UInt64Col(pos=3)
deltaX = UInt64Col(pos=4)
deltaY = Float64Col(pos=5)
class SSReadoutTableDs(IsDescription):
iro = UInt64Col() # readout numner/index
time = UInt64Col() # TACK timestamp
cpu_t = Float64Col() # native python timestamp float64
cpu_t_s = UInt64Col() # seconds time stamp uint64
cpu_t_ns = UInt64Col() # nano seconds time stamp uint64
data = Float32Col((N_TM, N_TM_PIX)) # 2D data array containing
class PopulationTable(tables.IsDescription): """ 2013.3.8 """ id = UInt64Col(pos=0) name = StringCol(512, pos=1) #name should like 'species_name.population_name' size = UInt64Col(pos=2) species_id = UInt64Col(pos=3)
class TwoAssociationLocusOverlapTable(tables.IsDescription):
"""
2012.12.24 new PyTables-based table definition
"""
id = UInt64Col(pos=0)
chromosome = StringCol(64, pos=1)
start = UInt64Col(pos=2)
stop = UInt64Col(pos=3)
fractionCoveredByAssociation2 = Float64Col(pos=4)
class RecombinationTable(tables.IsDescription): """ 2013.3.7 in case a pedigree is stored in IndividualTable, this table records the recombination events in meiosis. Used in SimulatePedigreeHaplotype.py """ id = UInt64Col(pos=0) parent_id = UInt64Col(pos=1) child_id = UInt64Col(pos=2) position = UInt64Col(pos=3, dflt=0)
class IndividualTable(tables.IsDescription): id = UInt64Col(pos=0) family_id = StringCol(512, pos=1) #64 byte-long name = StringCol(512, pos=2) # name should look like 'species_name.population_name.individual_name' to ensure uniqueness father_name = StringCol(512, pos=3) mother_name = StringCol(512, pos=4) sex = UInt64Col(pos=5) #0 is unknown, 1=male, 2=female phenotype = Float64Col(pos=6) population_id = UInt64Col(pos=7)
class LocusMapTable(tables.IsDescription): """ 2013.1.9 a pytable to store the locus_id and (chromosome, start, stop) used by Association.py to find chr, start, stop for input loci """ id = UInt64Col(pos=0) locus_id = UInt64Col(pos=1, dflt=0) chromosome = StringCol(64, pos=2, dflt="") #64 byte-long start = UInt64Col(pos=3, dflt=0) stop = UInt64Col(pos=4, dflt=0)
class ChromosomeTable(tables.IsDescription): """ 2013.3.8 """ id = UInt64Col(pos=0) name = StringCol(512, pos=1) #should be unique, but not 100% enforced #name should look like species_name.chromosome_name, if species_name is available length = UInt64Col(pos=2) sex_chromosome = StringCol(4, pos=3) #A=autosome, X=chr X, Y=chr Y species_id = UInt64Col(pos=4) path = StringCol(4512, pos=5) #path to the file containing the chromosome sequences, should be unique,
class StrokelitudeDataDescription(IsDescription):
frame = UInt64Col(pos=0)
trigger_timestamp = FloatCol(pos=1) # when the image trigger happened
processing_timestamp = FloatCol(pos=2) # when the analysis was done
left = Float32Col(pos=3) # angle, degrees
right = Float32Col(pos=4) # angle, degrees
left_antenna = Float32Col(pos=5) # angle, degrees
right_antenna = Float32Col(pos=6) # angle, degrees
head = Float32Col(pos=7) # angle, degrees
pulse_width = Float32Col(
pos=8) # random pulse width, 0 = short, 1 = long, 2 = unknown
pulse_frame = UInt64Col(pos=9) # frame number associated with random pulse
class AssociationLocusTable(tables.IsDescription): """ 2013.1.28 bugfix, was pos=3 for no_of_peaks (same as stop), now change it to pos=4, and increment others accordingly """ id = UInt64Col(pos=0) chromosome = StringCol(64, pos=1) #64 byte-long start = UInt64Col(pos=2) stop = UInt64Col(pos=3) no_of_peaks = UInt64Col(pos=4) connectivity = Float64Col(pos=5) no_of_results = UInt64Col(pos=6) phenotype_id_ls_in_str = StringCol(1000, pos=7)
class PolymorphismTable(tables.IsDescription): """ 2013.3.6 table that records the polymorphism alleles of individuals at relevant loci """ id = UInt64Col(pos=0) name = StringCol(512, pos=1) #name should be "individualName.locusName.chromosome_copy" to ensure uniqueness individual_id = UInt64Col(pos=2) locus_id = UInt64Col(pos=3) chromosome_copy = UInt64Col(pos=4, dflt=0) #starting from 0 allele_sequence = StringCol(512, pos=5) allele_sequence_length = UInt64Col(pos=6) allele_type = StringCol(512, pos=7)
class CountAssociationLocusTable(tables.IsDescription):
"""
2012.12.24 new PyTables-based table definition
"""
id = UInt64Col(pos=0)
min_score = Float64Col(pos=1)
min_overlap_ratio = Float64Col(pos=2)
total_no_of_results = UInt64Col(pos=3)
no_of_association_loci = UInt64Col(pos=4)
call_method_id_ls = StringCol(1000, pos=5)
cnv_method_id_ls = StringCol(1000, pos=6)
phenotype_method_id_ls = StringCol(1000, pos=7)
analysis_method_id_ls = StringCol(1000, pos=8)
class AssociationTable(tables.IsDescription): """ 2012.12.18 pytable class to store the genome-wide association result """ id = UInt64Col(pos=0) locus_id = UInt64Col(pos=1, dflt=0) chromosome = StringCol(64, pos=2, dflt='') #64 byte-long start = UInt64Col(pos=3, dflt=0) stop = UInt64Col(pos=4, dflt=0) score = Float64Col(pos=5, dflt=-1) mac = Int64Col(pos=6, dflt=-1) maf = Float64Col(pos=7, dflt=-1) genotype_var_perc = Float64Col(pos=8, dflt=-1) beta_list = Float64Col(shape=(5,), pos=9, dflt=-1) #2013.1.9 beta_pvalue_list = Float64Col(shape=(5,), pos=9, dflt=-1)
def setup_spec_table(self):
if not hasattr(self.node, "ms1_spec_table"):
for level in (1, 2):
description = {}
description["pm_index"] = UInt32Col(pos=0)
description["rt"] = Float32Col(pos=1)
description["scan_number"] = Int32Col(pos=2)
description["start"] = UInt64Col(pos=3)
description["end"] = UInt64Col(pos=4)
t = self.file_.create_table(self.node, 'ms%d_spec_table' % level, description,
filters=filters)
# every colums which appears in a where method call should/must be indexed !
# this is not only for performance but for correct lookup as well (I had strange bugs
# else)
t.cols.pm_index.create_index()
t.cols.rt.create_index()
class TimeseriesDescription(IsDescription):
frame = UInt64Col(pos=0) # the most recent data frame
trigger_timestamp = FloatCol(pos=1) # when the image for this frame happened
receive_timestamp = FloatCol(pos=2) # when the image trigger happened
start_display_timestamp = FloatCol(pos=3) # when the image trigger happened
stop_display_timestamp = FloatCol(pos=4) # when the image trigger happened
stripe_angle = FloatCol(pos=5) # degrees
stripe_vel = FloatCol(pos=6) # degrees/second
class SpeciesTable(tables.IsDescription): """ 2013.3.8 """ id = UInt64Col(pos=0) name = StringCol(512, pos=1) scientific_name = StringCol(512, pos=2) ploidy = UInt16Col(pos=3)
class StrokelitudeDataDescription(IsDescription):
frame = UInt64Col(pos=0)
trigger_timestamp = FloatCol(pos=1) # when the image trigger happened
processing_timestamp = FloatCol(pos=2) # when the analysis was done
left = Float32Col(pos=3) # angle, degrees
right = Float32Col(pos=4) # angle, degrees
left_antenna = Float32Col(pos=5) # angle, degrees
right_antenna = Float32Col(pos=6) # angle, degrees
head = Float32Col(pos=7) # angle, degrees
def _setup_for_table(self, table):
file_ = self.file_
self.meta_table = file_.create_table(
file_.root,
"meta_index",
description=dict(index=UInt64Col()),
filters=filters)
self.missing_values_table = file_.create_table(
file_.root,
"missing_values",
description=dict(row_index=UInt64Col(pos=0),
col_index=UInt64Col(pos=1)),
filters=filters)
def store_meta(what):
meta_index = self.manager.store_object(what)
row = self.meta_table.row
row["index"] = meta_index
row.append()
col_names = table.getColNames()
col_types = table.getColTypes()
col_formats = table.getColFormats()
store_meta(table.meta)
store_meta(col_names)
store_meta(col_types)
store_meta(col_formats)
self.meta_table.flush()
description = {}
for (pos, name, type_) in zip(itertools.count(), col_names, col_types):
tables_type = self.basic_type_map.get(type_)
if tables_type is None:
tables_type = UInt32Col
description[name] = tables_type(pos=pos)
self.row_table = file_.create_table(file_.root,
"rows",
description=description,
filters=filters)
class AssociationPeakTable(tables.IsDescription):
id = UInt64Col(pos=0)
chromosome = StringCol(64, pos=1) #64 byte-long
start = UInt64Col(pos=2)
stop = UInt64Col(pos=3)
start_locus_id = UInt64Col(pos=4)
stop_locus_id = UInt64Col(pos=5)
no_of_loci = UInt64Col(pos=6)
peak_locus_id = UInt64Col(pos=7)
peak_score = Float64Col(pos=8)
class TwoGenomeWideAssociationLocusMapTable(tables.IsDescription):
"""
2013.1.26 further attributes associated with this table:
input1_fname, input2_fname,
gw_association_locus1 (id, call-method, analysis-method-ls), gw_association_locus2 (id, cm, am)
if "pos=.." is not added, they are sorted alphabetically by their names.
"""
id = UInt64Col(pos=0)
#2013.2.24 overall position of the locus
chromosome = StringCol(64, pos=1) #64 byte-long
start = UInt64Col(pos=2)
stop = UInt64Col(pos=3)
input1_locus_id = UInt64Col(pos=4)
input1_chromosome = StringCol(64, pos=5)
input1_start = UInt64Col(pos=6)
input1_stop = UInt64Col(pos=7)
input2_locus_id = UInt64Col(pos=8)
input2_chromosome = StringCol(64, pos=9)
input2_start = UInt64Col(pos=10)
input2_stop = UInt64Col(pos=11)
locusOverlapFraction = Float64Col(pos=12)
no_of_total_phenotypes = UInt32Col(pos=13) #all significant phenotypes
total_phenotype_ls_in_str = StringCol(1000, pos=14)
fraction_of_total_phenotypes = Float32Col(
pos=15) #divided by all phenotypes with association
no_of_overlap_phenotypes = UInt32Col(pos=16)
overlap_phenotype_ls_in_str = StringCol(1000, pos=17)
fraction_of_overlap_phenotypes = Float32Col(
pos=18
) #divided by the no_of_total_phenotypes (3 cols above, with significant hits)
no_of_input1_only_phenotypes = UInt32Col(pos=19)
input1_only_phenotype_ls_in_str = StringCol(1000, pos=20)
fraction_of_input1_only_phenotypes = Float32Col(pos=21)
no_of_input2_only_phenotypes = UInt32Col(pos=22)
input2_only_phenotype_ls_in_str = StringCol(1000, pos=23)
fraction_of_input2_only_phenotypes = Float32Col(pos=24)
class LocusTable(tables.IsDescription):
id = UInt64Col(pos=0)
name = StringCol(512, pos=1) #'species_name.chromosome_name.locus_name'
chromosome_id = UInt64Col(pos=2) #64 byte-long
start = UInt64Col(pos=3)
stop = UInt64Col(pos=4)
ref_allele = StringCol(64, pos=5)
ref_allele_length = UInt64Col(
pos=6
) #this could be larger than 64, good when ref_allele is simply too long
ref_allele_frequency = Float64Col(pos=7)
alt_allele = StringCol(64, pos=8)
alt_allele_length = UInt64Col(
pos=9
) #this could be larger than 64, good when ref_allele is simply too long
alt_allele_frequency = Float64Col(pos=10)
generation_mutation_arose = Int64Col(pos=11)
generation_mutation_fixed = Int64Col(pos=12)
mutation_type = StringCol(
4, pos=13
) #0=synonymous/non-coding, 1=non-synonymous, i=insertion, d=deletion, v=inversion
fitness = Float64Col(pos=14)
ancestral_amino_acid = StringCol(4, pos=15) #only for mutation_type 0/1
derived_amino_acid = StringCol(4, pos=16)
def setup_spec_table(self):
if not hasattr(self.node, "spec_table"):
description = {}
description["pm_index"] = UInt32Col(pos=0)
description["rt"] = Float32Col(pos=1)
description["ms_level"] = UInt8Col(pos=2)
description["start"] = UInt64Col(pos=3)
description["size"] = UInt32Col(pos=4)
spec_table = self.file_.create_table(self.node,
'spec_table',
description,
filters=filters)
# every colums which appears in a where method call should/must be indexed !
# this is not only for performance but for correct lookup as well (I had strange bugs
# else)
spec_table.cols.pm_index.create_index()
spec_table.cols.rt.create_index()
def _setup_for_table(self, table):
file_ = self.file_
self.meta_table = file_.create_table(
file_.root,
"meta_index",
description=dict(index=UInt64Col()),
filters=filters)
def store_meta(what):
meta_index = self.manager.store_object("meta", what, object)
row = self.meta_table.row
row["index"] = meta_index
row.append()
col_names = table.getColNames()
col_types = table.getColTypes()
col_formats = table.getColFormats()
store_meta(table.meta)
store_meta(col_names)
store_meta(col_types)
store_meta(col_formats)
self.missing_values_flags = BitMatrix(file_, "flags", len(col_names))
# this table vesion was last modified in:
store_meta(dict(hdf5_table_version=self.LATEST_HDF5_TABLE_VERSION))
self.meta_table.flush()
description = {}
for (pos, name, type_) in izip(itertools.count(), col_names,
col_types):
tables_type = basic_type_map.get(type_)
if tables_type is None:
tables_type = UInt32Col
description[name] = tables_type(pos=pos)
self.row_table = file_.create_table(file_.root,
"rows",
description=description,
filters=filters)
class TelData(IsDescription):
db_t = Float64Col()
db_t_s = UInt64Col()
db_t_ns = UInt64Col()
ra = Float64Col()
dec = Float64Col()
class AssociationLocus2PeakTable(tables.IsDescription): id = UInt64Col(pos=0) association_locus_id = UInt64Col(pos=1) association_peak = AssociationPeakTable()
def to_hdf5(table, path):
file_ = open_file(path, mode="w")
filters = Filters(complib="blosc", complevel=9)
store, fetch, flush = setup(file_)
meta_table = file_.create_table(file_.root, "meta_index",
description=dict(index=UInt64Col()),
filters=filters)
missing_values = file_.create_table(file_.root, "missing_values",
description=dict(row_index=UInt64Col(pos=0),
col_index=UInt64Col(pos=1)),
filters=filters)
def store_meta(what):
meta_index = store(what)
row = meta_table.row
row["index"] = meta_index
row.append()
col_names = table.getColNames()
col_types = table.getColTypes()
col_formats = table.getColFormats()
store_meta(table.meta)
store_meta(col_names)
store_meta(col_types)
store_meta(col_formats)
basic_type_map = {int: Int64Col, long: Int64Col, float: Float64Col, bool: BoolCol}
description = {}
for (pos, name, type_) in zip(itertools.count(), col_names, col_types):
tables_type = basic_type_map.get(type_)
if tables_type is None:
tables_type = UInt32Col
description[name] = tables_type(pos=pos)
row_table = file_.create_table(file_.root, "rows", description=description,
filters=filters)
for row_index, row in enumerate(table.rows):
hdf_row = row_table.row
for col_index, value, name, type_ in zip(itertools.count(), row, col_names, col_types):
if value is None:
m_row = missing_values.row
m_row["row_index"] = row_index
m_row["col_index"] = col_index
m_row.append()
else:
if type_ in basic_type_map:
hdf_row[name] = value
else:
hdf_row[name] = store(value)
hdf_row.append()
flush()
meta_table.flush()
missing_values.flush()
row_table.flush()
file_.close()
class SimSSTableDs(IsDescription):
iro = UInt64Col()
ro_time = UInt64Col()
src_pos = Float32Col((2))