def _test(n):
_generate(n, 20, "tmp.1", "here are some data")
import time
t0 = time.clock()
f = open("tmp.1", "r")
q = read(f)
t1 = time.clock()
print "read:", t1 - t0, "seconds\n", q[0:5, 0:5]
f.close()
t0 = time.clock()
f = open("tmp.2", "w")
write(f, q)
t1 = time.clock()
print "write:", t1 - t0, "seconds"
# compare with TableIO:
try:
import TableIO
except:
sys.exit(0) # exit silently
t0 = time.clock()
p = TableIO.readTableAsArray("tmp.1", "#")
t1 = time.clock()
print "TableIO.readTableAsArray:", t1 - t0, "seconds\n", p[0:5, 0:5]
t0 = time.clock()
TableIO.writeArray("tmp.3", p)
t1 = time.clock()
print "TableIO.writeArray:", t1 - t0, "seconds"
def data_fsave(arr, outputfile):
""" fast, but unprecise way to store arrays to disk
arr: array to store
outputfile:
no return value
suitable for integer and short float numbers.
"""
try:
import TableIO
except ImportError:
data_save(arr, outputfile)
else:
TableIO.writeArray(outputfile, arr)
def data_fsave(arr, outputfile):
""" fast, but unprecise way to store arrays to disk
arr: array to store
outputfile:
no return value
suitable for integer and short float numbers.
"""
try:
import TableIO
except ImportError:
data_save(arr,outputfile)
else:
TableIO.writeArray(outputfile, arr)
def EverythingOutput(data, z, approach, wdm="CDM"):
"""
Outputs most of the calculated values as a .dat file
Input: data: a list of lists. An array containing all of the data to write to a table.
"""
print " Outputting Data Tables"
if wdm is None:
wdm = "CDM"
mass_var_table = TableIO.TableFromColumns(data)
if wdm is "CDM":
mass_var_table['filename'] = config.out_pref+'/OutputTables/'+'z'+str(z)+'.CDM.'+approach+'.txt'
else:
mass_var_table['filename'] = config.out_pref+'/OutputTables/'+'z'+str(z)+'.WDM'+str(wdm)+'.'+approach+'.txt'
TableIO.writeTable(mass_var_table)
def get_data(self, sepchar="\t", skipchar="#"):
"""
Load data from a text file and returns an array of the data
"""
if HAVE_TABLEIO:
data = numpy.fliplr(
TableIO.readTableAsArray(self.filename, skipchar))
else:
myfile = open(self.filename, "r", DEFAULT_BUFFER_SIZE)
contents = myfile.readlines()
myfile.close()
data = []
header = True
idx = 0
while header and idx < len(contents):
if contents[idx][0] != skipchar:
header = False
break
idx += 1
for i in xrange(idx, len(contents)):
line = contents[i].strip().split(sepchar)
id = [float(line[-1])]
id += map(float, line[0:-1])
data.append(id)
logging.debug("Loaded %d lines of data from %s" %
(len(data), self))
data = numpy.array(data, numpy.float32)
return data
def get_data(self, sepchar = "\t", skipchar = "#"):
"""
Load data from a text file and returns an array of the data
"""
if HAVE_TABLEIO:
data = numpy.fliplr(TableIO.readTableAsArray(self.filename, skipchar))
else:
myfile = open(self.filename, "r", DEFAULT_BUFFER_SIZE)
contents = myfile.readlines()
myfile.close()
data = []
header = True
idx = 0
while header and idx < len(contents):
if contents[idx][0] != skipchar:
header = False
break
idx += 1
for i in xrange(idx, len(contents)):
line = contents[i].strip().split(sepchar)
id = [float(line[-1])]
id += map(float, line[0:-1])
data.append(id)
logging.debug("Loaded %d lines of data from %s" % (len(data), self))
data = numpy.array(data, numpy.float32)
return data
def EverythingOutput(data, z, approach, wdm="CDM"):
"""
Outputs most of the calculated values as a .dat file
Input: data: a list of lists. An array containing all of the data to write to a table.
"""
print " Outputting Data Tables"
if wdm is None:
wdm = "CDM"
mass_var_table = TableIO.TableFromColumns(data)
if wdm is "CDM":
mass_var_table[
'filename'] = config.out_pref + '/OutputTables/' + 'z' + str(
z) + '.CDM.' + approach + '.txt'
else:
mass_var_table[
'filename'] = config.out_pref + '/OutputTables/' + 'z' + str(
z) + '.WDM' + str(wdm) + '.' + approach + '.txt'
TableIO.writeTable(mass_var_table)
def data_fload(inputfile):
""" fast array loader, uses TableIO
inputfile: name of file to read data from
returns array (Numeric)
"""
# open file, transfer filtered content into list listarray
try:
import TableIO
except ImportError:
return data_load(inputfile, check=1)
else:
return TableIO.readTableAsArray(inputfile, '#')
def data_fload(inputfile):
""" fast array loader, uses TableIO
inputfile: name of file to read data from
returns array (Numeric)
"""
# open file, transfer filtered content into list listarray
try:
import TableIO
except ImportError:
return data_load(inputfile, check=1)
else:
return TableIO.readTableAsArray(inputfile,'#')
def get_data(self, sepchar = "\t", skipchar = "#"):
if HAVE_TABLEIO:
data = numpy.fliplr(TableIO.readTableAsArray(self.filename, skipchar))
else:
contents = self.fileobj.readlines()
self.fileobj.close()
for i in xrange(idx, len(contents)):
line = contents[i].strip().split(sepchar)
id = [float(line[-1])]
id += map(float, line[0:-1])
data.append(id)
logging.debug("Loaded %d lines of data from %s" % (len(data), self))
data = numpy.array(data, numpy.float32)
return data
def get_data(self, sepchar="\t", skipchar="#"):
if HAVE_TABLEIO:
data = numpy.fliplr(
TableIO.readTableAsArray(self.filename, skipchar))
else:
contents = self.fileobj.readlines()
self.fileobj.close()
for i in xrange(idx, len(contents)):
line = contents[i].strip().split(sepchar)
id = [float(line[-1])]
id += map(float, line[0:-1])
data.append(id)
logging.debug("Loaded %d lines of data from %s" %
(len(data), self))
data = numpy.array(data, numpy.float32)
return data
def BEDtoCount(bed,chr_list,bins,binsize):
bedF=TableIO.parse("./bed_file/"+bed,'bed')
total_bin_num=0
count={}
for c,length in chr_list.items():
count[c]=[0]*bins[c]
read_num=0
for i in bedF:
if i.chr not in chr_list.keys(): continue
start_bin=i.start/binsize
end_bin=i.stop/binsize
for n in range(start_bin,(end_bin+1)):
try:
count[i.chr][n]+=1
except IndexError:
continue
read_num+=1
return count,read_num
def BEDtoCount(bed, chr_list, bins, binsize):
bedF = TableIO.parse("./bed_file/" + bed, 'bed')
total_bin_num = 0
count = {}
for c, length in chr_list.items():
count[c] = [0] * bins[c]
read_num = 0
for i in bedF:
if i.chr not in chr_list.keys(): continue
start_bin = i.start / binsize
end_bin = i.stop / binsize
for n in range(start_bin, (end_bin + 1)):
try:
count[i.chr][n] += 1
except IndexError:
continue
read_num += 1
return count, read_num
def ImportTransferFunction(transfer_file):
"""
Imports the Transfer Function file to be analysed, and returns the pair ln(k), ln(T)
Input: "transfer_file": full path to the file containing the transfer function (from camb).
Output: ln(k), ln(T)
"""
print " Importing Transfer Function"
k,T = TableIO.readColumns(transfer_file,"!#",columns=[0,1])
k = np.log(k)
T = np.log(T)
if config.loud:
print " Minimum logk "+str( min(k))
print " Maximum logk "+str( max(k))
return k,T
def ImportTransferFunction(transfer_file):
"""
Imports the Transfer Function file to be analysed, and returns the pair ln(k), ln(T)
Input: "transfer_file": full path to the file containing the transfer function (from camb).
Output: ln(k), ln(T)
"""
print " Importing Transfer Function"
k, T = TableIO.readColumns(transfer_file, "!#", columns=[0, 1])
k = np.log(k)
T = np.log(T)
if config.loud:
print " Minimum logk " + str(min(k))
print " Maximum logk " + str(max(k))
return k, T
def get_data(self, sepchar = "\t", skipchar = "#"):
"""
Load data from a text file and returns a list of data
"""
if HAVE_TABLEIO:
data = TableIO.readTableAsArray(self.filename, skipchar)
else:
myfile = open(self.filename, "r", DEFAULT_BUFFER_SIZE)
contents = myfile.readlines()
myfile.close()
data = []
header = True
idx = 0
while header:
if contents[idx][0] != skipchar:
header = False
break
idx += 1
for i in xrange(idx, len(contents)):
line = contents[i].strip().split(sepchar)
id = [float(line[0])]
id += map(float, line[1:])
data.append(id)
return numpy.array(data)
def get_data(self, sepchar="\t", skipchar="#"):
"""
Load data from a text file and returns a list of data
"""
if HAVE_TABLEIO:
data = TableIO.readTableAsArray(self.filename, skipchar)
else:
myfile = open(self.filename, "r", DEFAULT_BUFFER_SIZE)
contents = myfile.readlines()
myfile.close()
data = []
header = True
idx = 0
while header:
if contents[idx][0] != skipchar:
header = False
break
idx += 1
for i in xrange(idx, len(contents)):
line = contents[i].strip().split(sepchar)
id = [float(line[0])]
id += map(float, line[1:])
data.append(id)
return numpy.array(data)
