def _test(n):
_generate(n, 20, "tmp.1", "here are some data")
import time
t0 = time.clock()
f = open("tmp.1", "r")
q = read(f)
t1 = time.clock()
print "read:", t1 - t0, "seconds\n", q[0:5, 0:5]
f.close()
t0 = time.clock()
f = open("tmp.2", "w")
write(f, q)
t1 = time.clock()
print "write:", t1 - t0, "seconds"
# compare with TableIO:
try:
import TableIO
except:
sys.exit(0) # exit silently
t0 = time.clock()
p = TableIO.readTableAsArray("tmp.1", "#")
t1 = time.clock()
print "TableIO.readTableAsArray:", t1 - t0, "seconds\n", p[0:5, 0:5]
t0 = time.clock()
TableIO.writeArray("tmp.3", p)
t1 = time.clock()
print "TableIO.writeArray:", t1 - t0, "seconds"
def get_data(self, sepchar="\t", skipchar="#"):
"""
Load data from a text file and returns an array of the data
"""
if HAVE_TABLEIO:
data = numpy.fliplr(
TableIO.readTableAsArray(self.filename, skipchar))
else:
myfile = open(self.filename, "r", DEFAULT_BUFFER_SIZE)
contents = myfile.readlines()
myfile.close()
data = []
header = True
idx = 0
while header and idx < len(contents):
if contents[idx][0] != skipchar:
header = False
break
idx += 1
for i in xrange(idx, len(contents)):
line = contents[i].strip().split(sepchar)
id = [float(line[-1])]
id += map(float, line[0:-1])
data.append(id)
logging.debug("Loaded %d lines of data from %s" %
(len(data), self))
data = numpy.array(data, numpy.float32)
return data
def get_data(self, sepchar = "\t", skipchar = "#"):
"""
Load data from a text file and returns an array of the data
"""
if HAVE_TABLEIO:
data = numpy.fliplr(TableIO.readTableAsArray(self.filename, skipchar))
else:
myfile = open(self.filename, "r", DEFAULT_BUFFER_SIZE)
contents = myfile.readlines()
myfile.close()
data = []
header = True
idx = 0
while header and idx < len(contents):
if contents[idx][0] != skipchar:
header = False
break
idx += 1
for i in xrange(idx, len(contents)):
line = contents[i].strip().split(sepchar)
id = [float(line[-1])]
id += map(float, line[0:-1])
data.append(id)
logging.debug("Loaded %d lines of data from %s" % (len(data), self))
data = numpy.array(data, numpy.float32)
return data
def data_fload(inputfile):
""" fast array loader, uses TableIO
inputfile: name of file to read data from
returns array (Numeric)
"""
# open file, transfer filtered content into list listarray
try:
import TableIO
except ImportError:
return data_load(inputfile, check=1)
else:
return TableIO.readTableAsArray(inputfile,'#')
def data_fload(inputfile):
""" fast array loader, uses TableIO
inputfile: name of file to read data from
returns array (Numeric)
"""
# open file, transfer filtered content into list listarray
try:
import TableIO
except ImportError:
return data_load(inputfile, check=1)
else:
return TableIO.readTableAsArray(inputfile, '#')
def get_data(self, sepchar = "\t", skipchar = "#"):
if HAVE_TABLEIO:
data = numpy.fliplr(TableIO.readTableAsArray(self.filename, skipchar))
else:
contents = self.fileobj.readlines()
self.fileobj.close()
for i in xrange(idx, len(contents)):
line = contents[i].strip().split(sepchar)
id = [float(line[-1])]
id += map(float, line[0:-1])
data.append(id)
logging.debug("Loaded %d lines of data from %s" % (len(data), self))
data = numpy.array(data, numpy.float32)
return data
def get_data(self, sepchar="\t", skipchar="#"):
if HAVE_TABLEIO:
data = numpy.fliplr(
TableIO.readTableAsArray(self.filename, skipchar))
else:
contents = self.fileobj.readlines()
self.fileobj.close()
for i in xrange(idx, len(contents)):
line = contents[i].strip().split(sepchar)
id = [float(line[-1])]
id += map(float, line[0:-1])
data.append(id)
logging.debug("Loaded %d lines of data from %s" %
(len(data), self))
data = numpy.array(data, numpy.float32)
return data
def get_data(self, sepchar="\t", skipchar="#"):
"""
Load data from a text file and returns a list of data
"""
if HAVE_TABLEIO:
data = TableIO.readTableAsArray(self.filename, skipchar)
else:
myfile = open(self.filename, "r", DEFAULT_BUFFER_SIZE)
contents = myfile.readlines()
myfile.close()
data = []
header = True
idx = 0
while header:
if contents[idx][0] != skipchar:
header = False
break
idx += 1
for i in xrange(idx, len(contents)):
line = contents[i].strip().split(sepchar)
id = [float(line[0])]
id += map(float, line[1:])
data.append(id)
return numpy.array(data)
def get_data(self, sepchar = "\t", skipchar = "#"):
"""
Load data from a text file and returns a list of data
"""
if HAVE_TABLEIO:
data = TableIO.readTableAsArray(self.filename, skipchar)
else:
myfile = open(self.filename, "r", DEFAULT_BUFFER_SIZE)
contents = myfile.readlines()
myfile.close()
data = []
header = True
idx = 0
while header:
if contents[idx][0] != skipchar:
header = False
break
idx += 1
for i in xrange(idx, len(contents)):
line = contents[i].strip().split(sepchar)
id = [float(line[0])]
id += map(float, line[1:])
data.append(id)
return numpy.array(data)