def CompareWriteDelim(delim):
sys.stdout.write("\nTesting Write delimiter '" + delim + "'\n")
sys.stdout.write('-' * 79 + '\n')
fname = TestFile(delim)
# this is the output
maxdiffs = {}
# write the data
r = records.Records(fname, "w", delim)
r.Write(data)
# Now read it back in and compare
r = records.Records(fname, "r", delim, data.dtype, data.size)
res = r.Read()
# Set up some formatting stuff
fwidth = 20
nhead = 4
dashes = '-' * (fwidth * nhead + (nhead - 1)) + '\n'
head_format = '%' + repr(fwidth) + 's'
format = ' '.join([head_format] * nhead) + '\n'
ii = 0
diffs = numpy.zeros(data.size * len(data.dtype.names))
for name in data.dtype.names:
sys.stdout.write('Comparing Field: %s\n' % name)
sys.stdout.write(format % ("row", "orig", "read", "%diff"))
sys.stdout.write(dashes)
for i in range(data.size):
orig = data[name][i]
read = res[name][i]
diff = DiffTestData(orig, read)
diffs[ii] = diff
ii += 1
oprint = str(orig).replace('\n', '')
rprint = str(read).replace('\n', '')
sys.stdout.write(format % (i, oprint, rprint, diff))
maxdiffs['all'] = diffs.max()
maxdiffs['delim'] = delim
maxdiffs['type'] = 'write'
PrintSummary(maxdiffs)
return maxdiffs
def write(self, data):
"""
Class:
Recfile
Method:
write
Purpose:
Write data to the opened file. The dtype of the data must match
for successive calls to write.
Calling Sequence:
r=recfile.Open(.....)
r.write(array1)
r.write(array2)
"""
if self.fobj is None:
raise ValueError("You have not yet opened a file")
if self.fobj.mode[0] != 'w' and '+' not in self.fobj.mode:
raise ValueError("You must open with 'w*' or 'r+' to write")
self.fobj.seek(0, 2) # Seek to end of file
dataview = data.view(numpy.ndarray)
if self.verbose:
stdout.write("Writing %s: %s\n" % \
(dataview.size,pprint.pformat(dataview.dtype.descr)))
if (self.delim is not None):
# let recfile deal with ascii writing
r = records.Records(self.fobj,
mode='u',
delim=self.delim,
bracket_arrays=self.bracket_arrays)
# make sure the data are in native format. This greatly
# simplifies the C code
to_native_inplace(dataview)
r.Write(dataview, padnull=self.padnull, ignorenull=self.ignorenull)
else:
# Write data out as a binary chunk
dataview.tofile(self.fobj)
# update nrows to reflect the write
self.nrows += dataview.size
def read_slice(self, arg, split=False):
if self.fobj is None:
raise ValueError("You have not yet opened a file")
if self.fobj.tell() != self.offset:
self.fobj.seek(self.offset)
robj = records.Records(self.fobj,
mode='r',
nrows=self.nrows,
dtype=self.dtype,
delim=self.delim)
result = robj.ReadSlice(long(arg.start), long(arg.stop),
long(arg.step))
if split:
return split_fields(result)
else:
return result
return result
def timing(delim, duplicate):
"""
Write the data to a file many times to get a nice big file and then
test reading it
"""
ext = extension(delim)
fname = os.path.expanduser('~/tmp/testbig' + ext)
if not os.path.exists(fname):
sys.stdout.write("Writing data to '"+delim+"' delimited file %s, "+\
"%s times" % (fname, duplicate))
WriteTestData(fname, delim, duplicate=duplicate)
nrows = data.size * duplicate
rows2read = [int(nrows * 0.25), int(nrows * 0.75), nrows - 10]
cols2read = ['id', 'u1', '2S4', 'i4', '2x2u4', 'f8']
sys.stdout.write('Reading rows: %s from file %s\n' % (rows2read, fname))
sys.stdout.write('Columns: %s\n' % cols2read)
r = records.Records(fname, "r", delim, data.dtype, nrows)
tmp = r.Read(rows=rows2read, fields=cols2read)
sys.stdout.write('%s\n' % tmp)
from functools import wraps
from uuid import uuid4
from flask import Flask, request, session, render_template, url_for, jsonify
from flask import abort, redirect, Markup
import records
# Application
# -----------
is_test_run = 'TEST' in os.environ
app = Flask(__name__)
app.debug = True
db = records.Records(offline=is_test_run)
# Routes
# ------
@app.route('/')
def index():
"""Index"""
return jsonify(message='Hello World!', methods=['GET'])
def respond_with_data(city, date, time, field=None):
"""Find and returns JSON data for a given city, date, time and data field."""
try:
timestamp = datetime.strptime(date + time, '%Y%m%d%H%M')
def _showRecords(self):
self._sound.stop()
recordScreen = RECORDS.Records(self._pygame, self, -1)
recordScreen.start()
def _showRecords(self):
recordScreen = RECORDS.Records(self._pygame, self._menu,
self._recordPosition)
recordScreen.start()
self._gameExit = True
self._showingRecords = True
def records():
records.Records()
def read(self,
rows=None,
fields=None,
columns=None,
view=None,
split=False,
**keys):
"""
Class:
Recfile
Method:
read
Purpose:
read records from the opened file.
Syntax:
r=recfile.Open(...)
data = r.read(rows=None,
fields=None, columns=None,
view=None,
split=False)
If no arguments are given, all data are read.
Inputs:
rows: A scalar, sequence or array indicating a subset
of rows to read.
fields or columns: A scalar, sequence, or array indicating
a subset of field to read. fields and columns mean the
same thing.
view: Specify an alternative view of the data.
split: Return a tuple of results rather than a rec array. Note
the data are still stored in one big chunk, this is just
an alternative access method. E.g.
# this might return a rec array with fields accessed
# such as data['x'] data['y'] data['index']
data = r.read()
# this returns a tuple with an element for each
x,y,index = r.read(split=True)
"""
if self.fobj is None:
raise ValueError("You have not yet opened a file")
if self.fobj.tell() != self.offset:
self.fobj.seek(self.offset)
rows2read = self._get_rows2read(rows)
fields2read = self._get_fields2read(fields, columns=columns)
if fields2read is None and rows2read is None and self.delim is None:
# Its binary and we are reading everything. Use fromfile.
result = numpy.fromfile(self.fobj,
dtype=self.dtype,
count=self.nrows)
else:
robj = records.Records(self.fobj,
mode='r',
nrows=self.nrows,
dtype=self.dtype,
delim=self.delim)
result = robj.Read(rows=rows2read, fields=fields2read)
if view is not None:
result = result.view(view)
if split:
return split_fields(result)
else:
return result
def CompareReadDelim(delim):
"""
Test reading a file with the given input delimiter
The following tests are run:
reading the whole file
reading each row,col separately
reading entire rows
reading entire columns
A dictionary is returned with the maximum differences
between the input data to the file (a numpy array
defined at the top of this file) and the data read
from the file. The differences should be consistent
with the precision with which the data was written
to the file. For delim="" the difference should be
zero.
"""
sys.stdout.write("\nTesting Read delimiter '" + delim + "'\n")
sys.stdout.write('-' * 79 + '\n')
sys.stdout.write("Writing data\n")
fname = TestFile(delim)
WriteTestData(fname, delim)
# Set up some formatting stuff
fwidth = 20
nhead = 4
dashes = '-' * (fwidth * nhead + (nhead - 1)) + '\n'
head_format = '%' + repr(fwidth) + 's'
format = ' '.join([head_format] * nhead) + '\n'
rows = list(range(data.size))
# this is the output
maxdiffs = {}
sys.stdout.write('\n**** Reading all data ****\n')
sys.stdout.write(dashes)
diffs = numpy.zeros(data.size * len(data.dtype.names))
r = records.Records(fname, "r", delim, data.dtype, data.size)
res = r.Read()
ii = 0
for name in res.dtype.names:
sys.stdout.write('Comparing Field: %s\n' % name)
sys.stdout.write(format % ("row", "orig", "read", "%diff"))
sys.stdout.write(dashes)
for i in range(len(rows)):
orig = data[name][i]
read = res[name][rows[i]]
diff = DiffTestData(orig, read)
diffs[ii] = diff
ii += 1
oprint = str(orig).replace('\n', '')
rprint = str(read).replace('\n', '')
sys.stdout.write(format % (rows[i], oprint, rprint, diff))
maxdiffs['all'] = diffs.max()
sys.stdout.write(dashes)
sys.stdout.write('max diff reading all at once: %s\n' % diffs.max())
# Now read each row/field separately
sys.stdout.write('\n**** Reading each row,column separately ****\n')
sys.stdout.write(dashes)
diffs[:] = 0
ii = 0
for name in data.dtype.names:
sys.stdout.write('Comparing Field: %s\n' % name)
sys.stdout.write(format % ("row", "orig", "read", "%diff"))
sys.stdout.write(dashes)
for row in range(data.size):
r = records.Records(fname, "r", delim, data.dtype, data.size)
res = r.Read(rows=row, fields=name)
r.Close()
orig = data[name][row]
read = res[name][0]
diff = DiffTestData(orig, read)
diffs[ii] = diff
ii += 1
oprint = str(orig).replace('\n', '')
rprint = str(read).replace('\n', '')
sys.stdout.write(format % (row, oprint, rprint, diff))
maxdiffs['rowcol'] = diffs.max()
sys.stdout.write(dashes)
sys.stdout.write('max diff reading each row,col separately: %s\n' %
diffs.max())
sys.stdout.write('\n**** Reading one row at at time ****\n')
sys.stdout.write(dashes)
diffs[:] = 0
ii = 0
for row in range(data.size):
r = records.Records(fname, "r", delim, data.dtype, data.size)
res = r.Read(rows=row)
sys.stdout.write('Comparing Field: %s\n' % name)
sys.stdout.write(format % ("row", "orig", "read", "%diff"))
sys.stdout.write(dashes)
for name in res.dtype.names:
orig = data[name][row]
read = res[name][0]
diff = DiffTestData(orig, read)
diffs[ii] = diff
ii += 1
oprint = str(orig).replace('\n', '')
rprint = str(read).replace('\n', '')
sys.stdout.write(format % (name, oprint, rprint, diff))
maxdiffs['row'] = diffs.max()
sys.stdout.write(dashes)
sys.stdout.write('max diff reading one row at a time: %s\n' % diffs.max())
sys.stdout.write('\n**** Reading one field at at time ****\n')
sys.stdout.write(dashes)
diffs[:] = 0
ii = 0
for name in data.dtype.names:
sys.stdout.write('Comparing Field: %s\n' % name)
sys.stdout.write(format % ("row", "orig", "read", "%diff"))
sys.stdout.write(dashes)
r = records.Records(fname, "r", delim, data.dtype, data.size)
res = r.Read(fields=name)
for row in range(data.size):
orig = data[name][row]
read = res[name][row]
diff = DiffTestData(orig, read)
diffs[ii] = diff
ii += 1
oprint = str(orig).replace('\n', '')
rprint = str(read).replace('\n', '')
sys.stdout.write(format % (row, oprint, rprint, diff))
maxdiffs['col'] = diffs.max()
sys.stdout.write(dashes)
sys.stdout.write('max diff reading one field at a time: %s\n' %
diffs.max())
maxdiffs['delim'] = delim
maxdiffs['type'] = 'read'
PrintSummary(maxdiffs)
return maxdiffs