def test02_NewFileWithExpectedSize(self):
"Creation of a new file node with 'expectedsize' argument."
try:
filenode.newNode(
self.h5file, where = '/', name = 'test', expectedsize = 100000)
except TypeError:
self.fail("\
filenode.newNode() failed to accept 'expectedsize' argument.")
def setUp(self):
"""
This method sets the following instance attributes:
* ``h5fname``: the name of the temporary HDF5 file.
* ``h5file``: the writable, temporary HDF5 file with a ``/test`` node.
* ``fnode``: the readable file node in ``/test``, with text in it.
"""
super(ReadlineTestCase, self).setUp()
linesep = self.lineSeparator
# Fill the node file with some text.
fnode = filenode.newNode(self.h5file, where = '/', name = 'test')
fnode.lineSeparator = linesep
fnode.write(linesep)
fnode.write('short line%sshort line%s%s' % ((linesep,) * 3))
fnode.write('long line ' * 20 + linesep)
fnode.write('unterminated')
fnode.close()
# Re-open it for reading.
self.fnode = filenode.openNode(self.h5file.getNode('/test'))
self.fnode.lineSeparator = linesep
def writeFile(self, fileToImport):
"""
Used to import a binary file into the HDFFile.
"""
fnode = filenode.newNode(self.hdfFile, where=self.run, name=self._makesafekey(os.path.basename(fileToImport)))
with open(fileToImport, "r") as fdisk:
contents = fdisk.read()
fnode.write(contents)
fnode.close()
def setUp(self):
"""setUp() -> None
This method sets the following instance attributes:
* 'h5fname', the name of the temporary HDF5 file
* 'h5file', the writable, temporary HDF5 file with a '/test' node
* 'fnode', the writable file node in '/test'
"""
super(AttrsTestCase, self).setUp()
self.fnode = filenode.newNode(self.h5file, where = '/', name = 'test')
def test00_NewFile(self):
"Creation of a brand new file node."
try:
fnode = filenode.newNode(self.h5file, where = '/', name = 'test')
node = self.h5file.getNode('/test')
except LookupError:
self.fail("filenode.newNode() failed to create a new node.")
else:
self.assertEqual(
fnode.node, node,
"filenode.newNode() created a node in the wrong place.")
def testEnum(self):
"""
Write four rows into file enum.h5 and read it back.
Note that the actual enum is stored inside the file
"""
h5f = tables.openFile('enum.h5', 'w')
fnode = filenode.newNode(h5f, where='/', name='timeMaskHdr')
fnode.attrs.beginTime = 1234
fnode.attrs.endTime = 5678
fnode.close()
tbl = h5f.createTable('/','timeMask',TimeMask.TimeMask,"Time Mask")
b = [1010000, 1020000, 1010001, 1030000]
e = [1020000, 1030000, 1020001, 1040000]
r = ["Flash in r0", "Flash in r2", "Flash in r7", "Merged Flash"]
for i in range(len(b)):
row = tbl.row
row['tBegin'] = b[i]
row['tEnd'] = e[i]
row['reason'] = TimeMask.timeMaskReason[r[i]]
row.append()
tbl.flush()
tbl.close()
h5f.close()
fid = tables.openFile("enum.h5", mode='r')
node = fid.root.timeMaskHdr
self.assertEqual(node.attrs.beginTime, 1234)
self.assertEqual(node.attrs.endTime, 5678)
table = fid.getNode('/timeMask')
self.assertEqual(len(b), table.nrows)
enum = table.getEnum('reason')
for i in range(table.nrows):
self.assertEqual(table[i]['tBegin'], b[i])
self.assertEqual(table[i]['tEnd'], e[i])
self.assertEqual(enum(table[i]['reason']),r[i])
fid.close()
def setUp(self):
"""setUp() -> None
This method sets the following instance attributes:
* 'datafile', the opened data file
* 'h5fname', the name of the temporary HDF5 file
* 'h5file', the writable, temporary HDF5 file with a '/test' node
* 'fnode', the readable file node in '/test', with data in it
"""
self.datafname = self._testFilename(self.datafname)
self.datafile = file(self.datafname)
super(ReadFileTestCase, self).setUp()
fnode = filenode.newNode(self.h5file, where = '/', name = 'test')
copyFileToFile(self.datafile, fnode)
fnode.close()
self.datafile.seek(0)
self.fnode = filenode.openNode(self.h5file.getNode('/test'))
def writeFlashesToHdf5(self,overwrite=1):
"""
write intervals with flashes to the timeMask file
"""
# get the output file name, and make the directory if you need to
cmFileName = self.fn.cosmicMask()
(cmDir,name) = os.path.split(cmFileName)
if not os.path.exists(cmDir):
os.makedirs(cmDir)
# write parameters used to find flashes
h5f = tables.openFile(cmFileName, 'w')
fnode = filenode.newNode(h5f, where='/', name='timeMaskHdr')
fnode.attrs.beginTime = self.beginTime
fnode.attrs.endTime = self.endTime
fnode.attrs.nBinsPerSec = self.nBinsPerSec
fnode.attrs.flashMergeTime = self.flashMergeTime
fnode.close();
# write the times where flashes are located
tbl = h5f.createTable('/','timeMask',TimeMask.TimeMask,"Time Mask")
rlAll = list(self.roachList)
rlAll.append("all")
for roach in rlAll:
extrema = self.flashInterval[roach].extrema
for i in range(len(extrema)/2):
row = tbl.row
row['tBegin'] = int(extrema[2*i][0])
row['tEnd'] = int(extrema[2*i+1][0])
if (roach == "all"):
reason = "Merged Flash"
else:
reason = "Flash in %s" % roach
row['reason'] = TimeMask.timeMaskReason[reason]
row.append()
tbl.flush()
tbl.close()
h5f.close()
# # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # # Author: Vicent Mas - [email protected] # # This script is based on a script by Ivan Vilata. """How to use the filenode module.""" from tables.nodes import filenode import tables h5file = tables.openFile('fnode.h5', 'w') fnode = filenode.newNode(h5file, where='/', name='fnode_test') print >> fnode, "This is a test text line." print >> fnode, "And this is another one." print >> fnode fnode.write("Of course, file methods can also be used.") fnode.close() node = h5file.root.fnode_test fnode = filenode.openNode(node, 'a+') print >> fnode, "This is a new line." fnode.attrs.content_type = 'text/plain; charset=us-ascii' fnode.attrs.author = "Ivan Vilata i Balaguer" fnode.attrs.creation_date = '2004-10-20T13:25:25+0200' fnode.attrs.keywords_en = ["FileNode", "test", "metadata"]
def dir_to_h5(directory, h5_obj, h5_path="/"):
# Use the openDIR object to deal with the ugliness
dir_listing = openDIR( directory )
# You know what they say about assertions
assert (len(dir_listing.exp_paths) == 1), "This function only does 1 run at a time:" + str(dir_listing.exp_paths)
print dir_listing.total_trial_count, "blockfiles detected, compressing . . ."
# Create a leaf in the h5 for HTB and LOG file related data
htb_leaf = h5_obj.createGroup(h5_path, "htb")
log_leaf = h5_obj.createGroup(h5_path, "log")
# Create filenodes
htb_fn = filenode.newNode(h5_obj, where=htb_leaf, name='htbfile')
log_fn = filenode.newNode(h5_obj, where=log_leaf, name='logfile')
# Get file paths
log_file = dir_listing.get_LOG(dir_listing.exp_paths[0])
htb_file = dir_listing.get_HTB(dir_listing.exp_paths[0])
# Store the raw files (they are pretty small, so why not)
htb_fn.write(open(htb_file).read())
log_fn.write(open(log_file).read())
# Close to force flush
htb_fn.close()
log_fn.close()
# Open the HTB file, and break out and write each array
htb_file = openHTB(htb_file)
for a_name, htb_arr in zip( htb_file.db_names, htb_file.db_array ):
# Figure out the type in terms of tables symantics
atm = tables.Atom.from_dtype(htb_arr.dtype)
# Takes spaces out of the name, so we can use natural naming
nat_name = a_name.replace(" ","")
# Create the array and throw the data in it
leaf_arr = h5_obj.createCArray(htb_leaf, nat_name,
atm, htb_arr.shape,
a_name)
leaf_arr[:] = htb_arr
# Block file bits
# Conv. lists
condition = []
trial_num = []
# Pack in the block files
array_created = False
for n, (cn, bn, tn, f) in enumerate(dir_listing.BLK_iter(dir_listing.exp_paths[0])):
print "Starting", n, "of", dir_listing.total_trial_count
print "\tLoading .BLK (%s) cn:%i tn:%i" % (f, cn, tn)
# Open the block file
bf = openBLK(f)
bf.load_data()
# Create a c-array
if not array_created:
new_shape = bf.data_array.shape + tuple([0])
ear = h5_obj.createEArray("/", "block", tables.Int32Atom(), new_shape, "Data from '%s'" % directory)
array_created = True
print "\tWriting to h5."
# Make the stings for the name and description of each data bloc
ear.append( bf.data_array[:,:,:,newaxis] )
# Mem clear necessary? TODO
ear.flush()
h5_obj.flush()
del bf
# For later conv.
condition.append(cn)
trial_num.append(tn)
print "\tDone."
# Create the condition and trial number arrays.
# (i.e avoid redoing that ugly file name munging each time)
h5_obj.createArray(h5_path, "condition", array(condition))
h5_obj.createArray(h5_path, "trial_num", array(trial_num))
# Yay done!
print "Done!"
from tables.nodes import filenode
import tables
h5file = tables.openFile('fnode.h5', 'w')
fnode = filenode.newNode(h5file, where='/', name='fnode_test')
print h5file.getNodeAttr('/fnode_test', 'NODE_TYPE')
print >> fnode, "This is a test text line."
print >> fnode, "And this is another one."
print >> fnode
fnode.write("Of course, file methods can also be used.")
fnode.seek(0) # Go back to the beginning of file.
for line in fnode:
print repr(line)
fnode.close()
print fnode.closed
node = h5file.root.fnode_test
fnode = filenode.openNode(node, 'a+')
print repr(fnode.readline())
def download_and_store(url, datastore, data_filename=None, overwrite=False,
download_reference=None):
# set data_filename from the URL if not explicitly defined
if not data_filename:
data_filename = url.split('/')[-1]
# if datastore exists, check if filename already exists in datastore
if os.access(datastore, os.R_OK):
store = pd.HDFStore(datastore, "a")
# pandas API change between 0.10.1 and 0.11
try:
_h = store.handle
except:
_h = store._handle
src_filenames = [n.__repr__().split("(")[0].strip()[1:].split("/")[1]
for n in _h.iterNodes('/{0}'.format(path_src))]
if data_filename in src_filenames:
if overwrite:
_h.removeNode("/{0}/{1}".format(path_src,
data_filename))
else:
raise AttributeError("Data file cannot be saved: filename "
"already exists in datastore")
# create datastore
else:
store = pd.HDFStore(datastore, "w", complevel=complevel,
complib=complib)
# pandas API change between 0.10.1 and 0.11
try:
_h = store.handle
except:
_h = store._handle
# prepare datastore
if path_data not in [n.__repr__().split("(")[0].strip()[1:] for n in
_h.iterNodes('/')]:
_h.createGroup('/', path_data)
if path_src not in [n.__repr__().split("(")[0].strip()[1:] for n in
_h.iterNodes('/')]:
_h.createGroup('/', path_src)
if path_doc not in [n.__repr__().split("(")[0].strip()[1:] for n in
_h.iterNodes('/')]:
_h.createGroup('/', path_doc)
# retrieve file
# TODO: find out if download was successful
localfile = tempfile.mktemp()
urlretrieve(url, localfile)
with open(localfile, "rb") as fd:
rawdata = fd.read()
# save retrieved file to datastore
fnode = filenode.newNode(_h, where="/{0}".format(path_src),
name=data_filename)
fnode.write(rawdata)
fnode.close()
# create metadata
srcnode = _h.getNode("/{0}".format(path_src), data_filename)
srcnode.attrs.DOWNLOAD_URL = url
srcnode.attrs.DOWNLOAD_DATE = datetime.datetime.now().isoformat()
if download_reference:
srcnode.attrs.DOWNLOAD_REFERENCE = download_reference
# TODO: create README
# cleanup
os.remove(localfile)
store.close()