def main(): parser = argparse.ArgumentParser( usage='%(prog)s [-h] <json_file> <h5_file>') parser.add_argument('in_filename', nargs='+', help='JSon file to be converted to h5') parser.add_argument('out_filename', nargs='+', help='name of HDF5 output file') args = parser.parse_args() # create logger log = logging.getLogger("h5serv") # log.setLevel(logging.WARN) log.setLevel(logging.INFO) # add log handler handler = logging.FileHandler('./jsontoh5.log') # add handler to logger log.addHandler(handler) text = open(args.in_filename[0]).read() # parse the json file h5json = json.loads(text) if "root" not in h5json: raise Exception("no root key in input file") root_uuid = h5json["root"] filename = args.out_filename[0] # create the file, will raise IOError if there's a problem Hdf5db.createHDF5File(filename) with Hdf5db(filename, root_uuid=root_uuid, update_timestamps=False, app_logger=log) as db: h5writer = Writeh5(db, h5json) h5writer.writeFile() # open with h5py and remove the _db_ group # Note: this will delete any anonymous (un-linked) objects f = h5py.File(filename, 'a') if "__db__" in f: del f["__db__"] f.close() print("done!")
def testReadCommittedType(self): filepath = getFile('committed_type.h5', 'readcommitted_type.h5') with Hdf5db(filepath, app_logger=self.log) as db: root_uuid = db.getUUIDByPath('/') type_uuid = db.getUUIDByPath('/Sensor_Type') item = db.getCommittedTypeItemByUuid(type_uuid) self.assertTrue('type' in item) item_type = item['type'] self.assertTrue(item_type['class'], 'H5T_COMPOUND') ds1_uuid = db.getUUIDByPath('/DS1') item = db.getDatasetItemByUuid(ds1_uuid) shape = item['shape'] self.failUnlessEqual(shape['class'], 'H5S_SIMPLE') dims = shape['dims'] self.failUnlessEqual(len(dims), 1) self.failUnlessEqual(dims[0], 4) item_type = item['type'] self.assertTrue('class' in item_type) self.failUnlessEqual(item_type['class'], 'H5T_COMPOUND') self.assertTrue('uuid' in item_type) self.failUnlessEqual(item_type['uuid'], type_uuid) item = db.getAttributeItem("groups", root_uuid, "attr1") # FIXME: fails on Py3 shape = item['shape'] self.failUnlessEqual(shape['class'], 'H5S_SCALAR') item_type = item['type'] self.assertTrue('class' in item_type) self.failUnlessEqual(item_type['class'], 'H5T_COMPOUND') self.assertTrue('uuid' in item_type) self.failUnlessEqual(item_type['uuid'], type_uuid)
def testCreateVlenReferenceAttribute(self): filepath = getFile('empty.h5', 'createreferenceattribute.h5') with Hdf5db(filepath, app_logger=self.log) as db: root_uuid = db.getUUIDByPath('/') dims = () # if no space in body, default to scalar rsp = db.createDataset("H5T_STD_I64LE", dims, max_shape=None, creation_props=None) dset_uuid = rsp['id'] db.linkObject(root_uuid, dset_uuid, 'DS1') dims = (1,) datatype = {"class": "H5T_VLEN", "base": { "class": "H5T_REFERENCE", "base": "H5T_STD_REF_OBJ"} } ds1_ref = "datasets/" + dset_uuid value = [[ds1_ref,],] db.createAttribute("groups", root_uuid, "A1", dims, datatype, value) item = db.getAttributeItem("groups", root_uuid, "A1") # FIXME: fails on Py3 attr_type = item['type'] self.failUnlessEqual(attr_type["class"], "H5T_VLEN") base_type = attr_type["base"] # todo - this should be H5T_REFERENCE, not H5T_OPAQUE # See h5py issue: https://github.com/h5py/h5py/issues/553 self.failUnlessEqual(base_type["class"], "H5T_OPAQUE")
def testWriteVlenUnicodeAttribute(self): # getAttributeItemByUuid item = None filepath = getFile('empty.h5', 'writevlenunicodeattribute.h5') with Hdf5db(filepath, app_logger=self.log) as db: root_uuid = db.getUUIDByPath('/') dims = () datatype = { 'charSet': 'H5T_CSET_UTF8', 'class': 'H5T_STRING', 'strPad': 'H5T_STR_NULLTERM', 'length': 'H5T_VARIABLE' } value = u'\u6b22\u8fce\u63d0\u4ea4\u5fae\u535a\u641c\u7d22\u4f7f\u7528\u53cd\u9988\uff0c\u8bf7\u76f4\u63a5' db.createAttribute("groups", root_uuid, "A1", dims, datatype, value) item = db.getAttributeItem("groups", root_uuid, "A1") # FIXME: fails on Py3 self.failUnlessEqual(item['name'], "A1") self.failUnlessEqual(item['value'], value) now = int(time.time()) self.assertTrue(item['ctime'] > now - 5) self.assertTrue(item['mtime'] > now - 5) shape = item['shape'] self.failUnlessEqual(shape['class'], 'H5S_SCALAR') item_type = item['type'] self.failUnlessEqual(item_type['class'], 'H5T_STRING') self.failUnlessEqual(item_type['strPad'], 'H5T_STR_NULLTERM') self.failUnlessEqual(item_type['charSet'], 'H5T_CSET_UTF8') self.failUnlessEqual(item_type['length'], 'H5T_VARIABLE')
def testWriteVlenStringAttribute(self): # getAttributeItemByUuid item = None filepath = getFile('empty.h5', 'writevlenstringattribute.h5') with Hdf5db(filepath, app_logger=self.log) as db: root_uuid = db.getUUIDByPath('/') dims = () datatype = { 'charSet': 'H5T_CSET_ASCII', 'class': 'H5T_STRING', 'strPad': 'H5T_STR_NULLTERM', 'length': 'H5T_VARIABLE' } value = "Hello, world!" db.createAttribute("groups", root_uuid, "A1", dims, datatype, value) # FIXME: fails on Py3 item = db.getAttributeItem("groups", root_uuid, "A1") self.failUnlessEqual(item['name'], "A1") self.failUnlessEqual(item['value'], "Hello, world!") now = int(time.time()) self.assertTrue(item['ctime'] > now - 5) self.assertTrue(item['mtime'] > now - 5) shape = item['shape'] self.failUnlessEqual(shape['class'], 'H5S_SCALAR') item_type = item['type'] self.failUnlessEqual(item_type['class'], 'H5T_STRING') self.failUnlessEqual(item_type['strPad'], 'H5T_STR_NULLTERM') self.failUnlessEqual(item_type['charSet'], 'H5T_CSET_ASCII') self.failUnlessEqual(item_type['length'], 'H5T_VARIABLE')
def testWriteFixedNullTermStringAttribute(self): # getAttributeItemByUuid item = None filepath = getFile('empty.h5', 'writefixednulltermstringattribute.h5') with Hdf5db(filepath, app_logger=self.log) as db: root_uuid = db.getUUIDByPath('/') dims = () datatype = { 'charSet': 'H5T_CSET_ASCII', 'class': 'H5T_STRING', 'strPad': 'H5T_STR_NULLTERM', 'length': 13} value = "Hello, world!" # write the attribute db.createAttribute("groups", root_uuid, "A1", dims, datatype, value) # FIXME: fails on Py3 # read it back item = db.getAttributeItem("groups", root_uuid, "A1") self.failUnlessEqual(item['name'], "A1") # the following compare fails - see issue #34 #self.failUnlessEqual(item['value'], "Hello, world!") now = int(time.time()) self.assertTrue(item['ctime'] > now - 5) self.assertTrue(item['mtime'] > now - 5) shape = item['shape'] self.failUnlessEqual(shape['class'], 'H5S_SCALAR') item_type = item['type'] self.failUnlessEqual(item_type['length'], 13) self.failUnlessEqual(item_type['class'], 'H5T_STRING') # NULLTERM get's converted to NULLPAD since the numpy dtype does not # support other padding conventions. self.failUnlessEqual(item_type['strPad'], 'H5T_STR_NULLPAD') self.failUnlessEqual(item_type['charSet'], 'H5T_CSET_ASCII')
def testRootAcl(self): filepath = getFile('tall.h5', 'rootacl.h5') user1 = 123 with Hdf5db(filepath, app_logger=self.log) as db: root_uuid = db.getUUIDByPath('/') d111_uuid = db.getUUIDByPath('/g1/g1.1/dset1.1.1') num_acls = db.getNumAcls(d111_uuid) self.failUnlessEqual(num_acls, 0) # add read/write acl for user1 at root acl_root = db.getAcl(root_uuid, 0) self.failUnlessEqual(acl_root['userid'], 0) acl_root['create'] = 0 acl_root['read'] = 1 acl_root['update'] = 0 acl_root['delete'] = 0 acl_root['readACL'] = 0 acl_root['updateACL'] = 0 num_acls = db.getNumAcls(root_uuid) self.failUnlessEqual(num_acls, 0) db.setAcl(root_uuid, acl_root) num_acls = db.getNumAcls(root_uuid) self.failUnlessEqual(num_acls, 1) acl = db.getAcl(d111_uuid, user1) num_acls = db.getNumAcls(d111_uuid) # this will fetch the root acl self.failUnlessEqual(num_acls, 0) self.failUnlessEqual(acl['userid'], 0) self.failUnlessEqual(acl['create'], 0) self.failUnlessEqual(acl['read'], 1) self.failUnlessEqual(acl['update'], 0) self.failUnlessEqual(acl['delete'], 0) self.failUnlessEqual(acl['readACL'], 0) self.failUnlessEqual(acl['updateACL'], 0)
def testCreateCommittedCompoundTypeDataset(self): filepath = getFile('empty.h5', 'createcommittedcompoundtypedataset.h5') with Hdf5db(filepath, app_logger=self.log) as db: root_uuid = db.getUUIDByPath('/') datatype = {'class': 'H5T_COMPOUND', 'fields': [] } type_fields = [] type_fields.append({'name': 'field_1', 'type': 'H5T_STD_I64BE' }) type_fields.append({'name': 'field_2', 'type': 'H5T_IEEE_F64BE' }) datatype['fields'] = type_fields creation_props = { "fillValue": [ 0, 0.0 ] } item = db.createCommittedType(datatype) # FIXME: fails on Py3 type_uuid = item['id'] dims = () # if no space in body, default to scalar rsp = db.createDataset(type_uuid, dims, max_shape=None, creation_props=creation_props) dset_uuid = rsp['id'] item = db.getDatasetItemByUuid(dset_uuid) type_item = item['type'] self.assertTrue('uuid' in type_item) self.failUnlessEqual(type_item['uuid'], type_uuid)
def testGetNumLinks(self): items = None filepath = getFile('tall.h5', 'getnumlinks.h5') with Hdf5db(filepath, app_logger=self.log) as db: g1= db.getObjByPath('/g1') numLinks = db.getNumLinksToObject(g1) self.assertEqual(numLinks, 1)
def main(): parser = argparse.ArgumentParser(usage='%(prog)s [-h] [-D|-d] <hdf5_file>') parser.add_argument('-D', action='store_true', help='surpress all data output') parser.add_argument('-d', action='store_true', help='surpress data output for' + ' datasets (but not attribute values)') parser.add_argument('filename', nargs='+', help='HDF5 to be converted to json') args = parser.parse_args() # create logger log = logging.getLogger("h5serv") # log.setLevel(logging.WARN) log.setLevel(logging.INFO) # add log handler handler = logging.FileHandler('./h5tojson.log') # add handler to logger log.addHandler(handler) filename = args.filename[0] if not op.isfile(filename): sys.exit("Cannot find file: %s" % filename) log.info("h5tojson " + filename) dbFilename = getTempFileName() log.info("Using dbFile: " + dbFilename) with Hdf5db(filename, dbFilePath=dbFilename, readonly=True, app_logger=log) as db: dumper = DumpJson(db, app_logger=log, options=args) dumper.dumpFile()
def testInvalidPath(self): filepath = "/tmp/thisisnotafile.h5" try: with Hdf5db(filepath, app_logger=self.log) as db: self.assertTrue(False) # shouldn't get here except IOError as e: self.failUnlessEqual(e.errno, errno.ENXIO) self.failUnlessEqual(e.strerror, "file not found")
def testReadAttribute(self): # getAttributeItemByUuid item = None filepath = getFile('tall.h5', 'readattribute.h5') with Hdf5db(filepath, app_logger=self.log) as db: rootUuid = db.getUUIDByPath('/') self.failUnlessEqual(len(rootUuid), UUID_LEN) item = db.getAttributeItem("groups", rootUuid, "attr1") # FIXME: fails on Py3
def testInvalidFile(self): filepath = getFile('notahdf5file.h5', 'notahdf5file.h5') try: with Hdf5db(filepath, app_logger=self.log) as db: self.assertTrue(False) # shouldn't get here except IOError as e: self.failUnlessEqual(e.errno, errno.EINVAL) self.failUnlessEqual(e.strerror, "not an HDF5 file")
def main(): if len(sys.argv) < 2: print "usage: h5tojson <filename>" sys.exit() filepath = sys.argv[1] with Hdf5db(filepath, readonly=True) as db: dumper = DumpJson(db) dumper.dumpFile()
def testReadAttribute(self): # getAttributeItemByUuid item = None getFile('tall.h5') with Hdf5db('tall.h5') as db: rootUuid = db.getUUIDByPath('/') self.failUnlessEqual(len(rootUuid), config.get('uuidlen')) item = db.getAttributeItem("groups", rootUuid, "attr1")
def testGetCounts(self): with Hdf5db('tall.h5') as db: cnt = db.getNumberOfGroups() self.failUnlessEqual(cnt, 6) cnt = db.getNumberOfDatasets() self.failUnlessEqual(cnt, 4) cnt = db.getNumberOfDatatypes() self.failUnlessEqual(cnt, 0)
def testReadZeroDimDataset(self): getFile('zerodim.h5') d111_values = None d112_values = None with Hdf5db('zerodim.h5') as db: dsetUuid = db.getUUIDByPath('/dset') self.failUnlessEqual(len(dsetUuid), config.get('uuidlen')) dset_value = db.getDatasetValuesByUuid(dsetUuid) self.assertEqual(dset_value, 42)
def testReadZeroDimDataset(self): filepath = getFile('zerodim.h5', 'readzerodeimdataset.h5') d111_values = None d112_values = None with Hdf5db(filepath, app_logger=self.log) as db: dsetUuid = db.getUUIDByPath('/dset') self.failUnlessEqual(len(dsetUuid), UUID_LEN) dset_value = db.getDatasetValuesByUuid(dsetUuid) self.assertEqual(dset_value, 42)
def testGetCounts(self): filepath = getFile('tall.h5', 'testgetcounts_tall.h5') with Hdf5db(filepath, app_logger=self.log) as db: cnt = db.getNumberOfGroups() self.failUnlessEqual(cnt, 6) cnt = db.getNumberOfDatasets() self.failUnlessEqual(cnt, 4) cnt = db.getNumberOfDatatypes() self.failUnlessEqual(cnt, 0) filepath = getFile('empty.h5', 'testgetcounts_empty.h5') with Hdf5db(filepath, app_logger=self.log) as db: cnt = db.getNumberOfGroups() self.failUnlessEqual(cnt, 1) cnt = db.getNumberOfDatasets() self.failUnlessEqual(cnt, 0) cnt = db.getNumberOfDatatypes() self.failUnlessEqual(cnt, 0)
def testToTuple(self): filepath = getFile('empty.h5', 'totuple.h5') with Hdf5db(filepath, app_logger=self.log) as db: self.assertEqual(db.toTuple( [1,2,3] ), (1,2,3) ) self.assertEqual(db.toTuple( [[1,2],[3,4]] ), ((1,2),(3,4)) ) self.assertEqual(db.toTuple( ([1,2],[3,4]) ), ((1,2),(3,4)) ) self.assertEqual(db.toTuple( [(1,2),(3,4)] ), ((1,2),(3,4)) ) self.assertEqual(db.toTuple( [[[1,2],[3,4]], [[5,6],[7,8]]] ), (((1,2),(3,4)), ((5,6),(7,8))) )
def testDeleteLink(self): # get test file filepath = getFile('tall.h5', 'deletelink.h5') with Hdf5db(filepath, app_logger=self.log) as db: rootUuid = db.getUUIDByPath('/') numRootChildren = len(db.getLinkItems(rootUuid)) self.assertEqual(numRootChildren, 2) db.unlinkItem(rootUuid, "g2") numRootChildren = len(db.getLinkItems(rootUuid)) self.assertEqual(numRootChildren, 1)
def testDeleteLink(self): # get test file getFile('tall.h5', 'tall_grpdelete.h5') with Hdf5db('tall_grpdelete.h5') as db: rootUuid = db.getUUIDByPath('/') numRootChildren = len(db.getLinkItems(rootUuid)) self.assertEqual(numRootChildren, 2) db.unlinkItem(rootUuid, "g2") numRootChildren = len(db.getLinkItems(rootUuid)) self.assertEqual(numRootChildren, 1)
def testGetItemSoftLink(self): with Hdf5db('tall.h5') as db: grpUuid = db.getUUIDByPath('/g1/g1.2/g1.2.1') item = db.getLinkItemByUuid(grpUuid, "slink") self.assertTrue('id' not in item) self.assertEqual(item['title'], 'slink') self.assertEqual(item['class'], 'H5L_TYPE_SOFT') self.assertEqual(item['h5path'], 'somevalue') self.assertTrue('mtime' in item) self.assertTrue('ctime' in item)
def main(): parser = argparse.ArgumentParser(usage='%(prog)s [-h] <json_file> <h5_file>') parser.add_argument('in_filename', nargs='+', help='JSon file to be converted to h5') parser.add_argument('out_filename', nargs='+', help='name of HDF5 output file') args = parser.parse_args() # create logger log = logging.getLogger("h5serv") # log.setLevel(logging.WARN) log.setLevel(logging.INFO) # add log handler handler = logging.FileHandler('./jsontoh5.log') # add handler to logger log.addHandler(handler) text = open(args.in_filename[0]).read() # parse the json file h5json = json.loads(text) if "root" not in h5json: raise Exception("no root key in input file") root_uuid = h5json["root"] filename = args.out_filename[0] # create the file, will raise IOError if there's a problem Hdf5db.createHDF5File(filename) with Hdf5db(filename, root_uuid=root_uuid, update_timestamps=False, app_logger=log) as db: h5writer = Writeh5(db, h5json) h5writer.writeFile() # open with h5py and remove the _db_ group # Note: this will delete any anonymous (un-linked) objects f = h5py.File(filename, 'a') if "__db__" in f: del f["__db__"] f.close() print("done!")
def testGetItemHardLink(self): with Hdf5db('tall.h5') as db: grpUuid = db.getUUIDByPath('/g1/g1.1') item = db.getLinkItemByUuid(grpUuid, "dset1.1.1") self.assertTrue('id' in item) self.assertEqual(item['title'], 'dset1.1.1') self.assertEqual(item['class'], 'H5L_TYPE_HARD') self.assertEqual(item['collection'], 'datasets') self.assertTrue('target' not in item) self.assertTrue('mtime' in item) self.assertTrue('ctime' in item)
def testDeleteUDLink(self): # get test file getFile('tall_with_udlink.h5') with Hdf5db('tall_with_udlink.h5') as db: g2Uuid = db.getUUIDByPath('/g2') numG2Children = len(db.getLinkItems(g2Uuid)) self.assertEqual(numG2Children, 3) rc = db.unlinkItem(g2Uuid, "udlink") self.assertFalse(rc) numG2Children = len(db.getLinkItems(g2Uuid)) self.assertEqual(numG2Children, 3)
def testGetItemHardLink(self): with Hdf5db('tall.h5') as db: grpUuid = db.getUUIDByPath('/g1/g1.1') item = db.getLinkItemByUuid(grpUuid, "dset1.1.1") self.assertTrue('id' in item) self.assertEqual(item['name'], 'dset1.1.1') self.assertEqual(item['class'], 'hard') self.assertEqual(item['className'], 'Dataset') self.assertTrue('target' not in item) self.assertTrue('mtime' in item) self.assertTrue('ctime' in item)
def testGetUUIDByPath(self): # get test file g1Uuid = None with Hdf5db('tall.h5') as db: g1Uuid = db.getUUIDByPath('/g1') self.failUnlessEqual(len(g1Uuid), config.get('uuidlen')) obj = db.getObjByPath('/g1') self.failUnlessEqual(obj.name, '/g1') for name in obj: g = obj[name] g1links = db.getLinkItems(g1Uuid) self.failUnlessEqual(len(g1links), 2) for item in g1links: self.failUnlessEqual(len(item['id']), config.get('uuidlen')) # end of with will close file # open again and verify we can get obj by name with Hdf5db('tall.h5') as db: obj = db.getGroupObjByUuid(g1Uuid) g1 = db.getObjByPath('/g1') self.failUnlessEqual(obj, g1)
def testGetItemExternalLink(self): getFile('tall_with_udlink.h5') with Hdf5db('tall_with_udlink.h5') as db: grpUuid = db.getUUIDByPath('/g1/g1.2') item = db.getLinkItemByUuid(grpUuid, "extlink") self.assertTrue('uuid' not in item) self.assertEqual(item['title'], 'extlink') self.assertEqual(item['class'], 'H5L_TYPE_EXTERNAL') self.assertEqual(item['h5path'], 'somepath') self.assertEqual(item['file'], 'somefile') self.assertTrue('mtime' in item) self.assertTrue('ctime' in item)
def testGetItemUDLink(self): getFile('tall_with_udlink.h5') with Hdf5db('tall_with_udlink.h5') as db: grpUuid = db.getUUIDByPath('/g2') item = db.getLinkItemByUuid(grpUuid, "udlink") self.assertTrue('uuid' not in item) self.assertEqual(item['title'], 'udlink') self.assertEqual(item['class'], 'H5L_TYPE_USER_DEFINED') self.assertTrue('h5path' not in item) self.assertTrue('file' not in item) self.assertTrue('mtime' in item) self.assertTrue('ctime' in item)
def testReadOnlyGetUUID(self): # get test file getFile('tall.h5', 'tall_ro.h5', True) # remove db file! removeFile('.tall_ro.h5') g1Uuid = None with Hdf5db('tall_ro.h5') as db: g1Uuid = db.getUUIDByPath('/g1') self.failUnlessEqual(len(g1Uuid), config.get('uuidlen')) obj = db.getObjByPath('/g1') self.failUnlessEqual(obj.name, '/g1') # end of with will close file # open again and verify we can get obj by name with Hdf5db('tall_ro.h5') as db: obj = db.getGroupObjByUuid(g1Uuid) g1 = db.getObjByPath('/g1') self.failUnlessEqual(obj, g1) g1links = db.getLinkItems(g1Uuid) self.failUnlessEqual(len(g1links), 2) for item in g1links: self.failUnlessEqual(len(item['id']), config.get('uuidlen'))
def main(): nargs = len(sys.argv) dumper = Dumph5() dumper.verbose = False dumper.endpoint = None dumper.port = 7253 dumper.noDsetData = False dumper.noAttrData = False endpoint_option = "-endpoint=" port_option = "-port=" option_count = 0 for arg in sys.argv: if arg.startswith(endpoint_option): endpoint = arg[len(endpoint_option):] if endpoint.startswith("http"): dumper.endpoint = endpoint else: dumper.endpoint = "http://" + endpoint option_count += 1 elif arg.startswith(port_option): port = arg[len(port_option):] dumper.port = int(port) option_count += 1 elif arg == "-v": dumper.verbose = True if nargs - option_count <= 2: printUsage() domain = sys.argv[nargs-2] filename = sys.argv[nargs-1] print "domain:", domain print "filename:", filename dumper.domain = domain domain_json = dumper.makeRequest("/") if "root" not in domain_json: raise Exception("no root key in domain response") root_uuid = domain_json["root"] # create the file, will raise IOError if there's a problem Hdf5db.createHDF5File(filename) with Hdf5db(filename, root_uuid=root_uuid) as db: dumper.writeFile(db) # open with h5py and remove the _db_ group # Note: this will delete any anonymous (un-linked) objects f = h5py.File(filename, 'a') del f["__db__"] f.close() print "done!"