def main():
parser = argparse.ArgumentParser(
usage='%(prog)s [-h] <json_file> <h5_file>')
parser.add_argument('in_filename',
nargs='+',
help='JSon file to be converted to h5')
parser.add_argument('out_filename',
nargs='+',
help='name of HDF5 output file')
args = parser.parse_args()
# create logger
log = logging.getLogger("h5serv")
# log.setLevel(logging.WARN)
log.setLevel(logging.INFO)
# add log handler
handler = logging.FileHandler('./jsontoh5.log')
# add handler to logger
log.addHandler(handler)
text = open(args.in_filename[0]).read()
# parse the json file
h5json = json.loads(text)
if "root" not in h5json:
raise Exception("no root key in input file")
root_uuid = h5json["root"]
filename = args.out_filename[0]
# create the file, will raise IOError if there's a problem
Hdf5db.createHDF5File(filename)
with Hdf5db(filename,
root_uuid=root_uuid,
update_timestamps=False,
app_logger=log) as db:
h5writer = Writeh5(db, h5json)
h5writer.writeFile()
# open with h5py and remove the _db_ group
# Note: this will delete any anonymous (un-linked) objects
f = h5py.File(filename, 'a')
if "__db__" in f:
del f["__db__"]
f.close()
print("done!")
def main():
parser = argparse.ArgumentParser(usage='%(prog)s [-h] [-D|-d] <hdf5_file>')
parser.add_argument('-D', action='store_true', help='surpress all data output')
parser.add_argument('-d', action='store_true', help='surpress data output for' +
' datasets (but not attribute values)')
parser.add_argument('filename', nargs='+', help='HDF5 to be converted to json')
args = parser.parse_args()
# create logger
log = logging.getLogger("h5serv")
# log.setLevel(logging.WARN)
log.setLevel(logging.INFO)
# add log handler
handler = logging.FileHandler('./h5tojson.log')
# add handler to logger
log.addHandler(handler)
filename = args.filename[0]
if not op.isfile(filename):
sys.exit("Cannot find file: %s" % filename)
log.info("h5tojson " + filename)
dbFilename = getTempFileName()
log.info("Using dbFile: " + dbFilename)
with Hdf5db(filename, dbFilePath=dbFilename, readonly=True, app_logger=log) as db:
dumper = DumpJson(db, app_logger=log, options=args)
json = dumper.dumpFile()
print(json)
def addTocEntry(domain, filePath, userid=None):
"""
Helper method - update TOC when a domain is created
If userid is provide, the acl will be checked to ensure userid has permissions
to modify the object.
"""
log = logging.getLogger("h5serv")
hdf5_ext = config.get('hdf5_ext')
dataPath = config.get('datapath')
log.info("addTocEntry - domain: " + domain + " filePath: " + filePath)
if not filePath.startswith(dataPath):
log.error("unexpected filepath: " + filePath)
raise HTTPError(500)
filePath = fileUtil.getUserFilePath(filePath)
tocFile = fileUtil.getTocFilePathForDomain(domain)
log.info("tocFile: " + tocFile)
acl = None
try:
with Hdf5db(tocFile, app_logger=log) as db:
group_uuid = db.getUUIDByPath('/')
pathNames = filePath.split('/')
for linkName in pathNames:
if not linkName:
continue
if linkName.endswith(hdf5_ext):
linkName = linkName[:-(len(hdf5_ext))]
print("linkName:", linkName)
if userid is not None:
acl = db.getAcl(group_uuid, userid)
if not acl['create']:
self.log.info("unauthorized access to group:" +
group_uuid)
raise IOError(errno.EACCES) # unauthorized
log.info(
"createExternalLink -- uuid %s, domain: %s, linkName: %s",
group_uuid, domain, linkName)
db.createExternalLink(group_uuid, domain, '/', linkName)
else:
subgroup_uuid = getSubgroupId(db, group_uuid, linkName)
if subgroup_uuid is None:
if userid is not None:
acl = db.getAcl(group_uuid, userid)
if not acl['create']:
self.log.info("unauthorized access to group:" +
group_uuid)
raise IOError(errno.EACCES) # unauthorized
# create subgroup and link to parent group
subgroup_uuid = db.createGroup()
# link the new group
log.info(
"linkObject -- uuid: %s, subgroup_uuid: %s, linkName: %s",
group_uuid, subgroup_uuid, linkName)
db.linkObject(group_uuid, subgroup_uuid, linkName)
group_uuid = subgroup_uuid
except IOError as e:
log.info("IOError: " + str(e.errno) + " " + e.strerror)
raise e
def mirror_objects_in_girder(folder, progress, user, assetstore, hdf5_path,
name, obj):
progress.update(message=name)
if isinstance(obj, h5py.Dataset):
with Hdf5db(hdf5_path, readonly=True) as db:
uuid = db.getUUIDByPath(name)
attrs = db.getAttributeItems("datasets", uuid)
attributes = [
db.getAttributeItem("datasets", uuid, i["name"]) for i in attrs
]
resolve_dataset(folder, obj, user, assetstore, hdf5_path, attributes)
elif isinstance(obj, h5py.Group):
with Hdf5db(hdf5_path, readonly=True) as db:
uuid = db.getUUIDByPath(name)
attrs = db.getAttributeItems("groups", uuid)
attributes = [
db.getAttributeItem("groups", uuid, i["name"]) for i in attrs
]
resolve_group(folder, obj, user, attributes=attributes)
def tojson(self):
db = Hdf5db(self.path, dbFilePath=self.path, app_logger=None)
# `options_dict` is used to surpress data outputs.
# If both set to `False`, operations takes a lot of time to copy all
# Dataset values to json.
options_dict = {'D': True, 'd': False}
args = munchify(options_dict)
dumper = DumpJson(db, app_logger=None, options=args)
dumper.dumpFile()
self.json = dumper.json
return self.json
def returnH5(inJsonFile, outH5File):
# create logger
log = logging.getLogger("h5serv")
# log.setLevel(logging.WARN)
log.setLevel(logging.INFO)
# add log handler
handler = logging.FileHandler('./jsontoh5.log')
# add handler to logger
log.addHandler(handler)
text = open(inJsonFile).read()
# parse the json file
h5json = json.loads(text)
if "root" not in h5json:
raise Exception("no root key in input file")
root_uuid = h5json["root"]
filename = outH5File
# create the file, will raise IOError if there's a problem
Hdf5db.createHDF5File(filename)
with Hdf5db(filename,
root_uuid=root_uuid,
update_timestamps=False,
app_logger=log) as db:
h5writer = Writeh5(db, h5json)
h5writer.writeFile()
# # open with h5py and remove the _db_ group
# # Note: this will delete any anonymous (un-linked) objects
# f = h5py.File(filename, 'a')
# if "__db__" in f:
# del f["__db__"]
# f.close()
return
def returnJsonString(h5fileName, jsonFileName):
a = argparse.Namespace(d=True, D=False, filename=h5fileName)
# create logger
log = logging.getLogger("h5serv")
# log.setLevel(logging.WARN)
log.setLevel(logging.INFO)
# add log handler
handler = logging.FileHandler('./h5tojson.log')
# add handler to logger
log.addHandler(handler)
log.info("Using dbFile: " + jsonFileName)
with Hdf5db(h5fileName, dbFilePath=jsonFileName, readonly=True, app_logger=log) as db:
dumper = DumpJson(db, app_logger=log, options=a)
jsonData = dumper.dumpFile()
return jsonData
def _get_layers(self):
dbFilename = h5tojson.getTempFileName()
h5_json = ""
with Hdf5db(self.h5_file, dbFilePath=dbFilename, readonly=True) as db:
dumper = h5tojson.DumpJson(db)
h5_json = dumper.dumpFile()
# find root node according to name
# for latest keras like 2.1.2, root node's alias = /model_weights
# for version before 2.1.2, root node's alias = /
# check it by order
self.isNewFormat = False
for _, v in h5_json["groups"].items():
if v["alias"][0] == "/model_weights":
self.isNewFormat = True
break
if self.isNewFormat is None:
for _, v in h5_json["groups"].items():
if v["alias"][0] == "/":
self.isNewFormat = False
break
if self.isNewFormat is None:
self.logger.error("Cannot found root node in h5 file, return")
return
layers = None
for _, v in h5_json["groups"].items():
if self.isNewFormat is True:
if v["alias"][0] == "/model_weights":
layers = v["attributes"][0]["value"]
else:
if v["alias"][0] == "/":
layers = v["attributes"][0]["value"]
return layers
def removeTocEntry(domain, filePath, userid=None):
log = logging.getLogger("h5serv")
hdf5_ext = config.get('hdf5_ext')
dataPath = config.get('datapath')
if not filePath.startswith(dataPath):
log.error("unexpected filepath: " + filePath)
raise HTTPError(500)
filePath = fileUtil.getUserFilePath(filePath)
tocFile = fileUtil.getTocFilePathForDomain(domain)
log.info("removeTocEntry - domain: " + domain + " filePath: " + filePath +
" tocfile: " + tocFile)
pathNames = filePath.split('/')
log.info("pathNames: " + str(pathNames))
try:
with Hdf5db(tocFile, app_logger=log) as db:
group_uuid = db.getUUIDByPath('/')
log.info("group_uuid:" + group_uuid)
for linkName in pathNames:
if not linkName:
continue
log.info("linkName:" + linkName)
if linkName.endswith(hdf5_ext):
linkName = linkName[:-(len(hdf5_ext))]
log.info("unklink " + group_uuid + ", " + linkName)
db.unlinkItem(group_uuid, linkName)
else:
subgroup_uuid = getSubgroupId(db, group_uuid, linkName)
if subgroup_uuid is None:
msg = "Didn't find expected subgroup: " + group_uuid
log.error(msg)
raise HTTPError(500, reason=msg)
group_uuid = subgroup_uuid
except IOError as e:
log.info("IOError: " + str(e.errno) + " " + e.strerror)
raise e
def addTocEntry(toc_file, domain, base_domain):
"""
Helper method - update TOC when a domain is created
"""
if not domain.endswith(base_domain):
sys.exit("unexpected domain value: " + domain)
# trim domain by base domain
try:
with Hdf5db(toc_file) as db:
group_uuid = db.getUUIDByPath('/')
names = domain.split('.')
base_names = base_domain.split('.')
indexes = list(range(len(names)))
indexes = indexes[::-1] # reverse
for i in indexes:
if i >= len(names) - len(base_names):
continue # still in the base domain
linkName = names[i]
if not linkName:
continue
if i == 0:
db.createExternalLink(group_uuid, domain, '/', linkName)
else:
subgroup_uuid = getSubgroupId(db, group_uuid, linkName)
if subgroup_uuid is None:
# create subgroup and link to parent group
subgroup_uuid = db.createGroup()
# link the new group
db.linkObject(group_uuid, subgroup_uuid, linkName)
group_uuid = subgroup_uuid
except IOError as e:
print("IOError: " + str(e.errno) + " " + e.strerror)
sys.exit(-1)
def main():
h5path = None
filename = None
req_userids = []
if len(sys.argv) == 1 or sys.argv[1] == "-h":
printUsage()
sys.exit(0)
argn = 1
while argn < len(sys.argv):
arg = sys.argv[argn]
if arg == '-file':
filename = getNextArg(argn)
argn += 2
elif arg == '-path':
h5path = getNextArg(argn)
argn += 2
else:
# process userids
try:
userid = int(arg)
req_userids.append(userid)
except ValueError:
print("Invalid userid:", userid)
sys.exit(1)
argn += 1
if not isfile(filename):
print(filename, "not found")
sys.exit(1)
if not h5py.is_hdf5(filename):
print(filename, "not an hdf5 file")
sys.exit(1)
if h5path is None:
h5path = '/'
fields = ('userid', 'create', 'read', 'update', 'delete', 'readACL',
'updateACL')
with Hdf5db(filename) as db:
try:
obj_uuid = db.getUUIDByPath(h5path)
except KeyError:
print("no object found at path:", h5path)
sys.exit(1)
acl_dset = db.getAclDataset(obj_uuid)
if acl_dset and acl_dset.shape[0] > 0:
acls = {}
items = acl_dset[...]
for item in items:
acls[item[0]] = item
userids = list(acls.keys())
userids.sort() # sort to print by userid
print("%8s %8s %8s %8s %8s %8s %8s " % fields)
for userid in userids:
if len(req_userids) > 0 and userid not in req_userids:
continue
acl = acls[userid]
format_args = [userid]
for field in ('create', 'read', 'update', 'delete', 'readACL',
'updateACL'):
format_args.append('Y' if acl[field] else 'N')
print("%8s %8s %8s %8s %8s %8s %8s " % tuple(format_args))
else:
print("no ACLs")
sys.exit()
# setup logger
log = logging.getLogger("rebuildIndex")
log.setLevel(logging.INFO)
handler = logging.StreamHandler(sys.stdout)
# create formatter
formatter = logging.Formatter(
"%(levelname)s:%(filename)s:%(lineno)d::%(message)s")
handler.setFormatter(formatter)
log.addHandler(handler)
log.propagate = False
filepath = sys.argv[1]
log.info("openining file: " + filepath)
# remove the old index
f = h5py.File(filepath, 'a')
if dbname in f:
log.info("deleting old db group")
del f[dbname]
f.close()
# now open with hdf5db
with Hdf5db(filepath, app_logger=log) as db:
# the actual index rebuilding will happen in the init function
root_uuid = db.getUUIDByPath('/')
print("root_uuid:", root_uuid)
print("done!")
def main():
perm_abvr = {'c':'create', 'r': 'read', 'u': 'update', 'd': 'delete', 'e': 'readACL', 'p':'updateACL'}
h5path = None
filename = None
userids = []
add_list = []
remove_list = []
if len(sys.argv) == 1 or sys.argv[1] == "-h":
printUsage();
sys.exit(1)
argn = 1
while argn < len(sys.argv):
arg = sys.argv[argn]
if arg == '-file':
filename = getNextArg(argn)
argn += 2
elif arg == '-path':
h5path = getNextArg(argn)
argn += 2
elif arg[0] in ('+', '-'):
to_list = None
for ch in arg:
if ch == '+':
to_list = add_list
elif ch == '-':
to_list = remove_list
elif ch in perm_abvr.keys():
to_list.append(perm_abvr[ch])
else:
printUsage()
sys.exit(1)
argn += 1
else:
# process userids
try:
userid = int(arg)
userids.append(userid)
except ValueError:
print("Invalid userid:", userid)
sys.exit(1)
argn += 1
conflicts = list(set(add_list) & set(remove_list))
if len(conflicts) > 0:
print("permission: ", conflicts[0], " set for both add and remove")
sys.exit(1)
if filename is None:
print("no filename specified")
sys.exit(1)
if not isfile(filename):
print(filename, "not found")
sys.exit(1)
if not h5py.is_hdf5(filename):
print(filename, "not an hdf5 file")
sys.exit(1)
if h5path is None:
h5path = '/'
if len(userids) == 0:
userids.append(0)
fields = ('userid', 'create', 'read', 'update', 'delete', 'readACL', 'updateACL')
with Hdf5db(filename) as db:
try:
obj_uuid = db.getUUIDByPath(h5path)
except KeyError:
print("no object found at path:", h5path)
sys.exit(1)
print("%8s %8s %8s %8s %8s %8s %8s " % fields)
for userid in userids:
acl = db.getAclByObjAndUser(obj_uuid, userid)
if acl is None and userid != 0:
acl = db.getAclByObjAndUser(obj_uuid, 0)
if acl is None:
acl = db.getDefaultAcl()
acl['userid'] = userid
for field in add_list:
acl[field] = True
for field in remove_list:
acl[field] = False
format_args = [userid]
for field in fields:
if field == 'userid':
continue
format_args.append('Y' if acl[field] else 'N')
print("%8s %8s %8s %8s %8s %8s %8s " % tuple(format_args))
db.setAcl(obj_uuid, acl)
def main():
nargs = len(sys.argv)
dumper = Dumph5()
dumper.verbose = False
dumper.endpoint = None
dumper.port = 7253
dumper.noDsetData = False
dumper.noAttrData = False
endpoint_option = "-endpoint="
port_option = "-port="
option_count = 0
for arg in sys.argv:
if arg.startswith(endpoint_option):
endpoint = arg[len(endpoint_option):]
if endpoint.startswith("http"):
dumper.endpoint = endpoint
else:
dumper.endpoint = "http://" + endpoint
option_count += 1
elif arg.startswith(port_option):
port = arg[len(port_option):]
dumper.port = int(port)
option_count += 1
elif arg == "-v":
dumper.verbose = True
if nargs - option_count <= 2:
printUsage()
domain = sys.argv[nargs-2]
filename = sys.argv[nargs-1]
print("domain:", domain)
print("filename:", filename)
dumper.domain = domain
domain_json = dumper.makeRequest("/")
if "root" not in domain_json:
raise Exception("no root key in domain response")
root_uuid = domain_json["root"]
# create the file, will raise IOError if there's a problem
Hdf5db.createHDF5File(filename)
with Hdf5db(filename, root_uuid=root_uuid) as db:
dumper.writeFile(db)
# open with h5py and remove the _db_ group
# Note: this will delete any anonymous (un-linked) objects
f = h5py.File(filename, 'a')
del f["__db__"]
f.close()
print("done!")
def main():
nargs = len(sys.argv)
dumper = Dumph5()
dumper.verbose = False
dumper.endpoint = None
dumper.port = 7253
dumper.noDsetData = False
dumper.noAttrData = False
endpoint_option = "-endpoint="
port_option = "-port="
option_count = 0
for arg in sys.argv:
if arg.startswith(endpoint_option):
endpoint = arg[len(endpoint_option):]
if endpoint.startswith("http"):
dumper.endpoint = endpoint
else:
dumper.endpoint = "http://" + endpoint
option_count += 1
elif arg.startswith(port_option):
port = arg[len(port_option):]
dumper.port = int(port)
option_count += 1
elif arg == "-v":
dumper.verbose = True
if nargs - option_count <= 2:
printUsage()
domain = sys.argv[nargs - 2]
filename = sys.argv[nargs - 1]
print("domain:", domain)
print("filename:", filename)
dumper.domain = domain
domain_json = dumper.makeRequest("/")
if "root" not in domain_json:
raise Exception("no root key in domain response")
root_uuid = domain_json["root"]
# create the file, will raise IOError if there's a problem
Hdf5db.createHDF5File(filename)
with Hdf5db(filename, root_uuid=root_uuid) as db:
dumper.writeFile(db)
# open with h5py and remove the _db_ group
# Note: this will delete any anonymous (un-linked) objects
f = h5py.File(filename, 'a')
del f["__db__"]
f.close()
print("done!")
def main():
parser = argparse.ArgumentParser(usage='%(prog)s [-h] <json_file> <h5_file>')
parser.add_argument('in_filename', nargs='+', help='JSon file to be converted to h5')
parser.add_argument('out_filename', nargs='+', help='name of HDF5 output file')
args = parser.parse_args()
# create logger
log = logging.getLogger("h5serv")
# log.setLevel(logging.WARN)
log.setLevel(logging.INFO)
# add log handler
handler = logging.FileHandler('./jsontoh5.log')
# add handler to logger
log.addHandler(handler)
text = open(args.in_filename[0]).read()
# parse the json file
h5json = json.loads(text)
if "root" not in h5json:
raise Exception("no root key in input file")
root_uuid = h5json["root"]
filename = args.out_filename[0]
# create the file, will raise IOError if there's a problem
Hdf5db.createHDF5File(filename)
with Hdf5db(filename, root_uuid=root_uuid, update_timestamps=False, app_logger=log) as db:
h5writer = Writeh5(db, h5json)
h5writer.writeFile()
# open with h5py and remove the _db_ group
# Note: this will delete any anonymous (un-linked) objects
f = h5py.File(filename, 'a')
if "__db__" in f:
del f["__db__"]
f.close()
print("done!")