def addTocEntry(domain, filePath, userid=None):
"""
Helper method - update TOC when a domain is created
If userid is provide, the acl will be checked to ensure userid has permissions
to modify the object.
"""
log = logging.getLogger("h5serv")
hdf5_ext = config.get('hdf5_ext')
dataPath = config.get('datapath')
log.info("addTocEntry - domain: " + domain + " filePath: " + filePath)
if not filePath.startswith(dataPath):
log.error("unexpected filepath: " + filePath)
raise HTTPError(500)
filePath = fileUtil.getUserFilePath(filePath)
tocFile = fileUtil.getTocFilePathForDomain(domain)
log.info("tocFile: " + tocFile)
acl = None
try:
with Hdf5db(tocFile, app_logger=log) as db:
group_uuid = db.getUUIDByPath('/')
pathNames = filePath.split('/')
for linkName in pathNames:
if not linkName:
continue
if linkName.endswith(hdf5_ext):
linkName = linkName[:-(len(hdf5_ext))]
print("linkName:", linkName)
if userid is not None:
acl = db.getAcl(group_uuid, userid)
if not acl['create']:
self.log.info("unauthorized access to group:" +
group_uuid)
raise IOError(errno.EACCES) # unauthorized
log.info(
"createExternalLink -- uuid %s, domain: %s, linkName: %s",
group_uuid, domain, linkName)
db.createExternalLink(group_uuid, domain, '/', linkName)
else:
subgroup_uuid = getSubgroupId(db, group_uuid, linkName)
if subgroup_uuid is None:
if userid is not None:
acl = db.getAcl(group_uuid, userid)
if not acl['create']:
self.log.info("unauthorized access to group:" +
group_uuid)
raise IOError(errno.EACCES) # unauthorized
# create subgroup and link to parent group
subgroup_uuid = db.createGroup()
# link the new group
log.info(
"linkObject -- uuid: %s, subgroup_uuid: %s, linkName: %s",
group_uuid, subgroup_uuid, linkName)
db.linkObject(group_uuid, subgroup_uuid, linkName)
group_uuid = subgroup_uuid
except IOError as e:
log.info("IOError: " + str(e.errno) + " " + e.strerror)
raise e
def isTocFilePath(filePath):
datapath = config.get('datapath')
toc_file_path = fileUtil.join(datapath, config.get('toc_name'))
if filePath == toc_file_path:
isTocFilePath = True
else:
isTocFilePath = False
return isTocFilePath
def getTocFilePath(user=None):
datapath = config.get('datapath')
if user is None:
#print("get default toc")
toc_file_path = fileUtil.join(datapath, config.get('toc_name'))
else:
#print("get user toc")
toc_file_path = fileUtil.join(datapath, config.get('home_dir'))
toc_file_path = fileUtil.join(toc_file_path, config.get('toc_name'))
return toc_file_path
def getUserFilePath(file_path):
data_path = config.get('datapath')
file_path = file_path[len(data_path):] # strip off base data path
if len(file_path) > 1 and file_path[0] == '/':
file_path = file_path[
1:] # don't include first slash if preseent- messes up the split
path_names = file_path.split('/')
if path_names[0] == config.get('home_dir') and len(path_names) > 1:
# return a path relative to user's base dir
file_path = '/'
path_names = path_names[2:] # skip home, userid
for path_name in path_names:
file_path = op.join(file_path, path_name)
return file_path
def __init__(self, mongouri):
self.log = logging.getLogger("h5serv")
self.log.info("AuthMongo class init(" + mongouri + ")")
self.client = MongoClient(mongouri)
db_name = config.get('mongo_dbname')
self.db = self.client[db_name]
self.username_cache = {}
self.userid_cache = {}
def getDomain(file_path, base_domain=None):
# Get domain given a file path
data_path = op.normpath(
config.get('datapath')) # base path for data directory
data_path = posixpath(data_path)
file_path = posixpath(file_path)
hdf5_ext = config.get("hdf5_ext")
if op.isabs(file_path):
# compare with absolute path if we're given an absolute path
data_path = posixpath(op.abspath(data_path))
if file_path == data_path:
return config.get('domain')
if file_path.endswith(hdf5_ext):
domain = op.basename(file_path)[:-(len(hdf5_ext))]
else:
domain = op.basename(file_path)
# replace dots with %2E in basename
domain = domain.replace('.', '%2E')
dirname = op.dirname(file_path)
while len(dirname) > 1 and dirname != data_path:
domain += '.'
domain += op.basename(dirname)
if len(op.dirname(dirname)) >= len(dirname):
break
dirname = op.dirname(dirname)
domain += '.'
if base_domain:
domain += base_domain
else:
domain += config.get('domain')
return domain
def removeTocEntry(domain, filePath, userid=None):
log = logging.getLogger("h5serv")
hdf5_ext = config.get('hdf5_ext')
dataPath = config.get('datapath')
if not filePath.startswith(dataPath):
log.error("unexpected filepath: " + filePath)
raise HTTPError(500)
filePath = fileUtil.getUserFilePath(filePath)
tocFile = fileUtil.getTocFilePathForDomain(domain)
log.info("removeTocEntry - domain: " + domain + " filePath: " + filePath +
" tocfile: " + tocFile)
pathNames = filePath.split('/')
log.info("pathNames: " + str(pathNames))
try:
with Hdf5db(tocFile, app_logger=log) as db:
group_uuid = db.getUUIDByPath('/')
log.info("group_uuid:" + group_uuid)
for linkName in pathNames:
if not linkName:
continue
log.info("linkName:" + linkName)
if linkName.endswith(hdf5_ext):
linkName = linkName[:-(len(hdf5_ext))]
log.info("unklink " + group_uuid + ", " + linkName)
db.unlinkItem(group_uuid, linkName)
else:
subgroup_uuid = getSubgroupId(db, group_uuid, linkName)
if subgroup_uuid is None:
msg = "Didn't find expected subgroup: " + group_uuid
log.error(msg)
raise HTTPError(500, reason=msg)
group_uuid = subgroup_uuid
except IOError as e:
log.info("IOError: " + str(e.errno) + " " + e.strerror)
raise e
def getAuthClient():
log = logging.getLogger("h5serv")
log.info("getAuthClient")
password_uri = config.get("password_uri")
log.info("password_uri:" + password_uri)
auth = None
if password_uri.startswith("mongo"):
# use mongodb user db
from h5serv.authMongo import AuthClient
auth = AuthClient(password_uri)
else:
# use HDF5 file-based user db
from h5serv.authFile import AuthClient
auth = AuthClient(password_uri)
return auth
def getFilePath(host_value, auth=None):
# logging.info('getFilePath[' + host_value + ']')
# strip off port specifier (if present)
npos = host_value.rfind(':')
if npos > 0:
host = host_value[:npos]
else:
host = host_value
topdomain = config.get('domain')
# check to see if this is an ip address
if isIPAddress(host):
host = topdomain # use topdomain
if host.lower() == topdomain:
# if host is the same as topdomain, return toc path
# filePath = getTocFilePath()
filePath = config.get('datapath')
filePath = join(filePath, config.get('toc_name'))
return filePath
print("host:", host, "topdomain:", topdomain)
if len(host) <= len(
topdomain) or host[-len(topdomain):].lower() != topdomain:
msg = "top-level domain is not valid"
print(msg)
raise HTTPError(403, message=msg)
if host[-(len(topdomain) + 1)] != '.':
# there needs to be a dot separator
raise HTTPError(400, message='domain name is not valid')
host = host[:-(len(topdomain) + 1)] # strip off top domain part
if len(host) == 0 or host[0] == '.' or host[-1] == '.':
# needs a least one character (which can't be '.', or have '.' as first or last char)
raise HTTPError(400, message='domain name is not valid')
dns_path = host.split('.')
dns_path.reverse() # flip to filesystem ordering
filePath = config.get('datapath')
num_parts = 0
for field in dns_path:
if len(field) == 0:
raise HTTPError(400) # Bad syntax
filePath = join(filePath, field)
num_parts += 1
# check to see if this is the user's home domain
if num_parts == 2 and dns_path[0] == config.get('home_dir'):
if auth is None:
auth = getAuthClient
user_info = auth.getUserInfo(dns_path[1])
if user_info is None:
raise HTTPError(404) # not found
makeDirs(filePath) # add user directory if it doesn't exist
filePath = join(filePath, config.get('toc_name'))
else:
filePath += config.get('hdf5_ext') # add extension
#print('getFilePath[' + host + '] -> "' + filePath + '"')
return filePath
def getTocFilePathForDomain(host_value, auth=None):
""" Return toc file path for given domain value.
Will return path "../data/.toc.h5" for public domains or
"../data/home/<user>/.toc.h5" for user domains.
"""
# logging.info('getFilePath[' + host_value + ']')
# strip off port specifier (if present)
npos = host_value.rfind(':')
if npos > 0:
host = host_value[:npos]
else:
host = host_value
topdomain = config.get('domain')
# check to see if this is an ip address
if isIPAddress(host):
host = topdomain # use topdomain
if host.lower() == topdomain:
# if host is the same as topdomain, return toc path
# filePath = getTocFilePath()
filePath = config.get('datapath')
filePath = join(filePath, config.get('toc_name'))
return filePath
if len(host) <= len(
topdomain) or host[-len(topdomain):].lower() != topdomain:
host = topdomain # use topdomain
else:
if host[-(len(topdomain) + 1)] != '.':
# there needs to be a dot separator
raise HTTPError(400, message='domain name is not valid')
host = host[:-(len(topdomain) + 1)] # strip off top domain part
if len(host) == 0 or host[0] == '.' or host[-1] == '.':
# needs a least one character (which can't be '.', or have '.' as first or last char)
raise HTTPError(400, message='domain name is not valid')
dns_path = host.split('.')
dns_path.reverse() # flip to filesystem ordering
filePath = config.get('datapath')
if dns_path[0] == config.get('home_dir'):
filePath = join(filePath, config.get('home_dir'))
filePath = join(filePath, dns_path[1])
if auth is None:
auth = getAuthClient()
user_info = auth.getUserInfo(dns_path[1])
if user_info is None:
raise HTTPError(404) # not found
makeDirs(filePath) # add user directory if it doesn't exist
filePath = join(filePath, config.get('toc_name'))
#print("return user toc filepath")
else:
# not home dir, just return top-level toc
filePath = join(filePath, config.get('toc_name'))
#print("return default toc filepath")
return filePath
def createTocFile(datapath):
log = logging.getLogger("h5serv")
log.info("createTocFile(" + datapath + ")")
data_dir = fileUtil.posixpath(op.normpath(config.get('datapath')))
home_dir = fileUtil.join(data_dir, config.get("home_dir"))
log.info("home dir: " + home_dir)
if datapath.startswith(home_dir):
log.info("user toc")
user_toc = True
else:
log.info("system toc")
user_toc = False
if datapath.endswith(config.get('toc_name')):
toc_dir = fileUtil.posixpath(op.normpath(op.dirname(datapath)))
toc_file = datapath
else:
toc_dir = fileUtil.posixpath(op.normpath(datapath))
toc_file = fileUtil.join(toc_dir, config.get("toc_name"))
log.info("toc_dir:[" + toc_dir + "]")
log.info("data_dir:[" + data_dir + "]")
log.info("home_dir:[" + home_dir + "]")
log.info("check toc with path: " + toc_file)
if op.exists(toc_file):
msg = "toc file already exists"
log.warn(msg)
raise IOError(msg)
base_domain = fileUtil.getDomain(toc_dir)
log.info("base domain: " + base_domain)
#if os.name == 'nt':
# toc_dir = toc_dir.replace('\\', '/') # use unix style to map to HDF5 convention
hdf5_ext = config.get('hdf5_ext')
f = h5py.File(toc_file, 'w')
for root, subdirs, files in os.walk(toc_dir):
root = fileUtil.posixpath(root)
log.info("toc walk: " + root)
if toc_dir == data_dir:
log.info(fileUtil.join(toc_dir, home_dir))
if root.startswith(home_dir):
log.info("skipping home dir")
continue
grppath = root[len(toc_dir):]
if not grppath:
grppath = '/'
if grppath[-1] == '.':
grppath = grppath[:-1]
log.info("grppath: " + grppath)
if os.name == 'nt':
grppath = grppath.replace('\\', '/') # match HDF5 convention
grp = None
if grppath == '/':
grp = f['/'] # use root group
domainpath = fileUtil.getDomain(grppath, base_domain=base_domain)
log.info("grppath: " + grppath)
log.info("base_domain: " + base_domain)
log.info("domainpath: " + domainpath)
for filename in os.listdir(root):
log.info("walk, file: " + filename)
if filename[0] == '.':
log.info("skip hidden")
continue # skip 'hidden' files
filepath = fileUtil.join(root, filename)
log.info("walk, filepath: " + filepath)
link_target = '/'
if op.islink(filepath):
log.info("symlink: " + filepath)
# todo - quick hack for now to set a symlink with to sub-folder of data dir
# todo - revamp to use os.readlink and do the proper thing with the link value
filedomain = config.get('domain')
link_target += filename
log.info("setting symbolic link domainpath to: " + filedomain +
" target: /" + filename)
else:
ext_len = len(hdf5_ext)
if len(filename) < ext_len or filename[-ext_len:] != hdf5_ext:
log.info("skip non-hdf5 extension")
continue
if not h5py.is_hdf5(filepath):
log.info("skip non-hdf5 file")
continue
filename = filename[:-ext_len]
# replace any dots with '%2E' to disambiguate from domain seperators
filename_encoded = filename.replace('.', '%2E')
log.info("filename (noext): " + filename)
if domainpath[0] == '.':
filedomain = filename_encoded + domainpath
else:
filedomain = filename_encoded + '.' + domainpath
# create the grp at grppath if it doesn't exist
if not grp:
log.info("tocfile - create_group: " + grppath)
grp = f.create_group(grppath)
# verify that we can convert the domain back to a file path
log.info("filedomain: " + filedomain)
try:
fileUtil.getFilePath(filedomain)
# ok - add the external link
log.info("tocFile - ExternalLink: " + domainpath)
grp[filename] = h5py.ExternalLink(filedomain, link_target)
except HTTPError:
log.info("file path: [" + filepath +
"] is not valid dns name, ignoring")