def testGetDomain(self):
filePath = "tall.h5"
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.' + config.get('domain'))
filePath = "somevalue"
domain = getDomain(filePath)
self.assertEqual(domain, 'somevalue.' + config.get('domain'))
def testGetDomain(self):
"""
filePath = "tall.h5"
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.' + config.get('domain'))
filePath = "somevalue"
domain = getDomain(filePath)
self.assertEqual(domain, 'somevalue.' + config.get('domain'))
filePath = "subdir/tall.h5"
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.subdir.' + config.get('domain'))
"""
filePath = os.path.join(config.get('datapath'), 'subdir/tall.h5')
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.subdir.' + config.get('domain'))
filePath = os.path.join(config.get('datapath'), 'subdir/tall.h5')
filePath = os.path.abspath(filePath)
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.subdir.' + config.get('domain'))
# verify backslashes are ok for windows...
if os.name == 'nt':
filePath = "subdir\\subsubdir\\tall.h5"
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.subsubdir.subdir.' + config.get('domain'))
def testGetDomain(self):
"""
filePath = "tall.h5"
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.' + config.get('domain'))
filePath = "somevalue"
domain = getDomain(filePath)
self.assertEqual(domain, 'somevalue.' + config.get('domain'))
filePath = "subdir/tall.h5"
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.subdir.' + config.get('domain'))
"""
filePath = os.path.join(config.get('datapath'), 'subdir/tall.h5')
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.subdir.' + config.get('domain'))
filePath = os.path.join(config.get('datapath'), 'subdir/tall.h5')
filePath = os.path.abspath(filePath)
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.subdir.' + config.get('domain'))
# verify backslashes are ok for windows...
if os.name == 'nt':
filePath = "subdir\\subsubdir\\tall.h5"
domain = getDomain(filePath)
self.assertEqual(domain,
'tall.subsubdir.subdir.' + config.get('domain'))
def testGetDomain(self):
filePath = "tall.h5"
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.' + config.get('domain'))
filePath = "somevalue"
domain = getDomain(filePath)
self.assertEqual(domain, 'somevalue.' + config.get('domain'))
filePath = "subdir/tall.h5"
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.subdir.' + config.get('domain'))
filePath = os.path.join(config.get('datapath'), 'subdir/tall.h5')
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.subdir.' + config.get('domain'))
filePath = os.path.join(config.get('datapath'), 'subdir/tall.h5')
filePath = os.path.abspath(filePath)
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.subdir.' + config.get('domain'))
filePath = os.path.join(config.get('datapath'), 'home/test_user1/tall.h5')
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.test_user1.home.' + config.get('domain'))
filePath = '../data/home/test_user1/tall.h5'
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.test_user1.home.' + config.get('domain'))
#domainpath = fileUtil.getDomain(grppath, base_domain=base_domain)
filePath = "../data"
domain = getDomain(filePath)
self.assertEqual(domain, config.get('domain'))
# verify backslashes are ok for windows...
if os.name == 'nt':
filePath = "subdir\\subsubdir\\tall.h5"
domain = getDomain(filePath)
self.assertEqual(domain, 'tall.subsubdir.subdir.' + config.get('domain'))
def createTocFile(dir_path):
log = logging.getLogger("h5serv")
if os.name == 'nt':
dir_path = dir_path.replace(
'\\', '/') # use unix style to map to HDF5 convention
log.info("createTocFile(" + dir_path + ")")
hdf5_ext = config.get('hdf5_ext')
if not op.exists(dir_path):
raise IOError("invalid path")
toc_path = op.join(dir_path, config.get('toc_name'))
if op.isfile(toc_path):
raise IOError("toc file exists")
f = h5py.File(toc_path, 'w')
for root, subdirs, files in os.walk(dir_path):
grp_path = root[len(dir_path):]
if not grp_path:
continue
if os.name == 'nt':
grp_path = grp_path.replace('\\', '/')
grp = None
for file_name in files:
if file_name[0] == '.':
continue # skip 'hidden' files
if len(file_name) < 4 or file_name[-3:] != hdf5_ext:
continue
file_path = op.join(root, file_name)
if os.name == 'nt':
file_path = file_path.replace(
'\\', '/') # use unix style to map to HDF5 convention
log.info("createTocFile, path: " + file_path)
file_name = file_name[:-3]
if h5py.is_hdf5(file_path):
if not grp:
log.info("createTocFile - create_group: " + grp_path)
grp = f.create_group(grp_path)
domain_path = fileUtil.getDomain(file_path)
# verify that we can convert the domain back to a file path
try:
fileUtil.getFilePath(domain_path)
# ok - add the external link
log.info("createTocFile - ExternalLink: " + domain_path)
grp[file_name] = h5py.ExternalLink(domain_path, "/")
except HTTPError:
log.info("file path: [" + file_path +
"] is not valid dns name, ignoring")
def createTocFile(dir_path):
log = logging.getLogger("h5serv")
if os.name == 'nt':
dir_path = dir_path.replace('\\', '/') # use unix style to map to HDF5 convention
log.info("createTocFile(" + dir_path + ")")
hdf5_ext = config.get('hdf5_ext')
if not op.exists(dir_path):
raise IOError("invalid path")
toc_path = op.join(dir_path, config.get('toc_name'))
if op.isfile(toc_path):
raise IOError("toc file exists")
f = h5py.File(toc_path, 'w')
for root, subdirs, files in os.walk(dir_path):
grp_path = root[len(dir_path):]
if not grp_path:
continue
if os.name == 'nt':
grp_path = grp_path.replace('\\', '/')
grp = None
for file_name in files:
if file_name[0] == '.':
continue # skip 'hidden' files
if len(file_name) < 4 or file_name[-3:] != hdf5_ext:
continue
file_path = op.join(root, file_name)
if os.name == 'nt':
file_path = file_path.replace('\\', '/') # use unix style to map to HDF5 convention
log.info("createTocFile, path: " + file_path)
file_name = file_name[:-3]
if h5py.is_hdf5(file_path):
if not grp:
log.info("createTocFile - create_group: " + grp_path)
grp = f.create_group(grp_path)
domain_path = fileUtil.getDomain(file_path)
# verify that we can convert the domain back to a file path
try:
fileUtil.getFilePath(domain_path)
# ok - add the external link
log.info("createTocFile - ExternalLink: " + domain_path)
grp[file_name] = h5py.ExternalLink(domain_path, "/")
except HTTPError:
log.info(
"file path: [" + file_path +
"] is not valid dns name, ignoring")
def createTocFile(datapath):
log = logging.getLogger("h5serv")
log.info("createTocFile(" + datapath + ")")
data_dir = fileUtil.posixpath(op.normpath(config.get('datapath')))
home_dir = fileUtil.join(data_dir, config.get("home_dir"))
log.info("home dir: " + home_dir)
if datapath.startswith(home_dir):
log.info("user toc")
user_toc = True
else:
log.info("system toc")
user_toc = False
if datapath.endswith(config.get('toc_name')):
toc_dir = fileUtil.posixpath(op.normpath(op.dirname(datapath)))
toc_file = datapath
else:
toc_dir = fileUtil.posixpath(op.normpath(datapath))
toc_file = fileUtil.join(toc_dir, config.get("toc_name"))
log.info("toc_dir:[" + toc_dir + "]")
log.info("data_dir:[" + data_dir + "]")
log.info("home_dir:[" + home_dir + "]")
log.info("check toc with path: " + toc_file)
if op.exists(toc_file):
msg = "toc file already exists"
log.warn(msg)
raise IOError(msg)
base_domain = fileUtil.getDomain(toc_dir)
log.info("base domain: " + base_domain)
#if os.name == 'nt':
# toc_dir = toc_dir.replace('\\', '/') # use unix style to map to HDF5 convention
hdf5_ext = config.get('hdf5_ext')
f = h5py.File(toc_file, 'w')
for root, subdirs, files in os.walk(toc_dir):
root = fileUtil.posixpath(root)
log.info( "toc walk: " + root)
if toc_dir == data_dir:
log.info(fileUtil.join(toc_dir, home_dir))
if root.startswith(home_dir):
log.info("skipping home dir")
continue
grppath = root[len(toc_dir):]
if not grppath:
grppath = '/'
if grppath[-1] == '.':
grppath = grppath[:-1]
log.info("grppath: " + grppath)
if os.name == 'nt':
grppath = grppath.replace('\\', '/') # match HDF5 convention
grp = None
if grppath == '/':
grp = f['/'] # use root group
domainpath = fileUtil.getDomain(grppath, base_domain=base_domain)
log.info("grppath: " + grppath)
log.info("base_domain: " + base_domain)
log.info("domainpath: " + domainpath)
for filename in os.listdir(root):
log.info("walk, file: " + filename)
if filename[0] == '.':
log.info("skip hidden")
continue # skip 'hidden' files
filepath = fileUtil.join(root, filename)
log.info("walk, filepath: " + filepath)
link_target = '/'
if op.islink(filepath):
log.info("symlink: " + filepath)
# todo - quick hack for now to set a symlink with to sub-folder of data dir
# todo - revamp to use os.readlink and do the proper thing with the link value
filedomain = config.get('domain')
link_target += filename
log.info("setting symbolic link domainpath to: " + filedomain + " target: /" + filename)
else:
if len(filename) < 4 or filename[-3:] != hdf5_ext:
log.info("skip non-hdf5 extension")
continue
if not h5py.is_hdf5(filepath):
log.info("skip non-hdf5 file")
continue
filename = filename[:-(len(hdf5_ext))]
# replace any dots with '%2E' to disambiguate from domain seperators
filename_encoded = filename.replace('.', '%2E')
log.info("filename (noext): " + filename)
if domainpath[0] == '.':
filedomain = filename_encoded + domainpath
else:
filedomain = filename_encoded + '.' + domainpath
# create the grp at grppath if it doesn't exist
if not grp:
log.info("tocfile - create_group: " + grppath)
grp = f.create_group(grppath)
# verify that we can convert the domain back to a file path
log.info("filedomain: " + filedomain)
try:
fileUtil.getFilePath(filedomain)
# ok - add the external link
log.info("tocFile - ExternalLink: " + domainpath)
grp[filename] = h5py.ExternalLink(filedomain, link_target)
except HTTPError:
log.info("file path: [" + filepath + "] is not valid dns name, ignoring")