Example #1
0
def createTocFile(dir_path):
    log = logging.getLogger("h5serv")
    if os.name == 'nt':
        dir_path = dir_path.replace(
            '\\', '/')  # use unix style to map to HDF5 convention
    log.info("createTocFile(" + dir_path + ")")
    hdf5_ext = config.get('hdf5_ext')
    if not op.exists(dir_path):
        raise IOError("invalid path")
    toc_path = op.join(dir_path, config.get('toc_name'))
    if op.isfile(toc_path):
        raise IOError("toc file exists")
    f = h5py.File(toc_path, 'w')
    for root, subdirs, files in os.walk(dir_path):
        grp_path = root[len(dir_path):]
        if not grp_path:
            continue
        if os.name == 'nt':
            grp_path = grp_path.replace('\\', '/')
        grp = None
        for file_name in files:
            if file_name[0] == '.':
                continue  # skip 'hidden' files
            if len(file_name) < 4 or file_name[-3:] != hdf5_ext:
                continue
            file_path = op.join(root, file_name)
            if os.name == 'nt':
                file_path = file_path.replace(
                    '\\', '/')  # use unix style to map to HDF5 convention
            log.info("createTocFile, path: " + file_path)
            file_name = file_name[:-3]
            if h5py.is_hdf5(file_path):
                if not grp:
                    log.info("createTocFile - create_group: " + grp_path)
                    grp = f.create_group(grp_path)
                domain_path = fileUtil.getDomain(file_path)
                # verify that we can convert the domain back to a file path
                try:
                    fileUtil.getFilePath(domain_path)
                    # ok - add the external link
                    log.info("createTocFile - ExternalLink: " + domain_path)
                    grp[file_name] = h5py.ExternalLink(domain_path, "/")
                except HTTPError:
                    log.info("file path: [" + file_path +
                             "] is not valid dns name, ignoring")
Example #2
0
def createTocFile(dir_path):
    log = logging.getLogger("h5serv")
    if os.name == 'nt':
        dir_path = dir_path.replace('\\', '/')  # use unix style to map to HDF5 convention
    log.info("createTocFile(" + dir_path + ")")
    hdf5_ext = config.get('hdf5_ext')
    if not op.exists(dir_path):
        raise IOError("invalid path")
    toc_path = op.join(dir_path, config.get('toc_name'))
    if op.isfile(toc_path):
        raise IOError("toc file exists")
    f = h5py.File(toc_path, 'w')
    for root, subdirs, files in os.walk(dir_path):
        grp_path = root[len(dir_path):]
        if not grp_path:
            continue
        if os.name == 'nt':
            grp_path = grp_path.replace('\\', '/')
        grp = None
        for file_name in files:
            if file_name[0] == '.':
                continue  # skip 'hidden' files
            if len(file_name) < 4 or file_name[-3:] != hdf5_ext:
                continue
            file_path = op.join(root, file_name)
            if os.name == 'nt':
                file_path = file_path.replace('\\', '/')  # use unix style to map to HDF5 convention
            log.info("createTocFile, path: " + file_path)
            file_name = file_name[:-3]
            if h5py.is_hdf5(file_path): 
                if not grp:
                    log.info("createTocFile - create_group: " + grp_path)
                    grp = f.create_group(grp_path)
                domain_path = fileUtil.getDomain(file_path)
                # verify that we can convert the domain back to a file path
                try:
                    fileUtil.getFilePath(domain_path)
                    # ok - add the external link
                    log.info("createTocFile - ExternalLink: " + domain_path)
                    grp[file_name] = h5py.ExternalLink(domain_path, "/")
                except HTTPError:
                    log.info(
                        "file path: [" + file_path +
                        "] is not valid dns name, ignoring")
Example #3
0
 def testDomaintoFilePath(self):
     domain = 'tall.' + config.get('domain')  
     filePath = getFilePath(domain)
     self.assertEqual(filePath, "../data/tall.h5")
     # dot in front
     domain = '.tall.' + config.get('domain')  
     self.assertRaises(HTTPError, getFilePath, domain)
     # two dots
     domain = 'two..dots.' + config.get('domain')  
     self.assertRaises(HTTPError, getFilePath, domain)
     # no dot before domain
     domain = 'nodot' + config.get('domain')  
     self.assertRaises(HTTPError, getFilePath, domain)
Example #4
0
def createTocFile(datapath):
    log = logging.getLogger("h5serv")
    log.info("createTocFile(" + datapath + ")")
    data_dir = fileUtil.posixpath(op.normpath(config.get('datapath')))
    home_dir = fileUtil.join(data_dir, config.get("home_dir"))
    log.info("home dir: " + home_dir)
    if datapath.startswith(home_dir):
        log.info("user toc")
        user_toc = True
    else:
        log.info("system toc")
        user_toc = False
    
    if datapath.endswith(config.get('toc_name')):
        toc_dir = fileUtil.posixpath(op.normpath(op.dirname(datapath)))
        toc_file = datapath
    else:
        toc_dir = fileUtil.posixpath(op.normpath(datapath))
        toc_file = fileUtil.join(toc_dir, config.get("toc_name"))
   
           
    log.info("toc_dir:[" + toc_dir + "]")
    log.info("data_dir:[" + data_dir + "]") 
    log.info("home_dir:[" + home_dir + "]")
    log.info("check toc with path: " + toc_file)    
    if op.exists(toc_file):
        msg = "toc file already exists"
        log.warn(msg)
        raise IOError(msg)
        
    base_domain = fileUtil.getDomain(toc_dir)
    log.info("base domain: " + base_domain)
    
    #if os.name == 'nt':
    #    toc_dir = toc_dir.replace('\\', '/')  # use unix style to map to HDF5 convention
    
    hdf5_ext = config.get('hdf5_ext')  
    
    f = h5py.File(toc_file, 'w')
     
    for root, subdirs, files in os.walk(toc_dir):
        root = fileUtil.posixpath(root)
        log.info( "toc walk: " + root)
        
        if toc_dir == data_dir:
            log.info(fileUtil.join(toc_dir, home_dir))
            if root.startswith(home_dir):
                log.info("skipping home dir")
                continue
         
        grppath = root[len(toc_dir):]
        if not grppath:
            grppath = '/'
        if grppath[-1] == '.':
            grppath = grppath[:-1]
        log.info("grppath: " + grppath)
         
        if os.name == 'nt':
            grppath = grppath.replace('\\', '/')  # match HDF5 convention
        grp = None
        if grppath == '/':
            grp = f['/']  # use root group
         
        domainpath = fileUtil.getDomain(grppath, base_domain=base_domain)
        log.info("grppath: " + grppath)
        log.info("base_domain: " + base_domain)
        log.info("domainpath: " + domainpath)
        for filename in os.listdir(root):
            log.info("walk, file: " + filename)
            if filename[0] == '.':
                log.info("skip hidden")
                continue  # skip 'hidden' files
            
            filepath = fileUtil.join(root, filename)
            log.info("walk, filepath: " + filepath)
            link_target = '/'
            
            if op.islink(filepath):
                log.info("symlink: " + filepath)
                # todo - quick hack for now to set a symlink with to sub-folder of data dir
                # todo - revamp to use os.readlink and do the proper thing with the link value
                filedomain = config.get('domain')
                link_target += filename
                log.info("setting symbolic link domainpath to: " + filedomain + " target: /" + filename)
            else:
                if len(filename) < 4 or filename[-3:] != hdf5_ext:
                    log.info("skip non-hdf5 extension")
                    continue
                if not h5py.is_hdf5(filepath):
                    log.info("skip non-hdf5 file")
                    continue
                filename = filename[:-(len(hdf5_ext))]
                # replace any dots with '%2E' to disambiguate from domain seperators
                filename_encoded = filename.replace('.', '%2E')
                log.info("filename (noext): " + filename)
                if domainpath[0] == '.':        
                    filedomain = filename_encoded + domainpath
                else:
                    filedomain = filename_encoded + '.' + domainpath
                    
            # create the grp at grppath if it doesn't exist
            if not grp:
                log.info("tocfile - create_group: " + grppath)
                grp = f.create_group(grppath)           
                
            # verify that we can convert the domain back to a file path
            log.info("filedomain: " + filedomain)
            try:
                fileUtil.getFilePath(filedomain)
                # ok - add the external link
                log.info("tocFile - ExternalLink: " + domainpath)
                grp[filename] = h5py.ExternalLink(filedomain, link_target)
            except HTTPError:
                log.info("file path: [" + filepath + "] is not valid dns name, ignoring")