def main():
parser = _get_parser()
args = parser.parse_args(namespace=options)
# Get the options
if isinstance(args.rng, str):
try:
options.rng = eval("slice(" + args.rng + ")")
except Exception:
parser.error("Error when getting the range parameter.")
else:
args.dump = 1
# Catch the files passed as the last arguments
src = args.src.rsplit(':', 1)
if len(src) == 1:
filename, nodename = src[0], "/"
else:
filename, nodename = src
if nodename == "":
# case where filename == "filename:" instead of "filename:/"
nodename = "/"
try:
h5file = open_file(filename, 'r')
except Exception as e:
return 'Cannot open input file: ' + str(e)
with h5file:
# Check whether the specified node is a group or a leaf
nodeobject = h5file.get_node(nodename)
if isinstance(nodeobject, Group):
# Close the file again and reopen using the root_uep
dump_group(nodeobject, args.sort)
elif isinstance(nodeobject, Leaf):
# If it is not a Group, it must be a Leaf
dump_leaf(nodeobject)
else:
# This should never happen
print("Unrecognized object:", nodeobject)
def main():
parser = _get_parser()
args = parser.parse_args(namespace=options)
# Get the options
if isinstance(args.rng, six.string_types):
try:
options.rng = eval("slice(" + args.rng + ")")
except Exception:
parser.error("Error when getting the range parameter.")
else:
args.dump = 1
# Catch the files passed as the last arguments
src = args.src.split(':')
if len(src) == 1:
filename, nodename = src[0], "/"
else:
filename, nodename = src
if nodename == "":
# case where filename == "filename:" instead of "filename:/"
nodename = "/"
try:
h5file = open_file(filename, 'r')
except Exception as e:
return 'Cannot open input file: ' + str(e)
with h5file:
# Check whether the specified node is a group or a leaf
nodeobject = h5file.get_node(nodename)
if isinstance(nodeobject, Group):
# Close the file again and reopen using the root_uep
dump_group(nodeobject, args.sort)
elif isinstance(nodeobject, Leaf):
# If it is not a Group, it must be a Leaf
dump_leaf(nodeobject)
else:
# This should never happen
print("Unrecognized object:", nodeobject)
def copy_leaf(srcfile, dstfile, srcnode, dstnode, title, filters,
copyuserattrs, overwritefile, overwrtnodes, stats, start, stop,
step, chunkshape, sortby, check_CSI, propindexes,
upgradeflavors):
# Open the source file
srcfileh = open_file(srcfile, 'r')
# Get the source node (that should exist)
srcnode = srcfileh.get_node(srcnode)
# Get the destination node and its parent
last_slash = dstnode.rindex('/')
if last_slash == len(dstnode) - 1:
# print("Detected a trailing slash in destination node. "
# "Interpreting it as a destination group.")
dstgroup = dstnode[:-1]
elif last_slash > 0:
dstgroup = dstnode[:last_slash]
else:
dstgroup = "/"
dstleaf = dstnode[last_slash + 1:]
if dstleaf == "":
dstleaf = srcnode.name
# Check whether the destination group exists or not
if os.path.isfile(dstfile) and not overwritefile:
dstfileh = open_file(dstfile, 'a', pytables_sys_attrs=createsysattrs)
try:
dstgroup = dstfileh.get_node(dstgroup)
except:
# The dstgroup does not seem to exist. Try creating it.
dstgroup = newdst_group(dstfileh, dstgroup, title, filters)
else:
# The node exists, but it is really a group?
if not isinstance(dstgroup, Group):
# No. Should we overwrite it?
if overwrtnodes:
parent = dstgroup._v_parent
last_slash = dstgroup._v_pathname.rindex('/')
dstgroupname = dstgroup._v_pathname[last_slash + 1:]
dstgroup.remove()
dstgroup = dstfileh.create_group(parent,
dstgroupname,
title=title,
filters=filters)
else:
raise RuntimeError("Please check that the node names are "
"not duplicated in destination, and "
"if so, add the --overwrite-nodes "
"flag if desired.")
else:
# The destination file does not exist or will be overwritten.
dstfileh = open_file(dstfile,
'w',
title=title,
filters=filters,
pytables_sys_attrs=createsysattrs)
dstgroup = newdst_group(dstfileh, dstgroup, title="", filters=filters)
# Finally, copy srcnode to dstnode
try:
dstnode = srcnode.copy(dstgroup,
dstleaf,
filters=filters,
copyuserattrs=copyuserattrs,
overwrite=overwrtnodes,
stats=stats,
start=start,
stop=stop,
step=step,
chunkshape=chunkshape,
sortby=sortby,
check_CSI=check_CSI,
propindexes=propindexes)
except:
(type_, value, traceback) = sys.exc_info()
print("Problems doing the copy from '%s:%s' to '%s:%s'" %
(srcfile, srcnode, dstfile, dstnode))
print("The error was --> %s: %s" % (type_, value))
print("The destination file looks like:\n", dstfileh)
# Close all the open files:
srcfileh.close()
dstfileh.close()
raise RuntimeError("Please check that the node names are not "
"duplicated in destination, and if so, add "
"the --overwrite-nodes flag if desired.")
# Upgrade flavors in dstnode, if required
if upgradeflavors:
if srcfileh.format_version.startswith("1"):
# Remove original flavor in case the source file has 1.x format
dstnode.del_attr('FLAVOR')
elif srcfileh.format_version < "2.1":
if dstnode.get_attr('FLAVOR') in numpy_aliases:
dstnode.set_attr('FLAVOR', internal_flavor)
# Recreate possible old indexes in destination node
if srcnode._c_classid == "TABLE":
recreate_indexes(srcnode, dstfileh, dstnode)
# Close all the open files:
srcfileh.close()
dstfileh.close()
def main():
global verbose
global regoldindexes
global createsysattrs
parser = _get_parser()
args = parser.parse_args()
# check arguments
if args.rng:
try:
args.rng = eval("slice(" + args.rng + ")")
except Exception:
parser.error("Error when getting the range parameter.")
if args.chunkshape.isdigit() or args.chunkshape.startswith('('):
args.chunkshape = eval(args.chunkshape)
if args.complevel < 0 or args.complevel > 9:
parser.error(
'invalid "complevel" value, it sould be in te range [0, 9]')
# Catch the files passed as the last arguments
src = args.src.rsplit(':', 1)
dst = args.dst.rsplit(':', 1)
if len(src) == 1:
srcfile, srcnode = src[0], "/"
else:
srcfile, srcnode = src
if len(dst) == 1:
dstfile, dstnode = dst[0], "/"
else:
dstfile, dstnode = dst
if srcnode == "":
# case where filename == "filename:" instead of "filename:/"
srcnode = "/"
if dstnode == "":
# case where filename == "filename:" instead of "filename:/"
dstnode = "/"
# Ignore the warnings for tables that contains oldindexes
# (these will be handled by the copying routines)
warnings.filterwarnings("ignore", category=OldIndexWarning)
# Ignore the flavors warnings during upgrading flavor operations
if args.upgradeflavors:
warnings.filterwarnings("ignore", category=FlavorWarning)
# Build the Filters instance
filter_params = (
args.complevel,
args.complib,
args.shuffle,
args.bitshuffle,
args.fletcher32,
)
if (filter_params == (None, ) * 4 or args.keepfilters):
filters = None
else:
if args.complevel is None:
args.complevel = 0
if args.shuffle is None:
if args.complevel > 0:
args.shuffle = True
else:
args.shuffle = False
if args.bitshuffle is None:
args.bitshuffle = False
if args.bitshuffle:
# Shuffle and bitshuffle are mutually exclusive
args.shuffle = False
if args.complib is None:
args.complib = "zlib"
if args.fletcher32 is None:
args.fletcher32 = False
filters = Filters(complevel=args.complevel,
complib=args.complib,
shuffle=args.shuffle,
bitshuffle=args.bitshuffle,
fletcher32=args.fletcher32)
# The start, stop and step params:
start, stop, step = None, None, 1 # Defaults
if args.rng:
start, stop, step = args.rng.start, args.rng.stop, args.rng.step
# Set globals
verbose = args.verbose
regoldindexes = args.regoldindexes
createsysattrs = args.createsysattrs
# Some timing
t1 = time.time()
cpu1 = cputime()
# Copy the file
if verbose:
print("+=+" * 20)
print("Recursive copy:", args.recursive)
print("Applying filters:", filters)
if args.sortby is not None:
print("Sorting table(s) by column:", args.sortby)
print("Forcing a CSI creation:", args.checkCSI)
if args.propindexes:
print("Recreating indexes in copied table(s)")
print("Start copying %s:%s to %s:%s" %
(srcfile, srcnode, dstfile, dstnode))
print("+=+" * 20)
# Check whether the specified source node is a group or a leaf
h5srcfile = open_file(srcfile, 'r')
srcnodeobject = h5srcfile.get_node(srcnode)
# Close the file again
h5srcfile.close()
stats = {'groups': 0, 'leaves': 0, 'links': 0, 'bytes': 0, 'hardlinks': 0}
if isinstance(srcnodeobject, Group):
copy_children(srcfile,
dstfile,
srcnode,
dstnode,
title=args.title,
recursive=args.recursive,
filters=filters,
copyuserattrs=args.copyuserattrs,
overwritefile=args.overwritefile,
overwrtnodes=args.overwrtnodes,
stats=stats,
start=start,
stop=stop,
step=step,
chunkshape=args.chunkshape,
sortby=args.sortby,
check_CSI=args.checkCSI,
propindexes=args.propindexes,
upgradeflavors=args.upgradeflavors,
use_hardlinks=True)
else:
# If not a Group, it should be a Leaf
copy_leaf(srcfile,
dstfile,
srcnode,
dstnode,
title=args.title,
filters=filters,
copyuserattrs=args.copyuserattrs,
overwritefile=args.overwritefile,
overwrtnodes=args.overwrtnodes,
stats=stats,
start=start,
stop=stop,
step=step,
chunkshape=args.chunkshape,
sortby=args.sortby,
check_CSI=args.checkCSI,
propindexes=args.propindexes,
upgradeflavors=args.upgradeflavors)
# Gather some statistics
t2 = time.time()
cpu2 = cputime()
tcopy = round(t2 - t1, 3)
cpucopy = round(cpu2 - cpu1, 3)
try:
tpercent = int(round(cpucopy / tcopy, 2) * 100)
except ZeroDivisionError:
tpercent = 'NaN'
if verbose:
ngroups = stats['groups']
nleaves = stats['leaves']
nlinks = stats['links']
nhardlinks = stats['hardlinks']
nbytescopied = stats['bytes']
nnodes = ngroups + nleaves + nlinks + nhardlinks
print(
"Groups copied:",
ngroups,
", Leaves copied:",
nleaves,
", Links copied:",
nlinks,
", Hard links copied:",
nhardlinks,
)
if args.copyuserattrs:
print("User attrs copied")
else:
print("User attrs not copied")
print("KBytes copied:", round(nbytescopied / 1024., 3))
print("Time copying: %s s (real) %s s (cpu) %s%%" %
(tcopy, cpucopy, tpercent))
print("Copied nodes/sec: ", round((nnodes) / float(tcopy), 1))
print("Copied KB/s :", int(nbytescopied / (tcopy * 1024)))
def copy_children(srcfile,
dstfile,
srcgroup,
dstgroup,
title,
recursive,
filters,
copyuserattrs,
overwritefile,
overwrtnodes,
stats,
start,
stop,
step,
chunkshape,
sortby,
check_CSI,
propindexes,
upgradeflavors,
use_hardlinks=True):
"""Copy the children from source group to destination group"""
# Open the source file with srcgroup as root_uep
srcfileh = open_file(srcfile, 'r', root_uep=srcgroup)
# Assign the root to srcgroup
srcgroup = srcfileh.root
created_dstgroup = False
# Check whether the destination group exists or not
if os.path.isfile(dstfile) and not overwritefile:
dstfileh = open_file(dstfile, 'a', pytables_sys_attrs=createsysattrs)
try:
dstgroup = dstfileh.get_node(dstgroup)
except NoSuchNodeError:
# The dstgroup does not seem to exist. Try creating it.
dstgroup = newdst_group(dstfileh, dstgroup, title, filters)
created_dstgroup = True
else:
# The node exists, but it is really a group?
if not isinstance(dstgroup, Group):
# No. Should we overwrite it?
if overwrtnodes:
parent = dstgroup._v_parent
last_slash = dstgroup._v_pathname.rindex('/')
dstgroupname = dstgroup._v_pathname[last_slash + 1:]
dstgroup.remove()
dstgroup = dstfileh.create_group(parent,
dstgroupname,
title=title,
filters=filters)
else:
raise RuntimeError("Please check that the node names are "
"not duplicated in destination, and "
"if so, add the --overwrite-nodes "
"flag if desired.")
else:
# The destination file does not exist or will be overwritten.
dstfileh = open_file(dstfile,
'w',
title=title,
filters=filters,
pytables_sys_attrs=createsysattrs)
dstgroup = newdst_group(dstfileh, dstgroup, title="", filters=filters)
created_dstgroup = True
# Copy the attributes to dstgroup, if needed
if created_dstgroup and copyuserattrs:
srcgroup._v_attrs._f_copy(dstgroup)
# Finally, copy srcgroup children to dstgroup
try:
srcgroup._f_copy_children(dstgroup,
recursive=recursive,
filters=filters,
copyuserattrs=copyuserattrs,
overwrite=overwrtnodes,
stats=stats,
start=start,
stop=stop,
step=step,
chunkshape=chunkshape,
sortby=sortby,
check_CSI=check_CSI,
propindexes=propindexes,
use_hardlinks=use_hardlinks)
except:
(type_, value, traceback) = sys.exc_info()
print("Problems doing the copy from '%s:%s' to '%s:%s'" %
(srcfile, srcgroup, dstfile, dstgroup))
print("The error was --> %s: %s" % (type_, value))
print("The destination file looks like:\n", dstfileh)
# Close all the open files:
srcfileh.close()
dstfileh.close()
raise RuntimeError("Please check that the node names are not "
"duplicated in destination, and if so, add the "
"--overwrite-nodes flag if desired. In "
"particular, pay attention that root_uep is not "
"fooling you.")
# Upgrade flavors in dstnode, if required
if upgradeflavors:
for dstnode in dstgroup._f_walknodes("Leaf"):
if srcfileh.format_version.startswith("1"):
# Remove original flavor in case the source file has 1.x format
dstnode.del_attr('FLAVOR')
elif srcfileh.format_version < "2.1":
if dstnode.get_attr('FLAVOR') in numpy_aliases:
dstnode.set_attr('FLAVOR', internal_flavor)
# Convert the remaining tables with old indexes (if any)
for table in srcgroup._f_walknodes("Table"):
dsttable = dstfileh.get_node(dstgroup, table._v_pathname)
recreate_indexes(table, dstfileh, dsttable)
# Close all the open files:
srcfileh.close()
dstfileh.close()
def copy_leaf(srcfile, dstfile, srcnode, dstnode, title,
filters, copyuserattrs, overwritefile, overwrtnodes, stats,
start, stop, step, chunkshape, sortby, check_CSI,
propindexes, upgradeflavors):
# Open the source file
srcfileh = open_file(srcfile, 'r')
# Get the source node (that should exist)
srcnode = srcfileh.get_node(srcnode)
# Get the destination node and its parent
last_slash = dstnode.rindex('/')
if last_slash == len(dstnode)-1:
# print("Detected a trailing slash in destination node. "
# "Interpreting it as a destination group.")
dstgroup = dstnode[:-1]
elif last_slash > 0:
dstgroup = dstnode[:last_slash]
else:
dstgroup = "/"
dstleaf = dstnode[last_slash + 1:]
if dstleaf == "":
dstleaf = srcnode.name
# Check whether the destination group exists or not
if os.path.isfile(dstfile) and not overwritefile:
dstfileh = open_file(dstfile, 'a', pytables_sys_attrs=createsysattrs)
try:
dstgroup = dstfileh.get_node(dstgroup)
except:
# The dstgroup does not seem to exist. Try creating it.
dstgroup = newdst_group(dstfileh, dstgroup, title, filters)
else:
# The node exists, but it is really a group?
if not isinstance(dstgroup, Group):
# No. Should we overwrite it?
if overwrtnodes:
parent = dstgroup._v_parent
last_slash = dstgroup._v_pathname.rindex('/')
dstgroupname = dstgroup._v_pathname[last_slash + 1:]
dstgroup.remove()
dstgroup = dstfileh.create_group(parent, dstgroupname,
title=title,
filters=filters)
else:
raise RuntimeError("Please check that the node names are "
"not duplicated in destination, and "
"if so, add the --overwrite-nodes "
"flag if desired.")
else:
# The destination file does not exist or will be overwritten.
dstfileh = open_file(dstfile, 'w', title=title, filters=filters,
pytables_sys_attrs=createsysattrs)
dstgroup = newdst_group(dstfileh, dstgroup, title="", filters=filters)
# Finally, copy srcnode to dstnode
try:
dstnode = srcnode.copy(
dstgroup, dstleaf, filters=filters,
copyuserattrs=copyuserattrs, overwrite=overwrtnodes,
stats=stats, start=start, stop=stop, step=step,
chunkshape=chunkshape,
sortby=sortby, check_CSI=check_CSI, propindexes=propindexes)
except:
(type_, value, traceback) = sys.exc_info()
print("Problems doing the copy from '%s:%s' to '%s:%s'" %
(srcfile, srcnode, dstfile, dstnode))
print("The error was --> %s: %s" % (type_, value))
print("The destination file looks like:\n", dstfileh)
# Close all the open files:
srcfileh.close()
dstfileh.close()
raise RuntimeError("Please check that the node names are not "
"duplicated in destination, and if so, add "
"the --overwrite-nodes flag if desired.")
# Upgrade flavors in dstnode, if required
if upgradeflavors:
if srcfileh.format_version.startswith("1"):
# Remove original flavor in case the source file has 1.x format
dstnode.del_attr('FLAVOR')
elif srcfileh.format_version < "2.1":
if dstnode.get_attr('FLAVOR') in numpy_aliases:
dstnode.set_attr('FLAVOR', internal_flavor)
# Recreate possible old indexes in destination node
if srcnode._c_classid == "TABLE":
recreate_indexes(srcnode, dstfileh, dstnode)
# Close all the open files:
srcfileh.close()
dstfileh.close()
def copy_children(srcfile, dstfile, srcgroup, dstgroup, title,
recursive, filters, copyuserattrs, overwritefile,
overwrtnodes, stats, start, stop, step,
chunkshape, sortby, check_CSI, propindexes,
upgradeflavors, use_hardlinks=True):
"""Copy the children from source group to destination group"""
# Open the source file with srcgroup as root_uep
srcfileh = open_file(srcfile, 'r', root_uep=srcgroup)
# Assign the root to srcgroup
srcgroup = srcfileh.root
created_dstgroup = False
# Check whether the destination group exists or not
if os.path.isfile(dstfile) and not overwritefile:
dstfileh = open_file(dstfile, 'a', pytables_sys_attrs=createsysattrs)
try:
dstgroup = dstfileh.get_node(dstgroup)
except NoSuchNodeError:
# The dstgroup does not seem to exist. Try creating it.
dstgroup = newdst_group(dstfileh, dstgroup, title, filters)
created_dstgroup = True
else:
# The node exists, but it is really a group?
if not isinstance(dstgroup, Group):
# No. Should we overwrite it?
if overwrtnodes:
parent = dstgroup._v_parent
last_slash = dstgroup._v_pathname.rindex('/')
dstgroupname = dstgroup._v_pathname[last_slash + 1:]
dstgroup.remove()
dstgroup = dstfileh.create_group(parent, dstgroupname,
title=title,
filters=filters)
else:
raise RuntimeError("Please check that the node names are "
"not duplicated in destination, and "
"if so, add the --overwrite-nodes "
"flag if desired.")
else:
# The destination file does not exist or will be overwritten.
dstfileh = open_file(dstfile, 'w', title=title, filters=filters,
pytables_sys_attrs=createsysattrs)
dstgroup = newdst_group(dstfileh, dstgroup, title="", filters=filters)
created_dstgroup = True
# Copy the attributes to dstgroup, if needed
if created_dstgroup and copyuserattrs:
srcgroup._v_attrs._f_copy(dstgroup)
# Finally, copy srcgroup children to dstgroup
try:
srcgroup._f_copy_children(
dstgroup, recursive=recursive, filters=filters,
copyuserattrs=copyuserattrs, overwrite=overwrtnodes,
stats=stats, start=start, stop=stop, step=step,
chunkshape=chunkshape,
sortby=sortby, check_CSI=check_CSI, propindexes=propindexes,
use_hardlinks=use_hardlinks)
except:
(type_, value, traceback) = sys.exc_info()
print("Problems doing the copy from '%s:%s' to '%s:%s'" %
(srcfile, srcgroup, dstfile, dstgroup))
print("The error was --> %s: %s" % (type_, value))
print("The destination file looks like:\n", dstfileh)
# Close all the open files:
srcfileh.close()
dstfileh.close()
raise RuntimeError("Please check that the node names are not "
"duplicated in destination, and if so, add the "
"--overwrite-nodes flag if desired. In "
"particular, pay attention that root_uep is not "
"fooling you.")
# Upgrade flavors in dstnode, if required
if upgradeflavors:
for dstnode in dstgroup._f_walknodes("Leaf"):
if srcfileh.format_version.startswith("1"):
# Remove original flavor in case the source file has 1.x format
dstnode.del_attr('FLAVOR')
elif srcfileh.format_version < "2.1":
if dstnode.get_attr('FLAVOR') in numpy_aliases:
dstnode.set_attr('FLAVOR', internal_flavor)
# Convert the remaining tables with old indexes (if any)
for table in srcgroup._f_walknodes("Table"):
dsttable = dstfileh.get_node(dstgroup, table._v_pathname)
recreate_indexes(table, dstfileh, dsttable)
# Close all the open files:
srcfileh.close()
dstfileh.close()
def main():
global verbose
global regoldindexes
global createsysattrs
parser = _get_parser()
args = parser.parse_args()
# check arguments
if args.rng:
try:
args.rng = eval("slice(" + args.rng + ")")
except Exception:
parser.error("Error when getting the range parameter.")
if args.chunkshape.isdigit() or args.chunkshape.startswith('('):
args.chunkshape = eval(args.chunkshape)
if args.complevel < 0 or args.complevel > 9:
parser.error(
'invalid "complevel" value, it sould be in te range [0, 9]'
)
# Catch the files passed as the last arguments
src = args.src.rsplit(':', 1)
dst = args.dst.rsplit(':', 1)
if len(src) == 1:
srcfile, srcnode = src[0], "/"
else:
srcfile, srcnode = src
if len(dst) == 1:
dstfile, dstnode = dst[0], "/"
else:
dstfile, dstnode = dst
if srcnode == "":
# case where filename == "filename:" instead of "filename:/"
srcnode = "/"
if dstnode == "":
# case where filename == "filename:" instead of "filename:/"
dstnode = "/"
# Ignore the warnings for tables that contains oldindexes
# (these will be handled by the copying routines)
warnings.filterwarnings("ignore", category=OldIndexWarning)
# Ignore the flavors warnings during upgrading flavor operations
if args.upgradeflavors:
warnings.filterwarnings("ignore", category=FlavorWarning)
# Build the Filters instance
filter_params = (
args.complevel,
args.complib,
args.shuffle,
args.bitshuffle,
args.fletcher32,
)
if (filter_params == (None,) * 4 or args.keepfilters):
filters = None
else:
if args.complevel is None:
args.complevel = 0
if args.shuffle is None:
if args.complevel > 0:
args.shuffle = True
else:
args.shuffle = False
if args.bitshuffle is None:
args.bitshuffle = False
if args.bitshuffle:
# Shuffle and bitshuffle are mutually exclusive
args.shuffle = False
if args.complib is None:
args.complib = "zlib"
if args.fletcher32 is None:
args.fletcher32 = False
filters = Filters(complevel=args.complevel, complib=args.complib,
shuffle=args.shuffle, bitshuffle=args.bitshuffle,
fletcher32=args.fletcher32)
# The start, stop and step params:
start, stop, step = None, None, 1 # Defaults
if args.rng:
start, stop, step = args.rng.start, args.rng.stop, args.rng.step
# Set globals
verbose = args.verbose
regoldindexes = args.regoldindexes
createsysattrs = args.createsysattrs
# Some timing
t1 = time.time()
cpu1 = time.clock()
# Copy the file
if verbose:
print("+=+" * 20)
print("Recursive copy:", args.recursive)
print("Applying filters:", filters)
if args.sortby is not None:
print("Sorting table(s) by column:", args.sortby)
print("Forcing a CSI creation:", args.checkCSI)
if args.propindexes:
print("Recreating indexes in copied table(s)")
print("Start copying %s:%s to %s:%s" % (srcfile, srcnode,
dstfile, dstnode))
print("+=+" * 20)
# Check whether the specified source node is a group or a leaf
h5srcfile = open_file(srcfile, 'r')
srcnodeobject = h5srcfile.get_node(srcnode)
# Close the file again
h5srcfile.close()
stats = {'groups': 0, 'leaves': 0, 'links': 0, 'bytes': 0, 'hardlinks': 0}
if isinstance(srcnodeobject, Group):
copy_children(
srcfile, dstfile, srcnode, dstnode,
title=args.title, recursive=args.recursive, filters=filters,
copyuserattrs=args.copyuserattrs, overwritefile=args.overwritefile,
overwrtnodes=args.overwrtnodes, stats=stats,
start=start, stop=stop, step=step, chunkshape=args.chunkshape,
sortby=args.sortby, check_CSI=args.checkCSI,
propindexes=args.propindexes,
upgradeflavors=args.upgradeflavors,
use_hardlinks=True)
else:
# If not a Group, it should be a Leaf
copy_leaf(
srcfile, dstfile, srcnode, dstnode,
title=args.title, filters=filters,
copyuserattrs=args.copyuserattrs,
overwritefile=args.overwritefile, overwrtnodes=args.overwrtnodes,
stats=stats, start=start, stop=stop, step=step,
chunkshape=args.chunkshape,
sortby=args.sortby, check_CSI=args.checkCSI,
propindexes=args.propindexes,
upgradeflavors=args.upgradeflavors)
# Gather some statistics
t2 = time.time()
cpu2 = time.clock()
tcopy = round(t2 - t1, 3)
cpucopy = round(cpu2 - cpu1, 3)
try:
tpercent = int(round(cpucopy / tcopy, 2) * 100)
except ZeroDivisionError:
tpercent = 'NaN'
if verbose:
ngroups = stats['groups']
nleaves = stats['leaves']
nlinks = stats['links']
nhardlinks = stats['hardlinks']
nbytescopied = stats['bytes']
nnodes = ngroups + nleaves + nlinks + nhardlinks
print(
"Groups copied:", ngroups,
", Leaves copied:", nleaves,
", Links copied:", nlinks,
", Hard links copied:", nhardlinks,
)
if args.copyuserattrs:
print("User attrs copied")
else:
print("User attrs not copied")
print("KBytes copied:", round(nbytescopied / 1024., 3))
print("Time copying: %s s (real) %s s (cpu) %s%%" % (
tcopy, cpucopy, tpercent))
print("Copied nodes/sec: ", round((nnodes) / float(tcopy), 1))
print("Copied KB/s :", int(nbytescopied / (tcopy * 1024)))
def main():
usage = \
"""usage: %s [-d] [-v] [-a] [-c] [-i] [-R start,stop,step] [-h] file[:nodepath]
-d -- Dump data information on leaves
-v -- Dump more metainformation on nodes
-a -- Show attributes in nodes (only useful when -v or -d are active)
-c -- Show info of columns in tables (only useful when -v or -d are active)
-i -- Show info of indexed columns (only useful when -v or -d are active)
-R RANGE -- Select a RANGE of rows in the form "start,stop,step"
-h -- Print help on usage
\n""" \
% os.path.basename(sys.argv[0])
try:
opts, pargs = getopt.getopt(sys.argv[1:], 'R:ahdvci')
except:
sys.stderr.write(usage)
sys.exit(0)
# if we pass too much parameters, abort
if len(pargs) != 1:
sys.stderr.write(usage)
sys.exit(0)
# Get the options
for option in opts:
if option[0] == '-R':
options.dump = 1
try:
options.rng = eval("slice(" + option[1] + ")")
except:
print "Error when getting the range parameter."
(type, value, traceback) = sys.exc_info()
print " The error was:", value
sys.stderr.write(usage)
sys.exit(0)
elif option[0] == '-a':
options.showattrs = 1
elif option[0] == '-h':
sys.stderr.write(usage)
sys.exit(0)
elif option[0] == '-v':
options.verbose = 1
elif option[0] == '-d':
options.dump = 1
elif option[0] == '-c':
options.colinfo = 1
elif option[0] == '-i':
options.idxinfo = 1
else:
print option[0], ": Unrecognized option"
sys.stderr.write(usage)
sys.exit(0)
# Catch the files passed as the last arguments
src = pargs[0].split(':')
if len(src) == 1:
filename, nodename = src[0], "/"
else:
filename, nodename = src
if nodename == "":
# case where filename == "filename:" instead of "filename:/"
nodename = "/"
# Check whether the specified node is a group or a leaf
h5file = open_file(filename, 'r')
nodeobject = h5file.get_node(nodename)
if isinstance(nodeobject, Group):
# Close the file again and reopen using the root_uep
dump_group(nodeobject)
elif isinstance(nodeobject, Leaf):
# If it is not a Group, it must be a Leaf
dump_leaf(nodeobject)
else:
# This should never happen
print "Unrecognized object:", nodeobject
# Close the file
h5file.close()
def test_pytables_dummytable(self):
'''Test some pytable casting stuff NOT clearly documented :( '''
with open_file(self.output_file, 'a') as h5file:
table = h5file.create_table("/",
'table',
description=DummyTable)
row = table.row
# define set and get to avoid pylint flase positives
# everywhere:
def set(field, val): # @ReservedAssignment
'''sets a value on the row'''
row[field] = val # pylint: disable=unsupported-assignment-operation
def get(field):
'''gets a value from the row'''
return row[field] # pylint: disable=unsubscriptable-object
# assert the value is the default:
self.assertTrue(np.isnan(get('floatcol')))
# what if we supply a string? TypeError
with self.assertRaises(TypeError):
set('floatcol', 'a')
# same if string empty: TypeError
with self.assertRaises(TypeError):
set('floatcol', '')
# assert the value is still the default:
self.assertTrue(np.isnan(get('floatcol')))
# what if we supply a castable string instead? it is casted
set('floatcol', '5.5')
self.assertEqual(get('floatcol'), 5.5)
# what if we supply a castable string instead WITH SPACES? casted
set('floatcol', '1.0 ')
self.assertEqual(get('floatcol'), 1.0)
# what if we supply a scalr instead of an array?
# the value is broadcasted:
set('arraycol', 5)
self.assertTrue(np.allclose([5] * 10, get('arraycol')))
# what if arraycol string?
set('arraycol', '5')
self.assertTrue(np.allclose([5] * 10, get('arraycol')))
# what if arraycol array of strings?
set('arraycol', [str(_) for _ in [5] * 10])
self.assertTrue(np.allclose([5] * 10, get('arraycol')))
# what if arraycol array of strings with one nan?
aaa = [str(_) for _ in [5] * 10]
aaa[3] = 'asd'
with self.assertRaises(ValueError):
set('arraycol', aaa)
# what if we supply a float out of bound? no error
# but value is saved differently!
maxfloat32 = 3.4028235e+38
val = maxfloat32 * 10
set('floatcol', val)
val2 = get('floatcol')
# assert they are not the same
self.assertTrue(not np.isclose(val2, val))
# now restore val to the max Float32, and assert they are the same:
val = maxfloat32
set('floatcol', val)
val2 = get('floatcol')
self.assertTrue(np.isclose(val2, val))
# write to the table nan and see if we can select it later:
set('floatcol', float('nan'))
set('arraycol', [1, 2, 3, 4, float('nan'), 6.6, 7.7, 8.8, 9.9,
10.00045])
# setting ascii str in stringcol is safe, we do not need to convert
set('stringcol', "abc")
# However, returned value is bytes
self.assertEqual(get('stringcol'), b'abc')
# test WHY enumcol IS USELESS:
eval = get('ecol')
# here is the point about enumcols: WE CANNOT SET the LABEL!
# What's the point of having an enum if we actually need to
# set/get the associated int?
with self.assertRaises(Exception):
set('ecol' , 'a')
set('ecol' , -5)
self.assertEqual(get('ecol'), -5)
#test time64 col:
tme = get('tcol')
self.assertTrue(np.isnan(tme))
dtime = datetime.utcnow()
with self.assertRaises(TypeError):
set('tcol', dtime)
tme = get('tcol')
self.assertTrue(np.isnan(tme))
# now set a numpy datetim64. ERROR! WTF!
tme = np.datetime64('2007-02-01T00:01:04')
with self.assertRaises(TypeError):
set('tcol', tme)
# OK conclusion: pytables TimeCol is absolutely USELESS
row.append() # pylint: disable=no-member
table.flush()
# test selections:
with open_file(self.output_file, 'a') as h5file:
tbl = h5file.get_node("/", 'table')
###########################
# TEST NAN SELECTION:
###########################
# this does not work:
with self.assertRaises(NameError):
[r['floatcol'] for r in tbl.where('floatcol == nan')]
# what if we provide the nan as string?
# incompatible types (NotImplementedError)
with self.assertRaises(NotImplementedError):
[r['floatcol'] for r in tbl.where("floatcol == 'nan'")]
# we should use the condvars dict:
vals = [r['floatcol'] for r in
tbl.where('floatcol == nan',
condvars={'nan': float('nan')})]
# does not raise, but we did not get what we wanted:
self.assertTrue(not vals)
# we should actually test nan equalisty with this weird
# test: (https://stackoverflow.com/a/10821267)
vals = [r['floatcol'] for r in tbl.where('floatcol != floatcol')]
# now it works:
self.assertEqual(len(vals), 1)
###########################
# TEST ARRAY SELECTION:
###########################
# this does not work. Array selection not yet supported:
with self.assertRaises(NotImplementedError):
vals = [r['arraycol'] for r in
tbl.where('arraycol == 2')]
###########################
# TEST STRING SELECTION:
###########################
# this does not work, needs quotes:
with self.assertRaises(NameError):
vals = [r['stringcol'] for r in
tbl.where("stringcol == %s" % 'abc')]
# this SHOULD NOT WORK either (not binary strings), BUT IT DOES:
vals = [r['stringcol'] for r in
tbl.where("stringcol == 'abc'")]
self.assertTrue(len(vals) == 1)
def main():
global verbose
global regoldindexes
global createsysattrs
usage = """usage: %s [-h] [-v] [-o] [-R start,stop,step] [--non-recursive] [--dest-title=title] [--dont-create-sysattrs] [--dont-copy-userattrs] [--overwrite-nodes] [--complevel=(0-9)] [--complib=lib] [--shuffle=(0|1)] [--fletcher32=(0|1)] [--keep-source-filters] [--chunkshape=value] [--upgrade-flavors] [--dont-regenerate-old-indexes] [--sortby=column] [--checkCSI] [--propindexes] sourcefile:sourcegroup destfile:destgroup
-h -- Print usage message.
-v -- Show more information.
-o -- Overwrite destination file.
-R RANGE -- Select a RANGE of rows (in the form "start,stop,step")
during the copy of *all* the leaves. Default values are
"None,None,1", which means a copy of all the rows.
--non-recursive -- Do not do a recursive copy. Default is to do it.
--dest-title=title -- Title for the new file (if not specified,
the source is copied).
--dont-create-sysattrs -- Do not create sys attrs (default is to do it).
--dont-copy-userattrs -- Do not copy the user attrs (default is to do it).
--overwrite-nodes -- Overwrite destination nodes if they exist. Default is
to not overwrite them.
--complevel=(0-9) -- Set a compression level (0 for no compression, which
is the default).
--complib=lib -- Set the compression library to be used during the copy.
lib can be set to "zlib", "lzo", "bzip2" or "blosc". Defaults to
"zlib".
--shuffle=(0|1) -- Activate or not the shuffling filter (default is active
if complevel>0).
--fletcher32=(0|1) -- Whether to activate or not the fletcher32 filter
(not active by default).
--keep-source-filters -- Use the original filters in source files. The
default is not doing that if any of --complevel, --complib, --shuffle
or --fletcher32 option is specified.
--chunkshape=("keep"|"auto"|int|tuple) -- Set a chunkshape. A value
of "auto" computes a sensible value for the chunkshape of the
leaves copied. The default is to "keep" the original value.
--upgrade-flavors -- When repacking PyTables 1.x files, the flavor of
leaves will be unset. With this, such a leaves will be serialized
as objects with the internal flavor ('numpy' for 2.x series).
--dont-regenerate-old-indexes -- Disable regenerating old indexes. The
default is to regenerate old indexes as they are found.
--sortby=column -- Do a table copy sorted by the index in "column".
For reversing the order, use a negative value in the "step" part of
"RANGE" (see "-R" flag). Only applies to table objects.
--checkCSI -- Force the check for a CSI index for the --sortby column.
--propindexes -- Propagate the indexes existing in original tables. The
default is to not propagate them. Only applies to table objects.
\n""" % os.path.basename(sys.argv[0])
try:
opts, pargs = getopt.getopt(sys.argv[1:], 'hvoR:', [
'non-recursive',
'dest-title=',
'dont-create-sysattrs',
'dont-copy-userattrs',
'overwrite-nodes',
'complevel=',
'complib=',
'shuffle=',
'fletcher32=',
'keep-source-filters',
'chunkshape=',
'upgrade-flavors',
'dont-regenerate-old-indexes',
'sortby=',
'checkCSI',
'propindexes',
])
except:
(type, value, traceback) = sys.exc_info()
print "Error parsing the options. The error was:", value
sys.stderr.write(usage)
sys.exit(0)
# default options
overwritefile = False
keepfilters = False
chunkshape = "keep"
complevel = None
complib = None
shuffle = None
fletcher32 = None
title = ""
copyuserattrs = True
rng = None
recursive = True
overwrtnodes = False
upgradeflavors = False
sortby = None
checkCSI = False
propindexes = False
# Get the options
for option in opts:
if option[0] == '-h':
sys.stderr.write(usage)
sys.exit(0)
elif option[0] == '-v':
verbose = True
elif option[0] == '-o':
overwritefile = True
elif option[0] == '-R':
try:
rng = eval("slice(" + option[1] + ")")
except:
print "Error when getting the range parameter."
(type, value, traceback) = sys.exc_info()
print " The error was:", value
sys.stderr.write(usage)
sys.exit(0)
elif option[0] == '--dest-title':
title = option[1]
elif option[0] == '--dont-create-sysattrs':
createsysattrs = False
elif option[0] == '--dont-copy-userattrs':
copyuserattrs = False
elif option[0] == '--non-recursive':
recursive = False
elif option[0] == '--overwrite-nodes':
overwrtnodes = True
elif option[0] == '--keep-source-filters':
keepfilters = True
elif option[0] == '--chunkshape':
chunkshape = option[1]
if chunkshape.isdigit() or chunkshape.startswith('('):
chunkshape = eval(chunkshape)
elif option[0] == '--upgrade-flavors':
upgradeflavors = True
elif option[0] == '--dont-regenerate-old-indexes':
regoldindexes = False
elif option[0] == '--complevel':
complevel = int(option[1])
elif option[0] == '--complib':
complib = option[1]
elif option[0] == '--shuffle':
shuffle = int(option[1])
elif option[0] == '--fletcher32':
fletcher32 = int(option[1])
elif option[0] == '--sortby':
sortby = option[1]
elif option[0] == '--propindexes':
propindexes = True
elif option[0] == '--checkCSI':
checkCSI = True
else:
print option[0], ": Unrecognized option"
sys.stderr.write(usage)
sys.exit(0)
# if we pass a number of files different from 2, abort
if len(pargs) != 2:
print "You need to pass both source and destination!."
sys.stderr.write(usage)
sys.exit(0)
# Catch the files passed as the last arguments
src = pargs[0].split(':')
dst = pargs[1].split(':')
if len(src) == 1:
srcfile, srcnode = src[0], "/"
else:
srcfile, srcnode = src
if len(dst) == 1:
dstfile, dstnode = dst[0], "/"
else:
dstfile, dstnode = dst
if srcnode == "":
# case where filename == "filename:" instead of "filename:/"
srcnode = "/"
if dstnode == "":
# case where filename == "filename:" instead of "filename:/"
dstnode = "/"
# Ignore the warnings for tables that contains oldindexes
# (these will be handled by the copying routines)
warnings.filterwarnings("ignore", category=OldIndexWarning)
# Ignore the flavors warnings during upgrading flavor operations
if upgradeflavors:
warnings.filterwarnings("ignore", category=FlavorWarning)
# Build the Filters instance
if ((complevel, complib, shuffle, fletcher32) == (None, ) * 4
or keepfilters):
filters = None
else:
if complevel is None:
complevel = 0
if shuffle is None:
if complevel > 0:
shuffle = True
else:
shuffle = False
if complib is None:
complib = "zlib"
if fletcher32 is None:
fletcher32 = False
filters = Filters(complevel=complevel,
complib=complib,
shuffle=shuffle,
fletcher32=fletcher32)
# The start, stop and step params:
start, stop, step = None, None, 1 # Defaults
if rng:
start, stop, step = rng.start, rng.stop, rng.step
# Some timing
t1 = time.time()
cpu1 = time.clock()
# Copy the file
if verbose:
print "+=+" * 20
print "Recursive copy:", recursive
print "Applying filters:", filters
if sortby is not None:
print "Sorting table(s) by column:", sortby
print "Forcing a CSI creation:", checkCSI
if propindexes:
print "Recreating indexes in copied table(s)"
print "Start copying %s:%s to %s:%s" % (srcfile, srcnode, dstfile,
dstnode)
print "+=+" * 20
# Check whether the specified source node is a group or a leaf
h5srcfile = open_file(srcfile, 'r')
srcnodeobject = h5srcfile.get_node(srcnode)
# Close the file again
h5srcfile.close()
stats = {'groups': 0, 'leaves': 0, 'links': 0, 'bytes': 0}
if isinstance(srcnodeobject, Group):
copy_children(srcfile,
dstfile,
srcnode,
dstnode,
title=title,
recursive=recursive,
filters=filters,
copyuserattrs=copyuserattrs,
overwritefile=overwritefile,
overwrtnodes=overwrtnodes,
stats=stats,
start=start,
stop=stop,
step=step,
chunkshape=chunkshape,
sortby=sortby,
checkCSI=checkCSI,
propindexes=propindexes,
upgradeflavors=upgradeflavors)
else:
# If not a Group, it should be a Leaf
copy_leaf(srcfile,
dstfile,
srcnode,
dstnode,
title=title,
filters=filters,
copyuserattrs=copyuserattrs,
overwritefile=overwritefile,
overwrtnodes=overwrtnodes,
stats=stats,
start=start,
stop=stop,
step=step,
chunkshape=chunkshape,
sortby=sortby,
checkCSI=checkCSI,
propindexes=propindexes,
upgradeflavors=upgradeflavors)
# Gather some statistics
t2 = time.time()
cpu2 = time.clock()
tcopy = round(t2 - t1, 3)
cpucopy = round(cpu2 - cpu1, 3)
tpercent = int(round(cpucopy / tcopy, 2) * 100)
if verbose:
ngroups = stats['groups']
nleaves = stats['leaves']
nlinks = stats['links']
nbytescopied = stats['bytes']
nnodes = ngroups + nleaves + nlinks
print \
"Groups copied:", ngroups, \
" Leaves copied:", nleaves, \
" Links copied:", nlinks
if copyuserattrs:
print "User attrs copied"
else:
print "User attrs not copied"
print "KBytes copied:", round(nbytescopied / 1024., 3)
print "Time copying: %s s (real) %s s (cpu) %s%%" % \
(tcopy, cpucopy, tpercent)
print "Copied nodes/sec: ", round((nnodes) / float(tcopy), 1)
print "Copied KB/s :", int(nbytescopied / (tcopy * 1024))
def main():
global verbose
global regoldindexes
global createsysattrs
usage = """usage: %s [-h] [-v] [-o] [-R start,stop,step] [--non-recursive] [--dest-title=title] [--dont-create-sysattrs] [--dont-copy-userattrs] [--overwrite-nodes] [--complevel=(0-9)] [--complib=lib] [--shuffle=(0|1)] [--fletcher32=(0|1)] [--keep-source-filters] [--chunkshape=value] [--upgrade-flavors] [--dont-regenerate-old-indexes] [--sortby=column] [--checkCSI] [--propindexes] sourcefile:sourcegroup destfile:destgroup
-h -- Print usage message.
-v -- Show more information.
-o -- Overwrite destination file.
-R RANGE -- Select a RANGE of rows (in the form "start,stop,step")
during the copy of *all* the leaves. Default values are
"None,None,1", which means a copy of all the rows.
--non-recursive -- Do not do a recursive copy. Default is to do it.
--dest-title=title -- Title for the new file (if not specified,
the source is copied).
--dont-create-sysattrs -- Do not create sys attrs (default is to do it).
--dont-copy-userattrs -- Do not copy the user attrs (default is to do it).
--overwrite-nodes -- Overwrite destination nodes if they exist. Default is
to not overwrite them.
--complevel=(0-9) -- Set a compression level (0 for no compression, which
is the default).
--complib=lib -- Set the compression library to be used during the copy.
lib can be set to "zlib", "lzo", "bzip2" or "blosc". Defaults to
"zlib".
--shuffle=(0|1) -- Activate or not the shuffling filter (default is active
if complevel>0).
--fletcher32=(0|1) -- Whether to activate or not the fletcher32 filter
(not active by default).
--keep-source-filters -- Use the original filters in source files. The
default is not doing that if any of --complevel, --complib, --shuffle
or --fletcher32 option is specified.
--chunkshape=("keep"|"auto"|int|tuple) -- Set a chunkshape. A value
of "auto" computes a sensible value for the chunkshape of the
leaves copied. The default is to "keep" the original value.
--upgrade-flavors -- When repacking PyTables 1.x files, the flavor of
leaves will be unset. With this, such a leaves will be serialized
as objects with the internal flavor ('numpy' for 2.x series).
--dont-regenerate-old-indexes -- Disable regenerating old indexes. The
default is to regenerate old indexes as they are found.
--sortby=column -- Do a table copy sorted by the index in "column".
For reversing the order, use a negative value in the "step" part of
"RANGE" (see "-R" flag). Only applies to table objects.
--checkCSI -- Force the check for a CSI index for the --sortby column.
--propindexes -- Propagate the indexes existing in original tables. The
default is to not propagate them. Only applies to table objects.
\n""" % os.path.basename(sys.argv[0])
try:
opts, pargs = getopt.getopt(sys.argv[1:], 'hvoR:',
['non-recursive',
'dest-title=',
'dont-create-sysattrs',
'dont-copy-userattrs',
'overwrite-nodes',
'complevel=',
'complib=',
'shuffle=',
'fletcher32=',
'keep-source-filters',
'chunkshape=',
'upgrade-flavors',
'dont-regenerate-old-indexes',
'sortby=',
'checkCSI',
'propindexes',
])
except:
(type, value, traceback) = sys.exc_info()
print "Error parsing the options. The error was:", value
sys.stderr.write(usage)
sys.exit(0)
# default options
overwritefile = False
keepfilters = False
chunkshape = "keep"
complevel = None
complib = None
shuffle = None
fletcher32 = None
title = ""
copyuserattrs = True
rng = None
recursive = True
overwrtnodes = False
upgradeflavors = False
sortby = None
checkCSI = False
propindexes = False
# Get the options
for option in opts:
if option[0] == '-h':
sys.stderr.write(usage)
sys.exit(0)
elif option[0] == '-v':
verbose = True
elif option[0] == '-o':
overwritefile = True
elif option[0] == '-R':
try:
rng = eval("slice("+option[1]+")")
except:
print "Error when getting the range parameter."
(type, value, traceback) = sys.exc_info()
print " The error was:", value
sys.stderr.write(usage)
sys.exit(0)
elif option[0] == '--dest-title':
title = option[1]
elif option[0] == '--dont-create-sysattrs':
createsysattrs = False
elif option[0] == '--dont-copy-userattrs':
copyuserattrs = False
elif option[0] == '--non-recursive':
recursive = False
elif option[0] == '--overwrite-nodes':
overwrtnodes = True
elif option[0] == '--keep-source-filters':
keepfilters = True
elif option[0] == '--chunkshape':
chunkshape = option[1]
if chunkshape.isdigit() or chunkshape.startswith('('):
chunkshape = eval(chunkshape)
elif option[0] == '--upgrade-flavors':
upgradeflavors = True
elif option[0] == '--dont-regenerate-old-indexes':
regoldindexes = False
elif option[0] == '--complevel':
complevel = int(option[1])
elif option[0] == '--complib':
complib = option[1]
elif option[0] == '--shuffle':
shuffle = int(option[1])
elif option[0] == '--fletcher32':
fletcher32 = int(option[1])
elif option[0] == '--sortby':
sortby = option[1]
elif option[0] == '--propindexes':
propindexes = True
elif option[0] == '--checkCSI':
checkCSI = True
else:
print option[0], ": Unrecognized option"
sys.stderr.write(usage)
sys.exit(0)
# if we pass a number of files different from 2, abort
if len(pargs) != 2:
print "You need to pass both source and destination!."
sys.stderr.write(usage)
sys.exit(0)
# Catch the files passed as the last arguments
src = pargs[0].split(':')
dst = pargs[1].split(':')
if len(src) == 1:
srcfile, srcnode = src[0], "/"
else:
srcfile, srcnode = src
if len(dst) == 1:
dstfile, dstnode = dst[0], "/"
else:
dstfile, dstnode = dst
if srcnode == "":
# case where filename == "filename:" instead of "filename:/"
srcnode = "/"
if dstnode == "":
# case where filename == "filename:" instead of "filename:/"
dstnode = "/"
# Ignore the warnings for tables that contains oldindexes
# (these will be handled by the copying routines)
warnings.filterwarnings("ignore", category=OldIndexWarning)
# Ignore the flavors warnings during upgrading flavor operations
if upgradeflavors:
warnings.filterwarnings("ignore", category=FlavorWarning)
# Build the Filters instance
if ((complevel, complib, shuffle, fletcher32) == (None,)*4 or keepfilters):
filters = None
else:
if complevel is None:
complevel = 0
if shuffle is None:
if complevel > 0:
shuffle = True
else:
shuffle = False
if complib is None:
complib = "zlib"
if fletcher32 is None:
fletcher32 = False
filters = Filters(complevel=complevel, complib=complib,
shuffle=shuffle, fletcher32=fletcher32)
# The start, stop and step params:
start, stop, step = None, None, 1 # Defaults
if rng:
start, stop, step = rng.start, rng.stop, rng.step
# Some timing
t1 = time.time()
cpu1 = time.clock()
# Copy the file
if verbose:
print "+=+"*20
print "Recursive copy:", recursive
print "Applying filters:", filters
if sortby is not None:
print "Sorting table(s) by column:", sortby
print "Forcing a CSI creation:", checkCSI
if propindexes:
print "Recreating indexes in copied table(s)"
print "Start copying %s:%s to %s:%s" % (srcfile, srcnode,
dstfile, dstnode)
print "+=+"*20
# Check whether the specified source node is a group or a leaf
h5srcfile = open_file(srcfile, 'r')
srcnodeobject = h5srcfile.get_node(srcnode)
# Close the file again
h5srcfile.close()
stats = {'groups': 0, 'leaves': 0, 'links': 0, 'bytes': 0}
if isinstance(srcnodeobject, Group):
copy_children(
srcfile, dstfile, srcnode, dstnode,
title=title, recursive=recursive, filters=filters,
copyuserattrs=copyuserattrs, overwritefile=overwritefile,
overwrtnodes=overwrtnodes, stats=stats,
start=start, stop=stop, step=step, chunkshape=chunkshape,
sortby=sortby, checkCSI=checkCSI, propindexes=propindexes,
upgradeflavors=upgradeflavors)
else:
# If not a Group, it should be a Leaf
copy_leaf(
srcfile, dstfile, srcnode, dstnode,
title=title, filters=filters, copyuserattrs=copyuserattrs,
overwritefile=overwritefile, overwrtnodes=overwrtnodes,
stats=stats, start=start, stop=stop, step=step,
chunkshape=chunkshape,
sortby=sortby, checkCSI=checkCSI, propindexes=propindexes,
upgradeflavors=upgradeflavors)
# Gather some statistics
t2 = time.time()
cpu2 = time.clock()
tcopy = round(t2-t1, 3)
cpucopy = round(cpu2-cpu1, 3)
tpercent = int(round(cpucopy/tcopy, 2)*100)
if verbose:
ngroups = stats['groups']
nleaves = stats['leaves']
nlinks = stats['links']
nbytescopied = stats['bytes']
nnodes = ngroups + nleaves + nlinks
print \
"Groups copied:", ngroups, \
" Leaves copied:", nleaves, \
" Links copied:", nlinks
if copyuserattrs:
print "User attrs copied"
else:
print "User attrs not copied"
print "KBytes copied:", round(nbytescopied/1024., 3)
print "Time copying: %s s (real) %s s (cpu) %s%%" % \
(tcopy, cpucopy, tpercent)
print "Copied nodes/sec: ", round((nnodes) / float(tcopy), 1)
print "Copied KB/s :", int(nbytescopied / (tcopy * 1024))
from tables.file import File, open_file
from tables import Filters
from tables import Atom
import numpy as np
# Check that options are present, else print help msg
parser = OptionParser()
parser.add_option("-i", "--input", dest="infile", help="read input h5 from here")
parser.add_option("-f", "--filters", dest="filters", help="read the filters from here")
parser.add_option("-o", "--filename", dest="filename", help="specify the .h5 filename that will contain all the filtered data")
(options, args) = parser.parse_args()
# Open and prepare input and output hdf5 files
filename = options.filename
h5output = open_file(filename, mode = "w", title = "Filtered Data File")
zlib_filters = Filters(complib='zlib', complevel=5)
h5input = open_file(options.infile, mode = "r")
# Create a new group under "/" (root)
plates_group = h5output.createGroup("/", 'plates', 'the plates for this replicate')
all_plates = [p._v_name for p in h5input.walk_groups("/plates")]
all_plates = all_plates[1:]
# Create a group for each plate in the output file
for plate in all_plates:
desc = "plate number " + plate
h5output.create_group("/plates/",plate,desc)
