def unpack(self, buff):
"""
Unpack the given binary buffer into the fields. The result
is a dictionary mapping field names to values.
"""
args = struct.unpack_from(self._fmt, buff[:self._size])
return dict(izip(self._names, args))
def unpack(self, buff):
"""
Unpack the given binary buffer into the fields. The result
is a dictionary mapping field names to values.
"""
args = struct.unpack_from(self._fmt, buff[:self._size])
return dict(izip(self._names, args))
def _build_tree(self):
"""
Construct tree from grouping done by astropy table
Because the tree is constructed
"""
grouped = self.table.group_by(self.tree_keys)
columns = list(self.tree_keys)
columns.append(self.index_key)
for group_key, members in izip(grouped.groups.keys, grouped.groups):
key_list = list(group_key) # table rows can't be indexed w/slice
indexes = list(members[self.index_key])
self.add_keys(key_list, value=indexes)
def _build_tree(self):
"""
Construct tree from grouping done by astropy table
Because the tree is constructed
"""
grouped = self.table.group_by(self.tree_keys)
columns = list(self.tree_keys)
columns.append(self.index_key)
for group_key, members in izip(grouped.groups.keys,
grouped.groups):
key_list = list(group_key) # table rows can't be indexed w/slice
indexes = list(members[self.index_key])
self.add_keys(key_list, value=indexes)
def sort_directory(directory,
verbose=False,
destination=None,
no_log_destination=False,
script_name=None,
move=False):
"""
Sort files in a directory into a tree
Parameters
----------
directory : str
Directory whose files are to be sorted
verbose : bool
If True, increase logging verbosity
destination : str, optional
Directory into which the sorted files/directories should be placed. If
omitted, sorting is done in the source ``directory``.
no_log_destination : bool, optional
Suppress logging in the destination directory. Logging cannot be
suppressed if you are running in the destination directory.
script_name : str, optional, default is 'sort_files'
Name of the script calling this function; used to set the name of the
log file in the destination.
"""
script_name = script_name or 'sort_files'
if destination is not None:
working_dir = destination
if not os.path.exists(destination):
os.makedirs(destination)
else:
working_dir = directory
if (not no_log_destination) and (destination is not None):
add_file_handlers(logger, working_dir, script_name)
if move:
copy_or_move = shutil.move
else:
copy_or_move = shutil.copy2
logger.info("Working on directory: %s", directory)
logger.info("Destination directory is: %s", destination)
default_keys = ['imagetyp', 'exptime', 'filter', 'object']
images = ImageFileCollection(directory, keywords=default_keys)
if not images.files:
return
full_table = images.summary
bias = 'BIAS'
dark = 'DARK'
flat = 'FLAT'
light = 'LIGHT'
image_types = {
bias: None,
dark: ['exptime'],
flat: ['filter', 'exptime'],
light: ['object', 'filter', 'exptime']
}
table_by_type = full_table.group_by('imagetyp')
prepend_path = lambda path, file_list: \
[os.path.join(path, f) for f in file_list]
for im_type, table in izip(table_by_type.groups.keys,
table_by_type.groups):
image_type = im_type['imagetyp']
tree_keys = image_types[image_type]
dest_dir = os.path.join(working_dir, image_type)
if not tree_keys:
# bias
source_files = prepend_path(directory, table['file'])
this_dest = os.makedirs(dest_dir)
copy_files(source_files, dest_dir, copy_or_move)
continue
mask = [False] * len(table)
for key in tree_keys:
mask |= table[key].mask
if any(mask):
source_files = prepend_path(directory, table['file'][mask])
this_dest = os.path.join(dest_dir, UNSORTED_DIR)
os.makedirs(this_dest)
copy_files(source_files, this_dest, copy_or_move)
clean_table = table[~mask]
try:
tree = TableTree(clean_table, tree_keys, 'file')
except IndexError:
continue
for parents, children, files in tree.walk():
if files:
str_parents = [str(p) for p in parents]
this_dest = os.path.join(dest_dir, *str_parents)
os.makedirs(this_dest)
source_files = prepend_path(directory, files)
copy_files(source_files, this_dest, copy_or_move)
def sort_directory(directory, verbose=False,
destination=None,
no_log_destination=False,
script_name=None,
move=False):
"""
Sort files in a directory into a tree
Parameters
----------
directory : str
Directory whose files are to be sorted
verbose : bool
If True, increase logging verbosity
destination : str, optional
Directory into which the sorted files/directories should be placed. If
omitted, sorting is done in the source ``directory``.
no_log_destination : bool, optional
Suppress logging in the destination directory. Logging cannot be
suppressed if you are running in the destination directory.
script_name : str, optional, default is 'sort_files'
Name of the script calling this function; used to set the name of the
log file in the destination.
"""
script_name = script_name or 'sort_files'
if destination is not None:
working_dir = destination
if not os.path.exists(destination):
os.makedirs(destination)
else:
working_dir = directory
if (not no_log_destination) and (destination is not None):
add_file_handlers(logger, working_dir, script_name)
if move:
copy_or_move = shutil.move
else:
copy_or_move = shutil.copy2
logger.info("Working on directory: %s", directory)
logger.info("Destination directory is: %s", destination)
default_keys = ['imagetyp', 'exptime', 'filter', 'object']
images = ImageFileCollection(directory, keywords=default_keys)
if not images.files:
return
full_table = images.summary
bias = 'BIAS'
dark = 'DARK'
flat = 'FLAT'
light = 'LIGHT'
image_types = {bias: None,
dark: ['exptime'],
flat: ['filter', 'exptime'],
light: ['object', 'filter', 'exptime']}
table_by_type = full_table.group_by('imagetyp')
prepend_path = lambda path, file_list: \
[os.path.join(path, f) for f in file_list]
for im_type, table in izip(table_by_type.groups.keys,
table_by_type.groups):
image_type = im_type['imagetyp']
tree_keys = image_types[image_type]
dest_dir = os.path.join(working_dir, image_type)
if not tree_keys:
# bias
source_files = prepend_path(directory, table['file'])
this_dest = os.makedirs(dest_dir)
copy_files(source_files, dest_dir, copy_or_move)
continue
mask = [False] * len(table)
for key in tree_keys:
mask |= table[key].mask
if any(mask):
source_files = prepend_path(directory, table['file'][mask])
this_dest = os.path.join(dest_dir, UNSORTED_DIR)
os.makedirs(this_dest)
copy_files(source_files, this_dest, copy_or_move)
clean_table = table[~mask]
try:
tree = TableTree(clean_table, tree_keys, 'file')
except IndexError:
continue
for parents, children, files in tree.walk():
if files:
str_parents = [str(p) for p in parents]
this_dest = os.path.join(dest_dir, *str_parents)
os.makedirs(this_dest)
source_files = prepend_path(directory, files)
copy_files(source_files, this_dest, copy_or_move)
