def set_root(self, root_fn):
if self._root.get_n_children():
self.emit("rf-directory-tree-remove-node", tree_dir_path_to_string(
self._root.get_nth_child(0).get_path_tuple()
))
root = DirectoryTreeNode(root_fn, self._root)
self._root.child_list = [root]
self.set_cur_node(root)
self.emit("rf-directory-tree-add-node", tree_dir_path_to_string(root.get_path_tuple()))
self.get_cur_node().set_expanded()
for line in get_ftp().list_current_dir().split("\n"):
if not line:
continue
fn = filenode_new_from_line(line)
self.append_child_fn(fn)
def __buildFileTree(self):
tree = DirectoryTreeNode()
for _file in self.files():
tree.addFile(_file)
return tree
def generate_tree(filename, short):
path = check_header(filename)
filetypes = {}
# Extract the UDI from the passed starting path.
udi_pattern = re.compile('([A-Za-z0-9]{2}.x[0-9]{3}.[0-9]{3})[.:]([0-9]{4})')
udi_parts = udi_pattern.findall(path)
udi = udi_parts[0][0] + ':' + udi_parts[0][1]
if (path is not None):
sizes = OrderedDict()
with open(filename, 'rb') as f:
reader = csv.reader(f)
rownum = 1
last = None
for row in reader:
# skip header
if (rownum > 5):
object_filename = re.sub(path + '/', '', row[3])
object_size = row[0]
# Find file's filetype, add to count by filetype.
filetype = os.path.splitext(row[3])[1]
try:
filetypes[filetype] += 1
except KeyError:
filetypes[filetype] = 1
# Split out paths to keep track, by dir, of totals.
parts = splitall(object_filename)
for i in range (0, len(parts), 1):
if (i == 0):
my_str = parts[i]
elif (i < len(parts)-1):
my_str = my_str + '/' + parts[i]
else:
# Appending '|EOL:' to ends of non-file paths, so this indicates directories.
# elegance-- but works.
my_str = my_str + '/' + parts[i] + '|EOL:'
try:
sizes[my_str] += int(object_size)
except KeyError:
sizes[my_str] = int(object_size)
rownum += 1
# Output Section
if (short):
print "Dataset Directory Summary for " + udi
else:
print "Dataset File Manifest for " + udi
print textwrap.dedent("""\
This dataset is greater than 25 GB and therefore too large to be downloaded
through direct download. In order to obtain this dataset, please email
[email protected] to make arrangements. If you would like a subset of the
dataset files, please indicate which directories and/or files.
""")
# Display filetype summary in short mode.
if (short):
extensions = []
for file_type, type_count in filetypes.iteritems():
if (file_type == ''):
file_type = 'no extension'
extensions.append(file_type)
print("File Extensions:")
extensions.sort()
print(','.join(extensions))
print
# Sort by count in each type, descending.
for file_type, type_count in sorted(filetypes.iteritems(), reverse=True, key=lambda (k,v): (v,k)):
if(file_type == ''):
file_type = '<no extension>'
formatted_line = '%10s %15s' % (str(type_count), file_type)
print formatted_line
print
print("Total Files - " + str(rownum-5-1))
print
if (short):
print('Directories Structure:')
else:
print('File Listing:')
print
for path, size in sizes.iteritems():
if (short):
# Display directories only in short mode.
if(re.search("\|EOL:$", path)):
pass
else:
opPath = re.sub('\|EOL:', '', path)
DirectoryTreeNode.buildTree(directoryTreeNodeRoot, opPath, size)
else:
opPath = re.sub('\|EOL:', '', path)
DirectoryTreeNode.buildTree(directoryTreeNodeRoot, opPath, size)
# print the tree starting with the node(s) that
# are children of the root. The root does not contain data.
rootChildren = directoryTreeNodeRoot.getChildren()
for child in rootChildren:
child.printTree(0)
else:
print("Error in header. Stopping")
if(re.search("\|EOL:$", path)):
pass
else:
opPath = re.sub('\|EOL:', '', path)
DirectoryTreeNode.buildTree(directoryTreeNodeRoot, opPath, size)
else:
opPath = re.sub('\|EOL:', '', path)
DirectoryTreeNode.buildTree(directoryTreeNodeRoot, opPath, size)
# print the tree starting with the node(s) that
# are children of the root. The root does not contain data.
rootChildren = directoryTreeNodeRoot.getChildren()
for child in rootChildren:
child.printTree(0)
else:
print("Error in header. Stopping")
directoryTreeNodeRoot = DirectoryTreeNode('root',0)
def main(argv, script_name):
parser = argparse.ArgumentParser()
# Stores args.d boolean, true if -d is set, false otherwise.
parser.add_argument('-d', action='store_true', help='Print only directories.')
parser.add_argument('hashfile')
args = parser.parse_args()
generate_tree(args.hashfile, args.d)
if __name__ == "__main__":
main(sys.argv[1:], sys.argv[0])
def __init__(self):
gobject.GObject.__init__(self)
self._root = DirectoryTreeNode()
self._root.set_expanded()
class DirectoryTree(gobject.GObject):
def __init__(self):
gobject.GObject.__init__(self)
self._root = DirectoryTreeNode()
self._root.set_expanded()
def set_cur_node(self, node):
self.cur_node = node
get_ftp().change_dir(dir_tuple_to_path(node.get_dir_tuple()))
self.emit("rf-directory-tree-current-node-changed",
tree_dir_path_to_string(node.get_path_tuple()))
def get_cur_node(self):
return self.cur_node
def set_root(self, root_fn):
if self._root.get_n_children():
self.emit("rf-directory-tree-remove-node", tree_dir_path_to_string(
self._root.get_nth_child(0).get_path_tuple()
))
root = DirectoryTreeNode(root_fn, self._root)
self._root.child_list = [root]
self.set_cur_node(root)
self.emit("rf-directory-tree-add-node", tree_dir_path_to_string(root.get_path_tuple()))
self.get_cur_node().set_expanded()
for line in get_ftp().list_current_dir().split("\n"):
if not line:
continue
fn = filenode_new_from_line(line)
self.append_child_fn(fn)
def get_root(self):
return self._root.get_nth_child(0)
def get_node(self, path):
ret = self._root
for ind in path:
ret = ret.get_nth_child(ind)
return ret
def expand_node(self, node):
dir_tuple = node.get_dir_tuple()
node.set_expanded()
import os
path = os.path.join(*dir_tuple)
for line in get_ftp().list_dir(path).split("\n"):
if not line:
continue
fn = filenode_new_from_line(line)
self.append_child_fn(fn, node=node)
def change_into_child(self, dir_name):
for ch in self.cur_node.get_child_list():
if ch.get_filenode().get_name() == dir_name and ch.get_filenode().get_type() == FILE_TYPE_DIR:
self.set_cur_node(ch)
return
raise exp.DirectoryTreeChangeIntoChildFailed(dir_name)
def change_into_parent(self):
if self.cur_node.get_parent().get_parent():
self.set_cur_node(self.cur_node.get_parent())
return
raise exp.DirectoryTreeChangeIntoParentFailed()
def is_under_current_dir(self, node):
if self.cur_node.index_child(node) == -1:
return False
return True
def is_on_current_dir_path(self, node):
path = node.get_dir_tuple()
cur_path = self.cur_node.get_dir_tuple()
if len(path) <= len(cur_path):
len_path = len(path)
for index in range(0, len_path):
if cur_path[index] != path[index]:
return False
return True
return False
@std_path_notify_function("rf-directory-tree-add-node")
def append_child(self, cur_node, ch):
return cur_node.append_child(ch)
@std_path_notify_function("rf-directory-tree-add-node")
def append_child_fn(self, cur_node, fn):
return cur_node.append_child_fn(fn)
@std_path_notify_function("rf-directory-tree-add-node")
def prepend_child(self, cur_node, ch):
return cur_node.prepend_child(ch)
@std_path_notify_function("rf-directory-tree-add-node")
def prepend_child_fn(self, cur_node, fn):
return cur_node.prepend_child_fn(fn)
@std_path_notify_function("rf-directory-tree-add-node")
def insert_child(self, cur_node, index, ch):
return cur_node.insert_child(index, ch)
@std_path_notify_function("rf-directory-tree-add-node")
def insert_child_fn(self, cur_node, index, fn):
return cur_node.insert_child(index, fn)
@std_path_notify_function("rf-directory-tree-remove-node")
def remove_child(self, cur_node, index):
return cur_node.remove_child(index)
@std_path_notify_function("rf-directory-tree-change-node")
def change_child(self, cur_node, index, new_ch):
return cur_node.change_child(index, new_ch)
@std_path_notify_function("rf-directory-tree-change-node")
def change_child_fn(self, cur_node, index, new_fn):
return cur_node.change_child_fn(index, new_fn)
