def _expand_dir(self, index):
expanding = self.view.itemFromIndex(index)
if expanding in self.expanded:
return
dir_ = self._get_path(index)
output = subprocess.check_output(shlex.split(
"ssh -o PasswordAuthentication=no {user}{machine} ls -l {dir_}".\
format(user=(self.args.user + "@" if self.args.user else ""),
machine=self.args.control_ip,
dir_=dir_)))
for line in output.split("\n")[1:]: # first line is total line
match = self.ls_re.match(line)
if match:
item = Tree_Widget_Item(expanding)
groups = match.groupdict()
name = groups["name"]
item.setText(self.header_labels.index("File"), name)
if groups["type"] == "d":
item.setChildIndicatorPolicy(
Tree_Widget_Item.ShowIndicator)
else:
size = int(groups["size"])
item.setText(self.header_labels.index("Size"),
format_bytes(size, self.bytes_print_size))
self.file_sizes[os.path.join(dir_, name)] = size
else:
print "warning, failed to match:", line
self.expanded.add(expanding)
# Qt wouldn't show the scroll bar when the above actions would expand
# the expand the size of the view beyond the viewport,
# resizing seems to force this to happen
self.resize(self.size())
def _display_bank_info(self, root):
data = self.background_data[root]
root.setText(self.header_labels.index("#scan"), str(len(data.scans)))
root.setText(self.header_labels.index("Size"),
format_bytes(data.size, self.bytes_print_size))
for index, (recording, size) in enumerate(data.scans):
item = Tree_Widget_Item(root)
item.setText(self.header_labels.index("#scan"), str(index + 1))
item.setText(self.header_labels.index("Recording"), recording)
item.setText(self.header_labels.index("Size"),
format_bytes(size, self.bytes_print_size))
class File_View(Abstract_Machine_View):
header_labels = ["File", "Size"]
check_text = "File check"
def _get_selection(self):
indices = self.view.selectedIndexes()
if any([self.view.itemFromIndex(index).childIndicatorPolicy() == \
Tree_Widget_Item.ShowIndicator for index in indices]):
raise Invalid_Selection_Exception("Can only operate on files, "
"not directories.")
return [self._get_path(index) for index in indices \
if index.column() == 0]
def _check_selection(self, selection):
text = ""
with contextlib.closing(self._create_socket()) as s:
for filepath in selection:
text += send_query(s, "file_check?::{f}".format(f=filepath))
return text
def _emit_copy(self, selection):
self.copy_from.emit(self.args.control_ip,
[("file://{host}:{port}/".format(
host=self.args.control_ip,
port=self.args.port), \
filename)
for filename in selection])
def _get_m5copy_to(self):
indices = self.view.selectedIndexes()
items = list(set(self.view.itemFromIndex(index) for index in indices))
if (len(items) != 1) or \
(items[0].childIndicatorPolicy() != \
Tree_Widget_Item.ShowIndicator):
raise Invalid_Selection_Exception("Only one directory has to be "
"selected as the target directory to copy to.")
return (self.args.control_ip,
"file://{host}:{port}:{data_ip}/{dirname}/".format(
host=self.args.control_ip,
port=self.args.port,
data_ip=self.args.data_ip,
dirname=self._get_path(indices[0])))
def _selection_size(self, selection):
return sum([self.file_sizes[filepath] for filepath in selection])
def _create_view_widget(self):
self.view = QtGui.QTreeWidget(self)
self.root_item = Tree_Widget_Item(self.view)
self.root_item.setText(self.header_labels.index("File"),
self.args.root_path)
self.root_item.setChildIndicatorPolicy(
Tree_Widget_Item.ShowIndicator)
return self.view
def _create_socket(self, timeout=10):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(timeout)
s.connect((self.args.control_ip, self.args.port))
return s
def _get_path(self, index):
paths = [str(self.view.itemFromIndex(index).text(
self.header_labels.index("File")))]
parent = index.parent()
while parent.isValid():
paths.insert(0, str(self.view.itemFromIndex(parent).text(
self.header_labels.index("File"))))
parent = parent.parent()
return os.path.join(*paths)
#-rw-rw-r-- 1 jops jive 700 Sep 9 14:07 time_conversion.py
# for now only handle directories and regular files
ls_re = re.compile(
"\s*(?P<type>[d-])"
"(?P<permissions>([r-][w-][xTtsS-]){3})\s+"
"(?P<links>\d+)\s+"
"(?P<owner>\S+)\s+"
"(?P<group>\S+)\s+"
"(?P<size>\d+)\s+"
"(?P<date_time>\S+\s+\S+\s+\S+)\s+"
"(?P<name>.+)")
def _expand_dir(self, index):
expanding = self.view.itemFromIndex(index)
if expanding in self.expanded:
return
dir_ = self._get_path(index)
output = subprocess.check_output(shlex.split(
"ssh -o PasswordAuthentication=no {user}{machine} ls -l {dir_}".\
format(user=(self.args.user + "@" if self.args.user else ""),
machine=self.args.control_ip,
dir_=dir_)))
for line in output.split("\n")[1:]: # first line is total line
match = self.ls_re.match(line)
if match:
item = Tree_Widget_Item(expanding)
groups = match.groupdict()
name = groups["name"]
item.setText(self.header_labels.index("File"), name)
if groups["type"] == "d":
item.setChildIndicatorPolicy(
Tree_Widget_Item.ShowIndicator)
else:
size = int(groups["size"])
item.setText(self.header_labels.index("Size"),
format_bytes(size, self.bytes_print_size))
self.file_sizes[os.path.join(dir_, name)] = size
else:
print "warning, failed to match:", line
self.expanded.add(expanding)
# Qt wouldn't show the scroll bar when the above actions would expand
# the expand the size of the view beyond the viewport,
# resizing seems to force this to happen
self.resize(self.size())
def __init__(self, args, parent=None):
"""
@args: Bunch(user, machine, port, root_path)
"""
self.args = args
if self.args.root_path[-1] != '/':
self.args.root_path = self.args.root_path + '/'
super(File_View, self).__init__(parent)
# get directory contents on expand, but only once
self.view.expanded.connect(self._expand_dir)
self.expanded = set()
self.file_sizes = {} # {absolute path: number of bytes}
def _expand_scan(self, index):
level = 0
parent = index.parent()
while parent.isValid():
level += 1
parent = parent.parent()
if level != 2:
# no need to do work on experiment or station level indices
return
scan_item = self.view.itemFromIndex(index)
if scan_item in self.expanded:
# already scanned for the chunks
return
scan = str(scan_item.text(self.header_labels.index("Scan")))
station_item = scan_item.parent()
station = str(station_item.text(self.header_labels.index("Station")))
experiment = str(station_item.parent().text(
self.header_labels.index("Experiment")))
if len(self.flexbuffs) > 1:
machines = str(scan_item.text(
self.header_labels.index("FlexBuff"))).split(", ")
else:
machines = self.flexbuffs.keys()
for flexbuff in [self.flexbuffs[machine] for machine in machines]:
chunks = {}
errors = {}
is_mark6_data_format = self.mark6_format.isChecked()
check_flexbuff(flexbuff, chunks, None, errors,
self.data[experiment][station][scan].recording,
is_mark6_data_format)
display = set()
if is_mark6_data_format:
for chunk, size in chunks.items():
disk = "/".join(chunk.split("/")[3:5])
display.add((disk, size))
for (disk, size) in sorted(display):
self.chunks[experiment][station][scan][disk] = \
Hashable_Bunch(recording=chunk, size=size)
item = Tree_Widget_Item(scan_item)
item.setText(self.header_labels.index("Chunk"), disk)
item.setText(self.header_labels.index("Size"),
format_bytes(size, self.bytes_print_size))
if len(self.flexbuffs) > 1:
item.setText(self.header_labels.index("FlexBuff"),
flexbuff.machine)
else:
for chunk, size in chunks.items():
disk = chunk.split("/")[2]
index = chunk.split(".")[-1]
display.add((index, disk, size))
self.chunks[experiment][station][scan][index] = \
Hashable_Bunch(recording=chunk, size=size)
for (index, disk, size) in sorted(display):
item = Tree_Widget_Item(scan_item)
item.setText(self.header_labels.index("Chunk"),
index + " on " + disk)
item.setText(self.header_labels.index("Size"),
format_bytes(size, self.bytes_print_size))
if len(self.flexbuffs) > 1:
item.setText(self.header_labels.index("FlexBuff"),
flexbuff.machine)
self.expanded.add(scan_item)
def _display_data(self):
data = self.background_data[self.view]
# clear the disk usage layout
while True:
item = self.disk_usage_layout.takeAt(0)
if not item:
break
widget = item.widget()
if widget:
widget.deleteLater()
for index, text in enumerate(["Total", "Used", "Free"]):
if index > 0:
self.disk_usage_layout.addStretch(1)
try:
size = format_bytes(sum([l[index] \
for l in data.available.values()]),
self.bytes_print_size)
except IndexError:
size = "?"
self.disk_usage_layout.addWidget(
QtGui.QLabel("{text}: {n}".format(text=text, n=size)))
policy = Tree_Widget_Item.ShowIndicator \
if self.show_file_chunks.isChecked() \
else Tree_Widget_Item.DontShowIndicator
multi_flexbuff = (len(self.flexbuffs) > 1)
experiments = sorted(data.presence.keys())
for experiment in experiments:
experiment_item = Tree_Widget_Item(self.view)
experiment_item.setText(self.header_labels.index("Experiment"),
experiment)
experiment_size = sum(
[scan_size for experiment_data in data.usage.values() \
for station_data in experiment_data[experiment].values() \
for scan_size in station_data.values()])
experiment_item.setText(
self.header_labels.index("Size"),
format_bytes(experiment_size, self.bytes_print_size))
if multi_flexbuff:
station_flexbuffs = [set.union(*scan_data.values()) \
for scan_data in data.presence[experiment].values()]
experiment_flexbuffs = set.union(*station_flexbuffs)
experiment_item.setText(self.header_labels.index("FlexBuff"),
", ".join(experiment_flexbuffs))
stations = sorted(data.presence[experiment].keys())
for station in stations:
station_item = Tree_Widget_Item(experiment_item)
station_item.setText(self.header_labels.index("Station"),
station)
station_size = sum(
[scan_size for experiment_data in data.usage.values() \
for scan_size in \
experiment_data[experiment][station].values()])
station_item.setText(
self.header_labels.index("Size"),
format_bytes(station_size, self.bytes_print_size))
if multi_flexbuff:
station_flexbuffs = set.union(
*data.presence[experiment][station].values())
station_item.setText(self.header_labels.index("FlexBuff"),
", ".join(station_flexbuffs))
scans = sorted(data.presence[experiment][station].keys())
for (scan, recording) in scans:
scan_item = Tree_Widget_Item(station_item)
scan_item.setText(self.header_labels.index("Scan"), scan)
scan_size = sum(
[experiment_data[experiment][station] \
[(scan, recording)] \
for experiment_data in data.usage.values()])
scan_item.setText(
self.header_labels.index("Size"),
format_bytes(scan_size, self.bytes_print_size))
scan_item.setChildIndicatorPolicy(policy)
if multi_flexbuff:
scan_item.setText(
self.header_labels.index("FlexBuff"),
", ".join(data.presence[experiment]\
[station][(scan, recording)]))
self.view.collapseAll()
self.mark6_format.setEnabled(True)