def test_load_data(dataset_finder_class, file_pattern, maxfiles):
file_finder = FileFinder(file_pattern, maxfiles)
print("Constructing dataset...")
start = time.time()
datasets = dataset_finder_class(file_finder.get_files())
print("... done in {:.3f}.".format(time.time() - start))
print("Loading views...")
start = time.time()
views = datasets.get_datasets()
print("... done in {:.3f}. Total views {}.".format(time.time() - start,
len(views)))
print("Processing images in batches...")
start = time.time()
batch_size = 128
n_batches = len(views) // batch_size
for batch_id in range(n_batches):
start_index = batch_id * batch_size
end_index = start_index + batch_size
print("batch_id {}/{}, start_index {}, end_index {}".format(
batch_id, n_batches, start_index, end_index))
images, labels = load_h5view_data(views[start_index:end_index])
print(len(images), images[0].dtype, type(images[0]))
print("... done in {:.3f}.".format(time.time() - start))
def setUp(self):
arg1 = "targeted_file_extensions"
arg2 = "list_of_all_drives"
arg3 = "ignore_dirs"
arg4 = "file_storage"
"""Initialise DriveFinder class
args:
arg1 (str) : mock value
arg2 (str) : mock value
arg3 (str) : mock value
arg4 (str) : mock value
"""
self.file_finder = FileFinder(arg1, arg2, arg3, arg4)
def run():
parser = get_parser()
if version_info[1] < 7:
args = parser.parse_args()
else:
args = parser.parse_intermixed_args()
try:
file_finder = FileFinder(**vars(args))
except ValueError as e:
print("Error: {}".format(e))
return
if file_finder.run():
print("Done")
else:
print("An error occurred. No files have been modfied")
def _doit(): try: self.finder = FileFinder(self.opt.base_path, path_filter=self.opt.path_filter, quit_indicator=QUITTING_TIME) self.finder.populate() curses.wrapper(self._run) finally: QUITTING_TIME.set()
def test_finder(dataset_finder_class, file_pattern, maxfiles):
file_finder = FileFinder(file_pattern, maxfiles)
start = time.time()
print("Constructing dataset...")
dataset_finder = dataset_finder_class(file_finder.get_files())
print("Done. Time to construct dataset {:.3f}".format(time.time() - start))
start = time.time()
print("Loading views...")
dataset_finder.load_dataset_views()
print("Done. Time to load views {:.3f}".format(time.time() - start))
start = time.time()
views = dataset_finder.get_datasets()
print("Time to get datasets {:.3f}, size {}".format(
time.time() - start, len(views)))
def _find(self, data_type, id):
file = FileFinder.resolve("{}/{}{}.json".format(
self._data_directory, data_type, id))
if not file or not path.exists(file):
return None
with open(file) as json_file:
return json.load(json_file)
def _run(self):
self.finder = FileFinder(self.opt.base_path, path_filter=self.opt.path_filter, quit_indicator=QUITTING_TIME)
logging.info("getting file list...")
self.finder.populate()
try:
while True:
self._loop()
except (KeyboardInterrupt, EOFError):
print
return 0
except Exception:
import traceback
traceback.print_exc()
finally:
QUITTING_TIME.set()
def __init__(self, config_file='app_config.cfg'):
self.parser = configparser.ConfigParser(
interpolation=EnvInterpolation())
self.parser.read(FileFinder.resolve(config_file))
def configure_logging(cls, logging_ini='logging.ini'):
"""
Configures logging
"""
located_at = FileFinder.resolve(logging_ini)
logging.config.fileConfig(located_at, disable_existing_loggers=False)
def file_finder():
config = new_config()
return FileFinder(**config)
def test_find_file(self):
file = FileFinder.resolve("app_config.cfg")
assert path.exists(file)
assert path.isfile(file)
class TestDriveFinder(unittest.TestCase):
# The setup before each test method
def setUp(self):
arg1 = "targeted_file_extensions"
arg2 = "list_of_all_drives"
arg3 = "ignore_dirs"
arg4 = "file_storage"
"""Initialise DriveFinder class
args:
arg1 (str) : mock value
arg2 (str) : mock value
arg3 (str) : mock value
arg4 (str) : mock value
"""
self.file_finder = FileFinder(arg1, arg2, arg3, arg4)
# This method tests DriveFinder constructor
def test_constructor(self):
# Initialise variable to expected values of DriveFinder
# class fields
expected_targeted_file_extensions = "targeted_file_extensions"
expected_list_of_all_drives = "list_of_all_drives"
expected_ignore_dirs = "ignore_dirs"
expected_file_storage = "file_storage"
# Get actual values of DriveFinder class fields
actual_targeted_file_extensions = self.file_finder.targeted_file_extensions
actual_list_of_all_drives = self.file_finder.list_of_all_drives
actual_ignore_dirs = self.file_finder.ignore_dirs
actual_file_storage = self.file_finder.file_storage
# Evaluate actual against expected
self.assertEqual(expected_targeted_file_extensions,
actual_targeted_file_extensions)
self.assertEqual(expected_list_of_all_drives,
actual_list_of_all_drives)
self.assertEqual(expected_ignore_dirs, actual_ignore_dirs)
self.assertEqual(expected_file_storage, actual_file_storage)
# This method test DriveFinder.break_source_into_substrings function
def test_break_source_into_substrings(self):
# Mock file path
source = "C:/Users/sgore/Desktop/look_up_with_entropy/test_files"
# Break down mock file path into substring.
# This mimics the internals of the break_source_into_substrings
# function.
expected_file_name = os.path.basename(source)
expected_file_path = os.path.splitext(source)[0]
expected_file_extension = os.path.splitext(source)[1]
expected_absolute_path = source
# Get actual return values of the function
actual = json.loads(
self.file_finder.break_source_into_substrings(source))
actual_file_name = actual['file_name']
actual_file_path = actual['file_path']
actual_file_extension = actual['file_extension']
actual_absolute_path = actual['absolute_path']
# Evaluate expected - manually computed values aginst
# actual values returned by the function
self.assertEqual(expected_file_name, actual_file_name)
self.assertEqual(expected_file_path, actual_file_path)
self.assertEqual(expected_file_extension, actual_file_extension)
self.assertEqual(expected_absolute_path, actual_absolute_path)
"--size-and-time",
help="assume files with same size and mtime are equal",
action='store_true')
parser.add_argument("--dry-run", help="don't copy files", action='store_true')
args = parser.parse_args()
root_dir = "/"
proc = subprocess.Popen(args.command,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=sys.stderr)
client = SyncClient(proc)
server_files = client.get_file_db()
file_finder = FileFinder()
with open(args.file_list, "r") as filters_file:
file_finder.add_from_text(root_dir, filters_file)
created_dirs = {}
local_dirs = {}
local_files = {}
total_uploaded_size = 0
def create_parent_dirs(path):
path_dir = os.path.dirname(path)
dirs_to_create = []
while not path_dir in created_dirs and path_dir:
dirs_to_create.append(path_dir)
path_dir = os.path.dirname(path_dir)
class CursesUI(object):
def __init__(self, options):
self.opt = options
self.status = ""
self.ui_lock = threading.Lock()
self.status_queue = Queue()
self._num_files = 0
self.query = None
def run(self):
rootLogger = logging.getLogger()
rootLogger.addHandler(QueueHandler(self.status_queue, level=logging.INFO))
logging.info("scanning ...")
def _doit():
try:
self.finder = FileFinder(self.opt.base_path, path_filter=self.opt.path_filter, quit_indicator=QUITTING_TIME)
self.finder.populate()
curses.wrapper(self._run)
finally:
QUITTING_TIME.set()
work_thread = threading.Thread(target=log_exceptions(_doit), name="[curses] master")
work_thread.start()
# the main thread is just going to wait till someone tells it to quit
try:
QUITTING_TIME.wait()
except KeyboardInterrupt:
# somehow the main thread fails to exit when it is the one
# to receive KeyboardInterrupt !
QUITTING_TIME.set()
def refresh_results(self):
self.set_query(self.query)
@log_exceptions
def results_loop(self):
while True:
search = self.finder.results()
self.ui_lock.acquire()
self.set_results(search)
self.update()
self.ui_lock.release()
@log_exceptions
def status_loop(self):
if(QUITTING_TIME.is_set()): return
def _stat(msg):
self.ui_lock.acquire()
self.status = msg
self.update()
self.ui_lock.release()
while True:
try:
status_msg = self.status_queue.get(timeout=1.2)
_stat(status_msg)
sleep(1)
except Empty: pass
if self.status_queue.empty():
num_files = self.update_files_indexed()
_stat("%s files indexed" % (num_files,))
def update_files_indexed(self):
new_num_files = self.finder.file_count
if self._num_files != new_num_files:
self.requery_if_doing_nothing()
self._num_files = new_num_files
return new_num_files
def requery_if_doing_nothing(self):
if not self.finder.has_pending_queries:
logging.debug("resubmitting query: %s" % (self.query,))
self.set_query(self.query, is_repeat = True)
@log_exceptions
def _run(self, mainscr):
self.mainscr = mainscr
self._init_colors()
self._init_screens()
self._init_input()
self.update()
display_thread = threading.Thread(target=self.results_loop, name="[curses] results handler")
status_thread = threading.Thread(target=self.status_loop, name="[curses] status updater")
display_thread.daemon = True
status_thread.daemon = True
display_thread.start()
status_thread.start()
self._input_loop()
import time
#time.sleep(0.1) # random sleep, otherwise curses sometimes calls knickers.twist()
def _init_colors(self):
global A_INPUT, A_FILENAME, A_PATH, A_HIGHLIGHT, A_ERR, A_PROMPT, A_STATUS
curses.use_default_colors()
curses.curs_set(1) # line (input) cursor
A_INPUT = curses.A_REVERSE
n_filename = 1
n_path = 2
n_hi = 3
n_err = 4
n_prompt = 5
n_status = 6
bg_index = -1
curses.init_pair(n_filename, curses.COLOR_WHITE, bg_index)
curses.init_pair(n_path, curses.COLOR_BLUE, bg_index)
curses.init_pair(n_hi, curses.COLOR_GREEN, bg_index)
curses.init_pair(n_err, curses.COLOR_WHITE, curses.COLOR_RED)
curses.init_pair(n_prompt, curses.COLOR_BLUE, bg_index)
curses.init_pair(n_status, curses.COLOR_BLACK, bg_index)
A_FILENAME = curses.color_pair(n_filename)
A_PATH = curses.color_pair(n_path)
A_HIGHLIGHT = curses.color_pair(n_hi) | curses.A_BOLD
A_ERR = curses.color_pair(n_err) | curses.A_BOLD
A_PROMPT = curses.color_pair(n_prompt)
A_STATUS = curses.color_pair(n_status)
def _init_screens(self):
self.win_height, self.win_width = self.mainscr.getmaxyx()
self.input_win = curses.newwin(1, self.win_width, 0, 0)
self.results_win = curses.newpad(MAX_RESULTS, self.win_width)
self.status_win = curses.newwin(1, self.win_width, self.win_height-1, 0)
#IMPORTANT: input_win *must* be the last, so that it gets redrawed
# last (and therefore gets the cursor)
self.screens = (self.results_win, self.status_win, self.input_win)
def resize(self):
self._init_screens()
def _init_input(self):
curses.raw()
self.results = []
self.input_position = 0
self.selected = 0
self.results_scroll = 0
self.set_query("")
def update(self):
if (self.win_height, self.win_width) != self.mainscr.getmaxyx():
logging.debug("resizing...")
self.resize()
self.draw_input()
self.draw_results()
self.draw_status()
self._redraw()
def draw_input(self):
self.input_win.clear()
find_text = "Find: "
self.input_win.addnstr(0,0, find_text, self.win_width, A_PROMPT)
self.input_win.addnstr(0, len(find_text), self.query, self.win_width, A_INPUT)
self.input_win.bkgdset(' ', curses.A_REVERSE)
self.input_win.move(0,self.input_position + len(find_text))
def draw_results(self):
#TODO: scroll results buffer
linepos = 0
indent_width = 6
filename_len = min(int(self.win_width / 1.5), 50)
path_len = self.win_width - filename_len - 1 - indent_width
self.results_win.clear()
for file, path in self.results:
attr_mod = curses.A_REVERSE if linepos == self.selected else curses.A_NORMAL
drawn_chars = 0
remaining_chars = filename_len
for highlighted, segment in self.highlight(file):
attrs = A_FILENAME | A_HIGHLIGHT if highlighted else A_FILENAME
self.results_win.insnstr(linepos, indent_width + drawn_chars, segment, remaining_chars, attrs | attr_mod)
drawn_chars += len(segment)
remaining_chars -= len(segment)
if remaining_chars <= 0:
break
# now draw the path
relpath = os.path.split(path)[0]
explanation = ''
if relpath:
explanation = "(in %s)" % (relpath,)
self.results_win.insnstr(linepos, indent_width + filename_len + 1, explanation, path_len, A_PATH)
linepos += 1
if linepos >= MAX_RESULTS:
break
if linepos == 0 and len(self.query) >= MIN_QUERY and not self.finder.has_pending_queries:
self.results_win.insnstr(linepos, indent_width, 'No Matches...', self.win_width - indent_width, A_ERR)
def draw_status(self):
self.status_win.clear()
self.status_win.insnstr(0, 0, self.status, self.win_width, A_STATUS)
def with_selected(self, func):
index = self.selected
if len(self.results) <= index:
logging.warning("no such index: %s" % (index,))
self.clear_status()
return
filepath = self.results[index][-1]
func(filepath)
def open_selected(self):
def action(path):
self.opt.open(path)
self.flash(" ** opened **")
self.with_selected(action)
def select(self, amount):
self.ui_lock.acquire()
if amount == NEXT or amount == PREVIOUS:
self.selected += amount
elif amount == START:
self.selected = 0
elif amount == END:
self.selected = len(self.results)-1
self.selected = min(self.selected, len(self.results)-1)
self.selected = max(self.selected, 0)
self.ui_lock.release()
def set_query(self, new_query, is_repeat = False):
if len(new_query) >= MIN_QUERY:
self.finder.find(Search(new_query, is_repeat=is_repeat))
else:
self.finder.find(None)
self.ui_lock.acquire()
self.query = new_query
if self.input_position > len(self.query):
self.input_position = len(self.query)
self.ui_lock.release()
def set_results(self, search):
self.highlight = Highlight(search.text)
self.results = list(search.results)
if search.is_repeat:
self.selected = min(self.selected, len(self.results)-1)
else:
self.selected = 0
def add_char(self, ch):
new_query = self.modify_query_as_list(lambda q: q.insert(self.input_position, ch))
self.input_position += 1
self.set_query(new_query)
logging.debug("query = %s" % (self.query, ))
def modify_query_as_list(self, proc):
query_list = list(self.query)
proc(query_list)
return ''.join(query_list)
def remove_char(self, forwards=False):
letter_index = self.input_position if forwards else self.input_position - 1
if letter_index >= len(self.query) or letter_index < 0:
return
new_query = self.modify_query_as_list(lambda q: q.pop(letter_index))
if forwards:
self.input_position = max(self.input_position, len(self.query))
elif self.input_position > 0:
self.input_position -= 1
self.set_query(new_query)
def move_cursor(self, backwards=False):
offset = -1 if backwards else 1
self.input_position += offset
self.input_position = max(0, min(self.input_position, len(self.query)))
def move_cursor_to(self, index):
self.input_position = index
def flash(self, str):
self.status_queue.put(str)
def clear_status(self):
self.status_queue.put("")
def _redraw(self, *screens):
if QUITTING_TIME.is_set(): return
logging.debug("redrawing...")
if not screens:
screens = self.screens
for scr in screens:
if scr is self.results_win:
scr.noutrefresh(
self.results_scroll, 0, 1, 0,
self.win_height-2, self.win_width)
else:
scr.noutrefresh()
curses.doupdate()
def copy_selected_path_to_clipboard(self):
def action(filepath):
try:
import pyperclip
pyperclip.copy(self.opt.full_path(filepath))
self.flash(" ** copied **")
except StandardError, e:
logging.warn("error: %s" % (e,))
logging.exception("error copying to clipboard")
self.with_selected(action)
""" This file is meant to be a prototype for defining ROI's to be used in later applications """ import matplotlib.pyplot as plt from matplotlib.widgets import Slider, RadioButtons, Button from matplotlib.colorbar import ColorbarBase from matplotlib.colors import Normalize import numpy as np from roi_definer import ROI from file_finder import FileFinder start = FileFinder() start.get_name() plt.figure(1) axpic = plt.subplot2grid((20, 20), (0, 0), rowspan=14, colspan=14) axps = plt.subplot2grid((20, 20), (19, 10), rowspan=1, colspan=10) axrb = plt.subplot2grid((20, 20), (15, 10), rowspan=1, colspan=10) axre = plt.subplot2grid((20, 20), (16, 10), rowspan=1, colspan=10) axcb = plt.subplot2grid((20, 20), (17, 10), rowspan=1, colspan=10) axce = plt.subplot2grid((20, 20), (18, 10), rowspan=1, colspan=10) axgray = plt.subplot2grid((20, 20), (0, 15), rowspan=6, colspan=5) axskipf = plt.subplot2grid((20, 20), (19, 5), rowspan=1, colspan=2) axskipb = plt.subplot2grid((20, 20), (19, 3), rowspan=1, colspan=2) axvmin = plt.subplot2grid((20, 20), (15, 0), rowspan=2, colspan=5) axvmax = plt.subplot2grid((20, 20), (17, 0), rowspan=2, colspan=5) axbar = plt.subplot2grid((20, 20), (7, 14), rowspan=3, colspan=8) axzoom = plt.subplot2grid((20, 20), (11, 14), rowspan=3, colspan=8)
class Repl(object):
def __init__(self, options):
self.found_files = []
self.opt = options
def highlight_func(self, query_string):
if termstyle is None:
return lambda x: x
highlight = Highlight(query_string)
return lambda x: highlight.replace(x, green)
def summarise(self, result_iter, query_string):
subprocess.call(['clear'])
self.found_files = []
i = 0
highlight = self.highlight_func(query_string)
for filename, fullpath in result_iter:
self.found_files.append(fullpath)
relpath = os.path.split(fullpath)[0]
explanation = ''
if relpath:
explanation = "(in %s)" % (relpath,)
index = str(i+1).rjust(2)
filename = filename.ljust(30)
print " %s%s %s %s" % (yellow(index), yellow(":"), highlight(filename), black(explanation))
i += 1
def open(self, index):
index -= 1 # indexes start at 1 for readability
if len(self.found_files) <= index:
logging.warning("no such index: %s" % (index,))
return
filepath = self.found_files[index]
self.opt.open(filepath)
def _loop(self):
q = raw_input(blue("\nfind/open file: "))
if len(q) == 0:
q = 1 # open the first found file by default
try:
index = int(q)
except ValueError:
index = None
if index is not None:
self.open(index)
else:
self.finder.find(Search(q))
search = self.finder.results()
self.summarise(search.results, search.text)
def run(self):
work_thread = threading.Thread(target=self._run, name="repl")
work_thread.daemon = True
work_thread.start()
# the main thread is just going to wait till someone tells it to quit
try:
QUITTING_TIME.wait()
except KeyboardInterrupt:
# somehow the main thread fails to exit when it is the one
# to receive KeyboardInterrupt !
pass
def _run(self):
self.finder = FileFinder(self.opt.base_path, path_filter=self.opt.path_filter, quit_indicator=QUITTING_TIME)
logging.info("getting file list...")
self.finder.populate()
try:
while True:
self._loop()
except (KeyboardInterrupt, EOFError):
print
return 0
except Exception:
import traceback
traceback.print_exc()
finally:
QUITTING_TIME.set()
def run(srcDirectory, destDirectory, removeAfterCopy):
fileFinder = FileFinder(destDirectory, removeAfterCopy)
fileFinder.copyFiles(srcDirectory)
def unmarshal_gadget_file(gadget_file):
from file_finder import FileFinder
handle = FileFinder(gadget_file, "i386")
gadget_list = handle.find_gadgets()
return gadget_list