def test_load_data(dataset_finder_class, file_pattern, maxfiles): file_finder = FileFinder(file_pattern, maxfiles) print("Constructing dataset...") start = time.time() datasets = dataset_finder_class(file_finder.get_files()) print("... done in {:.3f}.".format(time.time() - start)) print("Loading views...") start = time.time() views = datasets.get_datasets() print("... done in {:.3f}. Total views {}.".format(time.time() - start, len(views))) print("Processing images in batches...") start = time.time() batch_size = 128 n_batches = len(views) // batch_size for batch_id in range(n_batches): start_index = batch_id * batch_size end_index = start_index + batch_size print("batch_id {}/{}, start_index {}, end_index {}".format( batch_id, n_batches, start_index, end_index)) images, labels = load_h5view_data(views[start_index:end_index]) print(len(images), images[0].dtype, type(images[0])) print("... done in {:.3f}.".format(time.time() - start))
def setUp(self): arg1 = "targeted_file_extensions" arg2 = "list_of_all_drives" arg3 = "ignore_dirs" arg4 = "file_storage" """Initialise DriveFinder class args: arg1 (str) : mock value arg2 (str) : mock value arg3 (str) : mock value arg4 (str) : mock value """ self.file_finder = FileFinder(arg1, arg2, arg3, arg4)
def run(): parser = get_parser() if version_info[1] < 7: args = parser.parse_args() else: args = parser.parse_intermixed_args() try: file_finder = FileFinder(**vars(args)) except ValueError as e: print("Error: {}".format(e)) return if file_finder.run(): print("Done") else: print("An error occurred. No files have been modfied")
def _doit(): try: self.finder = FileFinder(self.opt.base_path, path_filter=self.opt.path_filter, quit_indicator=QUITTING_TIME) self.finder.populate() curses.wrapper(self._run) finally: QUITTING_TIME.set()
def test_finder(dataset_finder_class, file_pattern, maxfiles): file_finder = FileFinder(file_pattern, maxfiles) start = time.time() print("Constructing dataset...") dataset_finder = dataset_finder_class(file_finder.get_files()) print("Done. Time to construct dataset {:.3f}".format(time.time() - start)) start = time.time() print("Loading views...") dataset_finder.load_dataset_views() print("Done. Time to load views {:.3f}".format(time.time() - start)) start = time.time() views = dataset_finder.get_datasets() print("Time to get datasets {:.3f}, size {}".format( time.time() - start, len(views)))
def _find(self, data_type, id): file = FileFinder.resolve("{}/{}{}.json".format( self._data_directory, data_type, id)) if not file or not path.exists(file): return None with open(file) as json_file: return json.load(json_file)
def _run(self): self.finder = FileFinder(self.opt.base_path, path_filter=self.opt.path_filter, quit_indicator=QUITTING_TIME) logging.info("getting file list...") self.finder.populate() try: while True: self._loop() except (KeyboardInterrupt, EOFError): print return 0 except Exception: import traceback traceback.print_exc() finally: QUITTING_TIME.set()
def __init__(self, config_file='app_config.cfg'): self.parser = configparser.ConfigParser( interpolation=EnvInterpolation()) self.parser.read(FileFinder.resolve(config_file))
def configure_logging(cls, logging_ini='logging.ini'): """ Configures logging """ located_at = FileFinder.resolve(logging_ini) logging.config.fileConfig(located_at, disable_existing_loggers=False)
def file_finder(): config = new_config() return FileFinder(**config)
def test_find_file(self): file = FileFinder.resolve("app_config.cfg") assert path.exists(file) assert path.isfile(file)
class TestDriveFinder(unittest.TestCase): # The setup before each test method def setUp(self): arg1 = "targeted_file_extensions" arg2 = "list_of_all_drives" arg3 = "ignore_dirs" arg4 = "file_storage" """Initialise DriveFinder class args: arg1 (str) : mock value arg2 (str) : mock value arg3 (str) : mock value arg4 (str) : mock value """ self.file_finder = FileFinder(arg1, arg2, arg3, arg4) # This method tests DriveFinder constructor def test_constructor(self): # Initialise variable to expected values of DriveFinder # class fields expected_targeted_file_extensions = "targeted_file_extensions" expected_list_of_all_drives = "list_of_all_drives" expected_ignore_dirs = "ignore_dirs" expected_file_storage = "file_storage" # Get actual values of DriveFinder class fields actual_targeted_file_extensions = self.file_finder.targeted_file_extensions actual_list_of_all_drives = self.file_finder.list_of_all_drives actual_ignore_dirs = self.file_finder.ignore_dirs actual_file_storage = self.file_finder.file_storage # Evaluate actual against expected self.assertEqual(expected_targeted_file_extensions, actual_targeted_file_extensions) self.assertEqual(expected_list_of_all_drives, actual_list_of_all_drives) self.assertEqual(expected_ignore_dirs, actual_ignore_dirs) self.assertEqual(expected_file_storage, actual_file_storage) # This method test DriveFinder.break_source_into_substrings function def test_break_source_into_substrings(self): # Mock file path source = "C:/Users/sgore/Desktop/look_up_with_entropy/test_files" # Break down mock file path into substring. # This mimics the internals of the break_source_into_substrings # function. expected_file_name = os.path.basename(source) expected_file_path = os.path.splitext(source)[0] expected_file_extension = os.path.splitext(source)[1] expected_absolute_path = source # Get actual return values of the function actual = json.loads( self.file_finder.break_source_into_substrings(source)) actual_file_name = actual['file_name'] actual_file_path = actual['file_path'] actual_file_extension = actual['file_extension'] actual_absolute_path = actual['absolute_path'] # Evaluate expected - manually computed values aginst # actual values returned by the function self.assertEqual(expected_file_name, actual_file_name) self.assertEqual(expected_file_path, actual_file_path) self.assertEqual(expected_file_extension, actual_file_extension) self.assertEqual(expected_absolute_path, actual_absolute_path)
"--size-and-time", help="assume files with same size and mtime are equal", action='store_true') parser.add_argument("--dry-run", help="don't copy files", action='store_true') args = parser.parse_args() root_dir = "/" proc = subprocess.Popen(args.command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=sys.stderr) client = SyncClient(proc) server_files = client.get_file_db() file_finder = FileFinder() with open(args.file_list, "r") as filters_file: file_finder.add_from_text(root_dir, filters_file) created_dirs = {} local_dirs = {} local_files = {} total_uploaded_size = 0 def create_parent_dirs(path): path_dir = os.path.dirname(path) dirs_to_create = [] while not path_dir in created_dirs and path_dir: dirs_to_create.append(path_dir) path_dir = os.path.dirname(path_dir)
class CursesUI(object): def __init__(self, options): self.opt = options self.status = "" self.ui_lock = threading.Lock() self.status_queue = Queue() self._num_files = 0 self.query = None def run(self): rootLogger = logging.getLogger() rootLogger.addHandler(QueueHandler(self.status_queue, level=logging.INFO)) logging.info("scanning ...") def _doit(): try: self.finder = FileFinder(self.opt.base_path, path_filter=self.opt.path_filter, quit_indicator=QUITTING_TIME) self.finder.populate() curses.wrapper(self._run) finally: QUITTING_TIME.set() work_thread = threading.Thread(target=log_exceptions(_doit), name="[curses] master") work_thread.start() # the main thread is just going to wait till someone tells it to quit try: QUITTING_TIME.wait() except KeyboardInterrupt: # somehow the main thread fails to exit when it is the one # to receive KeyboardInterrupt ! QUITTING_TIME.set() def refresh_results(self): self.set_query(self.query) @log_exceptions def results_loop(self): while True: search = self.finder.results() self.ui_lock.acquire() self.set_results(search) self.update() self.ui_lock.release() @log_exceptions def status_loop(self): if(QUITTING_TIME.is_set()): return def _stat(msg): self.ui_lock.acquire() self.status = msg self.update() self.ui_lock.release() while True: try: status_msg = self.status_queue.get(timeout=1.2) _stat(status_msg) sleep(1) except Empty: pass if self.status_queue.empty(): num_files = self.update_files_indexed() _stat("%s files indexed" % (num_files,)) def update_files_indexed(self): new_num_files = self.finder.file_count if self._num_files != new_num_files: self.requery_if_doing_nothing() self._num_files = new_num_files return new_num_files def requery_if_doing_nothing(self): if not self.finder.has_pending_queries: logging.debug("resubmitting query: %s" % (self.query,)) self.set_query(self.query, is_repeat = True) @log_exceptions def _run(self, mainscr): self.mainscr = mainscr self._init_colors() self._init_screens() self._init_input() self.update() display_thread = threading.Thread(target=self.results_loop, name="[curses] results handler") status_thread = threading.Thread(target=self.status_loop, name="[curses] status updater") display_thread.daemon = True status_thread.daemon = True display_thread.start() status_thread.start() self._input_loop() import time #time.sleep(0.1) # random sleep, otherwise curses sometimes calls knickers.twist() def _init_colors(self): global A_INPUT, A_FILENAME, A_PATH, A_HIGHLIGHT, A_ERR, A_PROMPT, A_STATUS curses.use_default_colors() curses.curs_set(1) # line (input) cursor A_INPUT = curses.A_REVERSE n_filename = 1 n_path = 2 n_hi = 3 n_err = 4 n_prompt = 5 n_status = 6 bg_index = -1 curses.init_pair(n_filename, curses.COLOR_WHITE, bg_index) curses.init_pair(n_path, curses.COLOR_BLUE, bg_index) curses.init_pair(n_hi, curses.COLOR_GREEN, bg_index) curses.init_pair(n_err, curses.COLOR_WHITE, curses.COLOR_RED) curses.init_pair(n_prompt, curses.COLOR_BLUE, bg_index) curses.init_pair(n_status, curses.COLOR_BLACK, bg_index) A_FILENAME = curses.color_pair(n_filename) A_PATH = curses.color_pair(n_path) A_HIGHLIGHT = curses.color_pair(n_hi) | curses.A_BOLD A_ERR = curses.color_pair(n_err) | curses.A_BOLD A_PROMPT = curses.color_pair(n_prompt) A_STATUS = curses.color_pair(n_status) def _init_screens(self): self.win_height, self.win_width = self.mainscr.getmaxyx() self.input_win = curses.newwin(1, self.win_width, 0, 0) self.results_win = curses.newpad(MAX_RESULTS, self.win_width) self.status_win = curses.newwin(1, self.win_width, self.win_height-1, 0) #IMPORTANT: input_win *must* be the last, so that it gets redrawed # last (and therefore gets the cursor) self.screens = (self.results_win, self.status_win, self.input_win) def resize(self): self._init_screens() def _init_input(self): curses.raw() self.results = [] self.input_position = 0 self.selected = 0 self.results_scroll = 0 self.set_query("") def update(self): if (self.win_height, self.win_width) != self.mainscr.getmaxyx(): logging.debug("resizing...") self.resize() self.draw_input() self.draw_results() self.draw_status() self._redraw() def draw_input(self): self.input_win.clear() find_text = "Find: " self.input_win.addnstr(0,0, find_text, self.win_width, A_PROMPT) self.input_win.addnstr(0, len(find_text), self.query, self.win_width, A_INPUT) self.input_win.bkgdset(' ', curses.A_REVERSE) self.input_win.move(0,self.input_position + len(find_text)) def draw_results(self): #TODO: scroll results buffer linepos = 0 indent_width = 6 filename_len = min(int(self.win_width / 1.5), 50) path_len = self.win_width - filename_len - 1 - indent_width self.results_win.clear() for file, path in self.results: attr_mod = curses.A_REVERSE if linepos == self.selected else curses.A_NORMAL drawn_chars = 0 remaining_chars = filename_len for highlighted, segment in self.highlight(file): attrs = A_FILENAME | A_HIGHLIGHT if highlighted else A_FILENAME self.results_win.insnstr(linepos, indent_width + drawn_chars, segment, remaining_chars, attrs | attr_mod) drawn_chars += len(segment) remaining_chars -= len(segment) if remaining_chars <= 0: break # now draw the path relpath = os.path.split(path)[0] explanation = '' if relpath: explanation = "(in %s)" % (relpath,) self.results_win.insnstr(linepos, indent_width + filename_len + 1, explanation, path_len, A_PATH) linepos += 1 if linepos >= MAX_RESULTS: break if linepos == 0 and len(self.query) >= MIN_QUERY and not self.finder.has_pending_queries: self.results_win.insnstr(linepos, indent_width, 'No Matches...', self.win_width - indent_width, A_ERR) def draw_status(self): self.status_win.clear() self.status_win.insnstr(0, 0, self.status, self.win_width, A_STATUS) def with_selected(self, func): index = self.selected if len(self.results) <= index: logging.warning("no such index: %s" % (index,)) self.clear_status() return filepath = self.results[index][-1] func(filepath) def open_selected(self): def action(path): self.opt.open(path) self.flash(" ** opened **") self.with_selected(action) def select(self, amount): self.ui_lock.acquire() if amount == NEXT or amount == PREVIOUS: self.selected += amount elif amount == START: self.selected = 0 elif amount == END: self.selected = len(self.results)-1 self.selected = min(self.selected, len(self.results)-1) self.selected = max(self.selected, 0) self.ui_lock.release() def set_query(self, new_query, is_repeat = False): if len(new_query) >= MIN_QUERY: self.finder.find(Search(new_query, is_repeat=is_repeat)) else: self.finder.find(None) self.ui_lock.acquire() self.query = new_query if self.input_position > len(self.query): self.input_position = len(self.query) self.ui_lock.release() def set_results(self, search): self.highlight = Highlight(search.text) self.results = list(search.results) if search.is_repeat: self.selected = min(self.selected, len(self.results)-1) else: self.selected = 0 def add_char(self, ch): new_query = self.modify_query_as_list(lambda q: q.insert(self.input_position, ch)) self.input_position += 1 self.set_query(new_query) logging.debug("query = %s" % (self.query, )) def modify_query_as_list(self, proc): query_list = list(self.query) proc(query_list) return ''.join(query_list) def remove_char(self, forwards=False): letter_index = self.input_position if forwards else self.input_position - 1 if letter_index >= len(self.query) or letter_index < 0: return new_query = self.modify_query_as_list(lambda q: q.pop(letter_index)) if forwards: self.input_position = max(self.input_position, len(self.query)) elif self.input_position > 0: self.input_position -= 1 self.set_query(new_query) def move_cursor(self, backwards=False): offset = -1 if backwards else 1 self.input_position += offset self.input_position = max(0, min(self.input_position, len(self.query))) def move_cursor_to(self, index): self.input_position = index def flash(self, str): self.status_queue.put(str) def clear_status(self): self.status_queue.put("") def _redraw(self, *screens): if QUITTING_TIME.is_set(): return logging.debug("redrawing...") if not screens: screens = self.screens for scr in screens: if scr is self.results_win: scr.noutrefresh( self.results_scroll, 0, 1, 0, self.win_height-2, self.win_width) else: scr.noutrefresh() curses.doupdate() def copy_selected_path_to_clipboard(self): def action(filepath): try: import pyperclip pyperclip.copy(self.opt.full_path(filepath)) self.flash(" ** copied **") except StandardError, e: logging.warn("error: %s" % (e,)) logging.exception("error copying to clipboard") self.with_selected(action)
""" This file is meant to be a prototype for defining ROI's to be used in later applications """ import matplotlib.pyplot as plt from matplotlib.widgets import Slider, RadioButtons, Button from matplotlib.colorbar import ColorbarBase from matplotlib.colors import Normalize import numpy as np from roi_definer import ROI from file_finder import FileFinder start = FileFinder() start.get_name() plt.figure(1) axpic = plt.subplot2grid((20, 20), (0, 0), rowspan=14, colspan=14) axps = plt.subplot2grid((20, 20), (19, 10), rowspan=1, colspan=10) axrb = plt.subplot2grid((20, 20), (15, 10), rowspan=1, colspan=10) axre = plt.subplot2grid((20, 20), (16, 10), rowspan=1, colspan=10) axcb = plt.subplot2grid((20, 20), (17, 10), rowspan=1, colspan=10) axce = plt.subplot2grid((20, 20), (18, 10), rowspan=1, colspan=10) axgray = plt.subplot2grid((20, 20), (0, 15), rowspan=6, colspan=5) axskipf = plt.subplot2grid((20, 20), (19, 5), rowspan=1, colspan=2) axskipb = plt.subplot2grid((20, 20), (19, 3), rowspan=1, colspan=2) axvmin = plt.subplot2grid((20, 20), (15, 0), rowspan=2, colspan=5) axvmax = plt.subplot2grid((20, 20), (17, 0), rowspan=2, colspan=5) axbar = plt.subplot2grid((20, 20), (7, 14), rowspan=3, colspan=8) axzoom = plt.subplot2grid((20, 20), (11, 14), rowspan=3, colspan=8)
class Repl(object): def __init__(self, options): self.found_files = [] self.opt = options def highlight_func(self, query_string): if termstyle is None: return lambda x: x highlight = Highlight(query_string) return lambda x: highlight.replace(x, green) def summarise(self, result_iter, query_string): subprocess.call(['clear']) self.found_files = [] i = 0 highlight = self.highlight_func(query_string) for filename, fullpath in result_iter: self.found_files.append(fullpath) relpath = os.path.split(fullpath)[0] explanation = '' if relpath: explanation = "(in %s)" % (relpath,) index = str(i+1).rjust(2) filename = filename.ljust(30) print " %s%s %s %s" % (yellow(index), yellow(":"), highlight(filename), black(explanation)) i += 1 def open(self, index): index -= 1 # indexes start at 1 for readability if len(self.found_files) <= index: logging.warning("no such index: %s" % (index,)) return filepath = self.found_files[index] self.opt.open(filepath) def _loop(self): q = raw_input(blue("\nfind/open file: ")) if len(q) == 0: q = 1 # open the first found file by default try: index = int(q) except ValueError: index = None if index is not None: self.open(index) else: self.finder.find(Search(q)) search = self.finder.results() self.summarise(search.results, search.text) def run(self): work_thread = threading.Thread(target=self._run, name="repl") work_thread.daemon = True work_thread.start() # the main thread is just going to wait till someone tells it to quit try: QUITTING_TIME.wait() except KeyboardInterrupt: # somehow the main thread fails to exit when it is the one # to receive KeyboardInterrupt ! pass def _run(self): self.finder = FileFinder(self.opt.base_path, path_filter=self.opt.path_filter, quit_indicator=QUITTING_TIME) logging.info("getting file list...") self.finder.populate() try: while True: self._loop() except (KeyboardInterrupt, EOFError): print return 0 except Exception: import traceback traceback.print_exc() finally: QUITTING_TIME.set()
def run(srcDirectory, destDirectory, removeAfterCopy): fileFinder = FileFinder(destDirectory, removeAfterCopy) fileFinder.copyFiles(srcDirectory)
def unmarshal_gadget_file(gadget_file): from file_finder import FileFinder handle = FileFinder(gadget_file, "i386") gadget_list = handle.find_gadgets() return gadget_list