def test_load(self): """ Test loading Filters using the string/dict notation """ loaded = filters.get_filters(filter_dict={'subreddit.equals': 1}) self.assertEqual(len(loaded), 1, "Loaded incorrect number of Filters!") for f in self.fields: for op in filters.Operators: fi = filters.get_filters( filter_dict={"%s.%s" % (f, op.value): 1})[0] self.assertEqual( fi.operator, op, "Filter %s failed to load with operator %s" % (f, op))
def create_restructured_text(setup_file_path, input_file_path, output_folder_path=None): log.info("++++++++++ textstory-to-beautiful-latex-html ++++++++++") # Setup log.info("***************** Running setup *****************") setup = Setup(setup_file_path, input_file_path, output_folder_path) log.info("Done with setup.") # run_with_setup(setup) input_markup = DocumentReader(input_file_path).get_string() filters = get_filters(setup) # if not prepare_out_folder(setup): # return # set outfile path # outfile_restructured_text = os.path.normpath(os.path.join(setup.output_folder_path, OUTFILE_RESTRUCTURED_TEXT)) # Create reStructuredText log.info("***************** Creating reStructuredText *****************") restructured_text_generator = ReStructuredTextGenerator(setup, input_markup, filters, None) # restructured_text_generator.create_output() log.info("Done creating reStructuredText.") return restructured_text_generator.output_markup
def main(): from skimage import transform, io from filters import get_filters max_intensity = 7 lib = load_lib() [F, filters, width, height] = get_filters() im_w = im_h = 256 image = io.imread('images/fur_obs.jpg', as_grey=True) image = transform.resize(image, (im_w, im_h), mode='symmetric', preserve_range=True) image = (image * max_intensity).astype(np.int32) n = 7 mi, ma = get_histogram(lib, image, filters[n:n+1], width[n:n+1], height[n:n+1], min_resp=None, max_resp=None) print(mi, ma) h2 = get_histogram(lib, image, filters[n:n+1], width[n:n+1], height[n:n+1], min_resp=mi, max_resp=ma) print(h2) syn2, synresp = julesz(lib, h2.reshape(1, 15), filters[n:n+1], width[n:n+1], height[n:n+1], im_w, im_h, max_intensity=max_intensity, min_resp=mi, max_resp=ma) print(h2) print(synresp) print(np.linalg.norm(h2 - synresp))
def reload_layers(config): layers = [] for layer_filters in config.get("layers", []): assert (type(layer_filters) == dict) assert (len(layer_filters) == 1) layer_type = list(layer_filters.keys())[0] layer_filters = layer_filters[layer_type] layers.append((layer_type, filters.get_filters(config, layer_filters))) return layers
def api_get_sources(): ret = {'available': [], 'active': [], 'filters': {}} for s in sources.load_sources(): ret['available'].append(s.to_obj(for_webui=True)) for s in settings.get_sources(): ret['active'].append(s.to_obj(for_webui=True)) ret['filters']['available'] = [f.to_js_obj() for f in filters.get_filters()] ret['filters']['operators'] = [f.value for f in filters.Operators] return ret
def api_save_sources(new_obj): print('Saving new source list:') output_settings = [] for so in new_obj: print('\tType:', so['type'], 'Alias:', so['alias'], so['filters']) for s in sources.load_sources(): if s.type == so['type']: s.set_alias(so['alias']) for k, v in so['data'].items(): s.insert_data(k, v) for f in so['filters']: for fi in filters.get_filters(): if f['field'] == fi.field: fi.set_operator(f['operator']) fi.set_limit(f['limit']) s.add_filter(fi) break output_settings.append(s) for s in settings.get_sources(): settings.remove_source(s, save_after=False) for s in output_settings: settings.add_source(s, prevent_duplicate=False, save_after=False) return settings.save()
import numpy as np import numpy.ma as ma from filters import get_filters get_filters(globals()) # imports all filters at once def comp_gradient( blob): # compare g within sub blob, a component of intra_blob dert__ = ma.empty(shape=blob.dert__.shape, dtype=int) # initialize dert__ g__ = ma.array(blob.dert__[:, :, 3], mask=~blob.map) # apply mask = ~map dy__ = g__[ 2:, 1: -1] - g__[:-2, 1: -1] # vertical comp between rows -> dy, (1:-1): first and last column are discarded dx__ = g__[1:-1, 2:] - g__[ 1:-1, : -2] # lateral comp between columns -> dx, (1:-1): first and last row are discarded gg__ = np.hypot(dy__, dx__) - ave # deviation of gradient # pack all derts into dert__ dert__[:, :, 0] = g__ dert__[ 1:-1, 1:-1, 1] = dy__ # first row, last row, first column and last-column are discarded dert__[1:-1, 1:-1, 2] = dx__ dert__[1:-1, 1:-1, 3] = gg__ blob.new_dert__[0] = dert__ # pack dert__ into blob
def run_with_setup(setup): input_markup = DocumentReader(setup.input_file_path).get_string() filters = get_filters(setup) out_folder = prepare_out_folder(setup) if not prepare_out_folder(setup): return # create directory structure latex_output_path = os.path.join(setup.output_folder_path, 'latex') html_output_path = os.path.join(setup.output_folder_path, 'html') try: if not os.path.exists(latex_output_path): os.makedirs(latex_output_path) # preliminariesPath = os.path.join(latex_output_path, 'bookPreliminaries') # if not os.path.exists(preliminariesPath): # os.makedirs(preliminariesPath) # appendixPath = os.path.join(latex_output_path, 'bookAppendix') # if not os.path.exists(appendixPath): # os.makedirs(appendixPath) if not os.path.exists(html_output_path): os.makedirs(html_output_path) except Exception as e: log.info("Failed creating output subdirectories: " + type(e).__name__ + str(e.args)) log.info("Abort.") return # copy required files from latex template folders appendix_source_path = os.path.join(dir_path, 'latex', APPENDIX_PATH) if not os.path.exists(appendix_source_path): os.makedirs(appendix_source_path) dir_util.copy_tree(appendix_source_path, os.path.join(latex_output_path, APPENDIX_PATH), update=1) # TODO appendix files from config? preliminaries_source_path = os.path.join(dir_path, 'latex', PRELIMINARIES_PATH) if not os.path.exists(preliminaries_source_path): os.makedirs(preliminaries_source_path) dir_util.copy_tree(preliminaries_source_path, os.path.join(latex_output_path, PRELIMINARIES_PATH), update=1) # TODO preliminary files from config? dir_util.copy_tree(os.path.join(dir_path, 'latex', 'img'), os.path.join(latex_output_path, 'img'), update=1) latex_files = [ 'build.bash', 'build.bat', 'license.tex' # TODO get license from config, allow not using license ] for file_name in latex_files: copy_file(file_name, 'latex', latex_output_path) # copy required files from html template folders dir_util.copy_tree(os.path.join(dir_path, 'html', 'css'), os.path.join(html_output_path, 'css'), update=1) dir_util.copy_tree(os.path.join(dir_path, 'html', 'img'), os.path.join(html_output_path, 'img'), update=1) html_files = [ # TODO get license from config, allow not using license 'apple-touch-icon.png', 'browserconfig.xml', 'favicon.ico', 'humans.txt', 'tile.png', 'tile-wide.png' ] for file_name in html_files: copy_file(file_name, 'html', html_output_path) # set outfile paths outfile_latex_doc = os.path.normpath(os.path.join(setup.output_folder_path, OUTFILE_LATEX_DOC)) outfile_latex_body = os.path.normpath(os.path.join(setup.output_folder_path, OUTFILE_LATEX_BODY)) outfile_html = os.path.normpath(os.path.join(setup.output_folder_path, OUTFILE_HTML)) # Create LaTeX log.info("***************** Creating LaTeX *****************") latex_generator = LatexGenerator(setup, input_markup, filters, LATEX_TEMPLATE, outfile_latex_doc, outfile_latex_body) latex_generator.create_output() log.info("Done creating LaTeX.") # Create HTML log.info("***************** Creating HTML *****************") html_generator = HtmlGenerator(setup, input_markup, filters, HTML_TEMPLATE, HTML_LICENSE, outfile_html) html_generator.create_output() log.info("Done creating HTML.")
def available_filters(self): """ Returns a list of the available filters this Source can listen to. """ return filters.get_filters()
def _load_filters(self, data): """ Builds the list of filters this object needs. """ if 'filters' not in data: return False # !cover self.filters = filters.get_filters(data['filters'])
def test_default_fields(self): """ Assure that all filterable fields are properly loaded as Filters """ loaded = [f.field for f in filters.get_filters(filter_dict=None)] for field in self.fields: self.assertIn(field, loaded, "Missing filter for field: %s" % field)
def main(): # TODO consider adjusting this value max_intensity = 7 lib = load_lib() # (1) generate filter bank [F, filters, width, height] = get_filters() im_w = im_h = 256 # # Load all images images = [] for img_name in ['fur_obs.jpg', 'grass_obs.bmp', 'stucco.bmp']: img = io.imread(f'images/{img_name}', as_grey=True) img = transform.resize(img, (im_w, im_h), mode='symmetric', preserve_range=True) if img_name == 'grass_obs.bmp': img = (img / 255 * max_intensity).astype(np.int32) else: img = (img * max_intensity).astype(np.int32) images.append(img) err_threshold = 0 bin_weights = np.array([8, 7, 6, 5, 4, 3, 2, 1, 2, 3, 4, 5, 6, 7, 8]) image_num = 0 for image in images: image_num += 1 # get_histogram = filtered image (response) chosen_filters = [] chosen_width = [] chosen_height = [] unselected_filters = range(len(filters)) synth_errors = [] synth_image = np.random.randint(0, 8, size=(im_w, im_h)).astype(np.int32) # synth_image = np.rint(np.random.uniform(0, 7, (im_w, im_h))).astype(np.int32) print("================== Image ==================") filter_ranges = [] for filter_num in range(len(filters)): # if we don't specify a range here, it returns the range of the hist instead min_resp, max_resp = get_histogram(lib, image, filters[[filter_num], :], width[[filter_num]], height[[filter_num]], max_intensity=max_intensity, min_resp=None, max_resp=None) filter_ranges.append((min_resp, max_resp)) # while the filter bank is not empty while len(unselected_filters) > 0: selected_filter_num = len(filters) - len(unselected_filters) filter_errs = [] # calculate the filter response for each unselected filter for filter_num in unselected_filters: h1 = get_histogram(lib, image, filters[[filter_num], :], width[[filter_num]], height[[filter_num]], max_intensity=max_intensity, min_resp=filter_ranges[filter_num][0], max_resp=filter_ranges[filter_num][1]) h2 = get_histogram(lib, synth_image, filters[[filter_num], :], width[[filter_num]], height[[filter_num]], max_intensity=max_intensity, min_resp=filter_ranges[filter_num][0], max_resp=filter_ranges[filter_num][1]) # TODO: for the errors, match the tails more closely than the centers # should change code in jules.c with the stuff from 3b err = np.dot(np.abs(h1 - h2).flatten(), bin_weights) filter_errs.append(err) filter_idx = np.argmax(filter_errs) if filter_errs[filter_idx] < err_threshold: break # remove filter once it's selected chosen_filters.append(unselected_filters[filter_idx]) unselected_filters = np.delete(unselected_filters, filter_idx, 0) print(f"{chosen_filters} {unselected_filters}") # synthesize new image h = get_histogram(lib, image, filters[chosen_filters, :], width[chosen_filters], height[chosen_filters], max_intensity=max_intensity, min_resp=filter_ranges[filter_num][0], max_resp=filter_ranges[filter_num][1]) synth_image, err = julesz(lib, h.reshape(len(chosen_filters), -1), filters[chosen_filters, :], width[chosen_filters], height[chosen_filters], im_w, im_h, max_intensity=max_intensity) synth_image = synth_image.reshape((im_w, im_h)) synth_errors.append(err) print(f"Filter {selected_filter_num} uses idx: {filter_idx}") print(f" Synth error = {np.linalg.norm(synth_image - image)}") print(f" Synth error = {np.linalg.norm(err)}") io.imsave( f'output/temp_synth_{image_num}_{selected_filter_num}.png', (synth_image * 32).astype(np.uint)) # stop after 20 filters if selected_filter_num > 19: break # record which filters were chosen print(chosen_filters) with open(f"chosen_filters_{image_num}.txt", "w") as f: f.write(str(chosen_filters)) print("====== Done ===================")
def main(): parser = argparse.ArgumentParser( description='Run LDA on a given collection') # Required arguments required_args_parser = parser.add_argument_group('required arguments') required_args_parser.add_argument('-c', '--collection_name', help='Collection name', required=True) required_args_parser.add_argument( '-t', '--topics', nargs='+', help='Number of topics to run the model on', required=True, type=int) required_args_parser.add_argument('-p', '--preprocess', help='Preprocess data', action='store_true', default=False, required=False) required_args_parser.add_argument('--table_name', help='Table name for HBase', required=False) sourcefile = required_args_parser.add_mutually_exclusive_group( required=True) sourcefile.add_argument('-hb', '--hbase', help='Get collection from HBase', action='store_true') sourcefile.add_argument('-f', '--file', help='File name for tokens') # Optional arguments with defaults parser.add_argument('-l', '--logs', help='Log directory', default='/tmp/logs') parser.add_argument('-a', '--alpha', help='Alpha hyperparameter', default=0.1, type=float) parser.add_argument('-b', '--beta', help='Beta hyperparameter', default=0.01, type=float) parser.add_argument('-i', '--iter', help='Number of iterations', default=800, type=int) parser.add_argument('--save_dir', help='Save directory for topic models', default=None) # Preprocessing arguments with defaults preprocess_parser = parser.add_argument_group( 'preprocessing arguments to be added when using -p flag') preprocess_parser.add_argument('--tokenizer', help='Tokenizer to use', default='wordtokenizer') preprocess_parser.add_argument('--mappers', help='Mappers to use', nargs='+', default=['lowercasemapper']) preprocess_parser.add_argument( '--filters', help='Filters to use', nargs='+', default=['stopwordfilter', 'punctuationfilter']) preprocess_parser.add_argument('--filter_words', help='Filename with words to filter out', default=None) args = parser.parse_args() # Arguments for preprocessing pipeline logging.basicConfig(format='%(asctime)s-%(levelname)s: %(message)s', level=logging.INFO) logger = logging.getLogger('LDA') logger.info('Application is starting!') if args.preprocess: if args.tokenizer is not None and args.mappers is not None and args.filters is not None: pipeline = Pipeline(tokenizer=get_tokenizer(args.tokenizer), mappers=get_mappers(args.mappers), filters=get_filters(args.filters, args.filter_words)) else: logger.critical( 'Cannot preprocess data if the type of tokenizer, mappers and filters is not given' ) sys.exit(2) else: pipeline = None start = timer() if args.hbase: documents = HBaseReader(args.table_name, args.collection_name, pipeline=pipeline) else: documents = WebpageTokensReader(args.file, pipeline=pipeline) end = timer() logging.info('Preprocessing time = %s' % str(timedelta(seconds=end - start))) start = timer() dictionary = Dictionary(documents) corpus = Corpus(documents, dictionary) end = timer() logging.info('Time taken to build vocabulary = %s' % str(timedelta(seconds=end - start))) lda = LDA(corpus, dictionary) perplexity, coherence = lda.run_models(args.collection_name, args.topics, args.save_dir, alpha=args.alpha, beta=args.beta, iterations=args.iter) topics = ['Num Topics'] + args.topics perplexities = ['Log Perplexity'] + perplexity coherences = ['Coherence (UMass)'] + coherence table = [topics, perplexities, coherences] print(tabulate(table)) with open( 'results_{}.txt'.format(args.collection_name.lower().replace( ' ', '_')), 'w') as fwriter: fwriter.write(tabulate(table))
if values["T2_single"]: T2_vis = 3 for i in range(T2_num): window["T2_band"+str(i)].update(visible=False) for i in range(3): window["T2_band"+str(i)].update(visible=True) elif event == "T2_system": window["T2_filter"].update(disabled=not values["T2_system"]) for i in range(T2_num): window["T2_filter"+str(i)].update(disabled=not values["T2_system"]) window["T2_wavelength"+str(i)].update(disabled=values["T2_system"]) elif event == "T2_filter": for i in range(T2_num): window["T2_filter"+str(i)].update(values=filters.get_filters(values["T2_filter"])) elif event in ["T2_filter"+str(i) for i in range(T2_num)]: i = event[-1] window["T2_wavelength"+i].update(filters.get_param(values["T2_filter"], values["T2_filter"+i], "L_mean")) elif event == "T2_folder": window["T2_process"].update(disabled=False) elif event == "T2_+": window["T2_band"+str(T2_vis)].update(visible=True) T2_vis += 1 elif event == "T2_-": window["T2_band"+str(T2_vis-1)].update(visible=False) T2_vis -= 1