def test_load(self):
""" Test loading Filters using the string/dict notation """
loaded = filters.get_filters(filter_dict={'subreddit.equals': 1})
self.assertEqual(len(loaded), 1, "Loaded incorrect number of Filters!")
for f in self.fields:
for op in filters.Operators:
fi = filters.get_filters(
filter_dict={"%s.%s" % (f, op.value): 1})[0]
self.assertEqual(
fi.operator, op,
"Filter %s failed to load with operator %s" % (f, op))
def create_restructured_text(setup_file_path, input_file_path, output_folder_path=None):
log.info("++++++++++ textstory-to-beautiful-latex-html ++++++++++")
# Setup
log.info("***************** Running setup *****************")
setup = Setup(setup_file_path, input_file_path, output_folder_path)
log.info("Done with setup.")
# run_with_setup(setup)
input_markup = DocumentReader(input_file_path).get_string()
filters = get_filters(setup)
# if not prepare_out_folder(setup):
# return
# set outfile path
# outfile_restructured_text = os.path.normpath(os.path.join(setup.output_folder_path, OUTFILE_RESTRUCTURED_TEXT))
# Create reStructuredText
log.info("***************** Creating reStructuredText *****************")
restructured_text_generator = ReStructuredTextGenerator(setup, input_markup, filters, None)
# restructured_text_generator.create_output()
log.info("Done creating reStructuredText.")
return restructured_text_generator.output_markup
def main():
from skimage import transform, io
from filters import get_filters
max_intensity = 7
lib = load_lib()
[F, filters, width, height] = get_filters()
im_w = im_h = 256
image = io.imread('images/fur_obs.jpg', as_grey=True)
image = transform.resize(image, (im_w, im_h), mode='symmetric', preserve_range=True)
image = (image * max_intensity).astype(np.int32)
n = 7
mi, ma = get_histogram(lib, image, filters[n:n+1], width[n:n+1], height[n:n+1], min_resp=None, max_resp=None)
print(mi, ma)
h2 = get_histogram(lib, image, filters[n:n+1], width[n:n+1], height[n:n+1], min_resp=mi, max_resp=ma)
print(h2)
syn2, synresp = julesz(lib, h2.reshape(1, 15), filters[n:n+1], width[n:n+1], height[n:n+1], im_w, im_h, max_intensity=max_intensity, min_resp=mi, max_resp=ma)
print(h2)
print(synresp)
print(np.linalg.norm(h2 - synresp))
def reload_layers(config):
layers = []
for layer_filters in config.get("layers", []):
assert (type(layer_filters) == dict)
assert (len(layer_filters) == 1)
layer_type = list(layer_filters.keys())[0]
layer_filters = layer_filters[layer_type]
layers.append((layer_type, filters.get_filters(config, layer_filters)))
return layers
def api_get_sources():
ret = {'available': [], 'active': [], 'filters': {}}
for s in sources.load_sources():
ret['available'].append(s.to_obj(for_webui=True))
for s in settings.get_sources():
ret['active'].append(s.to_obj(for_webui=True))
ret['filters']['available'] = [f.to_js_obj() for f in filters.get_filters()]
ret['filters']['operators'] = [f.value for f in filters.Operators]
return ret
def api_save_sources(new_obj):
print('Saving new source list:')
output_settings = []
for so in new_obj:
print('\tType:', so['type'], 'Alias:', so['alias'], so['filters'])
for s in sources.load_sources():
if s.type == so['type']:
s.set_alias(so['alias'])
for k, v in so['data'].items():
s.insert_data(k, v)
for f in so['filters']:
for fi in filters.get_filters():
if f['field'] == fi.field:
fi.set_operator(f['operator'])
fi.set_limit(f['limit'])
s.add_filter(fi)
break
output_settings.append(s)
for s in settings.get_sources():
settings.remove_source(s, save_after=False)
for s in output_settings:
settings.add_source(s, prevent_duplicate=False, save_after=False)
return settings.save()
import numpy as np
import numpy.ma as ma
from filters import get_filters
get_filters(globals()) # imports all filters at once
def comp_gradient(
blob): # compare g within sub blob, a component of intra_blob
dert__ = ma.empty(shape=blob.dert__.shape, dtype=int) # initialize dert__
g__ = ma.array(blob.dert__[:, :, 3], mask=~blob.map) # apply mask = ~map
dy__ = g__[
2:, 1:
-1] - g__[:-2, 1:
-1] # vertical comp between rows -> dy, (1:-1): first and last column are discarded
dx__ = g__[1:-1, 2:] - g__[
1:-1, :
-2] # lateral comp between columns -> dx, (1:-1): first and last row are discarded
gg__ = np.hypot(dy__, dx__) - ave # deviation of gradient
# pack all derts into dert__
dert__[:, :, 0] = g__
dert__[
1:-1, 1:-1,
1] = dy__ # first row, last row, first column and last-column are discarded
dert__[1:-1, 1:-1, 2] = dx__
dert__[1:-1, 1:-1, 3] = gg__
blob.new_dert__[0] = dert__ # pack dert__ into blob
def run_with_setup(setup):
input_markup = DocumentReader(setup.input_file_path).get_string()
filters = get_filters(setup)
out_folder = prepare_out_folder(setup)
if not prepare_out_folder(setup):
return
# create directory structure
latex_output_path = os.path.join(setup.output_folder_path, 'latex')
html_output_path = os.path.join(setup.output_folder_path, 'html')
try:
if not os.path.exists(latex_output_path):
os.makedirs(latex_output_path)
# preliminariesPath = os.path.join(latex_output_path, 'bookPreliminaries')
# if not os.path.exists(preliminariesPath):
# os.makedirs(preliminariesPath)
# appendixPath = os.path.join(latex_output_path, 'bookAppendix')
# if not os.path.exists(appendixPath):
# os.makedirs(appendixPath)
if not os.path.exists(html_output_path):
os.makedirs(html_output_path)
except Exception as e:
log.info("Failed creating output subdirectories: " + type(e).__name__ + str(e.args))
log.info("Abort.")
return
# copy required files from latex template folders
appendix_source_path = os.path.join(dir_path, 'latex', APPENDIX_PATH)
if not os.path.exists(appendix_source_path):
os.makedirs(appendix_source_path)
dir_util.copy_tree(appendix_source_path,
os.path.join(latex_output_path, APPENDIX_PATH), update=1)
# TODO appendix files from config?
preliminaries_source_path = os.path.join(dir_path, 'latex', PRELIMINARIES_PATH)
if not os.path.exists(preliminaries_source_path):
os.makedirs(preliminaries_source_path)
dir_util.copy_tree(preliminaries_source_path,
os.path.join(latex_output_path, PRELIMINARIES_PATH), update=1)
# TODO preliminary files from config?
dir_util.copy_tree(os.path.join(dir_path, 'latex', 'img'),
os.path.join(latex_output_path, 'img'), update=1)
latex_files = [
'build.bash',
'build.bat',
'license.tex' # TODO get license from config, allow not using license
]
for file_name in latex_files:
copy_file(file_name, 'latex', latex_output_path)
# copy required files from html template folders
dir_util.copy_tree(os.path.join(dir_path, 'html', 'css'), os.path.join(html_output_path, 'css'), update=1)
dir_util.copy_tree(os.path.join(dir_path, 'html', 'img'), os.path.join(html_output_path, 'img'), update=1)
html_files = [ # TODO get license from config, allow not using license
'apple-touch-icon.png',
'browserconfig.xml',
'favicon.ico',
'humans.txt',
'tile.png',
'tile-wide.png'
]
for file_name in html_files:
copy_file(file_name, 'html', html_output_path)
# set outfile paths
outfile_latex_doc = os.path.normpath(os.path.join(setup.output_folder_path, OUTFILE_LATEX_DOC))
outfile_latex_body = os.path.normpath(os.path.join(setup.output_folder_path, OUTFILE_LATEX_BODY))
outfile_html = os.path.normpath(os.path.join(setup.output_folder_path, OUTFILE_HTML))
# Create LaTeX
log.info("***************** Creating LaTeX *****************")
latex_generator = LatexGenerator(setup, input_markup, filters,
LATEX_TEMPLATE, outfile_latex_doc, outfile_latex_body)
latex_generator.create_output()
log.info("Done creating LaTeX.")
# Create HTML
log.info("***************** Creating HTML *****************")
html_generator = HtmlGenerator(setup, input_markup, filters, HTML_TEMPLATE, HTML_LICENSE, outfile_html)
html_generator.create_output()
log.info("Done creating HTML.")
def available_filters(self):
""" Returns a list of the available filters this Source can listen to. """
return filters.get_filters()
def _load_filters(self, data):
""" Builds the list of filters this object needs. """
if 'filters' not in data:
return False # !cover
self.filters = filters.get_filters(data['filters'])
def test_default_fields(self):
""" Assure that all filterable fields are properly loaded as Filters """
loaded = [f.field for f in filters.get_filters(filter_dict=None)]
for field in self.fields:
self.assertIn(field, loaded,
"Missing filter for field: %s" % field)
def main():
# TODO consider adjusting this value
max_intensity = 7
lib = load_lib()
# (1) generate filter bank
[F, filters, width, height] = get_filters()
im_w = im_h = 256
# # Load all images
images = []
for img_name in ['fur_obs.jpg', 'grass_obs.bmp', 'stucco.bmp']:
img = io.imread(f'images/{img_name}', as_grey=True)
img = transform.resize(img, (im_w, im_h),
mode='symmetric',
preserve_range=True)
if img_name == 'grass_obs.bmp':
img = (img / 255 * max_intensity).astype(np.int32)
else:
img = (img * max_intensity).astype(np.int32)
images.append(img)
err_threshold = 0
bin_weights = np.array([8, 7, 6, 5, 4, 3, 2, 1, 2, 3, 4, 5, 6, 7, 8])
image_num = 0
for image in images:
image_num += 1
# get_histogram = filtered image (response)
chosen_filters = []
chosen_width = []
chosen_height = []
unselected_filters = range(len(filters))
synth_errors = []
synth_image = np.random.randint(0, 8,
size=(im_w, im_h)).astype(np.int32)
# synth_image = np.rint(np.random.uniform(0, 7, (im_w, im_h))).astype(np.int32)
print("================== Image ==================")
filter_ranges = []
for filter_num in range(len(filters)):
# if we don't specify a range here, it returns the range of the hist instead
min_resp, max_resp = get_histogram(lib,
image,
filters[[filter_num], :],
width[[filter_num]],
height[[filter_num]],
max_intensity=max_intensity,
min_resp=None,
max_resp=None)
filter_ranges.append((min_resp, max_resp))
# while the filter bank is not empty
while len(unselected_filters) > 0:
selected_filter_num = len(filters) - len(unselected_filters)
filter_errs = []
# calculate the filter response for each unselected filter
for filter_num in unselected_filters:
h1 = get_histogram(lib,
image,
filters[[filter_num], :],
width[[filter_num]],
height[[filter_num]],
max_intensity=max_intensity,
min_resp=filter_ranges[filter_num][0],
max_resp=filter_ranges[filter_num][1])
h2 = get_histogram(lib,
synth_image,
filters[[filter_num], :],
width[[filter_num]],
height[[filter_num]],
max_intensity=max_intensity,
min_resp=filter_ranges[filter_num][0],
max_resp=filter_ranges[filter_num][1])
# TODO: for the errors, match the tails more closely than the centers
# should change code in jules.c with the stuff from 3b
err = np.dot(np.abs(h1 - h2).flatten(), bin_weights)
filter_errs.append(err)
filter_idx = np.argmax(filter_errs)
if filter_errs[filter_idx] < err_threshold:
break
# remove filter once it's selected
chosen_filters.append(unselected_filters[filter_idx])
unselected_filters = np.delete(unselected_filters, filter_idx, 0)
print(f"{chosen_filters} {unselected_filters}")
# synthesize new image
h = get_histogram(lib,
image,
filters[chosen_filters, :],
width[chosen_filters],
height[chosen_filters],
max_intensity=max_intensity,
min_resp=filter_ranges[filter_num][0],
max_resp=filter_ranges[filter_num][1])
synth_image, err = julesz(lib,
h.reshape(len(chosen_filters), -1),
filters[chosen_filters, :],
width[chosen_filters],
height[chosen_filters],
im_w,
im_h,
max_intensity=max_intensity)
synth_image = synth_image.reshape((im_w, im_h))
synth_errors.append(err)
print(f"Filter {selected_filter_num} uses idx: {filter_idx}")
print(f" Synth error = {np.linalg.norm(synth_image - image)}")
print(f" Synth error = {np.linalg.norm(err)}")
io.imsave(
f'output/temp_synth_{image_num}_{selected_filter_num}.png',
(synth_image * 32).astype(np.uint))
# stop after 20 filters
if selected_filter_num > 19:
break
# record which filters were chosen
print(chosen_filters)
with open(f"chosen_filters_{image_num}.txt", "w") as f:
f.write(str(chosen_filters))
print("====== Done ===================")
def main():
parser = argparse.ArgumentParser(
description='Run LDA on a given collection')
# Required arguments
required_args_parser = parser.add_argument_group('required arguments')
required_args_parser.add_argument('-c',
'--collection_name',
help='Collection name',
required=True)
required_args_parser.add_argument(
'-t',
'--topics',
nargs='+',
help='Number of topics to run the model on',
required=True,
type=int)
required_args_parser.add_argument('-p',
'--preprocess',
help='Preprocess data',
action='store_true',
default=False,
required=False)
required_args_parser.add_argument('--table_name',
help='Table name for HBase',
required=False)
sourcefile = required_args_parser.add_mutually_exclusive_group(
required=True)
sourcefile.add_argument('-hb',
'--hbase',
help='Get collection from HBase',
action='store_true')
sourcefile.add_argument('-f', '--file', help='File name for tokens')
# Optional arguments with defaults
parser.add_argument('-l',
'--logs',
help='Log directory',
default='/tmp/logs')
parser.add_argument('-a',
'--alpha',
help='Alpha hyperparameter',
default=0.1,
type=float)
parser.add_argument('-b',
'--beta',
help='Beta hyperparameter',
default=0.01,
type=float)
parser.add_argument('-i',
'--iter',
help='Number of iterations',
default=800,
type=int)
parser.add_argument('--save_dir',
help='Save directory for topic models',
default=None)
# Preprocessing arguments with defaults
preprocess_parser = parser.add_argument_group(
'preprocessing arguments to be added when using -p flag')
preprocess_parser.add_argument('--tokenizer',
help='Tokenizer to use',
default='wordtokenizer')
preprocess_parser.add_argument('--mappers',
help='Mappers to use',
nargs='+',
default=['lowercasemapper'])
preprocess_parser.add_argument(
'--filters',
help='Filters to use',
nargs='+',
default=['stopwordfilter', 'punctuationfilter'])
preprocess_parser.add_argument('--filter_words',
help='Filename with words to filter out',
default=None)
args = parser.parse_args()
# Arguments for preprocessing pipeline
logging.basicConfig(format='%(asctime)s-%(levelname)s: %(message)s',
level=logging.INFO)
logger = logging.getLogger('LDA')
logger.info('Application is starting!')
if args.preprocess:
if args.tokenizer is not None and args.mappers is not None and args.filters is not None:
pipeline = Pipeline(tokenizer=get_tokenizer(args.tokenizer),
mappers=get_mappers(args.mappers),
filters=get_filters(args.filters,
args.filter_words))
else:
logger.critical(
'Cannot preprocess data if the type of tokenizer, mappers and filters is not given'
)
sys.exit(2)
else:
pipeline = None
start = timer()
if args.hbase:
documents = HBaseReader(args.table_name,
args.collection_name,
pipeline=pipeline)
else:
documents = WebpageTokensReader(args.file, pipeline=pipeline)
end = timer()
logging.info('Preprocessing time = %s' %
str(timedelta(seconds=end - start)))
start = timer()
dictionary = Dictionary(documents)
corpus = Corpus(documents, dictionary)
end = timer()
logging.info('Time taken to build vocabulary = %s' %
str(timedelta(seconds=end - start)))
lda = LDA(corpus, dictionary)
perplexity, coherence = lda.run_models(args.collection_name,
args.topics,
args.save_dir,
alpha=args.alpha,
beta=args.beta,
iterations=args.iter)
topics = ['Num Topics'] + args.topics
perplexities = ['Log Perplexity'] + perplexity
coherences = ['Coherence (UMass)'] + coherence
table = [topics, perplexities, coherences]
print(tabulate(table))
with open(
'results_{}.txt'.format(args.collection_name.lower().replace(
' ', '_')), 'w') as fwriter:
fwriter.write(tabulate(table))
if values["T2_single"]:
T2_vis = 3
for i in range(T2_num):
window["T2_band"+str(i)].update(visible=False)
for i in range(3):
window["T2_band"+str(i)].update(visible=True)
elif event == "T2_system":
window["T2_filter"].update(disabled=not values["T2_system"])
for i in range(T2_num):
window["T2_filter"+str(i)].update(disabled=not values["T2_system"])
window["T2_wavelength"+str(i)].update(disabled=values["T2_system"])
elif event == "T2_filter":
for i in range(T2_num):
window["T2_filter"+str(i)].update(values=filters.get_filters(values["T2_filter"]))
elif event in ["T2_filter"+str(i) for i in range(T2_num)]:
i = event[-1]
window["T2_wavelength"+i].update(filters.get_param(values["T2_filter"], values["T2_filter"+i], "L_mean"))
elif event == "T2_folder":
window["T2_process"].update(disabled=False)
elif event == "T2_+":
window["T2_band"+str(T2_vis)].update(visible=True)
T2_vis += 1
elif event == "T2_-":
window["T2_band"+str(T2_vis-1)].update(visible=False)
T2_vis -= 1
