def __init__(self): """ Initializes the application and persistent objects/modules (settings, plugins,...) """ super(KonTextWsgiApp, self).__init__() self.cleanup_runtime_modules() os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry') setup_plugins() translation.load_translations(settings.get('global', 'translations'))
def __init__(self): """ Initializes the application and persistent objects/modules (settings, plugins,...) """ setup_logger(settings) cleanup_runtime_modules() setup_plugins() translation.load_translations(settings.get('global', 'translations')) l10n.configure(settings.get('global', 'translations')) os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry')
def __init__(self): """ Initializes the application and persistent objects/modules (settings, plugins,...) """ super(KonTextWsgiApp, self).__init__() self.cleanup_runtime_modules() os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry') setup_plugins() translation.load_translations(settings.get('global', 'translations')) def signal_handler(signal, frame): for p in plugins.runtime: fn = getattr(p.instance, 'on_soft_reset', None) if callable(fn): fn() signal.signal(signal.SIGUSR1, signal_handler)
def __init__(self): super(MaintenanceWsgiApp, self).__init__() translation.load_translations(settings.get('global', 'translations'))
def setUp(self): self.mock_redis_plugin.clear() # these are needed to return anonymous user w/o errors: load_translations('en_US') activate('en_US')
os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry') initializer.init_plugin('db') initializer.init_plugin('sessions') initializer.init_plugin('auth') initializer.init_plugin('conc_cache') initializer.init_plugin('query_storage') initializer.init_plugin('conc_persistence') initializer.init_plugin('sessions') initializer.init_plugin('user_items') initializer.init_plugin('corparch') initializer.init_plugin('token_connect', optional=True) initializer.init_plugin('live_attributes', optional=True) initializer.init_plugin('dispatch_hook', optional=True) translation.load_translations(settings.get('global', 'translations')) translation.activate('en_US') # background jobs do not need localization import conclib.calc import conclib.calc.base from bgcalc import (freq_calc, subc_calc, coll_calc) def load_script_module(name, path): return imp.load_source(name, path) class WorkerTaskException(Exception): pass
def __init__(self): super(MaintenanceWsgiApp, self).__init__() translation.load_translations(settings.get('global', 'translations')) l10n.configure(settings.get('global', 'translations'))
def __init__(self): setup_logger(settings) translation.load_translations(settings.get('global', 'translations')) l10n.configure(settings.get('global', 'translations'))
sys.path.insert(0, settings.get('global', 'manatee_path')) import manatee os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry') initializer.init_plugin('db') initializer.init_plugin('sessions') initializer.init_plugin('auth') initializer.init_plugin('conc_cache') initializer.init_plugin('query_storage') initializer.init_plugin('conc_persistence') initializer.init_plugin('sessions') initializer.init_plugin('user_items') initializer.init_plugin('corparch') initializer.init_plugin('live_attributes', optional=True) translation.load_translations(settings.get('global', 'translations')) translation.activate('en_US') # background jobs do not need localization import concworker import task from bgcalc import freq_calc from bgcalc import subc_calc from bgcalc import coll_calc _, conf = settings.get_full('global', 'calc_backend') app = task.get_celery_app(conf['conf']) def load_script_module(name, path): return imp.load_source(name, path)
def perform_calculation(data_directory: AnyStr, output_directory: AnyStr, language_codes: List[AnyStr], length: int = 10e9): translations = load_translations(language_codes) translation_count = 0 for translation in translations: translation_count = translation_count + 1 output_file = Path( f"{output_directory}/{translation.language_code}.txt") if output_file.exists(): continue global current_model, current_man_vec, current_woman_vec print( f"Starting to process {translation.language} - {translation_count}/{len(translations)}" ) override_and_print(f"Loading {translation.language} into memory...") man, woman = get_man_and_woman_vectors(translation.man, translation.woman, data_directory, translation.language_code) current_model = model.load_vectors(data_directory, translation.language_code, length) override_and_print(f"Loaded {translation.language}") override_and_print(f"Processing {translation.language}") # Load the vectors of the translations current_man_vec = list(man) current_woman_vec = list(woman) amount_of_words = len(current_model) rank = 0 words = list() harmonic = harmonic_series(amount_of_words) with Pool() as pool: it = pool.imap(func=worker, iterable=current_model.items(), chunksize=1000) while True: try: word, diff = next(it) # Update and print percentage rank += 1 print_status(translation.language, rank, amount_of_words) if word is None: continue freq = 1 / (rank * harmonic) words.append((word, diff, freq)) except StopIteration: break # Mark the model for deletion del current_model, current_man_vec, current_woman_vec # override_and_print(f"Sorting result of {translation}") # words = sort_output(words) override_and_print(f"Writing result of {translation.language} to disk") write_result(output_directory, translation.language_code, words) print( f"Finished {translation.language}! Result in {output_directory}/{translation.language_code}.txt" ) output_file = Path(f"{output_directory}/all.txt") if not output_file.exists(): write_merged_file(output_directory, translations)