def __init__(self):
"""
Initializes the application and persistent objects/modules (settings, plugins,...)
"""
super(KonTextWsgiApp, self).__init__()
self.cleanup_runtime_modules()
os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry')
setup_plugins()
translation.load_translations(settings.get('global', 'translations'))
def __init__(self):
"""
Initializes the application and persistent objects/modules (settings, plugins,...)
"""
super(KonTextWsgiApp, self).__init__()
self.cleanup_runtime_modules()
os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry')
setup_plugins()
translation.load_translations(settings.get('global', 'translations'))
def __init__(self):
"""
Initializes the application and persistent objects/modules (settings, plugins,...)
"""
setup_logger(settings)
cleanup_runtime_modules()
setup_plugins()
translation.load_translations(settings.get('global', 'translations'))
l10n.configure(settings.get('global', 'translations'))
os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry')
def __init__(self):
"""
Initializes the application and persistent objects/modules (settings, plugins,...)
"""
setup_logger(settings)
cleanup_runtime_modules()
setup_plugins()
translation.load_translations(settings.get('global', 'translations'))
l10n.configure(settings.get('global', 'translations'))
os.environ['MANATEE_REGISTRY'] = settings.get('corpora',
'manatee_registry')
def __init__(self):
"""
Initializes the application and persistent objects/modules (settings, plugins,...)
"""
super(KonTextWsgiApp, self).__init__()
self.cleanup_runtime_modules()
os.environ['MANATEE_REGISTRY'] = settings.get('corpora',
'manatee_registry')
setup_plugins()
translation.load_translations(settings.get('global', 'translations'))
def signal_handler(signal, frame):
for p in plugins.runtime:
fn = getattr(p.instance, 'on_soft_reset', None)
if callable(fn):
fn()
signal.signal(signal.SIGUSR1, signal_handler)
def __init__(self):
super(MaintenanceWsgiApp, self).__init__()
translation.load_translations(settings.get('global', 'translations'))
def setUp(self):
self.mock_redis_plugin.clear()
# these are needed to return anonymous user w/o errors:
load_translations('en_US')
activate('en_US')
os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry')
initializer.init_plugin('db')
initializer.init_plugin('sessions')
initializer.init_plugin('auth')
initializer.init_plugin('conc_cache')
initializer.init_plugin('query_storage')
initializer.init_plugin('conc_persistence')
initializer.init_plugin('sessions')
initializer.init_plugin('user_items')
initializer.init_plugin('corparch')
initializer.init_plugin('token_connect', optional=True)
initializer.init_plugin('live_attributes', optional=True)
initializer.init_plugin('dispatch_hook', optional=True)
translation.load_translations(settings.get('global', 'translations'))
translation.activate('en_US') # background jobs do not need localization
import conclib.calc
import conclib.calc.base
from bgcalc import (freq_calc, subc_calc, coll_calc)
def load_script_module(name, path):
return imp.load_source(name, path)
class WorkerTaskException(Exception):
pass
def __init__(self):
super(MaintenanceWsgiApp, self).__init__()
translation.load_translations(settings.get('global', 'translations'))
l10n.configure(settings.get('global', 'translations'))
def __init__(self):
setup_logger(settings)
translation.load_translations(settings.get('global', 'translations'))
l10n.configure(settings.get('global', 'translations'))
sys.path.insert(0, settings.get('global', 'manatee_path'))
import manatee
os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry')
initializer.init_plugin('db')
initializer.init_plugin('sessions')
initializer.init_plugin('auth')
initializer.init_plugin('conc_cache')
initializer.init_plugin('query_storage')
initializer.init_plugin('conc_persistence')
initializer.init_plugin('sessions')
initializer.init_plugin('user_items')
initializer.init_plugin('corparch')
initializer.init_plugin('live_attributes', optional=True)
translation.load_translations(settings.get('global', 'translations'))
translation.activate('en_US') # background jobs do not need localization
import concworker
import task
from bgcalc import freq_calc
from bgcalc import subc_calc
from bgcalc import coll_calc
_, conf = settings.get_full('global', 'calc_backend')
app = task.get_celery_app(conf['conf'])
def load_script_module(name, path):
return imp.load_source(name, path)
def perform_calculation(data_directory: AnyStr,
output_directory: AnyStr,
language_codes: List[AnyStr],
length: int = 10e9):
translations = load_translations(language_codes)
translation_count = 0
for translation in translations:
translation_count = translation_count + 1
output_file = Path(
f"{output_directory}/{translation.language_code}.txt")
if output_file.exists():
continue
global current_model, current_man_vec, current_woman_vec
print(
f"Starting to process {translation.language} - {translation_count}/{len(translations)}"
)
override_and_print(f"Loading {translation.language} into memory...")
man, woman = get_man_and_woman_vectors(translation.man,
translation.woman,
data_directory,
translation.language_code)
current_model = model.load_vectors(data_directory,
translation.language_code, length)
override_and_print(f"Loaded {translation.language}")
override_and_print(f"Processing {translation.language}")
# Load the vectors of the translations
current_man_vec = list(man)
current_woman_vec = list(woman)
amount_of_words = len(current_model)
rank = 0
words = list()
harmonic = harmonic_series(amount_of_words)
with Pool() as pool:
it = pool.imap(func=worker,
iterable=current_model.items(),
chunksize=1000)
while True:
try:
word, diff = next(it)
# Update and print percentage
rank += 1
print_status(translation.language, rank, amount_of_words)
if word is None:
continue
freq = 1 / (rank * harmonic)
words.append((word, diff, freq))
except StopIteration:
break
# Mark the model for deletion
del current_model, current_man_vec, current_woman_vec
# override_and_print(f"Sorting result of {translation}")
# words = sort_output(words)
override_and_print(f"Writing result of {translation.language} to disk")
write_result(output_directory, translation.language_code, words)
print(
f"Finished {translation.language}! Result in {output_directory}/{translation.language_code}.txt"
)
output_file = Path(f"{output_directory}/all.txt")
if not output_file.exists():
write_merged_file(output_directory, translations)
def __init__(self):
setup_logger(settings)
translation.load_translations(settings.get('global', 'translations'))
l10n.configure(settings.get('global', 'translations'))
def setUp(self):
self.mock_redis_plugin.clear()
# these are needed to return anonymous user w/o errors:
load_translations('en_US')
activate('en_US')
