def run_tarsqi(data_dir, filelist, start, end, crash=False):
print("$ python3 %s" % ' '.join(sys.argv))
for n, fname in elements(filelist, start, end):
print_element(n, fname)
if crash:
run_tarsqi_for_file(data_dir, fname)
else:
try:
run_tarsqi_for_file(data_dir, fname)
except Exception as e:
print('ERROR:', Exception, e)
def create_documents(data_dir, filelist, start, end, crash=False):
print("$ python3 %s\n" % ' '.join(sys.argv))
ela_dir = os.path.join(data_dir, 'ela')
if not os.path.exists(ela_dir):
os.mkdir(ela_dir)
for n, fname in elements(filelist, start, end):
print_element(n, fname)
if crash:
create_document(data_dir, fname)
else:
try:
create_document(data_dir, fname)
except Exception as e:
print('ERROR:', Exception, e)
def generate_topics(data_dir, filelist, start, end, crash=False):
print("$ python3 %s\n" % ' '.join(sys.argv))
lda = load_model()
topic_idx = {topic_id: topic for topic_id, topic
in lda.print_topics(num_topics=NUM_TOPICS)}
dictionary = load_dictionary()
for n, fname in elements(filelist, start, end):
print_element(n, fname)
if crash:
generate_topics_for_file(data_dir, fname, lda, topic_idx, dictionary)
else:
try:
generate_topics_for_file(data_dir, fname, lda, topic_idx, dictionary)
except Exception as e:
print('ERROR:', Exception, e)
sys.stderr.write("ERROR on %07d %s\n" % (n, fname))
def process_filelist(source_dir,
data_dir,
filelist,
start,
end,
crash=False,
test=False):
print("$ python3 %s\n" % ' '.join(sys.argv))
for n, fname in elements(filelist, start, end):
print_element(n, fname)
if crash:
process_list_element(source_dir, data_dir, fname, test=test)
else:
try:
process_list_element(source_dir, data_dir, fname, test=test)
except Exception as e:
print('ERROR:', Exception, e)
def generate_topics(data_dir, filelist, start, end, crash=False):
lda = load_model()
topic_idx = {
topic_id: topic
for topic_id, topic in lda.print_topics(num_topics=NUM_TOPICS)
}
dictionary = load_dictionary()
for n, fname in elements(filelist, start, end):
print_element(n, fname)
if crash:
generate_topics_for_file(data_dir, fname, lda, topic_idx,
dictionary)
else:
try:
generate_topics_for_file(data_dir, fname, lda, topic_idx,
dictionary)
except Exception as e:
print('ERROR:', Exception, e)