def init(profile):
audio_level_filter = mlt.Filter(profile, "audiolevel")
global MONITORING_AVAILABLE
if audio_level_filter != None:
MONITORING_AVAILABLE = True
editorstate.audio_monitoring_available = True
else:
MONITORING_AVAILABLE = False
editorstate.audio_monitoring_available = False
global CONTROL_SLOT_H, METER_SLOT_H, METER_LIGHTS, METER_HEIGHT
if editorstate.screen_size_small_height() == True:
if editorstate.SCREEN_HEIGHT > 898:
METER_SLOT_H = 400
CONTROL_SLOT_H = 240
METER_LIGHTS = 123
METER_HEIGHT = METER_LIGHTS * DASH_INK + (METER_LIGHTS -
1) * DASH_SKIP
else:
METER_SLOT_H = 275
CONTROL_SLOT_H = 240
METER_LIGHTS = 82
METER_HEIGHT = METER_LIGHTS * DASH_INK + (METER_LIGHTS -
1) * DASH_SKIP
# We want this to be always present when closing app or we'll need to handle it being missing.
global _update_ticker
_update_ticker = utils.Ticker(_audio_monitor_update, 0.04)
_update_ticker.start_ticker()
_update_ticker.stop_ticker()
def init(profile):
audio_level_filter = mlt.Filter(profile, "audiolevel")
global MONITORING_AVAILABLE
if audio_level_filter != None:
MONITORING_AVAILABLE = True
editorstate.audio_monitoring_available = True
else:
MONITORING_AVAILABLE = False
editorstate.audio_monitoring_available = False
# We want this to be always present when closing app or we'll need to handle it being missing.
global _update_ticker
_update_ticker = utils.Ticker(_audio_monitor_update, 0.04)
_update_ticker.start_ticker()
_update_ticker.stop_ticker()
def init(profile):
audio_level_filter = mlt.Filter(profile, "audiolevel")
global MONITORING_AVAILABLE
if audio_level_filter != None:
MONITORING_AVAILABLE = True
editorstate.audio_monitoring_available = True
else:
MONITORING_AVAILABLE = False
editorstate.audio_monitoring_available = False
global CONTROL_SLOT_H, METER_SLOT_H
if editorstate.screen_size_small_height() == True:
METER_SLOT_H = 400
CONTROL_SLOT_H = 220
# We want this to be always present when closing app or we'll need to handle it being missing.
global _update_ticker
_update_ticker = utils.Ticker(_audio_monitor_update, 0.04)
_update_ticker.start_ticker()
_update_ticker.stop_ticker()
def __init__(self, profile):
self.init_for_profile(profile)
self.ticker = utils.Ticker(self._ticker_event, TICKER_DELAY)
def run(self):
Gdk.threads_enter()
updater.set_info_icon(Gtk.STOCK_OPEN)
dialog = dialogs.load_dialog()
persistance.load_dialog = dialog
Gdk.threads_leave()
ticker = utils.Ticker(_load_pulse_bar, 0.15)
ticker.start_ticker()
old_project = editorstate.project
try:
editorstate.project_is_loading = True
project = persistance.load_project(self.filename)
sequence.set_track_counts(project)
editorstate.project_is_loading = False
except persistance.FileProducerNotFoundError as e:
print "did not find file:", e
self._error_stop(dialog, ticker)
primary_txt = _("Media asset was missing!")
secondary_txt = _("Path of missing asset:") + "\n <b>" + e.value + "</b>\n\n" + \
_("Relative search for replacement file in sub folders of project file failed.") + "\n\n" + \
_("To load the project you will need to either:") + "\n" + \
u"\u2022" + " " + _("Use 'Media Linker' tool to relink media assets to new files, or") + "\n" + \
u"\u2022" + " " + _("Place a file with the same exact name and path on the hard drive")
dialogutils.warning_message(primary_txt, secondary_txt, None, is_info=False)
editorstate.project = old_project # persistance.load_project() changes this,
# we simply change it back as no GUI or other state is yet changed
return
except persistance.ProjectProfileNotFoundError as e:
self._error_stop(dialog, ticker)
primary_txt = _("Profile with Description: '") + e.value + _("' was not found on load!")
secondary_txt = _("It is possible to load the project by creating a User Profile with exactly the same Description\nas the missing profile. ") + "\n\n" + \
_("User Profiles can be created by selecting 'Edit->Profiles Manager'.")
dialogutils.warning_message(primary_txt, secondary_txt, None, is_info=False)
editorstate.project = old_project # persistance.load_project() changes this,
# we simply change it back as no GUI or other state is yet changed
return
Gdk.threads_enter()
dialog.info.set_text(_("Opening"))
Gdk.threads_leave()
time.sleep(0.3)
Gdk.threads_enter()
app.open_project(project)
if self.block_recent_files: # naming flipped ????
editorpersistance.add_recent_project_path(self.filename)
editorpersistance.fill_recents_menu_widget(gui.editor_window.uimanager.get_widget('/MenuBar/FileMenu/OpenRecent'), open_recent_project)
Gdk.threads_leave()
Gdk.threads_enter()
updater.set_info_icon(None)
dialog.destroy()
Gdk.threads_leave()
ticker.stop_ticker()
def create_render_ticker(self):
self.render_ticker = utils.Ticker(self.render_tick, 1.0)
def launch_render_ticker(self):
self.ticker = utils.Ticker(self.render_tick, 1.0)
self.ticker.start_ticker()
def _loadRowRankings(data_dir, model, use_large_rowRankingCache):
print(f"Loading row rankings into memory...")
rowRankingsMap = {}
# Row rankings source directory
rowRankings_dir = f"{data_dir}/rowRankings/{model}/"
# Row rankings cache directory
lg_sm_str = "lg" if use_large_rowRankingCache else "sm"
rowRankings_cache_addr = f"{data_dir}/rowRankings/rowRankings_{model}_cache_{lg_sm_str}.json"
refresh_cache = utils.isCacheOld(rowRankings_dir, rowRankings_cache_addr)
if refresh_cache:
print(
f"\tWarning! Cache not up to date, reparsing original row rankings files from \"{rowRankings_dir}\""
)
# Only keep the top relevant facts
if not use_large_rowRankingCache:
N = 200 if model == 'tfidf' else 100
N += 1 # header
# N = -2 # header
# Popuilate this map with info from every question file
# Hash by QID
rowRankingsMap = {}
for split in ["Train", "Test", "Dev"]:
rankingsDir = f"{rowRankings_dir}/rankings.{split}/"
questionFiles = os.listdir(rankingsDir)
# User info
numQs = len(questionFiles)
ticker = utils.Ticker(
numQs,
tick_rate=121,
message=f"Loading {numQs} {split} questions...")
for rowRankingFileName in questionFiles:
# info for user
ticker.tick()
with open(rankingsDir + rowRankingFileName, mode="r") as fp:
# question ID is in the file name rowRanking_{qid}.tsv
qid = rowRankingFileName[11:-4]
lines = fp.read().split("\n")
# Grab the header and the lines
header = lines[0].split("\t")
lines = lines[
1:-1] if use_large_rowRankingCache else lines[1:N + 1]
if use_large_rowRankingCache: N = len(lines)
# Make a JSON for the tsv
rankingJSON = {head: [""] * N for head in header}
rankingJSON['length'] = N
for i, line in enumerate(lines):
cols = line.split("\t")
for j, col in enumerate(cols):
rankingJSON[header[j]][i] = col
# Save the JSON in the full export obj
rowRankingsMap[qid] = rankingJSON
ticker.end()
# Convert stuff to floats
print("Converting")
for qid in rowRankingsMap.keys():
for i, isGoldRow in enumerate(rowRankingsMap[qid]['isGoldRow']):
rowRankingsMap[qid]['isGoldRow'][i] = int(isGoldRow)
for i, score in enumerate(rowRankingsMap[qid]['score_1']):
rowRankingsMap[qid]['score_1'][i] = float(score)
print(f"Caching row rankings to \"{rowRankings_cache_addr}\".")
with open(rowRankings_cache_addr, mode="w") as fp:
fp.write(json_dumps(rowRankingsMap))
else:
print(
f"\tReading cached row rankings from \"{rowRankings_cache_addr}\"."
)
with open(rowRankings_cache_addr, mode="r") as fp:
rowRankingsMap = json_load(fp)
# Convert stuff to floats
for qid in rowRankingsMap.keys():
for i, isGoldRow in enumerate(
rowRankingsMap[qid]['isGoldRow']):
rowRankingsMap[qid]['isGoldRow'][i] = int(isGoldRow)
for i, score in enumerate(rowRankingsMap[qid]['score_1']):
rowRankingsMap[qid]['score_1'][i] = float(score)
print("Done.")
return rowRankingsMap
def _load_questions(self):
questionsAddr = self.questionsAddr
questions_cached_addr = f"{self.cache_dir}/questions_annotated_cache.json"
# Question map
questionsList = []
questionsList_JSON = [] # Parallel structure that doesnt use classes so that we can cache
qids = []
# check to see if cached questions are on file
refresh_cache = utils.isCacheOld(questionsAddr, questions_cached_addr)
if refresh_cache:
print(f"Loading questions from \"{questionsAddr}\".")
# Load the questions table
questions = pd.read_csv(questionsAddr, sep="\t")
# Filter out incomplete questions
questions = questions[questions['flags'].notna()]
questions = questions[questions['explanation'].notna()]
questions = questions[questions['examName'].notna()]
questions = questions[questions.flags.str.contains("SUCCESS|READY", regex=True)]
# Annotate the questions and save them locally
# Prepare interface for user to see progress
numQs = questions.shape[0]
ticker = utils.Ticker(numQs, tick_rate=166, message=f"Annotating {numQs} questions...")
# Convert questions to a map
# store the question dataframe; the lemmatized quesdtion text; the lemmatized answer text
for i, question_df in questions.iterrows():
question = question_df.to_dict()
qid = question['QuestionID']
qids.append(qid)
# info for user
ticker.tick()
# Read the question text and split it up into question and all answers
qText = question['question']
qText = re.split(r"\([A-G1-7]\)", qText)
# Distinguish Q and A
q_text, q_anss = qText[0], qText[1:]
# Remove outer whitespace
q_text = q_text.strip()
q_anss = [ans.strip() for ans in q_anss]
# Use spacy to annotate the texts
# Question text annotation
[q_text_words, q_text_lemmas, q_text_pos] = self.tokenize(q_text, annotationTypes=['word', 'lemma', 'pos'])
# Answer text annotation
isAlphaAnswerKey = re.match(r'[A-Ga-g]', question['AnswerKey'])
answerSelections = ['A', 'B', 'C', 'D', 'E', 'F', 'G'] if isAlphaAnswerKey else ['1', '2', '3', '4', '5', '6', '7']
answers_annotation = {}
for i_ans, q_ans in enumerate(q_anss):
[ans_text_words, ans_text_lemmas, ans_text_pos] = self.tokenize(q_ans, annotationTypes=['word', 'lemma', 'pos'])
answers_annotation[answerSelections[i_ans]] = {
"words": ans_text_words,
"lemmas": ans_text_lemmas,
"pos": ans_text_pos
}
# Add the annotation to the question
question["question_annotation"] = {
"words": q_text_words,
"lemmas": q_text_lemmas,
"pos": q_text_pos
}
question["answers_annotation"] = answers_annotation
questionsList_JSON.append(question)
# Convert to a question object
question = Question(question)
# Add to the list
questionsList.append(question)
ticker.end()
# Cache the annotated questions locally
print(f"\tCaching annotated questions to \"{questions_cached_addr}\".")
with open(questions_cached_addr, mode="w") as fp:
fp.write(json_dumps(questionsList_JSON))
else:
print(f"\tLoading cached annotated questions from \"{questions_cached_addr}\".")
with open(questions_cached_addr) as fp:
questionsList = json_load(fp)
# Convert to question class
questionsList = [Question(q) for q in questionsList]
# Get the list of qids
qids = [q.id for q in questionsList]
return qids, questionsList
def _load_tables(self):
"""
Retrieves the tables from the disk
Will attempt to read the cache first
If the cache is old or missing then this function re-reads the tables and annotates them
return:
(tables, tableNames, tableMap)
tables -
"""
# location of tables
tablestoreTablesDir = self.tablestoreTablesDir
tableFileNames = os.listdir(tablestoreTablesDir)
# ======= #
# Loading #
# ======= #
# Check cache
tablestore_cached_addr = f"{self.cache_dir}/tablestore_annotated_cache.json"
refresh_cache = utils.isCacheOld(tablestoreTablesDir, tablestore_cached_addr)
# If cache old re-load and annotate the tablestore
if refresh_cache:
print(f"\tWarning! Cache not up to date, re-annotating tables from \"{tablestoreTablesDir}\"")
# Get the NLP tool for tokenizing
nlp = self.nlp
# For each table make a Table object
tables = {} # Dictionary for tableName based lookup
tableNames = [] # TableNames
tableMap = {} # Map for quick lookup by UID
tableMap_json = {} # Map for storage on file
numTables = len(tableFileNames)
ticker = utils.Ticker(numTables, tick_rate=1, message=f"Annotating {numTables} tables in tablestore...")
for tableFileName in tableFileNames:
ticker.tick()
if tableFileName[-4:] == ".tsv":
tableFile_addr = f"{tablestoreTablesDir}/{tableFileName}"
tablename = tableFileName[:-4]
# Write down tablename
tableNames.append(tablename)
# read the table file wth pandas
try:
with open(tableFile_addr) as fp:
lines = fp.read().split("\n")
header = lines[0].split("\t")
header = header[:-1] # fix extra tab in the table files
if header[-1] == "[SKIP] UID":
# Sanitized header
headerSanitized = []
for head in header:
head_sanitized = re.sub(r"[^A-Za-z0-9]", " ", head)
head_sanitized = head_sanitized.strip()
head_sanitized = re.sub(r" +", " ", head_sanitized)
head_sanitized = re.sub(r" ", "_", head_sanitized)
headerSanitized.append(head_sanitized)
tableRows = []
lines = lines[1:-1] # newline at end of file
for line in lines:
cells = line.split("\t")[:-1]
# UUID
uuid = cells[-1]
# Convert the cells into a sentence (skip the skip)
cells_clean = []
for cell, head in zip(cells, header):
if cell != "" and not head.startswith("[SKIP]"):
cell_sanitized = re.sub(r";", " ", cell)
cell_sanitized = cell_sanitized.strip()
cell_sanitized = re.sub(r" +", " ", cell_sanitized)
cells_clean.append(cell_sanitized)
sentence = " ".join(cells_clean)
tokens = nlp(sentence) #spacy
# tokens = nlp(sentence).sentences[0].words #coreNLP
# Track the token idx to sync the tokens into the 3d cell/alt/word structure
token_idx = 0
# Populate the 3d structures
cellWords = []
cellWordsLowerCase = []
cellLemmas = []
cellTags = []
for cell, head in zip(cells, header):
cell = re.sub(r" +", " ", cell) # remove double space typos
isSkipCol = head.startswith("[SKIP]") # Flag for when its skip column
altWords = []
altWordsLowerCase = []
altLemmas = []
altTags = []
if cell != '':
alts = cell.split(";")
alts = [alt.strip() for alt in alts]
for alt in alts:
word_tokens = nlp(alt) #spacy
# word_tokens = nlp(alt).sentences[0].words #coreNLP
words = []
wordsLowerCase = []
lemmas = []
tags = []
if not isSkipCol:
for word_token in word_tokens:
# Spacy
words.append(tokens[token_idx].text)
wordsLowerCase.append(tokens[token_idx].lower_)
lemmas.append(tokens[token_idx].lemma_.lower())
tags.append(tokens[token_idx].tag_)
token_idx += 1
altWords.append(words)
altWordsLowerCase.append(wordsLowerCase)
altLemmas.append(lemmas)
altTags.append(tags)
# Append an empty alt here
else:
altWords.append([])
altWordsLowerCase.append([])
altLemmas.append([])
altTags.append([])
cellWords.append(altWords)
cellWordsLowerCase.append(altWordsLowerCase)
cellLemmas.append(altLemmas)
cellTags.append(altTags)
row_obj = {
"uuid": uuid,
"tablename": tablename,
"header": header,
"headerSanitized": headerSanitized,
"cells": cells,
"cellWords": cellWords,
"cellWordsLowerCase": cellWordsLowerCase,
"cellLemmas": cellLemmas,
"cellTags": cellTags,
}
tableRow = TableRow(row_obj, self)
tableRows.append(row_obj)
tableMap[uuid] = tableRow
table = Table(tablename, tableRows, self)
tableMap_json[tablename] = tableRows
# Safty check (dont keep empty tables)
if table.isNonEmpty:
tables[tablename] = table
else:
tableNames.remove(tablename)
else:
raise IOError
except IOError:
print('\n', end='\r', flush=True)
print(f"\tWarning! Cannot parse table file \"{tableFile_addr}\".")
ticker.end()
print(f"\tCaching annotated tables to \"{tablestore_cached_addr}\".")
with open(tablestore_cached_addr, mode='w') as fp:
fp.write(json_dumps(tableMap_json))
return tables, tableNames, tableMap
else:
print(f"\tLoading cached annotated table from \"{tablestore_cached_addr}\".")
if os.path.exists(tablestore_cached_addr):
with open(tablestore_cached_addr, mode='r') as fp:
# Parse the JSON file
tablestoreJSON = json.load(fp)
# The keys will be each tableName
tableNames = tablestoreJSON.keys()
# For each table make a Table object
tables = {} # Dictionary for tableName based lookup
tableMap = {} # Map for quick lookup by UID
for tableName in tableNames:
rows = tablestoreJSON[tableName]
table = Table(tableName, rows, self)
# Safty check (dont keep empty tables)
if table.isNonEmpty:
tables[tableName] = table
# Add the rows to the tableMap
for row in table.rows:
uuid = row.uuid
tableMap[uuid] = row
else:
tableNames.remove(tableName)
return tables, tableNames, tableMap
else:
print(f"\tERROR! Failed to load tablestore from JSON \"{tablestore_cached_addr}\"")
return False, False, False
