def main():
[known_args, other_args] = do_argparse()
logger = Logger(K_LOG_FILE).get_logger()
logger.info("Logger is inited")
logger.info("known cmd args: {}".format(known_args))
logger.info("other cmd args: {}".format(other_args))
env = Env()
env['logger'] = logger
# adding required/mandatory variables
env['solution_dir'] = os.path.realpath(K_SOLUTION_DIR)
env['sl_config_dir'] = os.path.realpath(K_ENV_CONFIG_DIR)
env['cmd_known_args'] = known_args
env['cmd_other_args'] = other_args
env.append_env("{}/slang.env.json".format(env['sl_config_dir']))
try:
PluginLoader(env, "OpenProject").process()
PluginLoader(env, "LoadProjectEnv").process()
env.print_env()
logger.info("==> definition_lang = {}".format(env["definition_lang"]))
do_test(env)
except Exception as e:
logger.error("test, exception error: {}".format(e))
logger.error(traceback.format_exc())
except:
logger.error("test, Unexpected error")
logger.error(traceback.format_exc())
def get_next_note(self):
self.k_ratio = 1.0
items_count = 5
q_res = PluginLoader(
self.env, self.env["sl_db_query"]).process(queries_list=[{
"query":
self.env["sql_select_expired_notes"].format(count=items_count),
"data": ()
}])
# there are no expired notes, loading new ones
if not len(q_res[0]):
sql = self.env["sql_select_new_notes"].format(count=items_count)
q_res = PluginLoader(
self.env,
self.env["sl_db_query"]).process(queries_list=[{
"query": sql,
"data": ()
}])
for r in q_res[0]:
note = self.__get_note_from_result(r)
if note:
self.__current_note = note
return note
raise Exception("Can't select new note for this test")
def __create_subtitle_file(self, track_map_id):
self.logger.info("extracting subts track id: {}".format(track_map_id))
active_dir = os.path.dirname(self.env["last_played_file"])
in_file_name = os.path.basename(self.env["last_played_file"])
out_file = "{}/{}_subtitle_{}.srt".format(
active_dir, in_file_name, track_map_id.replace(':', '_'))
status = PluginLoader(self.env, "CmdRun").process(
run_file=self.env["video_file_commands"]
, cmd_name="command_extract_subtitles"
, in_file=self.env["last_played_file"]
, track_id=track_map_id
, out_file=out_file
, active_dir=active_dir)
if not status:
self.logger.warning("Can't extract subtitles track.")
return None
PluginLoader(self.env, "PreprocessSubtitleFile").process(
subt_file=out_file)
return out_file
def _show_stats(self):
sql_query_item = {}
sql_query_item["query"] = self.env["sql_select_speed_test_history"]
sql_query_item["data"] = ()
self.env["sql_queries_list"] = [sql_query_item]
q_res = PluginLoader(self.env, self.env["sl_db_query"]).process(
queries_list=[{
"query": self.env["sql_select_speed_test_history"],
"data": ()
}])
self.__stat_items = []
for i in q_res[0]:
item = {}
item["id"] = i[0]
item["start_time"] = i[1]
item["end_time"] = i[2]
item["notes_count"] = i[3]
item["symbols_count"] = i[4]
item["speed_wpm"] = i[5]
item["speed_spm"] = i[6]
item["err_words_count"] = i[7]
self.__stat_items.append(item)
if len(self.__stat_items):
li = self.__stat_items[-1] # last itew for show statistic
self._show_pwerv_stats(li["speed_spm"], li["speed_wpm"],
li["err_words_count"],
li["end_time"] - li["start_time"],
li["notes_count"], li["symbols_count"])
else:
self._show_pwerv_stats(0, 0, 0, 0, 0, 0)
def do_test(env):
logger = Logger(K_LOG_FILE).get_logger()
# env.print_env()
# subt_file_src = "{}/subt_src.srt".format(env["prj_temp_dir"])
# subt_file = "{}/subt.srt".format(env["prj_temp_dir"])
# if os.path.isfile(subt_file):
# os.remove(subt_file)
# shutil.copyfile(subt_file_src, subt_file)
# PluginLoader(env, "PreprocessSubtitleFile").process(subt_file=subt_file)
import wx
# env["last_played_file"] = ""
# env.append_env("{}/ui_open.env.json".format(env["prj_config_local_dir"]))
# env.append_env("{}/video.project.env.json".format(
# env["prj_config_local_dir"]))
# app = wx.App(False)
# env.print_env()
# PluginLoader(env, "InitVideoParams").process(
# source="https://youtu.be/KvE9j2vTpSg"
# , directory=env["prj_temp_dir"])
# result = PluginLoader(env, "YouglishLoad").process(
# url="https://youglish.com/getbyid/87917135/hilarious/english")
# logger.info("==> result = {}".format(result))
result = PluginLoader(env, "CheckDependencies").process()
def pre_process(self):
command = PluginLoader(self.env, "CmdRun").process(
run_file=self.env["video_file_commands"]
, cmd_name="command_get_media_file_info"
, media_file=self.env["last_played_file"]
, get_command_only=True)
cmd = subprocess.Popen(
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(out, err) = cmd.communicate()
cmd.wait()
search_text = (out if out else err).decode("utf-8")
def search_and_add(reg_name, text, out_map):
pattern = re.compile(self.env[reg_name], re.MULTILINE)
for m in re.finditer(pattern, search_text):
# self.logger.info("match({})=={}".format(reg_name, m))
out_map[m.group(1)] = m.group(0)
search_and_add("pre_mkv_reg_audios"
, search_text, self.env["audios_map"])
search_and_add("pre_mkv_reg_subtitles"
, search_text, self.env["subtitles_map"])
search_and_add("pre_mkv_reg_videos"
, search_text, self.env["videos_map"])
def __add_db_required_data(self):
if not not os.path.isfile(self.env["sl_slang_database_file"]):
self.logger.info("database {} is absent. See the doc/index.md "\
"### creating/extending database with known words:".format(
self.env["sl_slang_database_file"]))
sql_queries = []
for data_map in self.env["db_insert_or_ignore_data"]:
request = self.env["db_insert_or_ignore"]
table = data_map["table"]
columns = ", ".join("'{0}'".format(w) for w in data_map["columns"])
values = ()
for value in data_map["values"]:
values += value,
values_template = "({})".format(", ".join('?' for v in values))
request = request.format(table=table,
columns=columns,
values=values_template)
sql_queries.append({"query": request, "data": values})
#delete notes with empty term field.
sql_queries.append({
"query": self.env["sql_delete_empty_term_notes"],
"data": ()
})
PluginLoader(self.env,
self.env["sl_db_query"]).process(queries_list=sql_queries)
self.logger.info("preparing database is finished")
def __image_set_bitmap(self, bitmap):
out_bitmap = PluginLoader(self.env, "ImageResize").process(
original_bitmap=self.bitmap_resize(bitmap,
self.image_size.GetValue()),
image_size=self.image_window.GetSize())
self.image.SetBitmap(out_bitmap)
def __show_current_note(self, note):
self.env["current_note_id"] = note["id"]
PluginLoader(self.env, "keystroke/LoadNoteMedia").process()
self.__show_media()
self.definition_note.SetValue(note["definition_note"])
self.definition.SetValue(note["definition"])
self.st.process_entered_text(self.text_prompt, self.text)
def get_left_notes_count(self):
q_res = PluginLoader(self.env, self.env["sl_db_query"]).process(
queries_list=[{
"query": self.env["select_notes_left_count"],
"data": ()
}])
return q_res[0][0][0]
def __process(self):
for package in self.env["packages_list"]:
if not self.__check_package(package):
PluginLoader(self.env, "CmdRun").process(
run_file=self.env["cmd_install_package_file"],
package_name=package)
if not self.__check_package(package):
raise Exception(f"Can't find/install package: {package}")
def _show_media(self):
out_bitmap = PluginLoader(self.env, "ImageResize").process(
original_bitmap=wx.Image(self.env["image_blob_file"]
, wx.BITMAP_TYPE_ANY).ConvertToBitmap() \
if os.path.isfile(self.env["image_blob_file"]) \
else wx.Image(self.env["no_image_jpg"]
, wx.BITMAP_TYPE_ANY).ConvertToBitmap()
, image_size=self.image_window.GetSize() )
self.image.SetBitmap(out_bitmap)
def __build_gui(self):
self.__root_win = wx.Frame(None,
wx.ID_ANY,
pos=(self.env["ui_x_pos"],
self.env["ui_y_pos"]))
self.__root_win.SetTitle("{} ({}-{})".format(
self.env["prj_name"], self.env["term_lang"],
self.env["definition_lang"]))
self.__close_callback = None
self.__root_win.Bind(wx.EVT_CLOSE, self.__on_close)
self.env["win_panel"] = wx.Panel(self.__root_win)
if not "test" in self.env["cmd_other_args"]:
PluginLoader(self.env, "keystroke/StartSpeedGUI").process()
else:
PluginLoader(self.env, "keystroke/StartTestGUI").process()
self.__root_win.SetSize((self.env["ui_x_size"], self.env["ui_y_size"]))
self.__root_win.Show(True)
def load_audio_file(self):
if not "video_audio_file" in self.env \
or not self.env["video_audio_file"]:
return
dst_file = self.env["prj_temp_dir"] + "/temp_audio_file.mp3"
if os.path.isfile(dst_file):
os.remove(dst_file)
PluginLoader(self.env, "CmdRun").process(
run_file=self.env["audio_file_commands"], out_file=dst_file)
self.term_audio.SetValue(dst_file)
def __on_set_definition_focus(self, event):
term_text = self.term.GetValue()
if not self.definition.GetValue() and term_text \
and self.env["autotranslate_term_text"] :
try:
self.env["translate_text_term"] = term_text
PluginLoader(self.env, "TranslateText").process()
self.definition.SetValue(self.env["translate_text_definition"])
except Exception as e:
self.logger.error("translate, status: {}".format(e))
self.definition.SetFocus()
def on_load(self, evt):
dlg = wx.FileDialog(self.tab_panel,
"Choose a video file or youtube link",
os.path.expanduser('~'), "", "*.*", wx.FD_OPEN)
if dlg.ShowModal() == wx.ID_OK:
source = dlg.GetFilename()
directory = dlg.GetDirectory()
# finally destroy the dialog
dlg.Destroy()
# do prep settings here
PluginLoader(self.env, "InitVideoParams").process(source=source,
directory=directory)
self.__start_playing_video()
def __deserialize_flds(self, flds, creation_time, mod_time):
def time_value(t):
value = int(t)
now_time = time.time()
# must be in milliseconds. not epoch
#perfom adjustement to epoch format here
if value > now_time:
value = value / 1000
return value
if not flds:
return None
fields = flds.split("\x1f")
card_item = {}
examples = [{}, {}, {}, {}, {}]
notes = [
card_item, examples[0], examples[1], examples[2], examples[3],
examples[4]
]
i = 0
for itm in notes:
itm["term"] = fields[i]
i += 1
itm["term_note"] = fields[i]
i += 1
itm["term_audio"] = self.__get_sound_name(fields[i])
i += 1
itm["image"] = self.__get_image_name(fields[i])
i += 1
itm["definition"] = fields[i]
i += 1
itm["definition_note"] = fields[i]
i += 1
itm["definition_audio"] = self.__get_sound_name(fields[i])
i += 1
#adding ids and timestamps
itm["creation_time"] = time_value(creation_time)
itm["mod_time"] = time_value(mod_time)
card_item["examples"] = examples
self.env["card_item"] = card_item
PluginLoader(self.env, "PurgeCardItem").process()
return card_item
def match(self, directory_name, source_name):
m = re.match(self.env["reg_youtube_source"], source_name)
if m:
self.logger.info("Processing source with YoutubeLink handler")
self.__vid = m.group(1)
result = PluginLoader(self.env, "CmdRun").process(
run_file=self.env["youtube_load_commands"]
, vid=self.__vid, out_dir=directory_name)
file_name = "{}.mp4".format(self.__vid)
return super().match(directory_name, file_name)
return False
def __run_command(self, run_file, cmd_name, **kwargs):
command = PluginLoader(self.env,
"CmdRun").process(run_file=run_file,
cmd_name=cmd_name,
get_command_only=True,
**kwargs)
output_str = ""
p = subprocess.Popen(command,
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
retval = p.wait()
for line in p.stdout.readlines():
output_str += line.decode("utf-8", 'ignore')
return (retval, output_str.strip())
def __load_media(self):
def write_to_file(data, file_name):
if data:
with open(file_name, 'wb') as file:
file.write(data)
return
if os.path.isfile(file_name):
os.remove(file_name)
query_result = PluginLoader(self.env, self.env["sl_db_query"]).process(
queries_list= [
{ "query" : self.env["sql_select_note_media"]
, "data" : ( self.env["current_note_id"], ) } ] )
for row in query_result[0]:
write_to_file(row[0], self.env["image_blob_file"])
write_to_file(row[1], self.env["term_audio_blob_file"])
write_to_file(row[2], self.env["definigion_audio_blob_file"])
def __on_image_url_change(self, event):
url = self.image_url.GetValue()
dst_file = self.env["prj_temp_dir"] + "/temp_img_file"
result = PluginLoader(self.env,
"LoadImageFromUrl").process(url=url,
dst_file=dst_file)
if isinstance(result, dict):
# extended answer
if result["status"] == True:
self.term.SetValue(result["term_text"])
self.load_image_file(result["image_file"])
if "definition_text" in result:
self.definition.SetValue(result["definition_text"])
else:
self.definition.SetValue("")
self.term_audio.SetValue(result["audio_file"])
elif result:
self.load_image_file(dst_file)
def main():
[known_args, other_args] = do_argparse()
logger = Logger(K_LOG_FILE).get_logger()
logger.info("Logger is inited")
logger.info("known cmd args: {}".format(known_args))
logger.info("other cmd args: {}".format(other_args))
env = Env()
env['logger'] = logger
# adding required/mandatory variables
env['solution_dir'] = K_SOLUTION_DIR
env['sl_project_name'] = known_args.project_name
set_config_dir(env)
env['cmd_known_args'] = known_args
env['cmd_other_args'] = other_args
# adding common slang.env envirionment
env.append_env("{}/slang.env.json".format(env['sl_config_dir']))
env.print_env()
if env["cmd_known_args"].extract_mp3:
PluginLoader(env, "SplitSoundTrack").process()
return
if env["cmd_known_args"].update_known_words_db:
sys.path.insert(1, './plugins/known_words_db')
pepp = PluginEntryPointProcessor(env, "sl_pep_UpdateKnownWordsDb")
pepp.process()
return
if env["cmd_known_args"].keystroke:
sys.path.insert(1, './plugins/keystroke')
pepp = PluginEntryPointProcessor(env, "sl_pep_Keystroke")
pepp.process()
return
if env["cmd_known_args"].create_audio_track:
sys.path.insert(1, './plugins/audio_process')
pepp = PluginEntryPointProcessor(env, "sl_pep_AudioTrackCreator")
pepp.process()
return
def __save_achievement(self):
n = self.__current_note
sql_query_item = {}
sql_query_item["query"] = self.env["sql_update_achievement"]
sql_query_item["data"] = (
'note',
n["id"] # item_id
,
time.time() # last_used_time
,
n["pace_time"],
0 # learning_status_id
,
n["repeat_factor"],
n["pace_factor"],
n["reviews_count"] + 1)
PluginLoader(
self.env,
self.env["sl_db_query"]).process(queries_list=[sql_query_item])
def __get_notes_list(self):
results = PluginLoader(self.env, self._db_name).process(
query="sql_select_notes")["sql_select_notes"]
self.logger.info(f"notes found: {len(results)}")
notes = []
for res in results:
note = {}
note["id"] = res[0]
# last_used_time
# next_use_time
note["pace_time"] = res[3]
note["term"] = res[4]
note["term_audio_id"] = res[5]
note["definition"] = res[6]
note["definition_audio_id"] = res[7]
notes.append(note)
return notes
def process(self, **kwargs):
for plugin_name in self._env["plugins"]:
try:
PluginLoader(self._env, plugin_name).process(**kwargs)
except SlPluginStatus as status:
self.logger.warning(
"plugin '{}' has finished with status: {}".format(
plugin_name, status))
except SlPluginEntryPointStatus as status:
self.logger.warning(
"plugin '{}' has finished with status: {}".format(
plugin_name, status))
break
except SlProgramStatus as status:
self.logger.warning(
"plugin '{}' has finished with status: {}".format(
plugin_name, status))
sys.exit()
def __create_audio_file(self, track_map_id):
out_file = ""
self.logger.info("extracting audio track id: {}".format(track_map_id))
active_dir = os.path.dirname(self.env["last_played_file"])
in_file_name = os.path.basename(self.env["last_played_file"])
out_file = "{}/{}_audio_{}.mp3".format(
active_dir, in_file_name, track_map_id.replace(':', '_'))
status = PluginLoader(self.env, "CmdRun").process(
run_file=self.env["video_file_commands"]
, cmd_name="command_extract_audio_track"
, in_file=self.env["last_played_file"]
, track_id=track_map_id
, out_file=out_file
, active_dir=active_dir)
if not os.path.isfile(out_file):
self.logger.error("Can't extract audio track.")
return None
return out_file
def __save_achievements(self):
requests = []
for n in self.__notes:
self.__rate_note(n)
sql_query_item = {}
sql_query_item["query"] = self.env["sql_update_achievement"]
sql_query_item["data"] = (
'note',
n["id"] # item_id
,
time.time() # last_used_time
,
n["pace_time"],
0 # learning_status_id
,
n["repeat_factor"],
n["pace_factor"],
n["reviews_count"])
requests.append(sql_query_item)
PluginLoader(self.env,
self.env["sl_db_query"]).process(queries_list=requests)
def __insert_card(self, deque_id, card):
if not card or not deque_id:
return
# check the card id
q_res = PluginLoader(self.env, self.env["sl_db_query"]).process(
db_cursor=self.__cursor,
query=self.env["sql_select_existing_notes_by_time"],
data=(card["creation_time"], ))
if len(q_res[0]):
self.logger.warning(
"Note(& expamles) already present in database: "
"card_creation_time={}, term={}".format(
card["creation_time"], card["term"]))
# __update_card(self, deque_id, card)
return
self.__cursor.execute(
self.env["sql_insert_card"].format(deque_id=deque_id))
self.__cursor.execute(
self.env["sql_select_empty_card"].format(deque_id=deque_id))
records = self.__cursor.fetchall()
card_id = records[0][0]
notes = []
note_ids = []
notes.append(card)
notes.extend(card["examples"])
for note in notes:
prefix = self.env["words_db_note_prefix"]
if prefix in note["term_note"] or prefix in note["definition_note"]:
# skip the notes already imported from other deques
continue
# self.logger.debug("words_db adding: {}".format(note["term"]))
# insert blobs to database
term_audio_id = self.__insert_media(note["term_audio"])
image_id = self.__insert_media(note["image"])
definition_audio_id = self.__insert_media(note["definition_audio"])
self.__cursor.execute(
self.env["sql_insert_note"],
(card_id, note["creation_time"], note["mod_time"],
note["term"], note["term_note"], term_audio_id, image_id,
note["definition"], note["definition_note"],
definition_audio_id))
note_ids.append(self.__cursor.lastrowid)
note_len = len(note_ids)
self.__cursor.execute(
self.env["sql_update_card"],
(
deque_id # deque_id
,
note_ids[0] # base_note_id
,
note_ids[1] if note_len > 1 else 'NULL' # ex_1_note_id
,
note_ids[2] if note_len > 2 else 'NULL' # ex_2_note_id
,
note_ids[3] if note_len > 3 else 'NULL' # ex_3_note_id
,
note_ids[4] if note_len > 4 else 'NULL' # ex_4_note_id
,
note_ids[5] if note_len > 5 else 'NULL' # ex_5_note_id
,
card_id)) # card_id
def on_link_load(self, evt):
PluginLoader(self.env, "InitVideoParams").process(
source=evt, directory=self.env["sl_temp_dir"])
self.__start_playing_video()
def process_entered_text(self, text_prompt, text):
res = PluginLoader(self.env, self.__get_checker_plugin_name()).process(
source_text=self.get_source_text(),
prompt_edit=text_prompt,
text_edit=text)
return res