def test_map(self):
for test_files in self.test_files_list:
with open(os.path.join(TEST_FILES_PATH, test_files[0])) as infile:
intervention = json.load(infile)
with open(os.path.join(TEST_FILES_PATH, test_files[1])) as infile:
alignment = json.load(infile)
with open(os.path.join(TEST_FILES_PATH, test_files[2])) as infile:
m_alignment = json.load(infile)
map_test = Map(list(intervention.values())[0], alignment)
# test check
map_test.check()
# test find speaker
map_test.find_speaker()
self.assertEqual(test_files[3], map_test.target_speaker)
# test enrich alignment
map_test.enrich_alignment()
target_in_list = False
for al in map_test.alignment:
if al.get('target_speaker'):
target_in_list = True
self.assertTrue(target_in_list)
# test alignment with original words
map_test.align()
with open(os.path.join(TMP_PATH, test_files[2]), 'w') as out:
json.dump(map_test.alignment, out, indent=2)
self.assertEqual(map_test.alignment, m_alignment)
def load_config(env):
global config
if config is not None:
return config
config = Map()
_merge_configs(config, ["default", env])
return config
def main(log_dir):
args = Map(json.load(open(os.path.join(log_dir, 'params.json'), 'r')))
plt_dir = os.path.join(log_dir, 'plt')
os.makedirs(plt_dir, exist_ok=True)
try:
plot_unsupervised(args, log_dir, plt_dir)
except FileNotFoundError:
pass
plot_supervised(args, log_dir, plt_dir)
make_html(log_dir)
def plot_unsupervised(args, log_dir, plt_dir):
history_filename = os.path.join(log_dir, 'history.pkl')
history = pickle.load(open(history_filename, 'rb'))
args = Map(json.load(open(os.path.join(log_dir, 'params.json'), 'r')))
if 'point2d' in log_dir:
plot_tasks_all(args, history, plt_dir, skip_if_exists=True)
plot_coverage_all(args, history, plt_dir)
elif 'half-cheetah' in log_dir or 'ant' in log_dir:
pass
else:
raise ValueError
plt.close('all')
return
def __init__(self, log_dir, sub_dir='vis'):
self.args = Map(
json.load(open(os.path.join(log_dir, 'params.json'), 'r')))
if self.args.interface is None:
print('args.interface not set, default rl2')
self.args.interface = 'rl2'
if self.args.interface == 'contextual':
self.envs = ContextualEnvInterface(self.args, mode='val')
elif self.args.interface == 'rl2':
self.envs = RL2EnvInterface(self.args, mode='val')
self.tasks = self.envs.rewarder.get_assess_tasks()
self.sub_dir = sub_dir
os.makedirs(os.path.join(self.args.log_dir, self.sub_dir),
exist_ok=True)
from pyvirtualdisplay import Display
display = Display(visible=False, size=(256, 256))
display.start()
_ = self.envs.envs.get_images()
def __init__(self, log_dir, sub_dir='vis'):
self.args = Map(
json.load(open(os.path.join(log_dir, 'params.json'), 'r')))
# set seeds
torch.manual_seed(self.args.seed)
torch.cuda.manual_seed_all(self.args.seed)
np.random.seed(self.args.seed)
if self.args.interface is None:
print('args.interface not set, default rl2')
self.args.interface = 'rl2'
if self.args.interface == 'contextual':
self.envs = ContextualEnvInterface(self.args, mode='val')
elif self.args.interface == 'rl2':
self.envs = RL2EnvInterface(self.args, mode='val')
self.tasks = self.envs.rewarder.get_assess_tasks()
self.sub_dir = sub_dir
os.makedirs(os.path.join(self.args.log_dir, self.sub_dir),
exist_ok=True)
def login():
id = request.values['_id']
if id.startswith('16DRL_'):
if id not in PLAYER_DB['player']:
abort(404)
else:
# Init Map
map = Map(13)
player = Player(id)
map.reset(player)
showMap = map.show_map(player)
level = map.show_total_map(player)
# Save Map
save(player, map)
id = player.hashID
return redirect("/map/" + id, code=303)
from utils.map import Map
acadonline = Map({
"home":
"https://sistemas.uepg.br/academicoonline/login/index",
"auth":
"https://sistemas.uepg.br/academicoonline/login/authenticate",
"grades":
"https://sistemas.uepg.br/academicoonline/avaliacaoDesempenho/index",
"password":
"******",
"perfil_get":
"https://sistemas.uepg.br/academicoonline/academico_pessoa/show",
"perfil_set":
"https://sistemas.uepg.br/academicoonline/academico_pessoa/index",
"activities":
"https://sistemas.uepg.br/academicoonline/atividadeComplementar/index",
"remember_password":
"******",
})
pergamum = Map({
"home":
"https://sistemas.uepg.br/pergamum/biblioteca/index.php",
"auth":
"https://sistemas.uepg.br/pergamum///biblioteca/index.php?rs=ajax_valida_acesso_novo&rst=&rsargs[]={login}&rsargs[]={password}",
"renew":
"https://sistemas.uepg.br/pergamum///biblioteca_s/meu_pergamum/index.php?rs=ajax_renova&rsrnd=1508281073663&rsargs[]={book}&rsargs[]=4&rsargs[]=4&rsargs[]=15411",
"search":
"https://sistemas.uepg.br/pergamum///biblioteca/index.php",
"validate":
(14, 13),
(14, 12),
(14, 11),
(14, 10),
(14, 9),
(14, 8),
(15, 14),
(14, 14),
(13, 14),
(12, 14),
(11, 14),
(10, 14),
(9, 14),
(8, 14),
]
Map_obj = Map(height=25, width=25, barrier=barrier)
# Create an instance of A* search algorithm
Dijkstra_obj = Dijkstra(map_matrix=Map_obj.map)
# Search the shortest path
start = (0, 0)
stop = (24, 24)
start_time = time()
shortest_path = Dijkstra_obj.search_shortest_path(start=start, stop=stop)
finish_time = time()
print("Number of point:", len(shortest_path))
print("Runtime: %.3f [s]" % (finish_time - start_time))
def process_pipeline(intervention, outdir):
text = ' '.join([text for sp, text in intervention['text']])
# assumes there is a single audio uri
audiofile = intervention['urls'][0][1]
logging.info('* %s' % audiofile)
if not text:
msg = '%s has empty text' % audiofile
logging.error(msg)
return []
if not intervention.get('words'):
# create lm and convert audio
align = Align(audiofile, text, DICT_PATH)
if not align.results_exist():
# if tmp exists uses the decode results from there
align.create_textcorpus()
align.create_lm()
align.convert_audio()
# run pocketsphinx
cs = CMU(MODEL_PATH)
logging.info('decoding for long alignment')
try:
cs.decode(align.audio_raw, align.lm)
except Exception as e:
msg = '%s decoding failed' % audiofile
logging.error(msg)
logging.error(str(e))
return []
segs = cs.segs
# TODO call decode functions in Align object
decode_align = Text(align.sentences, segs, align.align_outfile)
decode_align.align()
alignment = decode_align.align_results
else:
with open(align.align_outfile) as res_file:
alignment = json.load(res_file)
# get punctuation and speaker information
try:
m = Map(intervention, alignment)
m.prepare()
intervention['words'] = m.alignment
except Exception as e:
msg = 'mapping alignment failed for %s' % audiofile
logging.error(msg)
logging.error(str(e))
# unit segments from silences are combined into optimal segments btw 5-19 s
# exception handling needed since multiple block per speaker not implemented
# if the speaker does not speak most of his/her text in the first block it
# is possible to end up with 0 segments
try:
segmenter = get_optimal_segments(intervention, intervention['words'])
except Exception as e:
msg = 'segmentation not possible for %s' % audiofile
logging.error(msg)
logging.error(str(e))
return []
# segment audiofile
segmenter.segment_audio(audiofile, base_path=outdir)
# grammar evaluate each segment
geval = GEval(segmenter.best_segments, MODEL_PATH)
geval.evaluate()
# clean
baseaudio = os.path.basename(audiofile)
outpath = os.path.join(outdir, baseaudio[0], baseaudio[1], baseaudio[:-4])
subprocess.call('rm {0}*.jsgf {0}*.raw'.format(outpath), shell=True)
return segmenter.best_segments
def __init__(self, raw):
data = raw[4:].split('|')
map_id = data[0]
map_date = data[1]
map_key = data[2]
self.map = Map(map_id, map_date, map_key)
def test_segmenter(self):
for test_files in self.test_files_list:
test_file = os.path.join(TEST_FILES_PATH, test_files[0])
alignment_file = os.path.join(TEST_FILES_PATH, test_files[1])
comparison_mapped_alignment_file = os.path.join(
TEST_FILES_PATH, test_files[2])
with open(comparison_mapped_alignment_file) as infile:
self.comparison_mapped_alignment = json.load(infile)
self.base_segments_file = os.path.join(TEST_FILES_PATH,
test_files[3])
tmp_segments_file = os.path.join(
TMP_PATH, test_files[3].replace('_best', ''))
tmp_best_segments_file = os.path.join(TMP_PATH, test_files[3])
# get beginning and end of the target speaker segments
for token in self.comparison_mapped_alignment:
if token.get("target_speaker") and token.get('start'):
self.target_start = token['start']
break
for token in self.comparison_mapped_alignment[::-1]:
if token.get("target_speaker") and token.get('end'):
self.target_end = token['end']
break
with open(test_file) as infile:
intervention_full = json.load(infile)
for key, value in intervention_full.items():
self.intervention = value
with open(alignment_file) as infile:
self.alignment = json.load(infile)
# to get optimal segments
# get punctuation and speaker information
m = Map(self.intervention, self.alignment)
m.prepare()
# check map enrichment
#with open(comparison_mapped_alignment_file, 'w') as out:
# json.dump(m.alignment, out, indent=2)
self.assertEqual(m.alignment, self.comparison_mapped_alignment)
# initialize segmenter
segmenter = Segmenter(m.alignment)
# test shorten_segment
if test_files[1] == 'c3d9d2a15a76a9fbb591_align.json':
for long_segment in self.long_segments:
new_segments = segmenter.shorten_segment(long_segment)
max_new_length = max([nsegment['end']-nsegment['start']\
for nsegment in new_segments])
# check if it works
self.assertTrue(max_new_length <= segmenter.t_max,
msg='shorten segment not working\n%s' %
str(new_segments))
# check tokens are lost in the process
total_duration = sum([n['end']-n['start']\
for n in new_segments])
reference_duration = long_segment['end'] -\
long_segment['start']
self.assertEqual(' '.join(
[n['words'] for n in new_segments]),
long_segment['words'],
msg="shorten_segment loses tokens")
# check if there are many single tokens
singles = len([1 for n in new_segments\
if len(n['words'].split()) == 1])
self.assertTrue(singles < 2)
# get segments using silences
segmenter.get_segments()
# check if results are consistent with the
self.assertEqual(segmenter.segments[0]['start'], self.target_start)
self.assertEqual(segmenter.segments[-1]['end'], self.target_end,
msg='for %s last segment ends before the '\
'intervention probably bsc of multiple '\
'blocks of the speaker'%test_files[0])
# get segments before optimization
with open(tmp_segments_file, 'w') as out:
json.dump(segmenter.segments, out, indent=2)
max_unoptimized_length = max([segment['end']-segment['start']\
for segment in segmenter.segments])
self.assertTrue(max_unoptimized_length <= segmenter.t_max,
msg="for %s longest segment is longer than t_max:"\
" %2.1f vs %2.1f"%(tmp_segments_file,
max_unoptimized_length,
segmenter.t_max))
# assert that segments have original_words
self.assertTrue(
segmenter.segments[0].get('original_words') != None)
# optimize segments
segmenter.optimize()
with open(tmp_best_segments_file, 'w') as out:
json.dump(segmenter.best_segments, out, indent=2)
# check if optimized segments are > 94% of the whole duration
target_fraction = test_files[4]
segment_duration = 0
for segment in segmenter.best_segments:
segment_duration += (segment['end'] - segment['start'])
total_duration = self.target_end - self.target_start # not exactly
segment_fraction = segment_duration / total_duration
self.assertTrue(segment_fraction > target_fraction,
msg = 'total optimized segment duration is smaller than'\
' %1.2f: %1.2f for %s'%(target_fraction,
segment_fraction,
test_files[3]))
# check if there are unreasonably long segments
max_unoptimized_length = max([segment['end']-segment['start']\
for segment in segmenter.segments])
max_length = max([segment['end']-segment['start']\
for segment in segmenter.best_segments])
reference_length = max(30, max_unoptimized_length)
self.assertTrue(max_length < reference_length,
msg="unreasonably long optimized segments are "\
"present in %s with %f seconds wrt %f"%(test_files[3],
max_length,
reference_length))
from utils.map import Map
bb = Map()
def get_spider_type():
return bb.config._args["type"]
def get_spider_name():
return bb.config._args["name"]
def get_spider_imei():
return f"{get_spider_type()}-{get_spider_name()}"
def get_kwai_session_filepath():
return f"kwai/{get_spider_imei()}.session"
def get_xinkuai_cookies_filepath():
return "xinkuai/cookies.txt"