def base(d, player):
code, tag = d.menu("Vous etes au milieu du centre ville. il y a beaucoup de monde",
choices=[("(1)", "S'asseoir par terre"),
("(2)", "Aller dans le quartier commerçant"),
("(3)", "Aller au poste de police"),
("(4)", "Aller au chateau"),
("(5)", "Retourner à l'entrée de la ville")
],
ok_label="OK, je veux faire ca",
cancel_label="Préférances/Quitter")
if code == d.OK:
if tag == "(1)":
progress(d, 60, "Un policier vous prend pour un mendiant et vous envoie en prison", player)
database.players.changePref(player, "location", "posteDePolice")
elif tag == "(2)":
progress(d, 15, "Vous allez au quartier commerçant", player)
database.players.changePref(player, "location", "quartierCommercant")
return True
elif tag == "(3)":
progress(d, 15, "Vous allez au poste de police", player)
database.players.changePref(player, "location", "posteDePolice")
return True
elif tag == "(4)":
progress(d, 15, "Vous allez au chateau", player)
database.players.changePref(player, "location", "chateau")
return True
elif tag == "(5)":
progress(d, 15, "Vous allez à l'entrée de la ville", player)
database.players.changePref(player, "location", "villeDeNullepart")
return True
else:
return "pref"
def overlapping_cases(data: pd.DataFrame, instance: int, range: list,
out_q: Queue):
"""
This function is called by multiprocess_overlapping_cases(). It computes
the number of parallel executed case instances in a given subset.
:param data: case log
:param instance: current instace
:param range: subset
:param out_q: output queue
"""
# total number of cases
size = len(data)
# collect results
r = {}
for idx, row in data[range[0]:range[1]].iterrows():
exclude_cases = data.loc[(data['start'] > row['end']) |
(data['end'] < row['start'])]
counter = size - len(exclude_cases)
r[idx] = counter
# log progress
progress(len(data[range[0]:range[1]]), len(r),
'overlapping cases - instance: %s' % instance)
# collect final results
out_q.put(r)
def import_git_objects(self, remote_name=None, refs=None):
convert_list, commits = self.getnewgitcommits(refs)
# import each of the commits, oldest first
total = len(commits)
if total:
self.ui.status(_("importing git objects into hg\n"))
for i, csha in enumerate(commits):
util.progress(self.ui, 'importing', i, total=total, unit='commits')
commit = convert_list[csha]
self.import_git_commit(commit)
util.progress(self.ui, 'importing', None, total=total, unit='commits')
# Remove any dangling tag references.
for name, rev in self.repo.tags().items():
if not rev in self.repo:
if hasattr(self, 'tagscache') and self.tagscache and \
'name' in self.tagscache:
# Mercurial 1.4 and earlier.
del self.repo.tagscache[name]
elif hasattr(self, '_tags') and self._tags and \
'name' in self._tags:
# Mercurial 1.5 and later.
del self.repo._tags[name]
if name in self.repo._tagtypes:
del self.repo._tagtypes[name]
def reduce_leaf(net_set):
global dull_reduces
try:
fuses = extract_fuses(net_set)
except toolchain.FitterError:
return False
except subprocess.CalledProcessError as err:
if err.returncode == 245:
return False
raise
found = find_muxes(net_set, fuses)
if found == 0:
return False
elif found == 1:
return True
else: # found > 1
progress(1)
for net_name in det_random.sample(
sorted(net_set), len(net_set)):
if dull_reduces > 10:
break
if reduce_leaf(net_set.difference({net_name})):
found = find_muxes(net_set, fuses)
if found <= 1:
break
else:
dull_reduces += 1
return True
def base(d, player):
code, tag = d.menu("Vous etes chez le marchand de bombes.",
choices=[("(1)", "Achetez un bombes"),
("(2)", "Sortir du magasin"),
("(3)", "Afficher mon inventaire")],
ok_label="Ok, je veux faire ca",
cancel_label="Préférances/Quitter")
if code == d.OK:
if tag == "(1)":
if database.inventory.addToInventory(player, "bombe", 3):
database.inventory.addToInventory(player, "or", -40)
d.msgbox("Vous achetez 3 bombe, faites attention avec")
return True
else:
d.msgbox("Vous n'avez pas assez d'or")
return True
elif tag == "(2)":
progress(d, 15, "Vous sortez du magasin.", player)
database.players.changePref(player, "location", "quartierCommercant")
return True
elif tag == "(3)":
showInventory(d, player)
return True
else:
return "pref"
def update_weights(self,X):
sys.stderr.write("\tUpdating weights...")
LAMBDA = 0.01
for (i,artist) in enumerate(self.artists):
progress(i)
if len(artist.correct_similar) == 0:
continue
# this artist's features and current x vector
features = self.m_features[artist._id]
current_x = X[artist._id]
# calculate the average x vector for similar artists
avg_simil = np.average([X[simil_id] for simil_id in artist.correct_similar],axis=0)
# determine ideal weights for this artist
better_weights = self.np_divide(avg_simil,features)
# and adjust self.weights by LAMBDA * (better_weights - self.weights)
diff = np.subtract(better_weights,self.weights)
adjust_by = np.multiply(diff,LAMBDA)
self.weights = np.add(self.weights, adjust_by)
sys.stderr.write("\n")
def find_neighbors(self,X):
sys.stderr.write("\tFinding nearest neighbors...")
for i in xrange(len(X)):
progress(i)
(_,ind) = self.neigh.kneighbors(X[i].reshape(1,-1))
self.artists[i].predicted_similar = ind[0]
sys.stderr.write("\n")
def import_git_objects(self, remote_name=None, refs=None):
convert_list, commits = self.getnewgitcommits(refs)
# import each of the commits, oldest first
total = len(commits)
if total:
self.ui.status(_("importing git objects into hg\n"))
for i, csha in enumerate(commits):
util.progress(self.ui, 'importing', i, total=total, unit='commits')
commit = convert_list[csha]
self.import_git_commit(commit)
util.progress(self.ui, 'importing', None, total=total, unit='commits')
# Remove any dangling tag references.
for name, rev in self.repo.tags().items():
if not rev in self.repo:
if hasattr(self, 'tagscache') and self.tagscache and \
'name' in self.tagscache:
# Mercurial 1.4 and earlier.
del self.repo.tagscache[name]
elif hasattr(self, '_tags') and self._tags and \
'name' in self._tags:
# Mercurial 1.5 and later.
del self.repo._tags[name]
if name in self.repo._tagtypes:
del self.repo._tagtypes[name]
def do(label, lgsq, lgeq):
with open(jar(label), "rb") as brine:
ps = pickle.load(brine)
progress(label, "starting s/{}/e/{}".format(lgsq, lgeq))
f = tail_probability(ps.one_shot_s_quantile(lgsq), ps.threshold())
p(label, "s/{}/e/{}/cost".format(lgsq, lgeq), float(-log2(f)))
progress(label, "done s/{}/e/{}".format(lgsq, lgeq))
def dfs(pinyin='') -> List[Dict[str, str]]:
'Load recursively when the API limit is exceeded.'
progress()
results = stations(pinyin)
if len(results) < 100:
return results
else:
return sum((dfs(pinyin + c) for c in alphabet), [])
def dfs(name='') -> OrderedDict:
'Split bulk requests into chunks.'
results = tmis(name)
if len(results) == 50:
for i in range(1, 19):
progress()
results.update(tmis(name, i))
return results
def main():
global st
# read arguments "% ./create_lms [document dir] [output_dir]"
if len(sys.argv) != 3:
send_stdout("Usage: python3 {} [document_dir] [output_dir]".format(
sys.argv[0]))
return
# get filenames from the [document dir]
try:
DOC_DIR = sys.argv[1]
docs = [f for f in listdir(DOC_DIR) if isfile(join(DOC_DIR, f))]
except FileNotFoundError as e:
send_stdout('Error! No such file or directory "{}".'.format(DOC_DIR))
return
# check whether the index file already exist in the [output_dir]
LM_FILE = join(sys.argv[2], LM_NAME)
if isfile(LM_FILE):
send_stdout('Error! LM file "{}" already exist.'.format(LM_FILE))
return
# initialize stemmer (Lemmatizer)
if STEMMER:
st = Token_Preprocessing_Engine()
skipped_docs = []
invalid_filename_docs = []
f_num = len(docs)
for i in range(f_num):
fname = docs[i]
success, docID = filename_validation(fname)
if not success:
invalid_filename_docs.append(fname)
continue
try:
# read file, and create language models (calculate MLE)
read_file(join(DOC_DIR, fname), docID)
except Exception as e:
skipped_docs.append(fname)
continue
# update progress bar
progress(i + 1, f_num)
send_stdout()
# show invalid document name/format to stdout
if len(invalid_filename_docs) != 0:
send_stdout('Warning! Invalid document name format:')
send_stdout('{}, Skipped.'.format(invalid_filename_docs))
if len(skipped_docs) != 0:
send_stdout('Warning! Cannot process the following doc(s):')
send_stdout('{}, Skipped.'.format(skipped_docs))
# write index to file
f_out = open(LM_FILE, 'w')
f_out.write(str(LM_LMS))
f_out.close()
def tmis(name='', bureau=0) -> OrderedDict:
url = 'http://hyfw.12306.cn/hyinfo/action/FwcszsAction_getljcz'
params = 'limit timestamp sheng shi'
params = {k: '' for k in params.split()}
params.update(q=name, ljdm=format(bureau, '02'))
while True:
try:
response = requests.post(url, params, timeout=1).json()
except (requests.exceptions.Timeout, json.JSONDecodeError):
progress('X')
else:
break
return OrderedDict((d['HZZM'], d['TMISM']) for d in response)
def main():
global st
# read arguments
if len(sys.argv) != 2:
send_stdout("format: python {} [dir]".format(sys.argv[0]))
return
# get filenames from the [dir]
try:
path = sys.argv[1]
files = [f for f in listdir(path) if isfile(join(path, f))]
except FileNotFoundError as e:
send_stdout('Error! No such file or directory "{}".'.format(path))
return
# check whether the index file already exist
if isfile(INDEX_FILE):
send_stdout('Error! Index file "{}" already exist.'.format(INDEX_FILE))
return
# initialize stemmer (Lemmatizer)
if STEMMER:
st = Token_Preprocessing_Engine()
skipped_files = []
f_num = len(files)
for i in range(f_num):
fname = files[i]
finfo = fname.split(sep='_', maxsplit=2)
# filename validation
if finfo[0] != 'doc':
skipped_files.append(fname)
continue
try:
# read file, and create indexes
read_file(join(path, fname), int(finfo[1]))
except Exception as e:
skipped_files.append(fname)
continue
# update progress bar
progress(i + 1, f_num)
send_stdout()
if len(skipped_files) != 0:
send_stdout('Warning! Cannot index the following file(s):')
send_stdout('{}, Skipped.'.format(skipped_files))
# write index to file
f_out = open(INDEX_FILE, 'w')
for term in sorted(positional_index.keys()):
f_out.write('{term} {index}\n'.format(term=term,
index=positional_index[term]))
f_out.close()
def base(d, player):
code, tag = d.menu("Vous etes dans les quartiers pauvres de la ville.",
choices=[("(1)", "Aller dans les montagnes"),
("(2)", "Aller dans la taverne."),
("(3)", "Affronter des bandits(systeme de combat en cours de création)."),
("(4)", "Retourner dans le quartier commerçant."),
("(5)", "Afficher mon inventaire.")],
ok_label="Ok je veux faire ça",
cancel_label="Préférances/Quitter")
if code == d.OK:
if tag == "(1)":
progress(d, 40, "Vous allez dans les montagnes.", player)
database.players.changePref(player, "location", "montagne")
return True
elif tag == "(2)":
progress(d, 15, "Vous rentrez dans la taverne.", player)
database.players.changePref(player, "location", "taverne")
return True
elif tag == "(3)":
progress(d, 20, "Vous vous aventurez dans la ville pour trouver des bandits", player)
database.players.changePref(player, "location", "combatBandit")
return True
elif tag == "(4)":
progress(d, 30, "Vous retournez dans le quartier commerçant.", player)
database.players.changePref(player, "location", "quartierCommercant")
return True
elif tag == "(5)":
showInventory(d, player)
return True
else:
return "pref"
def download_file(file_obj, ftp_root, disk_work_root, show_progress):
download_dir = disk_work_root + '/.download'
file_name = file_obj['Name']
file_ext = os.path.splitext(file_name)[1]
file_folder = file_obj['Folder']
file_disk_dir = disk_work_root + '/' + file_folder
file_disk_path = file_disk_dir + '/' + file_name
file_ftp_path = ftp_root + '/' + file_folder + '/' + file_name
file_download_path = download_dir + '/' + file_folder + '/' + file_name
util.mkdir_p(os.path.dirname(file_download_path))
try:
response = urllib.urlopen(file_ftp_path)
size = int(response.info().getheader('Content-Length').strip())
downloaded_size = 0
chunk = min(size, 8192)
if show_progress:
util.progress(0, 'Downloading ' + file_name + ' ')
else:
print('Downloading ' + file_name + ' ...', end='')
with open(file_download_path, "wb") as local_file:
while True:
data_chunk = response.read(chunk)
if not data_chunk:
break
local_file.write(data_chunk)
downloaded_size += len(data_chunk)
if show_progress:
util.progress(int(100 * downloaded_size / size),
'Downloading ' + file_name + ' ')
print(' Done!')
except HTTPError as e:
raise Exception("Could not download file. [HTTP Error] {code}: {reason}."
.format(code=e.code, reason=e.reason))
except URLError as e:
raise Exception("Could not download file. [URL Error] {reason}."
.format(reason=e.reason))
except Exception as e:
raise e
if file_ext == '.zip':
update_zipfile(file_disk_dir, file_download_path)
else:
update_file(file_disk_path, file_download_path)
util.rmdir_p(download_dir)
def base(d, player):
code, tag = d.menu("Vous etes dans la pharmacie",
choices=[("(1)", "Acheter une potion de soin (15 or)"),
("(2)", "Acheter du chocolay noir (10 or)"),
("(3)", "Regarder les affiches de publicités"),
("(4)", "Retourner dans le quartier"),
("(5)", "Afficher mon inventaire")
],
ok_label="Ok, je veux faire ca",
cancel_label="Préférances/Quitter")
if code == d.OK:
if tag == "(1)":
if database.inventory.addToInventory(player, "or", -15):
database.inventory.addToInventory(player, "potion", 1)
d.msgbox("Vous avez acheté une potion de soin")
return True
else:
d.msgbox("Vous n'avez pas assez d'or.")
elif tag == "(2)":
if database.inventory.addToInventory(player, "or", -15):
database.inventory.addToInventory(player, "chocolat", 1)
d.msgbox("Vous avez acheté du chocolat,")
return True
else:
d.msgbox("Vous n'avez pas assez d'or.")
elif tag == "(3)":
code, tag = d.menu("Vous regardez les affiches",
choices=[("(1)", "Regarder l'affiche de droite"),
("(2)", "Regarder l'affiche de gauche")
],
ok_label="Regarder",
cancel_label="Ne pas regarder les affiches")
if code == d.OK:
if tag == "(1)":
d.msgbox("L'affiche est un publicité pour insectiside, le slogan est \"bugs everywhere\".")
elif tag == "(2)":
d.msgbox(
"L'affiche est celle d'unje société appelée Arperture Science Il y a un grand robot sur l'affiche qui s'appelle GlaDos")
else:
d.msgbox("Vous arrétez de regarder les affiches.")
elif tag == "(4)":
progress(d, 15, "Vous sortez du magasin", player)
database.players.changePref(player, "location", "quartierCommercant")
return True
else:
return "pref"
def _go(self):
# Similarity matrix
similar = zeros((self.N, self.N), float)
for i in range(self.N):
for j in range(self.N):
similar[i][j] = -1
#
# Compute similarity matrix of SW scores
#
progress = 0
for i in range(self.N):
for j in range(self.N):
if progress % (self.N * self.N / 100) == 0:
util.progress(100,
float(progress) / (self.N * self.N) * 100)
progress += 1
if similar[i][j] >= 0:
continue
seq1 = self.sequences[i][1]
seq2 = self.sequences[j][1]
(nseq1, nseq2, edits1, edits2, score, gaps) = \
align.SmithWaterman(seq1, seq2, self.smx, 0, 0)
similar[i][j] = similar[j][i] = score
util.progress(100, 100)
#
# Compute distance matrix of SW scores
#
for i in range(self.N):
for j in range(self.N):
if self.dmx[i][j] >= 0:
continue
#print similar[i][j], " ",similar[i][i]
if similar[i][i] != 0:
self.dmx[i][j] = 1 - (similar[i][j] / similar[i][i])
else:
self.dmx[i][j] = 1
self.dmx[j][i] = self.dmx[i][j]
def fetch_title(link_dir, link, timeout=TIMEOUT):
"""try to guess the page's title from its content"""
# if link already has valid title, skip it
if link['title'] and not link['title'].lower().startswith('http'):
return {'output': link['title'], 'status': 'skipped'}
end = progress(timeout, prefix=' ')
try:
title = fetch_page_title(link['url'], timeout=timeout, progress=False)
end()
output = title
except Exception as e:
end()
print(' {}Failed: {} {}{}'.format(ANSI['red'],
e.__class__.__name__, e,
ANSI['reset']))
output = e
# titles should show up in the global index immediatley for better UX,
# do a hacky immediate replacement to add them in as we're archiving
# TODO: figure out how to do this without gnarly string replacement
if title:
link['title'] = title
patch_index_title_hack(link['url'], title)
return {
'cmd': 'fetch_page_title("{}")'.format(link['url']),
'output': output,
}
def run_training(args):
out_dir = pathlib.Path(args.directory)
sentences = dataset.load(args.source)
if args.epoch is not None:
start = args.epoch + 1
storage = load(out_dir, args.epoch)
sentences = itertools.islice(sentences, start, None)
else:
start = 0
storage = init(args)
if (out_dir / meta_name).exists():
if input('Overwrite? [y/N]: ').strip().lower() != 'y':
exit(1)
with (out_dir / meta_name).open('wb') as f:
np.save(f, [storage])
batchsize = 5000
for i, sentence in enumerate(sentences, start):
if i % batchsize == 0:
print()
serializers.save_npz(str(out_dir / model_name(i)), storage.model)
serializers.save_npz(str(out_dir / optimizer_name(i)),
storage.optimizer)
else:
print(util.progress('batch {}'.format(i // batchsize),
(i % batchsize) / batchsize, 100),
end='')
train(storage.model, storage.optimizer, generate_data(sentence),
generate_label(sentence),
generate_attr(sentence, storage.mappings))
def run_training(args):
out_dir = pathlib.Path(args.directory)
sentences = [line.split() for line in args.source]
if args.epoch is not None:
start = args.epoch + 1
storage = load(out_dir, args.epoch)
sentences = itertools.islice(sentences, start, None)
else:
start = 1
storage = init(args)
if (out_dir / meta_name).exists():
if input('Overwrite? [y/N]: ').strip().lower() != 'y':
exit(1)
with (out_dir / meta_name).open('wb') as f:
np.save(f, [storage])
checkpoint = args.checkpoint
for i, sentence in enumerate(sentences, start):
train(storage.model, storage.optimizer, generate_data(sentence),
generate_label(sentence))
if i % checkpoint != 0:
print(util.progress('checkpoint {}'.format(i // checkpoint),
(i % checkpoint) / checkpoint, 100),
end='')
else:
print()
save_param(out_dir, i, storage)
print()
save_param(out_dir, i, storage)
def fetch_wget(link_dir,
link,
requisites=FETCH_WGET_REQUISITES,
timeout=TIMEOUT):
"""download full site using wget"""
domain_dir = os.path.join(link_dir, link['domain'])
existing_file = wget_output_path(link)
if os.path.exists(domain_dir) and existing_file:
return {'output': existing_file, 'status': 'skipped'}
CMD = [
# WGET CLI Docs: https://www.gnu.org/software/wget/manual/wget.html
*'wget -N -E -np -x -H -k -K -S --restrict-file-names=unix'.split(' '),
*(('-p', ) if FETCH_WGET_REQUISITES else ()),
*(('--user-agent={}'.format(WGET_USER_AGENT), ) if WGET_USER_AGENT else
()),
*((() if CHECK_SSL_VALIDITY else ('--no-check-certificate', ))),
link['url'],
]
end = progress(timeout, prefix=' ')
try:
result = run(CMD,
stdout=PIPE,
stderr=PIPE,
cwd=link_dir,
timeout=timeout + 1) # index.html
end()
output = wget_output_path(link, look_in=domain_dir)
# Check for common failure cases
if result.returncode > 0:
print(' got wget response code {}:'.format(
result.returncode))
if result.returncode != 8:
print('\n'.join(' ' + line
for line in (result.stderr or result.stdout
).decode().rsplit('\n', 10)[-10:]
if line.strip()))
if b'403: Forbidden' in result.stderr:
raise Exception('403 Forbidden (try changing WGET_USER_AGENT)')
if b'404: Not Found' in result.stderr:
raise Exception('404 Not Found')
if b'ERROR 500: Internal Server Error' in result.stderr:
raise Exception('500 Internal Server Error')
if result.returncode == 4:
raise Exception('Failed wget download')
except Exception as e:
end()
print(' Run to see full output:',
'cd {}; {}'.format(link_dir, ' '.join(CMD)))
print(' {}Failed: {} {}{}'.format(ANSI['red'],
e.__class__.__name__, e,
ANSI['reset']))
output = e
return {
'cmd': CMD,
'output': output,
}
def fetch_title(link_dir, link, timeout=TIMEOUT):
"""try to guess the page's title from its content"""
# if link already has valid title, skip it
if link['title'] and not link['title'].lower().startswith('http'):
return {
'output': link['title'],
'cmd': 'fetch_page_title("{}")'.format(link['url'])
}
end = progress(timeout, prefix=' ')
try:
title = fetch_page_title(link['url'], timeout=timeout, progress=False)
end()
output = title
except Exception as e:
end()
print(' {}Failed: {} {}{}'.format(ANSI['red'],
e.__class__.__name__, e,
ANSI['reset']))
output = e
return {
'cmd': 'fetch_page_title("{}")'.format(link['url']),
'output': output,
}
def fetch_favicon(link_dir, link, timeout=TIMEOUT):
"""download site favicon from google's favicon api"""
if os.path.exists(os.path.join(link_dir, 'favicon.ico')):
return {'output': 'favicon.ico', 'status': 'skipped'}
CMD = [
'curl',
'--max-time',
str(timeout),
'https://www.google.com/s2/favicons?domain={domain}'.format(**link),
]
fout = open('{}/favicon.ico'.format(link_dir), 'w')
end = progress(timeout, prefix=' ')
try:
run(CMD, stdout=fout, stderr=DEVNULL, cwd=link_dir,
timeout=timeout) # favicon.ico
fout.close()
end()
chmod_file('favicon.ico', cwd=link_dir)
output = 'favicon.ico'
except Exception as e:
fout.close()
end()
print(' {}Failed: {} {}{}'.format(ANSI['red'],
e.__class__.__name__, e,
ANSI['reset']))
print(' Run to see full output:')
print(' {}'.format(' '.join(CMD)))
output = e
return {
'cmd': CMD,
'output': output,
}
def fetch_pdf(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DIR):
"""print PDF of site to file using chrome --headless"""
if link['type'] in ('PDF', 'image'):
return {'output': wget_output_path(link)}
if os.path.exists(os.path.join(link_dir, 'output.pdf')):
return {'output': 'output.pdf', 'status': 'skipped'}
CMD = [
*chrome_headless(user_data_dir=user_data_dir),
'--print-to-pdf',
'--hide-scrollbars',
'--timeout={}'.format((timeout) * 1000),
*(() if CHECK_SSL_VALIDITY else ('--disable-web-security', '--ignore-certificate-errors')),
link['url']
]
end = progress(timeout, prefix=' ')
try:
result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=link_dir, timeout=timeout)
end()
if result.returncode:
print(' ', (result.stderr or result.stdout).decode())
raise Exception('Failed to print PDF')
chmod_file('output.pdf', cwd=link_dir)
output = 'output.pdf'
except Exception as e:
end()
output = e
print_error_hints(cmd=CMD, pwd=link_dir, err=e)
return {
'cmd': CMD,
'output': output,
}
def fetch_screenshot(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DIR, resolution=RESOLUTION):
"""take screenshot of site using chrome --headless"""
if link['type'] in ('PDF', 'image'):
return {'output': html_appended_url(link)}
if os.path.exists(os.path.join(link_dir, 'screenshot.png')):
return {'output': 'screenshot.png', 'status': 'skipped'}
CMD = [
*chrome_headless(user_data_dir=user_data_dir),
'--screenshot',
'--window-size={}'.format(resolution),
link['url']
]
end = progress(timeout, prefix=' ')
try:
result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=link_dir, timeout=timeout + 1) # sreenshot.png
end()
if result.returncode:
print(' ', (result.stderr or result.stdout).decode())
raise Exception('Failed to take screenshot')
chmod_file('screenshot.png', cwd=link_dir)
output = 'screenshot.png'
except Exception as e:
end()
print(' Run to see full output:', 'cd {}; {}'.format(link_dir, ' '.join(CMD)))
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
output = e
return {
'cmd': CMD,
'output': output,
}
def fetch_pdf(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DIR):
"""print PDF of site to file using chrome --headless"""
if link['type'] in ('PDF', 'image'):
return {'output': wget_output_path(link)}
if os.path.exists(os.path.join(link_dir, 'output.pdf')):
return {'output': 'output.pdf', 'status': 'skipped'}
CMD = [
*chrome_headless(user_data_dir=user_data_dir),
'--print-to-pdf',
link['url']
]
end = progress(timeout, prefix=' ')
try:
result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=link_dir, timeout=timeout + 1) # output.pdf
end()
if result.returncode:
print(' ', (result.stderr or result.stdout).decode())
raise Exception('Failed to print PDF')
chmod_file('output.pdf', cwd=link_dir)
output = 'output.pdf'
except Exception as e:
end()
print(' Run to see full output:', 'cd {}; {}'.format(link_dir, ' '.join(CMD)))
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
output = e
return {
'cmd': CMD,
'output': output,
}
def cmd_erase(self, ui, args):
"""erase flash"""
# check for erase all
if len(args) == 1 and args[0] == '*':
ui.put('erase all: ')
n_errors = self.driver.erase_all()
ui.put('done (%d errors)\n' % n_errors)
return
# memory region erase
x = util.mem_args(ui, args, self.device)
if x is None:
return
(adr, n) = x
if n is None:
ui.put('bad erase length\n')
return
r = mem.region(None, adr, n)
# build a list of regions to be erased
erase_list = [x for x in self.driver.sector_list() if r.overlap(x)]
if len(erase_list) == 0:
ui.put('nothing to erase\n')
return
# do the erase
ui.put('erasing : ')
progress = util.progress(ui, 1, len(erase_list))
n_erased = 0
n_errors = 0
for x in erase_list:
n_errors += self.driver.erase(x)
n_erased += 1
progress.update(n_erased)
progress.erase()
ui.put('done (%d errors)\n' % n_errors)
def fetch_favicon(link_dir, link, timeout=TIMEOUT):
"""download site favicon from google's favicon api"""
output = 'favicon.ico'
if os.path.exists(os.path.join(link_dir, output)):
return {'output': output, 'status': 'skipped'}
CMD = [
CURL_BINARY,
'--max-time',
str(timeout),
'--location',
'--output',
output,
*(() if CHECK_SSL_VALIDITY else ('--insecure', )),
'https://www.google.com/s2/favicons?domain={}'.format(
domain(link['url'])),
]
end = progress(timeout, prefix=' ')
try:
run(CMD, stdout=PIPE, stderr=PIPE, cwd=link_dir, timeout=timeout)
end()
chmod_file('favicon.ico', cwd=link_dir)
output = 'favicon.ico'
except Exception as e:
end()
output = e
print_error_hints(cmd=CMD, pwd=link_dir, err=e)
return {
'cmd': CMD,
'output': output,
}
def _go(self):
# Similarity matrix
similar = zeros((self.N, self.N), float)
for i in range(self.N):
for j in range(self.N):
similar[i][j] = -1
#
# Compute similarity matrix of SW scores
#
progress = 0
for i in range(self.N):
for j in range(self.N):
if progress % (self.N * self.N / 100) == 0:
util.progress(100, float(progress) / (self.N * self.N) * 100)
progress += 1
if similar[i][j] >= 0:
continue
seq1 = self.sequences[i][1]
seq2 = self.sequences[j][1]
(nseq1, nseq2, edits1, edits2, score, gaps) = \
align.SmithWaterman(seq1, seq2, self.smx, 0, 0)
similar[i][j] = similar[j][i] = score
util.progress(100,100)
#
# Compute distance matrix of SW scores
#
for i in range(self.N):
for j in range(self.N):
if self.dmx[i][j] >= 0:
continue
#print similar[i][j], " ",similar[i][i]
if similar[i][i] != 0:
self.dmx[i][j] = 1 - (similar[i][j] / similar[i][i])
else:
self.dmx[i][j] = 1
self.dmx[j][i] = self.dmx[i][j]
def archive_dot_org(link_dir, link, timeout=TIMEOUT):
"""submit site to archive.org for archiving via their service, save returned archive url"""
output = 'archive.org.txt'
archive_org_url = None
path = os.path.join(link_dir, output)
if os.path.exists(path):
archive_org_url = open(path, 'r').read().strip()
return {'output': archive_org_url, 'status': 'skipped'}
submit_url = 'https://web.archive.org/save/{}'.format(link['url'])
CMD = [
CURL_BINARY,
'--location',
'--head',
'--user-agent', 'ArchiveBox/{} (+https://github.com/pirate/ArchiveBox/)'.format(GIT_SHA), # be nice to the Archive.org people and show them where all this ArchiveBox traffic is coming from
'--max-time', str(timeout),
*(() if CHECK_SSL_VALIDITY else ('--insecure',)),
submit_url,
]
end = progress(timeout, prefix=' ')
try:
result = run(CMD, stdout=PIPE, stderr=DEVNULL, cwd=link_dir, timeout=timeout)
end()
content_location, errors = parse_archive_dot_org_response(result.stdout)
if content_location:
archive_org_url = 'https://web.archive.org{}'.format(content_location[0])
elif len(errors) == 1 and 'RobotAccessControlException' in errors[0]:
archive_org_url = None
# raise Exception('Archive.org denied by {}/robots.txt'.format(domain(link['url'])))
elif errors:
raise Exception(', '.join(errors))
else:
raise Exception('Failed to find "content-location" URL header in Archive.org response.')
except Exception as e:
end()
output = e
print_error_hints(cmd=CMD, pwd=link_dir, err=e)
if not isinstance(output, Exception):
# instead of writing None when archive.org rejects the url write the
# url to resubmit it to archive.org. This is so when the user visits
# the URL in person, it will attempt to re-archive it, and it'll show the
# nicer error message explaining why the url was rejected if it fails.
archive_org_url = archive_org_url or submit_url
with open(os.path.join(link_dir, output), 'w', encoding='utf-8') as f:
f.write(archive_org_url)
chmod_file('archive.org.txt', cwd=link_dir)
output = archive_org_url
return {
'cmd': CMD,
'output': output,
}
def setup(label, ps):
progress(label, "starting setup")
try:
with open(jar(label), "rb") as brine:
ps = pickle.load(brine)
except IOError:
progress(label, "generating distributions")
secret_l2_dist = ps.secret_l2_distribution()
query_l2_dist = ps.query_l2_distribution()
one_shot_dist = ps.one_shot_distribution()
f = tail_probability(one_shot_dist, ps.threshold())
p(label, "oneshot", float(-log2(f)))
with open(jar(label), "wb") as brine:
pickle.dump(ps, brine)
def export_git_objects(self):
self.ui.status(_("importing Hg objects into Git\n"))
self.init_if_missing()
nodes = [self.repo.lookup(n) for n in self.repo]
export = [node for node in nodes if not hex(node) in self._map_hg]
total = len(export)
for i, rev in enumerate(export):
util.progress(self.ui, 'importing', i, total=total)
ctx = self.repo.changectx(rev)
state = ctx.extra().get('hg-git', None)
if state == 'octopus':
self.ui.debug("revision %d is a part "
"of octopus explosion\n" % ctx.rev())
continue
self.export_hg_commit(rev)
util.progress(self.ui, 'importing', None, total=total)
def train(epoch, global_step):
optimizer = config.optimizer(steps=global_step, epoch=epoch)
# return {'accuracy': 10.0, 'loss': 10.0}
for batch in range(len(labels) // config.batch_size()):
L = batch * config.batch_size()
progress(epoch, (L * 10.0) / len(labels))
R = L + config.batch_size()
mini_batch_images = images[L:R]
mini_batch_labels = labels[L:R]
with tfe.GradientTape() as tape:
s = tf.concat([
tf.gather_nd(mini_batch_images, gather_map),
tf.zeros(list(np.shape(gather_map)[:2]) + [2])
], -1)
logits = model.flow(s)
input_labels_one_hot = tf.one_hot(mini_batch_labels, 10)
loss = tf.reduce_sum(
tf.nn.softmax_cross_entropy_with_logits_v2(
labels=input_labels_one_hot, logits=logits))
grads = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(
zip(grads, model.trainable_variables),
global_step=tf.train.get_or_create_global_step())
global_step += 1
loss = 0.0
success = 0.0
for batch in range(len(labels) // config.batch_size()):
L = batch * config.batch_size()
R = L + config.batch_size()
mini_batch_images = images[L:R]
mini_batch_labels = labels[L:R]
s = tf.concat([
tf.gather_nd(mini_batch_images, gather_map),
tf.zeros(list(np.shape(gather_map)[:2]) + [2])
], -1)
logits = model.flow(s)
classes = tf.argmax(logits, 1)
loss += tf.reduce_sum(
tf.nn.softmax_cross_entropy_with_logits_v2(
labels=input_labels_one_hot, logits=logits))
success += tf.reduce_sum(
tf.cast(tf.equal(classes, mini_batch_labels), tf.float32))
return {'accuracy': success * 100.0 / len(labels), 'loss': loss}
def export_git_objects(self):
self.ui.status(_("importing Hg objects into Git\n"))
self.init_if_missing()
nodes = [self.repo.lookup(n) for n in self.repo]
export = [node for node in nodes if not hex(node) in self._map_hg]
total = len(export)
for i, rev in enumerate(export):
util.progress(self.ui, 'import', i, total=total)
ctx = self.repo.changectx(rev)
state = ctx.extra().get('hg-git', None)
if state == 'octopus':
self.ui.debug("revision %d is a part "
"of octopus explosion\n" % ctx.rev())
continue
self.export_hg_commit(rev)
util.progress(self.ui, 'import', None, total=total)
def base(d, player):
code, tag = d.menu("Les policier vous ont enfermés dans la prison, mais le geôlier vous libère.",
choices=[("(1)", "Parler au geôlier"),
("(2)", "Tenter de libérer les prisonniers"),
("(3)", "Revenir a la ville"),
("(4)", "Afficher mon inventaire")],
ok_label="Ok, je veux faire ca",
cancel_label="Préférances/Quitter")
if code == d.OK:
if tag == "(1)":
d.msgbox("""C'est ici que nous gardons les criminels qui ont été arrétés.""")
code, tag = d.menu("Que dire ?",
choices=[
("(1)", "Les criminels?"),
("(2)", "D'accord ,donc sans vous ce serait l'anarchie.")
],
ok_label="Dire ça",
cancel_label="mettre fin à la conversation")
if code == d.OK:
if tag == "(1)":
d.msgbox("Oui, cette ville regorge de fripouille en tous genre. Alors faites attention.")
elif tag == "(2)":
d.msgbox("ça fais plaisir de voir que la jeune génération respecte mon travail.")
else:
d.msgbox("vous dites au revoir au geôlier")
elif tag == "(2)":
r = database.inventory.addToInventory(player, "or", -150)
if not r:
progress(d, 200,
"Votre tentative est stoppée net par le geôlier qui vous enferme. Vous n'avez pas assez d'or pour écourter votre peine.", player)
else:
progress(d, 120,
"Vous tenter de libérer les prisonniers, mais le geôlier vous en empeche et vous arrète. Vous avez une amende de 50 pièce d'or", player)
return True
elif tag == "(3)":
database.players.changePref(player, "location", "centreVille")
return True
elif tag == "(4)":
showInventory(d, player)
return True
else:
return "pref"
def __init__(self, ui, msg, name, size, mode = 'le'):
self.ui = ui
self.f = open(name, 'wb')
self.n = 0
self.fmt16 = ('>H', '<H')[mode == 'le']
self.fmt32 = ('>L', '<L')[mode == 'le']
# display output
self.ui.put('%s ' % msg)
self.progress = util.progress(ui, 8, size)
def archive_dot_org(link_dir, link, timeout=TIMEOUT):
"""submit site to archive.org for archiving via their service, save returned archive url"""
path = os.path.join(link_dir, 'archive.org.txt')
if os.path.exists(path):
archive_org_url = open(path, 'r').read().strip()
return {'output': archive_org_url, 'status': 'skipped'}
submit_url = 'https://web.archive.org/save/{}'.format(link['url'])
success = False
CMD = ['curl', '-L', '-I', '-X', 'GET', submit_url]
end = progress(timeout, prefix=' ')
try:
result = run(CMD, stdout=PIPE, stderr=DEVNULL, cwd=link_dir, timeout=timeout + 1) # archive.org.txt
end()
# Parse archive.org response headers
headers = defaultdict(list)
# lowercase all the header names and store in dict
for header in result.stdout.splitlines():
if b':' not in header or not header.strip():
continue
name, val = header.decode().split(':', 1)
headers[name.lower().strip()].append(val.strip())
# Get successful archive url in "content-location" header or any errors
content_location = headers['content-location']
errors = headers['x-archive-wayback-runtime-error']
if content_location:
saved_url = 'https://web.archive.org{}'.format(content_location[0])
success = True
elif len(errors) == 1 and 'RobotAccessControlException' in errors[0]:
output = submit_url
# raise Exception('Archive.org denied by {}/robots.txt'.format(link['domain']))
elif errors:
raise Exception(', '.join(errors))
else:
raise Exception('Failed to find "content-location" URL header in Archive.org response.')
except Exception as e:
end()
print(' Visit url to see output:', ' '.join(CMD))
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
output = e
if success:
with open(os.path.join(link_dir, 'archive.org.txt'), 'w', encoding='utf-8') as f:
f.write(saved_url)
chmod_file('archive.org.txt', cwd=link_dir)
output = saved_url
return {
'cmd': CMD,
'output': output,
}
def __init__(self, ui, msg, name, size, mode = 'le'):
self.ui = ui
self.f = open(name, 'wb')
self.n = 0
self.fmt16 = ('>H', '<H')[mode == 'le']
self.fmt32 = ('>L', '<L')[mode == 'le']
# display output
self.ui.put('%s ' % msg)
self.progress = util.progress(ui, 8, size)
def row_search(OPTS, m, line_tups):
"""Writes the scan results to a pickle file.
The pickled object is a dictionary with cne names as keys and list as value.
Each list contains tuples of (coord, index) where coord is a coord dict
and index is int index starting from 0.
"""
rows = {}
for i, l in enumerate(line_tups):
name, a, b, c, = l[0], l[OPTS.a-1], l[OPTS.b-1], parse_coords(l[OPTS.c-1])
m.fit([a])
hits = m.scan([b], n=None, reverse_complement=True, sort=False)[0]
scores = [score for index, score in hits]
rows[name] = scores
progress(50, (float(i)+1)/len(line_tups)*100, pre="Processing genes")
pkl_filename = OPTS.model + '.pkl'
sys.stdout.write("Writing pickle file %s. Do not exit!\n" % pkl_filename)
pkl = open(OPTS.model + '.pkl', 'wb')
pickle.dump(rows, pkl, pickle.HIGHEST_PROTOCOL)
def separate(lines, p, train_out, test_out):
dataset = list(mecab.load(lines))
random.shuffle(dataset)
for i, sentence in enumerate(dataset, 1):
if random.random() < p:
pickle.dump(sentence, train_out)
else:
pickle.dump(sentence, test_out)
if i % 100 == 0:
print(util.progress('write', i / len(dataset), 100), end='')
print()
def separate(lines, p, train_out, test_out):
dataset = list(mecab.load(lines))
random.shuffle(dataset)
for i, sentence in enumerate(dataset, 1):
if random.random() < p:
pickle.dump(sentence, train_out)
else:
pickle.dump(sentence, test_out)
if i % 100 == 0:
print(util.progress('write', i / len(dataset), 100), end='')
print()
def fetch_git(link_dir, link, timeout=TIMEOUT):
"""download full site using git"""
url_is_clonable = (domain(link['url']) in GIT_DOMAINS
or link['url'].endswith('.git')
or link['type'] == 'git')
if not url_is_clonable:
return {'output': None, 'status': 'skipped'}
git_dir = os.path.join(link_dir, 'git')
if os.path.exists(git_dir):
return {'output': 'git', 'status': 'skipped'}
os.makedirs(git_dir, exist_ok=True)
output = 'git'
CMD = [
GIT_BINARY,
'clone',
'--mirror',
'--recursive',
*(() if CHECK_SSL_VALIDITY else ('-c', 'http.sslVerify=false')),
without_query(without_fragment(link['url'])),
]
end = progress(timeout, prefix=' ')
try:
result = run(CMD,
stdout=PIPE,
stderr=PIPE,
cwd=git_dir,
timeout=timeout + 1) # git/<reponame>
end()
if result.returncode == 128:
# ignore failed re-download when the folder already exists
pass
elif result.returncode > 0:
print(' got git response code {}:'.format(
result.returncode))
raise Exception('Failed git download')
except Exception as e:
end()
print(' {}Failed: {} {}{}'.format(ANSI['red'],
e.__class__.__name__, e,
ANSI['reset']))
print(' Run to see full output:')
print(' cd {};'.format(link_dir))
print(' {}'.format(' '.join(CMD)))
output = e
return {
'cmd': CMD,
'output': output,
}
def base(d, player):
code, tag = d.menu("Vous entrez dans le chateau il est tres grand.",
choices=[("(1)", "Parler aux gardes"),
("(2)", "Aller à la salle du trone"),
("(3)", "Revenir a la ville"),
("(4)", "Afficher mon inventaire")],
ok_label="Ok, je veux faire ça",
cancel_label="Préférances/Quitter")
if code == d.OK:
if tag == "(1)":
d.msgbox(
"""Les gardes vous interpellent sans vous laiser le temps de parler "Vous ètes ici pour teter d'arretez le Sorcier? aller à la salle du trone""")
code, tag = d.menu("Que dire ?",
choices=[
("(1)", "Le sorcier ?"),
("(2)", "Oui c'est exactement cela!")
],
ok_label="Dire ça",
cancel_label="mettre fin à la conversation")
if code == d.OK:
if tag == "(1)":
d.msgbox("Oui le Sorcier qui est la cause de tous les problème. D'ou sortez vous? bon sang!")
elif tag == "(2)":
d.msgbox("Encore un prétentieux qui va mourir.")
else:
d.msgbox("Vous dites au revoir aux gardes")
return True
else:
return "pref"
elif tag == "(2)":
progress(d, 15, "Vous allez dans la salle du trone", player)
database.players.changePref(player, "location", "salleDuTrone")
elif tag == "(3)":
database.players.changePref(player, "location", "centreVille")
return True
elif tag == "(4)":
showInventory(d, player)
return True
else:
return "pref"
def base(d, player):
code, tag = d.menu("Vous ètes dans les montagnes, il y a un chateau et une caverne.",
choices=[("(1)", "Aller dans le chateau"),
("(2)", "Aller dans la caverne."),
("(3)", "Retourner dans la ville."),
("(4)", "Afficher mon inventaire.")],
ok_label="Ok je veux faire ça",
cancel_label="Préférance/Quitter")
if code == d.OK:
if tag == "(1)":
progress(d, 30, "Vous allez dans le chateau.", player)
database.players.changePref(player, "location", "chateauMontagne")
return True
elif tag == "(2)":
progress(d, 20, "Vous rentrez dans la grotte, mais il s'agit d'un repère de brigand", player)
database.players.changePref(player, "location", "caverne")
return True
elif tag == "(3)":
progress(d, 30, "Vous retournez dans la ville.", player)
database.players.changePref(player, "location", "quartierSud")
return True
elif tag == "(4)":
showInventory(d, player)
return True
else:
return "pref"
def base(d, player):
code, tag = d.menu("Vous êtes dans le petit bois. Que voulez-vous faire ?",
choices=[("(1)", "Fouiller la terre"),
("(2)", "Continuer sur ce sentier"),
("(3)", "Rentrer à la maison"),
("(4)", "Afficher mon inventaire")],
ok_label="Ok, je veux faire ca",
cancel_label="Préférances/Quitter")
if code == d.OK:
if tag == "(1)":
d.msgbox("Vous creusez dans ce bois.")
progress(d, 20, "Vous creusez la terre...", player)
database.inventory.addToInventory(player, "brindilles", 20)
database.inventory.addToInventory(player, "terre", 5)
d.msgbox("Vous trouvez 5 terre et 2 brindilles.")
return True
elif tag == "(2)":
progress(d, 25, "Vous continuez sur le sentier... Il semble en bien plus mauvais état !", player)
database.players.changePref(player, "location", "terreDeChaisPasOu")
return True
elif tag == "(3)":
progress(d, 10, "Vous rentrez à la maison...", player)
database.players.changePref(player, "location", "maison")
return True
elif tag == "(4)":
showInventory(d, player)
return True
else:
return "pref"
def base(d, player):
code, tag = d.menu(
"Vous etes dans une grande plaine. Un panneau indique : \"terre de chais pas où\". Le reste du panneau est effacé. Il semble y avoir une boutique un peu plus loin.",
choices=[("(1)", "Aller dans le magasin"),
("(2)", "Continuer sur le santier"),
("(3)", "Rentrer à la fôret"),
("(4)", "Afficher mon inventaire")
],
ok_label="Ok, je veux faire ca",
cancel_label="Préférances/Quitter")
if code == d.OK:
if tag == "(1)":
progress(d, 3, "Vous allez vers la boutique.", player)
database.players.changePref(player, "location", "magasinTDCPO")
return True
elif tag == "(2)":
if database.inventory.getItemNumber(player, "lunettes") != 0:
progress(d, 60, "Vous continuez sur le sentier. Il est quasiment invisible.", player)
database.players.changePref(player, "location", "villeDeNullepart")
return True
else:
d.msgbox("Impossible. Le chemin est presque indiscernable à partir d'ici !")
return True
elif tag == "(3)":
progress(d, 25, "Vous rebroussez chemain vers la forèt", player)
database.players.changePref(player, "location", "bois")
return True
elif tag == "(4)":
showInventory(d, player)
return True
else:
return "pref"
def base(d, player):
code, tag = d.menu("Vous etes arrivé a la ville de NullePart. La ville est grande. Il y a quelques passants. ",
choices=[("(1)", "Aller au centre ville"),
("(2)", "Prendre le bus jusqu'à la maison (2x or)"),
("(3)", "Rentrer vers la pleine"),
("(4)", "Afficher mon inventaire")
],
ok_label="Ok, je veux faire ca",
cancel_label="Préférances/Quitter")
if code == d.OK:
if tag == "(1)":
progress(d, 15, "Vous marchez jusqu'au centre ville.", player)
database.players.changePref(player, "location", "centreVille")
return True
elif tag == "(2)":
if database.inventory.addToInventory(player, "or", -2):
progress(d, 10, "Le bus vous ramene chez vous.", player)
database.players.changePref(player, "location", "maison")
return True
else:
d.msgbox("Impossible. Vous n'avez pas asser d'argent !")
return True
elif tag == "(3)":
progress(d, 60, "Vous rebroussez chemain vers la plaine", player)
database.players.changePref(player, "location", "terreDeChaisPasOu")
return True
elif tag == "(4)":
showInventory(d, player)
return True
else:
return "pref"
def computeFBANKDeltaDelta(sswi,NFilt=40,NDelta=2):
nframes = speakersent.countFrames(sswi)
NFilt=NFilt+1 # Energy values count as one extra
features = np.zeros(shape=(nframes,NFilt*3))
util.startprogress("FBANK Features")
frameCnt=0
for k,v in sswi.iteritems(): # k: speakerId, v: dict with sentenceId
speaker_=k
for k2,v2 in v.iteritems(): # k2: sentenceId, v: dict with frameId -> entry
sent_=k2
r,sig=getWavFile(speaker_,sent_)
fbank_frames,energy=fbank(sig,r,winlen=winlen,winstep=winstep,nfilt=NFilt-1)
fbank_frames=np.log(np.append(np.reshape(energy,(energy.shape[0],1)),fbank_frames,axis=1))
delta_1 = computeDelta(fbank_frames,N=NDelta)
delta_2 = computeDelta(delta_1,N=NDelta)
util.progress(float(frameCnt)/nframes*100.0)
for i in range(len(v2)):
frameCnt+=1
features[v2[i+1],:NFilt]=fbank_frames[i,:]
features[v2[i+1],NFilt:2*NFilt]=delta_1[i,:]
features[v2[i+1],2*NFilt:]=delta_2[i,:]
util.endprogress()
return features
def base(d, player):
code, tag = d.menu("Vous êtes à la mine. Que voulez-vous faire ?",
choices=[("(1)", "Miner dans la mine de pierre"),
("(2)", "Rentrer à la maison"),
("(3)", "Afficher mon inventaire")],
ok_label="Ok, je veux faire ca",
cancel_label="Préférances/Quitter")
if code == d.OK:
if tag == "(1)":
d.msgbox("Vous minez dans la grotte environnante")
progress(d, 20, "Vous minez...", player)
database.inventory.addToInventory(player, "pierre", 10)
d.msgbox("Vous trouvez 10 pierres.")
return True
elif tag == "(2)":
progress(d, 20, "Vous rentrez de la mine", player)
database.players.changePref(player, "location", "maison")
return True
elif tag == "(3)":
showInventory(d, player)
return True
else:
return "pref"
def progress(self, msg):
# 'Counting objects: 33640, done.\n'
# 'Compressing objects: 0% (1/9955) \r
msgs = re.split('[\r\n]', self.msgbuf + msg)
self.msgbuf = msgs.pop()
for msg in msgs:
td = msg.split(':', 1)
data = td.pop()
if not td:
self.flush(data)
continue
topic = td[0]
m = re.search('\((\d+)/(\d+)\)', data)
if m:
if self.lasttopic and self.lasttopic != topic:
self.flush()
self.lasttopic = topic
pos, total = map(int, m.group(1, 2))
util.progress(self.ui, topic, pos, total=total)
else:
self.flush(msg)
def __init__(self, width, ui, name, nmax, le=False):
self.width = width
self.ui = ui
self.file = open(name, "wb")
self.n = 0
# endian converion
self.convert = util.identity
if le:
if self.width == 32:
self.convert = util.btol32
elif self.width == 64:
self.convert = util.btol64
# display output
self.ui.put("writing to %s " % name)
self.progress = util.progress(ui, 7, nmax)
def export_git_objects(self):
self.init_if_missing()
nodes = [self.repo.lookup(n) for n in self.repo]
export = [node for node in nodes if not hex(node) in self._map_hg]
total = len(export)
if total:
self.ui.note(_("exporting hg objects to git\n"))
# By only exporting deltas, the assertion is that all previous objects
# for all other changesets are already present in the Git repository.
# This assertion is necessary to prevent redundant work.
exporter = hg2git.IncrementalChangesetExporter(self.repo)
for i, rev in enumerate(export):
util.progress(self.ui, 'exporting', i, total=total)
ctx = self.repo.changectx(rev)
state = ctx.extra().get('hg-git', None)
if state == 'octopus':
self.ui.debug("revision %d is a part "
"of octopus explosion\n" % ctx.rev())
continue
self.export_hg_commit(rev, exporter)
util.progress(self.ui, 'importing', None, total=total)
def base(d, player):
code, tag = d.menu("Vous êtes dans le quartier commerçant il énormément de gens.",
choices=[("(1)", "Aller chez le marchand de matériaux"),
("(2)", "Aller chez le pharmacien"),
("(3)", "Aller dans les quartier du sud"),
("(4)", "Aller chez le marchand d'armes"),
("(5)", "parler aux gens"),
("(6)", "Revenir a la ville"),
("(7)", "Afficher mon inventaire")],
ok_label="Ok, je veux faire ca",
cancel_label="Préférances/Quitter")
if code == d.OK:
if tag == "(1)":
progress(d, 15, "Vous entrez dans le marchand de matériaux", player)
database.players.changePref(player, "location", "marchandMateriaux")
return True
elif tag == "(2)":
progress(d, 15, "Vous entrez dans la Pharmacie", player)
database.players.changePref(player, "location", "pharmacie")
return True
elif tag == "(3)":
if database.inventory.getItemNumber(player, "gourdin") != 0:
progress(d, 30, "Vous allez dans les quartiers du sud", player)
database.players.changePref(player, "location", "quartierSud")
else:
d.msgbox("Des citadins vous empechent de passer.")
return True
elif tag == "(4)":
progress(d, 15, "Vous allez chez le marchand d'armes", player)
database.players.changePref(player, "location", "marchandArme")
return True
elif tag == "(5)":
d.msgbox(
"""Vous tentez de parler à un citadin dans une charrette mais il semble trop occupé a hurler "P***** avance!".""")
return True
elif tag == "(6)":
database.players.changePref(player, "location", "centreVille")
return True
elif tag == "(7)":
showInventory(d, player)
return True
else:
return "pref"
def run_training(args):
out_dir = pathlib.Path(args.directory)
sentences = dataset.load(args.source)
if args.epoch is not None:
start = args.epoch + 1
storage = load(out_dir, args.epoch)
sentences = itertools.islice(sentences, start, None)
else:
start = 0
storage = init(args)
if (out_dir/meta_name).exists():
if input('Overwrite? [y/N]: ').strip().lower() != 'y':
exit(1)
with (out_dir/meta_name).open('wb') as f:
np.save(f, [storage])
batchsize = 5000
for i, sentence in enumerate(sentences, start):
if i % batchsize == 0:
print()
serializers.save_npz(
str(out_dir/model_name(i)),
storage.model
)
serializers.save_npz(
str(out_dir/optimizer_name(i)),
storage.optimizer
)
else:
print(
util.progress(
'batch {}'.format(i // batchsize),
(i % batchsize) / batchsize, 100),
end=''
)
train(storage.model,
storage.optimizer,
generate_data(sentence),
generate_label(sentence),
generate_attr(
sentence,
storage.mappings
)
)
def run_training(args):
out_dir = pathlib.Path(args.directory)
sentences = [line.split() for line in args.source]
if args.epoch is not None:
start = args.epoch + 1
storage = load(out_dir, args.epoch)
sentences = itertools.islice(sentences, start, None)
else:
start = 1
storage = init(args)
if (out_dir/meta_name).exists():
if input('Overwrite? [y/N]: ').strip().lower() != 'y':
exit(1)
with (out_dir/meta_name).open('wb') as f:
np.save(f, [storage])
checkpoint = args.checkpoint
for i, sentence in enumerate(sentences, start):
train(storage.model,
storage.optimizer,
generate_data(sentence),
generate_label(sentence)
)
if i % checkpoint != 0:
print(
util.progress(
'checkpoint {}'.format(i // checkpoint),
(i % checkpoint) / checkpoint, 100),
end=''
)
else:
print()
save_param(out_dir, i, storage)
print()
save_param(out_dir, i, storage)
def import_git_objects(self, remote_name=None, refs=None):
self.ui.status(_("importing Git objects into Hg\n"))
self.init_if_missing()
# import heads and fetched tags as remote references
todo = []
done = set()
convert_list = {}
# get a list of all the head shas
seenheads = set()
if refs is None:
refs = self.git.refs.as_dict()
if refs:
for sha in refs.itervalues():
# refs contains all the refs in the server, not just the ones
# we are pulling
if sha in self.git.object_store:
obj = self.git.get_object(sha)
while isinstance(obj, Tag):
obj_type, sha = obj.object
obj = self.git.get_object(sha)
if isinstance (obj, Commit) and sha not in seenheads:
seenheads.add(sha)
todo.append(sha)
# sort by commit date
def commitdate(sha):
obj = self.git.get_object(sha)
return obj.commit_time-obj.commit_timezone
todo.sort(key=commitdate, reverse=True)
# traverse the heads getting a list of all the unique commits
commits = []
seen = set(todo)
while todo:
sha = todo[-1]
if sha in done:
todo.pop()
continue
assert isinstance(sha, str)
obj = self.git.get_object(sha)
assert isinstance(obj, Commit)
for p in obj.parents:
if p not in done:
todo.append(p)
break
else:
commits.append(sha)
convert_list[sha] = obj
done.add(sha)
todo.pop()
commits = [commit for commit in commits if not commit in self._map_git]
# import each of the commits, oldest first
total = len(commits)
for i, csha in enumerate(commits):
util.progress(self.ui, 'import', i, total=total, unit='commits')
commit = convert_list[csha]
self.import_git_commit(commit)
util.progress(self.ui, 'import', None, total=total, unit='commits')
def base(d, player):
code, tag = d.menu("Vous etes chez le marchand de matériaux. Les prix sont anormalement élevées.",
choices=[("(1)", "Demandez au marchand pourquoi les prix sont il si élevés."),
("(2)", "Vendre 10 pierre pour 30 or"),
("(3)", "Vendre 5 terre pour 15 or"),
("(4)", "Vendre 2 brindilles pour 10 or"),
("(5)", "Acheter 10 pierre pour 30 or"),
("(6)", "Acheter 5 terre pour 15 or"),
("(7)", "Acheter 2 brindilles pour 10 or"),
("(8)", "Revenir dans le quartier."),
("(9)", "Afficher mon inventaire")
],
ok_label="Ok, je veux faire ca",
cancel_label="Préférances/Quitter")
if code == d.OK:
if tag == "(1)":
d.msgbox(
"""Le marchand vous répond que la banque "Street Money" à racheté le fournisseur du magasin et qu'il rachète a prix d'or les matériaux. """)
return True
elif tag == "(2)":
if database.inventory.addToInventory(player, "pierre", -10):
database.inventory.addToInventory(player, "or", 30)
d.msgbox(
"Vous échangez 10 pierre contre 30 or. Le marchand dit : \"merci pour votre achat, cher voyageur.\"")
return True
else:
d.msgbox("Vous n'avez pas assez de pierre")
elif tag == "(3)":
if database.inventory.addToInventory(player, "terre", -5):
database.inventory.addToInventory(player, "or", 15)
d.msgbox(
"Vous échangez 5 terre contre 15 or. Le marchand dit : \"merci pour votre achat, cher voyageur.\"")
return True
else:
d.msgbox("Vous n'avez pas assez de terre")
elif tag == "(4)":
if database.inventory.addToInventory(player, "brindille", -2):
database.inventory.addToInventory(player, "or", 10)
d.msgbox(
"Vous échangez 2 brindilles contre 10 or. Le marchand dit : \"merci pour votre achat, cher voyageur.\"")
return True
else:
d.msgbox("Vous n'avez pas assez de terre")
return True
elif tag == "(5)":
if database.inventory.addToInventory(player, "pierre", 10):
database.inventory.addToInventory(player, "or", -30)
d.msgbox(
"Vous échangez 10 pierre contre 30 or. Le marchand dit : \"merci pour votre achat, cher voyageur.\"")
return True
else:
d.msgbox("Vous n'avez pas assez d'or")
elif tag == "(6)":
if database.inventory.addToInventory(player, "terre", 5):
database.inventory.addToInventory(player, "or", -15)
d.msgbox(
"Vous échangez 5 terre contre 15 or. Le marchand dit : \"merci pour votre achat, cher voyageur.\"")
return True
else:
d.msgbox("Vous n'avez pas assez d'or")
return True
elif tag == "(7)":
if database.inventory.addToInventory(player, "terre", 5):
database.inventory.addToInventory(player, "or", -10)
d.msgbox(
"Vous échangez 2 brindilles contre 10 or. Le marchand dit : \"merci pour votre achat, cher voyageur.\"")
return True
else:
d.msgbox("Vous n'avez pas assez d'or")
elif tag == "(8)":
progress(d, 15, "Vous retourner dans le quartier commerçant", player)
database.players.changePref(player, "location", "quartierCommercant")
return True
elif tag == "(9)":
showInventory(d, player)
return True
else:
return "pref"
def base(d, player):
code, tag = d.menu("Vous etes devant le magasin. Un homme vous propose :",
choices=[("(1)", "Vendre 5 terre contre 3 or"),
("(2)", "Vendre 10 pierres contre 4 or"),
("(3)", "Acheter 5 terre contre 3 or"),
("(4)", "Acheter 10 pierres contre 4 or"),
("(5)", "Parler à l'Homme."),
("(6)", "Repartir dans la plaine"),
("(7)", "Afficher mon inventaire")
],
ok_label="Ok, je veux faire ca",
cancel_label="Préférances/Quitter")
if code == d.OK:
if tag == "(1)":
if database.inventory.addToInventory(player, "terre", -5):
database.inventory.addToInventory(player, "or", 3)
d.msgbox(
"Vous échangez 5x terre contre 3x or. Le marchand dit : \"merci pour votre achat, cher voyageur.\"")
return True
else:
d.msgbox("Vous n'avez pas asser de terre.")
return True
elif tag == "(2)":
if database.inventory.addToInventory(player, "pierre", -10):
database.inventory.addToInventory(player, "or", 4)
d.msgbox(
"Vous échangez 10x pierres contre 4x or. Le marchand dit : \"merci pour votre achat, cher voyageur.\"")
return True
else:
d.msgbox("Vous n'avez pas asser de pierres.")
return True
elif tag == "(3)":
if database.inventory.addToInventory(player, "or", -3):
database.inventory.addToInventory(player, "terre", 5)
d.msgbox(
"Vous échangez 3x or contre 5x terre. Le marchand dit : \"merci pour votre achat, cher voyageur.\"")
return True
else:
d.msgbox("Vous n'avez pas assez d'or.")
elif tag == "(4)":
if database.inventory.addToInventory(player, "or", -4):
database.inventory.addToInventory(player, "pierre", 10)
d.msgbox(
"Vous échangez 4x or contre 10x terre. Le marchand dit : \"merci pour votre achat, cher voyageur.\"")
elif tag == "(5)":
if database.inventory.getItemNumber(player, "lunettes") <= 0:
d.msgbox("""Que faites vous dans ce coin perdu ? Vous ne voulez pas rejoindre la ville ?
Comment ca quelle ville ? Mais la ville de NullePart pardi. Tu n'as pas vu le panneau ?
Prendss mes lunettes et vas-y !""")
database.inventory.addToInventory(player, "lunettes", 1)
else:
d.msgbox("As-tu pu visiter la ville ?")
elif tag == "(6)":
progress(d, 3, "Vous rebroussez chemain vers la plaine", player)
database.players.changePref(player, "location", "terreDeChaisPasOu")
return True
elif tag == "(7)":
showInventory(d, player)
return True
else:
return "pref"
