def create_simulator(simulator_ready_future, command, path="./"):
print('creating sim process...', end=' ')
create = create_subprocess_exec(*command,
cwd=path,
stdout=PIPE,
stderr=STDOUT)
proc = yield from create
all_data = yield from get_line_until_timeout(proc, 1.5, False)
mat = re.findall('((?P<sim>\/tmp\/[-\w\d]+?)\s)|(?P<pts>\/dev\/pts\/\d+)',
"".join(all_data))
if not mat:
# ugly, need to catch these
assert bool(
mat
), '\033[91m' + 'This should find something... probably not building simulator. output: {}'.format(
repr("\n".join(all_data))) + '\033[0m'
_, sim, pts = [next(x for x in y if x) for y in zip(*mat)]
yield from _make_and_upload_sim_code(sim)
if pts:
simulator_ready_future.set_result(pts)
yield from proc.wait()
else:
return simulator_ready_future.set_exception(
Exception('no pts returned'))
def tf_check_call(self, *args, **kwargs):
proc = yield create_subprocess_exec(self.tf_bin,
*args,
**kwargs,
cwd=self.get_module_dir())
yield proc.wait()
if proc.returncode != 0:
raise CalledProcessError(proc.returncode, self.tf_bin)
async def run(*args, watch=False, **kwargs):
"""Run a command with asyncio.create_subprocess_exec."""
_env = environ
if watch:
args = ["watch", "-t", *args]
if "env" in kwargs:
_env.update(kwargs["env"])
kwargs["env"] = _env
proc = await (create_subprocess_shell(join(args), **kwargs)
if "shell" in kwargs and kwargs["shell"] else
create_subprocess_exec(*args, **kwargs))
return proc, _error_handler(proc, args)
def create_simulator(simulator_ready_future, command, path="./"):
print('creating sim process...', end=' ')
create = create_subprocess_exec(*command, cwd=path, stdout=PIPE, stderr=STDOUT)
proc = yield from create
all_data = yield from get_line_until_timeout(proc, 1.5, False)
mat = re.findall('((?P<sim>\/tmp\/[-\w\d]+?)\s)|(?P<pts>\/dev\/pts\/\d+)', "".join(all_data))
if not mat:
# ugly, need to catch these
assert bool(mat), '\033[91m' + 'This should find something... probably not building simulator. output: {}'.format(repr("\n".join(all_data))) + '\033[0m'
_, sim, pts = [next(x for x in y if x) for y in zip(*mat)]
yield from _make_and_upload_sim_code(sim)
if pts:
simulator_ready_future.set_result(pts)
yield from proc.wait()
else:
return simulator_ready_future.set_exception(Exception('no pts returned'))
def __init__(self, cmd, **kwargs):
self._cmd = cmd
self._proc = None
self._coro = create_subprocess_exec(
*split(cmd), **kwargs, stdout=PIPE)
def create_process_with_command(command, cwd=None):
return create_subprocess_exec(*command,
cwd=cwd,
stdout=PIPE,
stderr=STDOUT)
def __init__(self, cmd, **kwargs):
self._cmd = cmd
self._proc = None
self._coro = create_subprocess_exec(*split(cmd), **kwargs, stdout=PIPE)
def main():
args = parse_args()
target_id = args.target_id
mats = dict(
meta = [],
sentence_len_dist = [],
word_len_dist = [],
word_morph_len_dist = [],
unigrams = [],
bigrams = [],
trigrams = []
)
finnpos = loop.run_until_complete(create_subprocess_exec(
'ftb-label', stdin=PIPE, stdout=PIPE))
def proc_cont(cont):
sd, wd, wmd, u, b, t = process_content(finnpos, cont)
mats['sentence_len_dist'].append(sd)
mats['word_len_dist'].append(wd)
mats['word_morph_len_dist'].append(wmd)
mats['unigrams'].append(u)
mats['bigrams'].append(b)
mats['trigrams'].append(t)
# Project gutenberg
if args.gutenberg:
with open(args.gutenberg) as csvfile:
gutenberg = csv.DictReader(csvfile)
for row in gutenberg:
id = "{}:{}".format(row['id'], row['page'])
if target_id is not None and id != target_id:
continue
print(id)
try:
proc_cont(row['content'])
except BlankEntry:
continue
else:
mats['meta'].append([id, 'gutenberg', row['id']])
# Punk in Finland
if args.punkinfinland:
with open(args.punkinfinland) as csvfile:
punk = csv.DictReader(csvfile)
for row in punk:
id = row['id']
if target_id is not None and id != target_id:
continue
print(id)
try:
proc_cont(row['content'])
except BlankEntry:
continue
else:
mats['meta'].append([id, 'punkinfinland', deforeignify(row['author'])])
# Yle
if args.yle:
with open(args.yle) as csvfile:
yle = csv.DictReader(csvfile)
for row in yle:
id = row['id']
if target_id is not None and id != target_id:
continue
print(id)
if row['is_very_easy'] == "[True]":
subclass_name = 'tosihelppo'
else:
subclass_name = 'selkouutiset'
try:
proc_cont(row['content'])
except BlankEntry:
continue
else:
mats['meta'].append([id, 'yle', subclass_name])
# Everything must be ASCII clean or else Matlab with have a hissy fit
mats['meta'] = np.array(mats['meta'])
mats['meta'] = np.vectorize(force_ascii)(mats['meta'])
for k in mats:
if k != 'meta':
mats[k] = np.array(mats[k])
io.savemat(args.output, mats, appendmat=False)
def get_process(commands):
return create_subprocess_exec(*commands, stdout=PIPE, stderr=PIPE)
def create_process_with_command(command, cwd=None):
return create_subprocess_exec(*command, cwd=cwd,
stdout=PIPE, stderr=STDOUT)