def train_launch():
id = request.form.get('engine_id')
if user_utils.is_normal(): return url_for('index')
task_id, monitor_task_id = Trainer.launch(id, user_utils.is_admin())
return url_for('train.train_console', id=id)
def instances_datatables_feed():
if user_utils.is_normal(): return redirect(url_for('index'))
columns = [Engine.id, Engine.name]
dt = datatables.Datatables()
rows, rows_filtered, search = dt.parse(Engine, columns, request,
Engine.status.like("training"))
user_data = []
for engine in (rows_filtered if search else rows):
user_data.append([
engine.id, engine.name,
engine.uploader.email if engine.uploader else "", "Training", None
])
columns_translating = [RunningEngines.id]
rows_translating, rows_translating_filtered, search_translating = dt.parse(
RunningEngines, columns_translating, request)
for engine_entry in (rows_translating_filtered
if search_translating else rows_translating):
user_data.append([
engine_entry.engine.id, engine_entry.engine.name,
engine_entry.engine.uploader.email
if engine_entry.engine.uploader else "", "Translating", None
])
return dt.response(rows, rows_filtered, user_data)
def train_launching(task_id):
if user_utils.is_normal(): return redirect(url_for('index'))
return render_template('launching.html.jinja2',
page_name='train',
page_title='Launching training',
task_id=task_id)
def stop_engine():
if user_utils.is_normal(): return redirect(url_for('index'))
id = request.args.get('id')
try:
Trainer.stop(id, admin_stop=True)
except:
pass
return redirect(request.referrer)
def train_test():
if user_utils.is_normal(): return redirect(url_for('index'))
engine_id = request.form.get('engine_id')
task = tasks.test_training.apply_async(args=[engine_id])
engine = Engine.query.filter_by(id=engine_id).first()
engine.test_task_id = task.id
db.session.commit()
return task.id
def become(type, id):
if user_utils.is_normal(): return redirect(url_for('index'))
if user_utils.get_user().admin:
user = User.query.filter_by(id=id).first()
user.expert = (type == "expert")
user.admin = (type == "admin")
db.session.commit()
return redirect(request.referrer)
def train_attention(id):
if user_utils.is_normal():
return send_file(
os.path.join(app.config['BASE_CONFIG_FOLDER'], "attention.png"))
engine = Engine.query.filter_by(id=id).first()
files = glob.glob(os.path.join(engine.path, "*.att"))
if len(files) > 0:
return send_file(files[0])
else:
return send_file(
os.path.join(app.config['BASE_CONFIG_FOLDER'], "attention.png"))
def train_test_status():
if user_utils.is_normal(): return redirect(url_for('index'))
task_id = request.form.get('task_id')
task_success, task_value = utils.get_task_result(tasks.test_training,
task_id)
if task_success is not None:
if not task_success:
return jsonify({"result": -2})
return jsonify({"result": 200, "test": task_value})
else:
return jsonify({"result": -1})
def python_feed():
if user_utils.is_normal(): return redirect(url_for('index'))
draw = request.form.get('draw')
search = request.form.get('search[value]')
start = int(request.form.get('start'))
length = int(request.form.get('length'))
order = int(request.form.get('order[0][column]'))
dir = request.form.get('order[0][dir]')
ps = subprocess.Popen("ps aux | grep python",
shell=True,
stdout=subprocess.PIPE)
ps.wait()
rows = []
for line_raw in ps.stdout:
line = line_raw.decode("utf-8").strip()
if line:
data = line.split(None)
command = " ".join(data[10:])
if not any(x in command
for x in ["ps aux", "grep python", "<defunct>"]):
rows.append([data[1], data[8], data[2], data[3], command])
if start and length:
rows = rows[start:length]
if order is not None:
rows.sort(key=lambda row: row[order], reverse=(dir == "desc"))
rows_filtered = []
if search:
for row in rows:
found = False
for col in row:
if not found:
if search in col:
rows_filtered.append(row)
found = True
return jsonify({
"draw":
int(draw) + 1,
"recordsTotal":
len(rows),
"recordsFiltered":
len(rows_filtered) if search else len(rows),
"data":
rows_filtered if search else rows
})
def user_edit(id):
if user_utils.is_normal(): return redirect(url_for('index'))
user = User.query.filter_by(id=id).one()
if request.method == 'GET':
return render_template('users_edit.admin.html.jinja2',
page_name='admin_users',
page_title='Edit user',
user=user)
else:
notes = request.form.get('userNotes')
user.notes = escape(notes)
db.session.commit()
return redirect(request.referrer)
def data_upload_perform():
if user_utils.is_normal(): return redirect(url_for('index'))
try:
if request.method == 'POST':
task_id = data_utils.process_upload_request(user_utils.get_uid(), request.files.get('bitext_file'), request.files.get('source_file'),
request.files.get('target_file'), request.form.get('source_lang'), request.form.get('target_lang'),
request.form.get('name'), request.form.get('description'), request.form.get('topic'))
return jsonify({ "result": 200, "task_id": task_id })
else:
raise Exception("Sorry, but we couldn't handle your request.")
except Exception as e:
Flash.issue(e, Flash.ERROR)
return jsonify({ "result": -1 })
def user_datatables_feed():
if user_utils.is_normal(): return redirect(url_for('index'))
columns = [User.id, User.username, User.email, User.notes]
dt = datatables.Datatables()
rows, rows_filtered, search = dt.parse(User, columns, request)
user_data = []
for user in (rows_filtered if search else rows):
user_data.append([
user.id, user.username, user.email,
'Admin' if user.admin else 'Expert' if user.expert else 'Beginner',
user.notes, '', user.admin, user.expert
])
return dt.response(rows, rows_filtered, user_data)
def train_index():
if user_utils.is_normal(): return redirect(url_for('index'))
currently_training = Engine.query.filter_by(uploader_id = user_utils.get_uid()) \
.filter(Engine.status.like("training")).all()
if (len(currently_training) > 0):
return redirect(
url_for('train.train_console', id=currently_training[0].id))
currently_launching = Engine.query.filter_by(uploader_id = user_utils.get_uid()) \
.filter(Engine.status.like("launching")).all()
if (len(currently_launching) > 0):
return redirect(
url_for('train.train_launching',
task_id=currently_launching[0].bg_task_id))
random_name = namegenerator.gen()
tryout = 0
while len(Engine.query.filter_by(name=random_name).all()):
random_name = namegenerator.gen()
tryout += 1
if tryout >= 5:
random_name = ""
break
random_name = " ".join(random_name.split("-")[:2])
library_corpora = user_utils.get_user_corpora().filter(
LibraryCorpora.corpus.has(Corpus.type == "bilingual")).all()
corpora = [c.corpus for c in library_corpora]
languages = UserLanguage.query.filter_by(user_id=current_user.id).order_by(
UserLanguage.name).all()
return render_template('train.html.jinja2',
page_name='train',
page_title='Train',
corpora=corpora,
random_name=random_name,
languages=languages)
def train_start():
if user_utils.is_normal(): return url_for('index')
engine_path = os.path.join(
user_utils.get_user_folder("engines"),
utils.normname(user_utils.get_user().username,
request.form['nameText']))
task = tasks.launch_training.apply_async(args=[
user_utils.get_uid(), engine_path,
{
i[0]: i[1] if i[0].endswith('[]') else i[1][0]
for i in request.form.lists()
}
])
return jsonify({
"result":
200,
"launching_url":
url_for('train.train_launching', task_id=task.id)
})
def delete_user():
if user_utils.is_normal(): return redirect(url_for('index'))
id = request.args.get('id')
try:
assert int(id) != user_utils.get_uid()
user = User.query.filter_by(id=id).first()
for corpus in Corpus.query.filter_by(owner_id=id).all():
user_utils.library_delete("library_corpora", corpus.id, id)
for engine_entry in user.user_engines:
user_utils.library_delete("library_engines",
engine_entry.engine.id, id)
shutil.rmtree(user_utils.get_user_folder(user_id=id))
db.session.delete(user)
db.session.commit()
except:
pass
return redirect(request.referrer)
def admin_system():
if user_utils.is_normal(): return redirect(url_for('index'))
factor = 1073741824
vmem = psutil.virtual_memory()
ram = {
"percent": vmem.percent,
"used": round(vmem.used / factor, 2),
"total": round(vmem.total / factor, 2)
} # GB
hdd = psutil.disk_usage(app.config['USERSPACE_FOLDER'])
disk_usage = {
"percent": round((hdd.used / hdd.total) * 100, 2),
"used": round(hdd.used / factor, 2),
"total": round(hdd.total / factor, 2)
} # GB
gpus = []
pynvml.nvmlInit()
for i in range(0, pynvml.nvmlDeviceGetCount()):
handle = pynvml.nvmlDeviceGetHandleByIndex(i)
resources = pynvml.nvmlDeviceGetUtilizationRates(handle)
gpus.append({
"id": i,
"memory": resources.memory,
"proc": resources.gpu
})
return render_template('system.admin.html.jinja2',
page_name='admin_system',
page_title='System',
ram=ram,
cpu=round(psutil.cpu_percent(), 2),
gpus=gpus,
disk_usage=disk_usage)
def train_finish(id):
if user_utils.is_normal(): return redirect(url_for('index'))
return _train_stop(id, False)
def admin_instances():
if user_utils.is_normal(): return redirect(url_for('index'))
return render_template('instances.admin.html.jinja2',
page_name='admin_instances',
page_title='Instances')
def data_upload_perform():
if user_utils.is_normal(): return redirect(url_for('index'))
try:
if request.method == 'POST':
# Handle possible custom languages
def add_custom_language(code, name):
custom_language = UserLanguage.query.filter_by(
code=code, user_id=current_user.id).first()
if custom_language:
custom_language.name = custom_src_lang_name
db.session.commit()
else:
custom_language = UserLanguage(code=code,
name=name,
user_id=current_user.id)
db.session.add(custom_language)
db.session.commit()
return UserLanguage.query.filter_by(
code=code, user_id=current_user.id).first()
source_lang = request.form.get('source_lang')
target_lang = request.form.get('target_lang')
custom_src_lang_code = request.form.get('sourceCustomLangCode')
custom_trg_lang_code = request.form.get('targetCustomLangCode')
if custom_src_lang_code:
custom_src_lang_name = request.form.get('sourceCustomLangName')
custom_lang = add_custom_language(custom_src_lang_code,
custom_src_lang_name)
source_lang = custom_lang.id
else:
source_lang = UserLanguage.query.filter_by(
code=source_lang, user_id=current_user.id).one().id
if custom_trg_lang_code:
custom_trg_lang_name = request.form.get('targetCustomLangName')
custom_lang = add_custom_language(custom_trg_lang_code,
custom_trg_lang_name)
target_lang = custom_lang.id
else:
target_lang = UserLanguage.query.filter_by(
code=target_lang, user_id=current_user.id).one().id
task_id = data_utils.process_upload_request(
user_utils.get_uid(), request.files.get('bitext_file'),
request.files.get('source_file'),
request.files.get('target_file'), source_lang, target_lang,
request.form.get('name'), request.form.get('description'),
request.form.get('topic'))
return jsonify({"result": 200, "task_id": task_id})
else:
raise Exception("Sorry, but we couldn't handle your request.")
except Exception as e:
Flash.issue(e, Flash.ERROR)
return jsonify({"result": -1})
