def train_launch(): id = request.form.get('engine_id') if user_utils.is_normal(): return url_for('index') task_id, monitor_task_id = Trainer.launch(id, user_utils.is_admin()) return url_for('train.train_console', id=id)
def instances_datatables_feed(): if user_utils.is_normal(): return redirect(url_for('index')) columns = [Engine.id, Engine.name] dt = datatables.Datatables() rows, rows_filtered, search = dt.parse(Engine, columns, request, Engine.status.like("training")) user_data = [] for engine in (rows_filtered if search else rows): user_data.append([ engine.id, engine.name, engine.uploader.email if engine.uploader else "", "Training", None ]) columns_translating = [RunningEngines.id] rows_translating, rows_translating_filtered, search_translating = dt.parse( RunningEngines, columns_translating, request) for engine_entry in (rows_translating_filtered if search_translating else rows_translating): user_data.append([ engine_entry.engine.id, engine_entry.engine.name, engine_entry.engine.uploader.email if engine_entry.engine.uploader else "", "Translating", None ]) return dt.response(rows, rows_filtered, user_data)
def train_launching(task_id): if user_utils.is_normal(): return redirect(url_for('index')) return render_template('launching.html.jinja2', page_name='train', page_title='Launching training', task_id=task_id)
def stop_engine(): if user_utils.is_normal(): return redirect(url_for('index')) id = request.args.get('id') try: Trainer.stop(id, admin_stop=True) except: pass return redirect(request.referrer)
def train_test(): if user_utils.is_normal(): return redirect(url_for('index')) engine_id = request.form.get('engine_id') task = tasks.test_training.apply_async(args=[engine_id]) engine = Engine.query.filter_by(id=engine_id).first() engine.test_task_id = task.id db.session.commit() return task.id
def become(type, id): if user_utils.is_normal(): return redirect(url_for('index')) if user_utils.get_user().admin: user = User.query.filter_by(id=id).first() user.expert = (type == "expert") user.admin = (type == "admin") db.session.commit() return redirect(request.referrer)
def train_attention(id): if user_utils.is_normal(): return send_file( os.path.join(app.config['BASE_CONFIG_FOLDER'], "attention.png")) engine = Engine.query.filter_by(id=id).first() files = glob.glob(os.path.join(engine.path, "*.att")) if len(files) > 0: return send_file(files[0]) else: return send_file( os.path.join(app.config['BASE_CONFIG_FOLDER'], "attention.png"))
def train_test_status(): if user_utils.is_normal(): return redirect(url_for('index')) task_id = request.form.get('task_id') task_success, task_value = utils.get_task_result(tasks.test_training, task_id) if task_success is not None: if not task_success: return jsonify({"result": -2}) return jsonify({"result": 200, "test": task_value}) else: return jsonify({"result": -1})
def python_feed(): if user_utils.is_normal(): return redirect(url_for('index')) draw = request.form.get('draw') search = request.form.get('search[value]') start = int(request.form.get('start')) length = int(request.form.get('length')) order = int(request.form.get('order[0][column]')) dir = request.form.get('order[0][dir]') ps = subprocess.Popen("ps aux | grep python", shell=True, stdout=subprocess.PIPE) ps.wait() rows = [] for line_raw in ps.stdout: line = line_raw.decode("utf-8").strip() if line: data = line.split(None) command = " ".join(data[10:]) if not any(x in command for x in ["ps aux", "grep python", "<defunct>"]): rows.append([data[1], data[8], data[2], data[3], command]) if start and length: rows = rows[start:length] if order is not None: rows.sort(key=lambda row: row[order], reverse=(dir == "desc")) rows_filtered = [] if search: for row in rows: found = False for col in row: if not found: if search in col: rows_filtered.append(row) found = True return jsonify({ "draw": int(draw) + 1, "recordsTotal": len(rows), "recordsFiltered": len(rows_filtered) if search else len(rows), "data": rows_filtered if search else rows })
def user_edit(id): if user_utils.is_normal(): return redirect(url_for('index')) user = User.query.filter_by(id=id).one() if request.method == 'GET': return render_template('users_edit.admin.html.jinja2', page_name='admin_users', page_title='Edit user', user=user) else: notes = request.form.get('userNotes') user.notes = escape(notes) db.session.commit() return redirect(request.referrer)
def data_upload_perform(): if user_utils.is_normal(): return redirect(url_for('index')) try: if request.method == 'POST': task_id = data_utils.process_upload_request(user_utils.get_uid(), request.files.get('bitext_file'), request.files.get('source_file'), request.files.get('target_file'), request.form.get('source_lang'), request.form.get('target_lang'), request.form.get('name'), request.form.get('description'), request.form.get('topic')) return jsonify({ "result": 200, "task_id": task_id }) else: raise Exception("Sorry, but we couldn't handle your request.") except Exception as e: Flash.issue(e, Flash.ERROR) return jsonify({ "result": -1 })
def user_datatables_feed(): if user_utils.is_normal(): return redirect(url_for('index')) columns = [User.id, User.username, User.email, User.notes] dt = datatables.Datatables() rows, rows_filtered, search = dt.parse(User, columns, request) user_data = [] for user in (rows_filtered if search else rows): user_data.append([ user.id, user.username, user.email, 'Admin' if user.admin else 'Expert' if user.expert else 'Beginner', user.notes, '', user.admin, user.expert ]) return dt.response(rows, rows_filtered, user_data)
def train_index(): if user_utils.is_normal(): return redirect(url_for('index')) currently_training = Engine.query.filter_by(uploader_id = user_utils.get_uid()) \ .filter(Engine.status.like("training")).all() if (len(currently_training) > 0): return redirect( url_for('train.train_console', id=currently_training[0].id)) currently_launching = Engine.query.filter_by(uploader_id = user_utils.get_uid()) \ .filter(Engine.status.like("launching")).all() if (len(currently_launching) > 0): return redirect( url_for('train.train_launching', task_id=currently_launching[0].bg_task_id)) random_name = namegenerator.gen() tryout = 0 while len(Engine.query.filter_by(name=random_name).all()): random_name = namegenerator.gen() tryout += 1 if tryout >= 5: random_name = "" break random_name = " ".join(random_name.split("-")[:2]) library_corpora = user_utils.get_user_corpora().filter( LibraryCorpora.corpus.has(Corpus.type == "bilingual")).all() corpora = [c.corpus for c in library_corpora] languages = UserLanguage.query.filter_by(user_id=current_user.id).order_by( UserLanguage.name).all() return render_template('train.html.jinja2', page_name='train', page_title='Train', corpora=corpora, random_name=random_name, languages=languages)
def train_start(): if user_utils.is_normal(): return url_for('index') engine_path = os.path.join( user_utils.get_user_folder("engines"), utils.normname(user_utils.get_user().username, request.form['nameText'])) task = tasks.launch_training.apply_async(args=[ user_utils.get_uid(), engine_path, { i[0]: i[1] if i[0].endswith('[]') else i[1][0] for i in request.form.lists() } ]) return jsonify({ "result": 200, "launching_url": url_for('train.train_launching', task_id=task.id) })
def delete_user(): if user_utils.is_normal(): return redirect(url_for('index')) id = request.args.get('id') try: assert int(id) != user_utils.get_uid() user = User.query.filter_by(id=id).first() for corpus in Corpus.query.filter_by(owner_id=id).all(): user_utils.library_delete("library_corpora", corpus.id, id) for engine_entry in user.user_engines: user_utils.library_delete("library_engines", engine_entry.engine.id, id) shutil.rmtree(user_utils.get_user_folder(user_id=id)) db.session.delete(user) db.session.commit() except: pass return redirect(request.referrer)
def admin_system(): if user_utils.is_normal(): return redirect(url_for('index')) factor = 1073741824 vmem = psutil.virtual_memory() ram = { "percent": vmem.percent, "used": round(vmem.used / factor, 2), "total": round(vmem.total / factor, 2) } # GB hdd = psutil.disk_usage(app.config['USERSPACE_FOLDER']) disk_usage = { "percent": round((hdd.used / hdd.total) * 100, 2), "used": round(hdd.used / factor, 2), "total": round(hdd.total / factor, 2) } # GB gpus = [] pynvml.nvmlInit() for i in range(0, pynvml.nvmlDeviceGetCount()): handle = pynvml.nvmlDeviceGetHandleByIndex(i) resources = pynvml.nvmlDeviceGetUtilizationRates(handle) gpus.append({ "id": i, "memory": resources.memory, "proc": resources.gpu }) return render_template('system.admin.html.jinja2', page_name='admin_system', page_title='System', ram=ram, cpu=round(psutil.cpu_percent(), 2), gpus=gpus, disk_usage=disk_usage)
def train_finish(id): if user_utils.is_normal(): return redirect(url_for('index')) return _train_stop(id, False)
def admin_instances(): if user_utils.is_normal(): return redirect(url_for('index')) return render_template('instances.admin.html.jinja2', page_name='admin_instances', page_title='Instances')
def data_upload_perform(): if user_utils.is_normal(): return redirect(url_for('index')) try: if request.method == 'POST': # Handle possible custom languages def add_custom_language(code, name): custom_language = UserLanguage.query.filter_by( code=code, user_id=current_user.id).first() if custom_language: custom_language.name = custom_src_lang_name db.session.commit() else: custom_language = UserLanguage(code=code, name=name, user_id=current_user.id) db.session.add(custom_language) db.session.commit() return UserLanguage.query.filter_by( code=code, user_id=current_user.id).first() source_lang = request.form.get('source_lang') target_lang = request.form.get('target_lang') custom_src_lang_code = request.form.get('sourceCustomLangCode') custom_trg_lang_code = request.form.get('targetCustomLangCode') if custom_src_lang_code: custom_src_lang_name = request.form.get('sourceCustomLangName') custom_lang = add_custom_language(custom_src_lang_code, custom_src_lang_name) source_lang = custom_lang.id else: source_lang = UserLanguage.query.filter_by( code=source_lang, user_id=current_user.id).one().id if custom_trg_lang_code: custom_trg_lang_name = request.form.get('targetCustomLangName') custom_lang = add_custom_language(custom_trg_lang_code, custom_trg_lang_name) target_lang = custom_lang.id else: target_lang = UserLanguage.query.filter_by( code=target_lang, user_id=current_user.id).one().id task_id = data_utils.process_upload_request( user_utils.get_uid(), request.files.get('bitext_file'), request.files.get('source_file'), request.files.get('target_file'), source_lang, target_lang, request.form.get('name'), request.form.get('description'), request.form.get('topic')) return jsonify({"result": 200, "task_id": task_id}) else: raise Exception("Sorry, but we couldn't handle your request.") except Exception as e: Flash.issue(e, Flash.ERROR) return jsonify({"result": -1})