def register():
""" Registers a new user. """
username = request.args.get('username')
email = request.args.get('email')
password = request.args.get('password')
if not all([username, email, password]):
msg = 'You must provide a username, email, and password to register.'
return jsonify(success=False, message=msg)
existing_accounts = db.session.query(User)\
.filter(or_(User.username == username,
User.email == email)).all()
if existing_accounts:
usernames = [u.username for u in existing_accounts]
msg = 'There is already an account with this '
if username in usernames:
msg += 'username'
else:
msg += 'email address'
return jsonify(success=False, message=msg)
new_user = User(username=username,
email=email,
active=False,
password=sha256_crypt.encrypt(password))
new_user.insert()
site_url = CONFIG.get('app', 'url')
verify_link = 'https://{0}/verify?id={1}'.format(site_url, new_user.id)
subject = "Welcome to {0}!".format(CONFIG.get('app', 'name'))
email_msg = '\n'.join([
'Welcome! Your account has been created!',
'Please click the link below to verify your email address.',
verify_link, '', '',
'Thank you for joining. We hope you enjoy your account.'
])
send_mail(new_user.email, subject, email_msg)
return jsonify(success=True,
message='Please check your email to verify your account.')
def create_user_and_db():
""" Creates the database and user in app.cfg.
Assumes that postgres is fresh with initial users/permissions.
ONLY FOR DEVELOPMENT PURPOSES. aws handles this for us in RDS creation."""
username = CONFIG.get('database', 'username')
password = CONFIG.get('database', 'password')
db_name = CONFIG.get('database', 'name')
def run_cmd(cmd):
sh.psql('-c', cmd, 'postgres')
create_user = "******".format(username, password)
run_cmd(create_user)
create_db = "CREATE DATABASE {0} WITH OWNER {1}".format(db_name, username)
run_cmd(create_db)
def logout():
""" Use the session to logout the user and redirect to index """
session.pop('username', None)
session.pop('userId', None)
session.pop('loggedIn', None)
flash('You have sucessfully logged out.', 'info')
return redirect('https://' + CONFIG.get('app', 'url'))
def admin():
if request.method == 'GET':
url = 'https://' + request.host + '/api/applications'
csv = r.get(url).json()
return render_template('admin.html', config=CONFIG(), players=PLAYERS(), csv=json.dumps(csv, indent=4), config_file_view=CONFIG_VIEW())
else:
return 'Invalid Method'
def logout():
""" Use the session to logout the user and redirect to index """
session.pop('username', None)
session.pop('userId', None)
session.pop('loggedIn', None)
flash('You have sucessfully logged out.', 'info')
return redirect('https://' + CONFIG.get('app', 'url'))
def register():
""" Registers a new user. """
username = request.args.get('username')
email = request.args.get('email')
password = request.args.get('password')
if not all([username, email, password]):
msg = 'You must provide a username, email, and password to register.'
return jsonify(success=False, message=msg)
existing_accounts = db.session.query(User)\
.filter(or_(User.username == username,
User.email == email)).all()
if existing_accounts:
usernames = [u.username for u in existing_accounts]
msg = 'There is already an account with this '
if username in usernames:
msg += 'username'
else:
msg += 'email address'
return jsonify(success=False, message=msg)
new_user = User(username=username, email=email, active=False,
password=sha256_crypt.encrypt(password))
new_user.insert()
site_url = CONFIG.get('app', 'url')
verify_link = '{0}/verify?id={1}'.format(site_url, new_user.id)
subject = "Welcome to {0}!".format(CONFIG.get('app', 'name'))
email_msg = '\n'.join([
'Welcome! Your account has been created!',
'Please click the link below to verify your email address.',
verify_link, '', '',
'Thank you for joining. We hope you enjoy your account.'
])
send_mail(new_user.email, subject, email_msg)
return jsonify(success=True,
message='Please check your email to verify your account.')
def index():
""" Directs logged in users to the dashboard
and others to the index. """
if session.get('loggedIn'):
return render_template('dashboard.html')
else:
google_client_id = CONFIG.get('google', 'client_id')
return render_template('index.html', google_client_id=google_client_id)
from flask import (Flask, flash, jsonify, redirect,
render_template, request, session)
from passlib.hash import sha256_crypt
from sqlalchemy import or_
sys.path.append(os.path.join(os.path.dirname(__file__), 'lib'))
import db
from models import User
from utils import CONFIG, send_mail
from faceInfo import FaceInfo
application = Flask(__name__)
# TODO do this with api app.register_blueprint(api)
application.secret_key = CONFIG.get('app', 'secret_key')
application.debug = True
@application.route("/")
@application.route("/dashboard")
def index():
""" Directs logged in users to the dashboard
and others to the index. """
if session.get('loggedIn'):
return render_template('dashboard.html')
else:
return render_template('index.html')
def inject_globals():
return dict(app_name=CONFIG.get('app', 'name'))
import tempfile
import os
from flask import (Flask, flash, jsonify, redirect,
render_template, request, session)
from passlib.hash import sha256_crypt
from sqlalchemy import or_
sys.path.append(os.path.join(os.path.dirname(__file__), 'lib'))
import db
from models import User
from utils import analyze_photo, CONFIG, send_mail
app = Flask(__name__)
# TODO do this with api app.register_blueprint(api)
app.secret_key = CONFIG.get('app', 'secret_key')
app.debug = True
@app.route("/")
@app.route("/dashboard")
def index():
""" Directs logged in users to the dashboard
and others to the index. """
if session.get('loggedIn'):
return render_template('dashboard.html')
else:
return render_template('index.html')
def stats():
return render_template('stats.html', IP=CONFIG()['IP'], **format_statistics(server().get_status()))
def inject_globals():
return dict(app_name=CONFIG.get('app', 'name'))
def github():
return redirect(CONFIG()['GITHUB'])
import os
from flask import (Flask, flash, jsonify, redirect, render_template, request,
session)
from passlib.hash import sha256_crypt
from sqlalchemy import or_
sys.path.append(os.path.join(os.path.dirname(__file__), 'lib'))
import db
from models import User
from utils import CONFIG, send_mail
from faceInfo import FaceInfo
application = Flask(__name__)
# TODO do this with api app.register_blueprint(api)
application.secret_key = CONFIG.get('app', 'secret_key')
application.debug = True
@application.route("/")
@application.route("/dashboard")
def index():
""" Directs logged in users to the dashboard
and others to the index. """
if session.get('loggedIn'):
return render_template('dashboard.html')
else:
return render_template('index.html')
def server():
return Server(CONFIG()['IP'])
"""
This file controls all DB setup and session logic.
"""
from sqlalchemy import create_engine
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import scoped_session, sessionmaker
from utils import CONFIG, uuid
db_url = CONFIG.get('database', 'url')
db_name = CONFIG.get('database', 'name')
db_user = CONFIG.get('database', 'username')
db_pass = CONFIG.get('database', 'password')
connection_fmt = '://{username}:{password}@{url}/{name}'
DRIVER = 'postgresql+psycopg2'
FULL_DB_URL = DRIVER + connection_fmt.format(username=db_user, password=db_pass,
url=db_url, name=db_name)
engine = create_engine(FULL_DB_URL, convert_unicode=True,
isolation_level="SERIALIZABLE")
session = scoped_session(sessionmaker(autocommit=False,
autoflush=False,
bind=engine))
Base = declarative_base()
class MyBase(object):
""" Our Mixin class for defining declarative table models
def addSamples(notebook_root, dataset_root, ip):
def create_notebook_pool():
notebook_pool = []
in_result = []
cursor, db = create_connection()
sql = 'select distinct notebook_id from result'
cursor.execute(sql)
sql_res = cursor.fetchall()
for row in sql_res:
in_result.append(int(row[0]))
sql = 'select pair.nid from pair,dataset where pair.did=dataset.id and dataset.server_ip = \'' + ip + '\''
cursor.execute(sql)
sql_res = cursor.fetchall()
for row in sql_res:
if int(row[0]) not in in_result:
continue
if int(row[0]) not in notebook_pool:
notebook_pool.append(int(row[0]))
return notebook_pool
notebook_pool = create_notebook_pool()
train_config = eval(CONFIG.get('train', 'train'))
nepisode = train_config['nepisode']
obs_dim = train_config['obs_dim']
ope_dic = eval(CONFIG.get('operators', 'operations'))
learning_rate = train_config['learning_rate']
gamma = train_config['gamma']
dense_dim = train_config['dense_dim']
column_num = train_config['column_num']
act_1_dim = 0
for item in ope_dic:
if ope_dic[item]['index'] > act_1_dim:
act_1_dim = ope_dic[item]['index'] # 27
# agent = PolicyGradient(act_1_dim=act_1_dim, act_2_dim=column_num, obs_dim=obs_dim, dense_dim=dense_dim, lr=learning_rate, gamma=gamma)
for i_episode in range(nepisode):
ep_rwd = 0
notebook_id = random.choice(notebook_pool)
print("\033[0;35;40m" + "notebook_id:" + str(notebook_id) + "\033[0m")
notebook_path = notebook_root + str(notebook_id) + '.ipynb'
notebook_code = get_code_txt(notebook_path)
res_line_number = -1
s_t = get_origin_state(notebook_id,
notebook_code,
column_num,
dataset_root=dataset_root)
print(s_t)
while s_t == 'run failed':
notebook_pool.remove(notebook_id)
notebook_id = random.choice(notebook_pool)
print("\033[0;34;40m" + "notebook_id:" + str(notebook_id) +
"\033[0m")
notebook_path = notebook_root + str(notebook_id) + '.ipynb'
notebook_code = get_code_txt(notebook_path)
s_t = get_origin_state(notebook_id,
notebook_code,
column_num,
dataset_root=dataset_root)
print(s_t)
while True:
s_t = np.ravel(s_t)
action1, action2 = agent.step(s_t) # 已知当前状态，通过网络预测预测下一步的动作(这里要改)
target_content = {
'operation': action1,
'data_object': action2,
}
act = (action1, action2)
s_t, action, reward, terminal, s_t_plus_1, notebook_code, res_line_number = do_an_action(
notebook_id, notebook_code, target_content, column_num,
res_line_number) # 执行动作，得到新状态，立即回报，是否终止
print("\033[0;36;40m" + "reward:" + str(reward) + "\033[0m")
print("\033[0;36;40m" + "terminal:" + str(terminal) + "\033[0m")
s_t = np.ravel(s_t)
s_t_plus_1 = np.ravel(s_t_plus_1)
agent.memory.store_transition(s_t, act, reward) # 放入采样池(这里要改)
s_t = s_t_plus_1
ep_rwd += reward
if terminal:
agent.learn() # 一个完整过程终止，开始优化网络
print('Ep: %i' % i_episode, "|Ep_r: %i" % ep_rwd)
break
s_t = np.ravel(s_t)
action1, action2 = agent.step(s_t) # 已知当前状态，通过网络预测预测下一步的动作(这里要改)
target_content = {
'operation': action1,
'data_object': action2,
}
act = (action1, action2)
s_t, action, reward, terminal, s_t_plus_1, notebook_code, res_line_number = do_an_action(
notebook_id, notebook_code, target_content, column_num,
res_line_number) # 执行动作，得到新状态，立即回报，是否终止
print("\033[0;36;40m" + "reward:" + str(reward) + "\033[0m")
print("\033[0;36;40m" + "terminal:" + str(terminal) + "\033[0m")
s_t = np.ravel(s_t)
s_t_plus_1 = np.ravel(s_t_plus_1)
agent.memory.store_transition(s_t, act, reward) # 放入采样池(这里要改)
s_t = s_t_plus_1
ep_rwd += reward
if terminal:
agent.learn() # 一个完整过程终止，开始优化网络
print('Ep: %i' % i_episode, "|Ep_r: %i" % ep_rwd)
break
if __name__ == '__main__':
ip = get_host_ip()
server_dic = eval(CONFIG.get('server', 'server'))
notebook_root = server_dic[ip]['npath']
dataset_root = server_dic[ip]['dpath']
addSamples(notebook_root, dataset_root, ip)
def apply_redirect():
return redirect(CONFIG()['APPLY_FORM'])
def api_config():
return json.dumps(
CONFIG(),
indent=4
)
def add_result(notebook_id, origin_code, walk_logs_path="../walklogs"):
func_def = ''
func_def += "from utils import add_result\n"
func_def += "def insert_result(type, content, code, model_type):\n"
func_def += " notebook_id = " + str(notebook_id) + '\n'
func_def += " add_result(notebook_id, type, content, code, model_type)\n"
try:
this_walk_logs = np.load(walk_logs_path + '/' + str(notebook_id) +
'.npy',
allow_pickle=True).item()
model_pred = this_walk_logs['models_pred']
except:
model_pred = []
origin_code = func_def + origin_code
model_dic = eval(CONFIG.get('models', 'model_dic'))
model_result_log = {}
line = 0
metirc_dic = eval(CONFIG.get('metrics', 'metrics'))
add = False
result = set()
ins_num = 0
while (line < len(origin_code.split('\n'))):
code_list = origin_code.split('\n')
code = code_list[line]
# print(code)
m_type = -1
now_key = ''
now_len = -1
now_name = ''
for key in metirc_dic.keys():
if key in code:
m_type = metirc_dic[key]['type']
now_key = key
now_len = metirc_dic[key]['len']
now_name = metirc_dic[key]['name']
break
if m_type == 1:
index = code.find(now_key)
head = index - 1
# print("index:",index)
while (code[head].isalpha() or code[head] == '_' \
or code[head] == ']' \
or code[head] == ')' \
or code[head] == '.' \
or code[head] == '\'' or code[head] == '\"' \
or code[head].isalnum()) \
and head != -1:
if code[head] == ']':
while code[head] != '[':
head -= 1
if code[head] == ')':
while code[head] != '(':
# print(code[head])
head -= 1
head -= 1
left_index = index + now_len
left_num = 1
right_index = 0
for ind in range(left_index + 1, len(code)):
if code[ind] == '(':
left_num += 1
elif code[ind] == ')':
left_num -= 1
if left_num == 0:
right_index = ind
break
model_id = -1
for item in model_pred:
if model_pred[item] in code:
model_id = model_dic[item]
elif item in code:
model_id = model_dic[item]
temp_code = code
temp_code = temp_code.replace(' ', '')
temp_code = temp_code.replace('\t', '')
temp_code = temp_code.replace('\'', '\\\'')
temp_code = temp_code.replace('"', '\\"')
need2print = "insert_result(" + str(model_id) + "," + code[
head + 1:right_index +
1] + ',"' + temp_code + '", "' + now_name + '")'
ctxt = code[head + 1:right_index + 1]
add_model_result = get_model_from_code(ctxt, origin_code, line,
ins_num)
result.add(add_model_result[0])
origin_code = add_model_result[1]
add_line = add_model_result[2]
line += 1
line += add_line
origin_code = insert_one_line_in_code(origin_code, line,
need2print)
ins_num += 1
line += 1
origin_code = insert_one_line_in_code(
origin_code, line, 'mdtypes_' + str(ins_num) + ' = set()')
add = True
elif m_type == 2:
index = code.find(now_key)
head = index - 2
while (code[head].isalpha() or code[head] == '_' \
or code[head] == ']' \
or code[head] == ')' \
or code[head] == '.' \
or code[head] == '\'' or code[head] == '\"' \
or code[head].isalnum()) \
and head != -1:
if code[head] == ']':
while code[head] != '[':
head -= 1
if code[head] == ')':
while code[head] != '(':
# print(code[head])
head -= 1
head -= 1
right_index = index + now_len
model_id = -1
for item in model_pred:
if model_pred[item] in code:
model_id = model_dic[item]
elif item in code:
model_id = model_dic[item]
temp_code = code
temp_code = temp_code.replace(' ', '')
temp_code = temp_code.replace('\t', '')
temp_code = temp_code.replace('\'', '\\\'')
temp_code = temp_code.replace('"', '\\"')
need2print = "insert_result(" + str(model_id) + "," + code[
head + 1:right_index +
1] + ',"' + temp_code + '", "' + now_name + '")'
ctxt = code[head + 1:right_index + 1]
add_model_result = get_model_from_code(ctxt, origin_code, line,
ins_num)
result.add(add_model_result[0])
origin_code = add_model_result[1]
add_line = add_model_result[2]
# print(add_model_result[1])
line += 1
line += add_line
origin_code = insert_one_line_in_code(origin_code, line,
need2print)
line += 1
ins_num += 1
origin_code = insert_one_line_in_code(
origin_code, line, 'mdtypes_' + str(ins_num) + ' = set()')
add = True
elif m_type == 3:
now_len = len(now_key) - 1
add = True
index = code.find(now_key)
if index == 0:
head = index
left_index = index + now_len
left_num = 1
right_index = 0
for ind in range(left_index + 1, len(code)):
if code[ind] == '(':
left_num += 1
elif code[ind] == ')':
left_num -= 1
if left_num == 0:
right_index = ind
break
model_id = -1
for item in model_pred:
if model_pred[item] in code:
model_id = model_dic[item]
elif item in code:
model_id = model_dic[item]
temp_code = code
temp_code = temp_code.replace(' ', '')
temp_code = temp_code.replace('\t', '')
temp_code = temp_code.replace('\'', '\\\'')
temp_code = temp_code.replace('"', '\\"')
need2print = "insert_result(" + str(model_id) + "," + code[
head:right_index +
1] + ',"' + temp_code + '", "' + now_name + '")'
ctxt = code[head:right_index + 1]
add_model_result = get_model_from_code(ctxt, origin_code, line,
ins_num)
result.add(add_model_result[0])
origin_code = add_model_result[1]
add_line = add_model_result[2]
# print(add_model_result[1])
line += 1
line += add_line
origin_code = insert_one_line_in_code(origin_code, line,
need2print)
line += 1
ins_num += 1
origin_code = insert_one_line_in_code(
origin_code, line, 'mdtypes_' + str(ins_num) + ' = set()')
elif code[index - 1] != '.':
if code[index -1].isalpha() or code[index-1] == '_' \
or code[index-1].isalnum():
line += 1
continue
head = index
left_index = index + now_len
left_num = 1
right_index = 0
for ind in range(left_index + 1, len(code)):
if code[ind] == '(':
left_num += 1
elif code[ind] == ')':
left_num -= 1
if left_num == 0:
right_index = ind
break
model_id = -1
for item in model_pred:
if model_pred[item] in code:
model_id = model_dic[item]
elif item in code:
model_id = model_dic[item]
temp_code = code
temp_code = temp_code.replace(' ', '')
temp_code = temp_code.replace('\t', '')
temp_code = temp_code.replace('\'', '\\\'')
temp_code = temp_code.replace('"', '\\"')
need2print = "insert_result(" + str(model_id) + "," + code[
head:right_index +
1] + ',"' + temp_code + '", "' + now_name + '")'
ctxt = code[head:right_index + 1]
add_model_result = get_model_from_code(ctxt, origin_code, line,
ins_num)
result.add(add_model_result[0])
origin_code = add_model_result[1]
add_line = add_model_result[2]
# print(add_model_result[1])
line += 1
line += add_line
origin_code = insert_one_line_in_code(origin_code, line,
need2print)
line += 1
ins_num += 1
origin_code = insert_one_line_in_code(
origin_code, line, 'mdtypes_' + str(ins_num) + ' = set()')
else:
head = index - 2
while (code[head].isalpha() or code[head] == '_' \
or code[head] == ']' \
or code[head] == ')' \
or code[head] == '.' \
or code[head] == '\'' or code[head] == '\"' \
or code[head].isalnum())\
and head != -1:
if code[head] == ']':
while code[head] != '[':
head -= 1
if code[head] == ')':
while code[head] != '(':
# print(code[head])
head -= 1
head -= 1
left_index = index + now_len
left_num = 1
right_index = 0
for ind in range(left_index + 1, len(code)):
if code[ind] == '(':
left_num += 1
elif code[ind] == ')':
left_num -= 1
if left_num == 0:
right_index = ind
break
model_id = -1
for item in model_pred:
if model_pred[item] in code:
model_id = model_dic[item]
elif item in code:
model_id = model_dic[item]
temp_code = code
temp_code = temp_code.replace(' ', '')
temp_code = temp_code.replace('\t', '')
temp_code = temp_code.replace('\'', '\\\'')
temp_code = temp_code.replace('"', '\\"')
need2print = "insert_result(" + str(model_id) + "," + code[
head + 1:right_index +
1] + ',"' + temp_code + '", "' + now_name + '")'
ctxt = code[head + 1:right_index + 1]
add_model_result = get_model_from_code(ctxt, origin_code, line,
ins_num)
result.add(add_model_result[0])
origin_code = add_model_result[1]
add_line = add_model_result[2]
# print(add_model_result[1])
# get_one_result
line += 1
line += add_line
origin_code = insert_one_line_in_code(origin_code, line,
need2print)
line += 1
ins_num += 1
origin_code = insert_one_line_in_code(
origin_code, line, 'mdtypes_' + str(ins_num) + ' = set()')
line += 1
# print(origin_code)
return add_model_result[1], add, result
def heroku():
return redirect(CONFIG()['HEROKU'])