def _check_git_branch():
branch = output("git rev-parse --abbrev-ref HEAD")
validate(
error="This program needs to be on master branch before to build.",
target="git repository",
valid=(branch == "master"))
def _check_ssh_connection():
script = "ssh -i ./deploy_key -p %i -q %s@%s [[ -d %s ]] && echo 1 || echo 0" % (
conf.ssh_port, conf.ssh_user, conf.ssh_host, conf.ssh_path)
validate(error="ssh connection could not be established.",
target="ssh address",
valid=(output(script) is "1"))
def _check_pkg_content():
folders = [f.name for f in os.scandir(paths.pkg) if f.is_dir()]
diff = set(folders) - set(conf.packages)
validate(error="No package.py was found in pkg subdirectories: %s" %
(", ".join(diff)),
target="content",
valid=(len(diff) == 0))
def _check_internet_up():
try:
socket.create_connection(("www.github.com", 80))
connected = True
except OSError:
connected = False
validate(error="This program needs to be connected to internet.",
target="internet",
valid=connected)
def _check_travis_openssl(content):
valid = False
error_msg = "No openssl statement could be found in your travis file.\nPlease make sure to execute: travis encrypt-file ./deploy_key --add"
if "before_install" in content:
for statement in content["before_install"]:
if statement.startswith("openssl"):
valid = True
validate(error=error_msg, target="openssl", valid=valid)
def _check_database():
valid = True
error_msg = ""
if conf.db in ("core", "extra", "community"):
valid = False
error_msg = "Database must be different than core, community and extra."
elif conf.db.isalnum() is False:
valid = False
error_msg = "Database must be an alphanumeric string."
validate(error=error_msg, target="database", valid=valid)
def _check_deploy_key():
valid = True
target = "deploy_key"
if IS_TRAVIS and os.path.isfile(os.path.join(paths.base,
"deploy_key.enc")) is False:
valid = False
target = "deploy_key.enc"
elif os.path.isfile(os.path.join(paths.base, "deploy_key")) is False:
valid = False
validate(error="%s could not been found." % target,
target=target,
valid=valid)
def _check_repository():
valid = True
target = "repository.yml"
if IS_TRAVIS and os.path.isfile(
os.path.join(paths.base, "repository.yml.enc")) is False:
valid = False
target = "repository.yml.enc"
elif os.path.isfile(os.path.join(paths.base, "repository.yml")) is False:
valid = False
validate(error="%s could not been found." % target,
target=target,
valid=valid)
def _check_content():
valid = True
content = ALIAS_CONFIGS
if not remote_repository():
content = set(content) - set(SSH_CONFIGS)
for name in content:
if not get_attr_value(conf, name):
valid = False
break
validate(error="%s must be defined in repository.yml" % name,
target="content",
valid=valid)
def _check_github_token():
valid = False
user = git_remote_path().split("/")[1]
response = output("curl -su %s:%s https://api.github.com/user" %
(user, conf.github_token))
content = json.loads(response)
if "login" in content:
valid = True
validate(
error=
"An error occured while trying to connect to your github repository with your encrypted token.\nPlease make sure that your token is working.",
target="github token api",
valid=valid)
def signup():
if request.method == 'GET':
return render_template('signup.html')
# [Do] Process data instead..
name = request.form['name']
username = request.form['username']
email = request.form['email']
passwd = request.form['passwd']
# [WIP] Check for sanity of fields, redirect if unclean (avoid SQLIA).
resp = validate(username, name)
if resp['is_valid'] == 0:
return render_template('signup.html', issue=resp['message'])
elif resp['is_valid'] == 1:
name = resp['name']
# [Do] Check for username availability (assuming clean data)
q = "SELECT `name` FROM `credentials` WHERE username='******';"
try:
cur.execute(q)
match = cur.fetchone()
except:
# Connection-Reset-On-Idle exception handler.
try:
conn = mysql.connect()
cur = conn.cursor()
cur.execute(q.format(username))
match = cur.fetchone()
except:
# [Done] Redirect to a separate template meant for errors.
return render_template('error.html')
if not match is None:
return render_template('signup.html', issue='Username already taken')
# Proceed with insertion otherwise.
q = '''
INSERT INTO `credentials` (username, name, hash, email)
VALUES (%s,%s,%s,%s)
'''
try:
cur.execute(q, (username, name,
generate_password_hash(passwd, method='sha1'), email))
conn.commit()
except:
return render_template('error.html')
# Log the user in.
login_user(User(username))
session['name'] = name
session['username'] = username
return redirect(url_for('dashboard', username=username))
def _check_pkg_testing():
if conf.environment is "prod":
return
valid = True
error_msg = ""
if conf.package_to_test is None:
valid = False
error_msg = "You need to define which package you want to test with this command: make package test=discord"
elif conf.package_to_test not in conf.packages:
valid = False
error_msg = "%s is not in pkg directory." % conf.package_to_test
elif has_git_changes(paths.pkg + "/" + conf.package_to_test):
valid = False
error_msg = "You need to commit your changes before to test your package."
validate(error=error_msg, target="testing", valid=valid)
def _check_mirror_connection():
token = secrets.token_hex(15)
source = os.path.join(paths.mirror, "validation_token")
with open(source, "w") as f:
f.write(token)
f.close()
os.system(
"rsync -aqvz -e 'ssh -i ./deploy_key -p %i' %s %s@%s:%s" %
(conf.ssh_port, source, conf.ssh_user, conf.ssh_host, conf.ssh_path))
try:
response = requests.get(conf.url + "/validation_token")
valid = True if response.status_code == 200 else False
except requests.RequestException:
valid = False
validate(error="This program can't connect to %s." % conf.url,
target="mirror host",
valid=valid)
def signup():
if request.method == 'GET':
return render_template('signup.html')
# [Do] Process data instead..
name = request.form['name']
username = request.form['username']
email = request.form['email']
passwd = request.form['passwd']
# [WIP] Check for sanity of fields, redirect if unclean (avoid SQLIA).
resp = validate(username, name)
if resp['is_valid'] == 0:
return render_template('signup.html', issue=resp['message'])
elif resp['is_valid'] == 1:
name = resp['name']
# [Do] Check for username availability (assuming clean data)
q = "SELECT `name` FROM `credentials` WHERE username='******';"
try:
cur.execute(q.format(username))
match = cur.fetchone()
except:
# Connection-Reset-On-Idle exception handler.
try:
conn = mysql.connect()
cur = conn.cursor()
cur.execute(q.format(username))
match = cur.fetchone()
except:
# [Done] Redirect to a separate route meant for errors.
global ERROR_EXISTS
ERROR_EXISTS = True
return redirect(url_for('error'))
if not match is None:
return render_template('signup.html', issue='Username already taken')
# Proceed with insertion otherwise.
q = '''
INSERT INTO `credentials` (username, name, hash, email)
VALUES (%s,%s,%s,%s)
'''
try:
cur.execute(q, (username, name,
generate_password_hash(passwd, method='sha1'), email))
conn.commit()
except:
# global ERROR_EXISTS
'''
Uncommenting above line gives the following SyntaxError:
name 'ERROR_EXISTS' is assigned to before global declaration
The error occurs before server starts.
But, this does not happen in other places where `global ...` is implemented.
Debugging required.
'''
ERROR_EXISTS = True
return redirect(url_for('error'))
# Log the user in.
login_user(User(username))
session['logged_in'] = True
session['name'] = name
session['username'] = username
return redirect(url_for('dashboard', username=username))
def trainer(X, Y, data, para):
'''
IMPORTANT: to see the plots in jupyter remember to invoke:
%matplotlib inline
(could be used as stand-alone but we call it through commands)
INPUT: X with one or more variables in float32 and Y with a single
binary value. These can be easily produced through
transform_data if you insist to bybass commands function.
OUTOUT: Trains a model and outputs the training results with a plot
comparing train and test. The predictions are loaded on to
a data object.
'''
ind_var = Y # this is used later for output
X_num = X
data = data.sample(frac=1)
X, Y = transform_data(data, para['flatten'], X, Y)
if para['validation'] is not False:
X, Y, X_val, Y_val = separate(X, Y, para['validation'])
try:
dims = X.shape[1]
except IndexError:
dims = X_num
para['dims'] = dims
if para['layers'] == 1:
para['shape'] = 'funnel'
if para['neuron_max'] == 'auto' and dims >= 4:
para['neuron_max'] = int(dims + (dims * 0.2))
elif para['neuron_max'] == 'auto':
para['neuron_max'] = 4
para['neuron_count'] = shapes(para)
if para['model'] is 'mlp':
model, history = mlp(X, Y, para)
if para['model'] is 'regression':
model, history = regression(X, Y, para['epoch'], para['reg_mode'])
network_scale = len(
X) * para['epoch'] * para['layers'] * para['neuron_max']
# train / test results
ex2 = pd.DataFrame({
'train_acc': history.history['acc'],
'train_loss': history.history['loss'],
'test_acc': history.history['val_acc'],
'test_loss': history.history['val_loss']
})
scores = model.evaluate(X, Y, verbose=para['verbose'])
if para['double_check'] is False or para['validation'] is False:
if para['hyperscan'] is False:
print("\n%s: %.2f%%" % (model.metrics_names[1], scores[1] * 100))
# calculate and round predictions
predictions = model.predict(X)
rounded = [round(x[0]) for x in predictions]
if para['double_check'] is True:
check(Y, rounded, scores)
if para['save_model'] is False and para['validation'] is not False:
para['save_model'] = 'saved_model'
if para['save_model'] is not False:
save_model_as(X_num, data.columns, model, para['save_model'],
para['flatten'])
# shuffling and separating the data
if para['validation'] is not False:
validate(X_val, Y_val, para['save_model'])
# model parameters
ex1 = pd.Series({
'ind_var': ind_var,
'y_transform': para['flatten'],
'n=': len(X),
'features': para['dims'],
'epochs': para['epoch'],
'layers': para['layers'],
'dropout': para['dropout'],
'batch_size': para['batch_size'],
'shape': para['shape'],
'max_neurons': para['neuron_max'],
'network_scale': network_scale
})
# prevent Tensorflow memory leakage
K.clear_session()
if para['hyperscan'] is True:
ex3 = pd.Series({
'optimizer': para['optimizer'],
'activation': para['activation'],
'activation_out': para['activation_out'],
'loss': para['loss'],
})
return ex1, ex2, ex3
else:
display(pd.DataFrame(ex1).transpose())
accuracy(ex2)
return
def _check_port():
validate(error="port must be an interger in repository.yml",
target="port",
valid=(type(conf.ssh_port) == int))
def _check_travis_lint(content):
error_msg = "An error occured while trying to parse your travis file.\nPlease make sure that the file is valid YAML.",
validate(error=error_msg, target="lint", valid=(type(content) is dict))
def _check_pkg_directory():
validate(error="No package was found in pkg directory.",
target="directory",
valid=(len(conf.packages) > 0))
def _check_operating_system():
validate(error="This program needs to be executed in Arch Linux.",
target="operating system",
valid=(platform.dist()[0] == "arch"))
def _check_is_docker_image():
validate(error="This program needs to be executed in a docker image.",
target="docker",
valid=(os.environ.get("IS_DOCKER", False)))
def _check_user_privileges():
validate(error="This program needs to be not execute as root.",
target="user privileges",
valid=(os.getuid() != 0))
def signup():
try:
if session['logged_in']:
return redirect(url_for('dashboard', username=session['username']))
except KeyError:
pass
if request.method == 'GET':
return render_template('signup.html')
name = request.form['name']
username = request.form['username']
email = request.form['email']
passwd = request.form['passwd']
# [WIP] Apply checks on fields, redirect if unclean (avoid SQLIA).
resp = validate(username, name)
if resp['is_valid'] == 0:
return render_template('signup.html', issue=resp['message'])
elif resp['is_valid'] == 1:
name = resp['name']
# Check username availability (assuming clean data)
q = "SELECT `name` FROM `credentials` WHERE username='******';"
try:
cur.execute(q.format(username))
match = cur.fetchone()
except:
# Connection-Reset-On-Idle exception handler
try:
conn = mysql.connect()
cur = conn.cursor()
cur.execute(q.format(username))
match = cur.fetchone()
except:
# Redirect to a separate route meant for errors
session['ERROR_EXISTS'] = True
return redirect(url_for('error'))
if match is not None:
return render_template('signup.html', issue='Username already taken')
# Proceed with insertion otherwise
q = '''
INSERT INTO `credentials` (username, name, hash, email)
VALUES (%s,%s,%s,%s)
'''
try:
cur.execute(q, (username, name,
generate_password_hash(passwd, method='sha1'), email))
conn.commit()
except:
session['ERROR_EXISTS'] = True
return redirect(url_for('error'))
# Log the user in
login_user(User(username))
session['logged_in'] = True
session['name'] = name
session['username'] = username
return redirect(url_for('dashboard', username=username))
