def _check_git_branch(): branch = output("git rev-parse --abbrev-ref HEAD") validate( error="This program needs to be on master branch before to build.", target="git repository", valid=(branch == "master"))
def _check_ssh_connection(): script = "ssh -i ./deploy_key -p %i -q %s@%s [[ -d %s ]] && echo 1 || echo 0" % ( conf.ssh_port, conf.ssh_user, conf.ssh_host, conf.ssh_path) validate(error="ssh connection could not be established.", target="ssh address", valid=(output(script) is "1"))
def _check_pkg_content(): folders = [f.name for f in os.scandir(paths.pkg) if f.is_dir()] diff = set(folders) - set(conf.packages) validate(error="No package.py was found in pkg subdirectories: %s" % (", ".join(diff)), target="content", valid=(len(diff) == 0))
def _check_internet_up(): try: socket.create_connection(("www.github.com", 80)) connected = True except OSError: connected = False validate(error="This program needs to be connected to internet.", target="internet", valid=connected)
def _check_travis_openssl(content): valid = False error_msg = "No openssl statement could be found in your travis file.\nPlease make sure to execute: travis encrypt-file ./deploy_key --add" if "before_install" in content: for statement in content["before_install"]: if statement.startswith("openssl"): valid = True validate(error=error_msg, target="openssl", valid=valid)
def _check_database(): valid = True error_msg = "" if conf.db in ("core", "extra", "community"): valid = False error_msg = "Database must be different than core, community and extra." elif conf.db.isalnum() is False: valid = False error_msg = "Database must be an alphanumeric string." validate(error=error_msg, target="database", valid=valid)
def _check_deploy_key(): valid = True target = "deploy_key" if IS_TRAVIS and os.path.isfile(os.path.join(paths.base, "deploy_key.enc")) is False: valid = False target = "deploy_key.enc" elif os.path.isfile(os.path.join(paths.base, "deploy_key")) is False: valid = False validate(error="%s could not been found." % target, target=target, valid=valid)
def _check_repository(): valid = True target = "repository.yml" if IS_TRAVIS and os.path.isfile( os.path.join(paths.base, "repository.yml.enc")) is False: valid = False target = "repository.yml.enc" elif os.path.isfile(os.path.join(paths.base, "repository.yml")) is False: valid = False validate(error="%s could not been found." % target, target=target, valid=valid)
def _check_content(): valid = True content = ALIAS_CONFIGS if not remote_repository(): content = set(content) - set(SSH_CONFIGS) for name in content: if not get_attr_value(conf, name): valid = False break validate(error="%s must be defined in repository.yml" % name, target="content", valid=valid)
def _check_github_token(): valid = False user = git_remote_path().split("/")[1] response = output("curl -su %s:%s https://api.github.com/user" % (user, conf.github_token)) content = json.loads(response) if "login" in content: valid = True validate( error= "An error occured while trying to connect to your github repository with your encrypted token.\nPlease make sure that your token is working.", target="github token api", valid=valid)
def signup(): if request.method == 'GET': return render_template('signup.html') # [Do] Process data instead.. name = request.form['name'] username = request.form['username'] email = request.form['email'] passwd = request.form['passwd'] # [WIP] Check for sanity of fields, redirect if unclean (avoid SQLIA). resp = validate(username, name) if resp['is_valid'] == 0: return render_template('signup.html', issue=resp['message']) elif resp['is_valid'] == 1: name = resp['name'] # [Do] Check for username availability (assuming clean data) q = "SELECT `name` FROM `credentials` WHERE username='******';" try: cur.execute(q) match = cur.fetchone() except: # Connection-Reset-On-Idle exception handler. try: conn = mysql.connect() cur = conn.cursor() cur.execute(q.format(username)) match = cur.fetchone() except: # [Done] Redirect to a separate template meant for errors. return render_template('error.html') if not match is None: return render_template('signup.html', issue='Username already taken') # Proceed with insertion otherwise. q = ''' INSERT INTO `credentials` (username, name, hash, email) VALUES (%s,%s,%s,%s) ''' try: cur.execute(q, (username, name, generate_password_hash(passwd, method='sha1'), email)) conn.commit() except: return render_template('error.html') # Log the user in. login_user(User(username)) session['name'] = name session['username'] = username return redirect(url_for('dashboard', username=username))
def _check_pkg_testing(): if conf.environment is "prod": return valid = True error_msg = "" if conf.package_to_test is None: valid = False error_msg = "You need to define which package you want to test with this command: make package test=discord" elif conf.package_to_test not in conf.packages: valid = False error_msg = "%s is not in pkg directory." % conf.package_to_test elif has_git_changes(paths.pkg + "/" + conf.package_to_test): valid = False error_msg = "You need to commit your changes before to test your package." validate(error=error_msg, target="testing", valid=valid)
def _check_mirror_connection(): token = secrets.token_hex(15) source = os.path.join(paths.mirror, "validation_token") with open(source, "w") as f: f.write(token) f.close() os.system( "rsync -aqvz -e 'ssh -i ./deploy_key -p %i' %s %s@%s:%s" % (conf.ssh_port, source, conf.ssh_user, conf.ssh_host, conf.ssh_path)) try: response = requests.get(conf.url + "/validation_token") valid = True if response.status_code == 200 else False except requests.RequestException: valid = False validate(error="This program can't connect to %s." % conf.url, target="mirror host", valid=valid)
def signup(): if request.method == 'GET': return render_template('signup.html') # [Do] Process data instead.. name = request.form['name'] username = request.form['username'] email = request.form['email'] passwd = request.form['passwd'] # [WIP] Check for sanity of fields, redirect if unclean (avoid SQLIA). resp = validate(username, name) if resp['is_valid'] == 0: return render_template('signup.html', issue=resp['message']) elif resp['is_valid'] == 1: name = resp['name'] # [Do] Check for username availability (assuming clean data) q = "SELECT `name` FROM `credentials` WHERE username='******';" try: cur.execute(q.format(username)) match = cur.fetchone() except: # Connection-Reset-On-Idle exception handler. try: conn = mysql.connect() cur = conn.cursor() cur.execute(q.format(username)) match = cur.fetchone() except: # [Done] Redirect to a separate route meant for errors. global ERROR_EXISTS ERROR_EXISTS = True return redirect(url_for('error')) if not match is None: return render_template('signup.html', issue='Username already taken') # Proceed with insertion otherwise. q = ''' INSERT INTO `credentials` (username, name, hash, email) VALUES (%s,%s,%s,%s) ''' try: cur.execute(q, (username, name, generate_password_hash(passwd, method='sha1'), email)) conn.commit() except: # global ERROR_EXISTS ''' Uncommenting above line gives the following SyntaxError: name 'ERROR_EXISTS' is assigned to before global declaration The error occurs before server starts. But, this does not happen in other places where `global ...` is implemented. Debugging required. ''' ERROR_EXISTS = True return redirect(url_for('error')) # Log the user in. login_user(User(username)) session['logged_in'] = True session['name'] = name session['username'] = username return redirect(url_for('dashboard', username=username))
def trainer(X, Y, data, para): ''' IMPORTANT: to see the plots in jupyter remember to invoke: %matplotlib inline (could be used as stand-alone but we call it through commands) INPUT: X with one or more variables in float32 and Y with a single binary value. These can be easily produced through transform_data if you insist to bybass commands function. OUTOUT: Trains a model and outputs the training results with a plot comparing train and test. The predictions are loaded on to a data object. ''' ind_var = Y # this is used later for output X_num = X data = data.sample(frac=1) X, Y = transform_data(data, para['flatten'], X, Y) if para['validation'] is not False: X, Y, X_val, Y_val = separate(X, Y, para['validation']) try: dims = X.shape[1] except IndexError: dims = X_num para['dims'] = dims if para['layers'] == 1: para['shape'] = 'funnel' if para['neuron_max'] == 'auto' and dims >= 4: para['neuron_max'] = int(dims + (dims * 0.2)) elif para['neuron_max'] == 'auto': para['neuron_max'] = 4 para['neuron_count'] = shapes(para) if para['model'] is 'mlp': model, history = mlp(X, Y, para) if para['model'] is 'regression': model, history = regression(X, Y, para['epoch'], para['reg_mode']) network_scale = len( X) * para['epoch'] * para['layers'] * para['neuron_max'] # train / test results ex2 = pd.DataFrame({ 'train_acc': history.history['acc'], 'train_loss': history.history['loss'], 'test_acc': history.history['val_acc'], 'test_loss': history.history['val_loss'] }) scores = model.evaluate(X, Y, verbose=para['verbose']) if para['double_check'] is False or para['validation'] is False: if para['hyperscan'] is False: print("\n%s: %.2f%%" % (model.metrics_names[1], scores[1] * 100)) # calculate and round predictions predictions = model.predict(X) rounded = [round(x[0]) for x in predictions] if para['double_check'] is True: check(Y, rounded, scores) if para['save_model'] is False and para['validation'] is not False: para['save_model'] = 'saved_model' if para['save_model'] is not False: save_model_as(X_num, data.columns, model, para['save_model'], para['flatten']) # shuffling and separating the data if para['validation'] is not False: validate(X_val, Y_val, para['save_model']) # model parameters ex1 = pd.Series({ 'ind_var': ind_var, 'y_transform': para['flatten'], 'n=': len(X), 'features': para['dims'], 'epochs': para['epoch'], 'layers': para['layers'], 'dropout': para['dropout'], 'batch_size': para['batch_size'], 'shape': para['shape'], 'max_neurons': para['neuron_max'], 'network_scale': network_scale }) # prevent Tensorflow memory leakage K.clear_session() if para['hyperscan'] is True: ex3 = pd.Series({ 'optimizer': para['optimizer'], 'activation': para['activation'], 'activation_out': para['activation_out'], 'loss': para['loss'], }) return ex1, ex2, ex3 else: display(pd.DataFrame(ex1).transpose()) accuracy(ex2) return
def _check_port(): validate(error="port must be an interger in repository.yml", target="port", valid=(type(conf.ssh_port) == int))
def _check_travis_lint(content): error_msg = "An error occured while trying to parse your travis file.\nPlease make sure that the file is valid YAML.", validate(error=error_msg, target="lint", valid=(type(content) is dict))
def _check_pkg_directory(): validate(error="No package was found in pkg directory.", target="directory", valid=(len(conf.packages) > 0))
def _check_operating_system(): validate(error="This program needs to be executed in Arch Linux.", target="operating system", valid=(platform.dist()[0] == "arch"))
def _check_is_docker_image(): validate(error="This program needs to be executed in a docker image.", target="docker", valid=(os.environ.get("IS_DOCKER", False)))
def _check_user_privileges(): validate(error="This program needs to be not execute as root.", target="user privileges", valid=(os.getuid() != 0))
def signup(): try: if session['logged_in']: return redirect(url_for('dashboard', username=session['username'])) except KeyError: pass if request.method == 'GET': return render_template('signup.html') name = request.form['name'] username = request.form['username'] email = request.form['email'] passwd = request.form['passwd'] # [WIP] Apply checks on fields, redirect if unclean (avoid SQLIA). resp = validate(username, name) if resp['is_valid'] == 0: return render_template('signup.html', issue=resp['message']) elif resp['is_valid'] == 1: name = resp['name'] # Check username availability (assuming clean data) q = "SELECT `name` FROM `credentials` WHERE username='******';" try: cur.execute(q.format(username)) match = cur.fetchone() except: # Connection-Reset-On-Idle exception handler try: conn = mysql.connect() cur = conn.cursor() cur.execute(q.format(username)) match = cur.fetchone() except: # Redirect to a separate route meant for errors session['ERROR_EXISTS'] = True return redirect(url_for('error')) if match is not None: return render_template('signup.html', issue='Username already taken') # Proceed with insertion otherwise q = ''' INSERT INTO `credentials` (username, name, hash, email) VALUES (%s,%s,%s,%s) ''' try: cur.execute(q, (username, name, generate_password_hash(passwd, method='sha1'), email)) conn.commit() except: session['ERROR_EXISTS'] = True return redirect(url_for('error')) # Log the user in login_user(User(username)) session['logged_in'] = True session['name'] = name session['username'] = username return redirect(url_for('dashboard', username=username))