def allowed_file(filename):
if len(filename.rsplit('.')) > 2:
return filename.rsplit('.')[1] + '.' + filename.rsplit(
'.')[2].lower() in SETTINGS.get('ALLOWED_EXTENSIONS')
else:
return '.' in filename and \
filename.rsplit('.', 1)[1].lower() in SETTINGS.get('ALLOWED_EXTENSIONS')
def create_execution(script_id, params, user):
logging.info('[SERVICE]: Creating execution')
script = ScriptService.get_script(script_id, user)
if not script:
raise ScriptNotFound(message='Script with id ' + script_id +
' does not exist')
if script.status != 'SUCCESS':
raise ScriptStateNotValid(message='Script with id ' + script_id +
' is not BUILT')
execution = Execution(script_id=script.id,
params=params,
user_id=user.id)
try:
logging.info('[DB]: ADD')
db.session.add(execution)
db.session.commit()
except Exception as error:
raise error
try:
environment = SETTINGS.get('environment', {})
environment['EXECUTION_ID'] = execution.id
param_serial = json.dumps(params).encode('utf-8')
param_serial = str(base64.b64encode(param_serial)).replace(
'\'', '')
logging.debug(param_serial)
docker_run.delay(execution.id, script.slug, environment,
param_serial)
except Exception as e:
raise e
return execution
def init_rollbar():
"""init rollbar module"""
rollbar.init(
SETTINGS.get('ROLLBAR_SERVER_TOKEN'),
# environment name
os.getenv('ENVIRONMENT'),
# server root directory, makes tracebacks prettier
root=os.path.dirname(os.path.realpath(__file__)),
# flask already sets up logging
allow_logging_basic_config=False)
# send exceptions from `app` to rollbar, using flask's signal system.
got_request_exception.connect(rollbar.contrib.flask.report_exception, app)
def download_script(script):
"""Download a script"""
logging.info('[ROUTER]: Download script '+script)
try:
script = ScriptService.get_script(script, current_identity)
return send_from_directory(directory=SETTINGS.get('SCRIPTS_FS'), filename=script.slug + '.tar.gz')
except ScriptNotFound as e:
logging.error('[ROUTER]: '+e.message)
return error(status=404, detail=e.message)
except NotAllowed as e:
logging.error('[ROUTER]: '+e.message)
return error(status=403, detail=e.message)
except Exception as e:
logging.error('[ROUTER]: '+str(e))
return error(status=500, detail='Generic Error')
def create_script(sent_file, user, script=None):
logging.info('[SERVICE]: Creating script')
if sent_file and allowed_file(sent_file.filename):
logging.info('[SERVICE]: Allowed format')
filename = secure_filename(sent_file.filename)
sent_file_path = os.path.join(SETTINGS.get('UPLOAD_FOLDER'),
filename)
logging.info('[SERVICE]: Saving file')
try:
if not os.path.exists(SETTINGS.get('UPLOAD_FOLDER')):
os.makedirs(SETTINGS.get('UPLOAD_FOLDER'))
sent_file.save(sent_file_path)
except Exception as e:
logging.error(e)
raise e
logging.info('[SERVICE]: File saved')
else:
raise InvalidFile(message='Invalid File')
try:
with tarfile.open(name=sent_file_path, mode='r:gz') as tar:
if 'configuration.json' not in tar.getnames():
raise InvalidFile(message='Invalid File')
config_file = tar.extractfile(member='configuration.json')
logging.info('[SERVICE]: Config file extracted')
config_content = config_file.read()
logging.info('[SERVICE]: Config file opened')
config = json.loads(config_content)
script_name = config.get('name', None)
except Exception as error:
raise error
if script is None:
# Creating new entity
name = script_name
slug = slugify(script_name)
currentScript = Script.query.filter_by(slug=slug).first()
if currentScript:
raise ScriptDuplicated(message='Script with name ' + name +
' generates an existing script slug')
script = Script(name=name, slug=slug, user_id=user.id)
else:
# Updating existing entity
logging.debug(script_name)
script.name = script_name
script.updated_at = datetime.datetime.utcnow()
# TO DB
try:
logging.info('[DB]: ADD')
db.session.add(script)
shutil.move(
sent_file_path,
os.path.join(SETTINGS.get('SCRIPTS_FS'),
script.slug + '.tar.gz'))
sent_file_path = os.path.join(SETTINGS.get('SCRIPTS_FS'),
script.slug + '.tar.gz')
with tarfile.open(name=sent_file_path, mode='r:gz') as tar:
tar.extractall(path=SETTINGS.get('SCRIPTS_FS') + '/' +
script.slug)
db.session.commit()
result = docker_build.delay(script.id,
path=SETTINGS.get('SCRIPTS_FS') + '/' +
script.slug,
tag_image=script.slug)
except Exception as error:
logging.error(error)
raise error
return script
from __future__ import division
from __future__ import print_function
import random
import datetime
import string
import logging
from uuid import UUID
from gefapi import db
from gefapi.models import User
from gefapi.errors import UserNotFound, UserDuplicated, AuthError, EmailError
from gefapi.services import EmailService
from gefapi.config import SETTINGS
ROLES = SETTINGS.get('ROLES')
class UserService(object):
"""User Class"""
@staticmethod
def create_user(user):
logging.info('[SERVICE]: Creating user')
email = user.get('email', None)
password = user.get('password', None)
password = ''.join(
random.choices(string.ascii_uppercase + string.digits,
k=6)) if password is None else password
role = user.get('role', 'USER')
name = user.get('name', 'notset')
country = user.get('country', None)
from __future__ import print_function
import os
import json
import logging
from flask import Flask, request, current_app
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate
from flask_cors import CORS, cross_origin
from gefapi.config import SETTINGS
from gefapi.celery import make_celery
from flask_jwt import JWTError
logging.basicConfig(
level=SETTINGS.get('logging', {}).get('level'),
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
datefmt='%Y%m%d-%H:%M%p',
)
# Flask App
app = Flask(__name__)
CORS(app)
# Config
app.config['SQLALCHEMY_DATABASE_URI'] = SETTINGS.get('SQLALCHEMY_DATABASE_URI')
app.config['SECRET_KEY'] = SETTINGS.get('SECRET_KEY')
app.config['UPLOAD_FOLDER'] = SETTINGS.get('UPLOAD_FOLDER')
app.config['JWT_AUTH_USERNAME_KEY'] = SETTINGS.get('JWT_AUTH_USERNAME_KEY')
app.config['JWT_AUTH_HEADER_PREFIX'] = SETTINGS.get('JWT_AUTH_HEADER_PREFIX')
app.config['JWT_EXPIRATION_DELTA'] = SETTINGS.get('JWT_EXPIRATION_DELTA')
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import logging
import os
from shutil import copyfile
import docker
import time
from gefapi import db, celery
from gefapi.models import Script, ScriptLog, Execution
from gefapi.config import SETTINGS
REGISTRY_URL = SETTINGS.get('REGISTRY_URL')
DOCKER_URL = SETTINGS.get('DOCKER_URL')
api_client = docker.APIClient(base_url=DOCKER_URL)
docker_client = docker.DockerClient(base_url=DOCKER_URL)
@celery.task()
def docker_build(script_id, path, tag_image):
logging.info('[THREAD] Running build')
logging.debug('Obtaining script with id %s' % (script_id))
script = Script.query.get(script_id)
script.status = 'BUILDING'
db.session.add(script)
db.session.commit()
logging.debug('Building...')