def create_app(package_name,
package_path,
settings_override=None,
register_security_blueprint=True,
config_name="development"):
"""Returns a :class:`Flask` application instance configured with common
functionality for the chatfirst platform.
:param package_name: application package name
:param package_path: application package path
:param settings_override: a dictionary of settings to override
:param register_security_blueprint: flag to specify if the Flask-Security
Blueprint should be registered. Defaults
to `True`.
"""
app = Flask(package_name, instance_relative_config=True)
app.config.from_object('webpanda.settings')
app.config.from_object(config[config_name])
app.config.from_pyfile('settings.cfg', silent=True)
app.config.from_object(settings_override)
if 'SQLALCHEMY_DATABASE_URI' in os.environ.keys():
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ['DATABASE_URL']
db.init_app(app)
#security.init_app(app, SQLAlchemyUserDatastore(db, User, Role),
# register_blueprint=register_security_blueprint)
register_blueprints(app, package_name, package_path)
app.wsgi_app = HTTPMethodOverrideMiddleware(app.wsgi_app)
app.log = NrckiLogger().getLogger(package_name)
# Prepare auth
lm.init_app(app)
lm.login_view = 'auth.main_auth'
lm.anonymous_user = AnonymousUser
@lm.user_loader
def load_user(id):
if id == 0:
return AnonymousUser()
return users_.get(id=id)
@app.before_request
def before_request():
g.user = current_user
g.user.last_seen = datetime.utcnow()
g.user.save()
values = request.form.to_dict()
app.log.info(
"incoming request: {method} {url}; Form: {form}; Data: {data}".
format(method=request.method,
url=request.full_path,
form=str(values),
data=str(request.get_json(silent=True))))
return app
# -*- coding: utf-8 -*-
from flask import render_template, jsonify, Blueprint
from webpanda.api import route
from webpanda.services import tasks_
from webpanda.common.NrckiLogger import NrckiLogger
bp = Blueprint('tasks', __name__, url_prefix="/tasks")
_logger = NrckiLogger().getLogger("api.tasks")
@route(bp, "/", methods=['GET'])
def list_all():
return tasks_.all()
@route(bp, "/<tag>", methods=['GET'])
def show(tag):
return tasks_.first(tag=tag)
@route(bp, "/<tag>/jobs", methods=['GET'])
def show_jobs(tag):
task = tasks_.first(tag=tag)
return task.jobs
# -*- coding: utf-8 -*-
from flask import jsonify, Blueprint, make_response
from webpanda.common.NrckiLogger import NrckiLogger
from webpanda.pilot import route
from webpanda.services import transfers_meta_
bp = Blueprint('task', __name__, url_prefix="/task")
_logger = NrckiLogger().getLogger("pilot.task")
@route(bp, '/<id>/info', methods=['GET'])
def task_info(id):
"""
GET: /pilot/task/<id>/info
Returns task status
:param id: Task id
:rtype: json
"""
n = transfers_meta_.find(task_id=id).count()
if n == 0:
data = {}
data['id'] = id
data['status'] = 'unknown'
data['date_done'] = ''
data['traceback'] = ''
return data
task = transfers_meta_.first(task_id=id)
# -*- coding: utf-8 -*-
from flask import render_template, jsonify, Blueprint
from webpanda.api import route
from webpanda.services import jobs_
from webpanda.common.NrckiLogger import NrckiLogger
bp = Blueprint('jobs', __name__, url_prefix="/jobs")
_logger = NrckiLogger().getLogger("api.jobs")
@route(bp, "/<id_>", methods=['GET'])
def show(id_):
return jobs_.get(id_)
@route(bp, "/tag/<tag>", methods=['GET'])
def bytag(tag):
return jobs_.find(tags=tag).all()
# -*- coding: utf-8 -*-
import json
from flask import current_app
from webpanda. async import celery
from webpanda.common.NrckiLogger import NrckiLogger
from webpanda.core import WebpandaError
from webpanda.files.scripts import cloneReplica, linkReplica, copyReplica, uploadContainer, setFileMeta, upload_dir
from webpanda.jobs import Job
from webpanda.jobs.scripts import killJobs, send_job, update_status, register_outputs
from webpanda.services import conts_, sites_
_logger = NrckiLogger().getLogger("async")
@celery.task
def async_send_job(*args, **kwargs):
_logger.debug("async_send_job: start")
jobid = kwargs.get('jobid', 0L)
siteid = kwargs.get('siteid', 0L)
if int(jobid) == 0:
raise Exception('Illegal argument: jobid')
if int(siteid) == 0:
raise Exception('Illegal argument: siteid')
return json.dumps(send_job(jobid, siteid))
@celery.task
def async_kill_job(pandaid):
if int(pandaid) == 0:
# -*- coding: utf-8 -*-
import commands
from flask import jsonify, Blueprint, make_response, g, current_app
import os
from webpanda.common.NrckiLogger import NrckiLogger
from webpanda.core import WebpandaError
from webpanda.files import Container
from webpanda.files.scripts import registerLocalFile
from webpanda.files.common import getScope
from webpanda.pilot import route
from webpanda.services import conts_
bp = Blueprint('container', __name__, url_prefix="/container")
_logger = NrckiLogger().getLogger("pilot.container")
@route(bp, '/', methods=['POST'])
def new_cont():
"""
POST: /pilot/container
Saves new container
:return: ftp/guid
:rtype: json
"""
cont = Container()
guid = 'job.' + commands.getoutput('uuidgen')
cont.guid = guid
# -*- coding: utf-8 -*-
from flask import jsonify, Blueprint, make_response, current_app, request, g, Response
import os
from webpanda.common.NrckiLogger import NrckiLogger
from webpanda.core import WebpandaError
from webpanda.files import Replica, File
from webpanda.files.scripts import getFtpLink, setFileMeta
from webpanda.files.common import getScope, getGUID
from webpanda.pilot import route
from webpanda.services import sites_, conts_, files_, replicas_
from webpanda. async .scripts import async_cloneReplica, async_copyReplica
from webpanda.fc import client as fc
bp = Blueprint('file', __name__, url_prefix="/file")
_logger = NrckiLogger().getLogger("pilot.file")
@route(bp, '/<container_guid>/<lfn>/makereplica/<se>', methods=['POST'])
def new_replica(container_guid, lfn, se):
"""
POST: /pilot/file/<container_guid>/<lfn>/makereplica/<se>
Creates task to make new file replica
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:param se: SE codename
:type se: str
import json
from flask import render_template, Blueprint, g, request, url_for, current_app
import os
from webpanda.common.NrckiLogger import NrckiLogger
from webpanda.common.utils import adler32, md5sum, fsize
from webpanda.dashboard import route_s
from webpanda.ddm.scripts import ddm_checkifexists
from webpanda.files import Container, File, Replica
from webpanda.files.scripts import setFileMeta
from webpanda.files.common import getGUID, getScope
from webpanda.services import sites_, conts_, files_, replicas_
from werkzeug.utils import redirect
from webpanda.fc import client as fc
bp = Blueprint('main', __name__)
_logger = NrckiLogger().getLogger("dashboard.main")
@route_s(bp, '/')
def index():
user = g.user
return render_template("dashboard/main/index.html",
title='Home',
user=user)
@route_s(bp, '/help')
def help():
return render_template("dashboard/main/help.html")
# -*- coding: utf-8 -*-
from flask import Blueprint, g
from webpanda.api import route
from webpanda.common.NrckiLogger import NrckiLogger
from webpanda.services import pipelines_
bp = Blueprint('pipelines', __name__, url_prefix="/pipelines")
_logger = NrckiLogger().getLogger("api.pipelines")
@route(bp, "/", methods=['GET'])
def list_all():
return pipelines_.find(owner_id=g.user.id).all()
# -*- coding: utf-8 -*-
from flask import Blueprint, request, render_template, current_app, session, jsonify
from flask import g
from flask import make_response
from webpanda.common.NrckiLogger import NrckiLogger
from webpanda.dashboard import route_s
from webpanda.services import tasks_, jobs_
from webpanda.tasks import Task
bp = Blueprint('tasks', __name__, url_prefix="/tasks")
_logger = NrckiLogger().getLogger("dashboard.tasks")
@route_s(bp, "/", methods=['GET'])
def list_all():
hours_limit = request.args.get('hours',
current_app.config['HOURS_LIMIT'],
type=int)
display_limit = request.args.get('display_limit',
current_app.config['DISPLAY_LIMIT'],
type=int)
session['hours_limit'] = hours_limit
session['display_limit'] = display_limit
return render_template("dashboard/tasks/list.html")
@route_s(bp, "/<id>", methods=['GET'])
def task_info(id):
"""
Task info view
from webpanda.common.NrckiLogger import NrckiLogger
__author__ = 'it'
logger = NrckiLogger().getLogger("pipeline.scripts")
import ldap
from flask import current_app
from flask_login import login_user, logout_user
from flask import Blueprint, render_template, request, url_for, flash, g
from werkzeug.utils import redirect
from webpanda.auth.forms import LoginForm
from webpanda.auth.forms import RegisterForm
from webpanda.auth.models import User
from webpanda.auth.scripts import get_token_by_code, sso_get_user, get_auth_endpoint
from webpanda.dashboard import route, route_s
from webpanda.services import users_
from webpanda.common.NrckiLogger import NrckiLogger
bp = Blueprint('auth', __name__)
_logger = NrckiLogger().getLogger("dashboard.auth")
@route(bp, '/auth', methods=['GET'])
def main_auth():
url = get_auth_endpoint(
) if current_app.config["AUTH_AUTH_ENDPOINT"] else url_for('auth.login')
return redirect(url)
@route(bp, '/login', methods=['GET', 'POST'])
def login():
user = g.user
_logger.debug("Trying to login user: {user}".format(user=user.username))
if user.is_authenticated:
# if user is logged in we get out of here
# -*- coding: utf-8 -*-
from flask import Blueprint, jsonify, request, render_template, url_for, make_response, g, current_app, session, \
Response
from webpanda.files import File
from webpanda.files.forms import NewFileForm
from werkzeug.utils import redirect
from webpanda.common.NrckiLogger import NrckiLogger
from webpanda. async .scripts import async_upload_dir
from webpanda.dashboard import route_s
from webpanda.services import files_, sites_
from webpanda.fc import client as fc
from webpanda.files.common import getScope
bp = Blueprint('files', __name__, url_prefix="/files")
_logger = NrckiLogger().getLogger("dashboard.files")
@route_s(bp, "/", methods=['GET'])
def files():
"""
List of user's files view
:return: Response obj
"""
hours_limit = request.args.get('hours',
current_app.config['HOURS_LIMIT'],
type=int)
display_limit = request.args.get('display_limit',
current_app.config['DISPLAY_LIMIT'],
type=int)
session['hours_limit'] = hours_limit
from flask import render_template
from flask_login import login_required
from werkzeug.security import gen_salt
from werkzeug.utils import redirect
from webpanda.services import clients_, grants_, tokens_, users_
from webpanda.api import route
from webpanda.auth import Grant
from webpanda.auth import Token
from webpanda.common.NrckiLogger import NrckiLogger
from webpanda.core import oauth
bp = Blueprint('auth', __name__)
_logger = NrckiLogger().getLogger("api.main")
@oauth.clientgetter
def load_client(client_id):
return clients_.first(client_id=client_id)
@oauth.grantgetter
def load_grant(client_id, code):
return grants_.first(client_id=client_id, code=code)
@oauth.grantsetter
def save_grant(client_id, code, request, *args, **kwargs):
# decide the expires time yourself
expires = datetime.utcnow() + timedelta(seconds=100)
grant = Grant(
from taskbuffer.FileSpec import FileSpec
import userinterface.Client as Client
from flask import current_app
import os
from webpanda.common import client_config
from webpanda.common.NrckiLogger import NrckiLogger
from webpanda.ddm.DDM import SEFactory
from webpanda.ddm.scripts import ddm_localextractfile
from webpanda.files import File
from webpanda.files.common import getScope, getGUID
from webpanda.jobs import Job
from webpanda.services import sites_, jobs_, replicas_, files_, users_
from webpanda.fc import client as fc
_logger = NrckiLogger().getLogger("files.scripts")
def submitJobs(jobList):
print 'Submit jobs'
_logger.debug('Submit jobs')
_logger.debug(str(jobList))
s, o = Client.submitJobs(jobList)
_logger.debug(o)
_logger.debug(s)
_logger.debug("---------------------")
for x in o:
_logger.debug("PandaID=%s" % x[0])
return o
import commands
from flask import Blueprint, g, request, make_response, jsonify, session, render_template, url_for, current_app
from webpanda.common.NrckiLogger import NrckiLogger
from webpanda.dashboard import route_s
from webpanda.files import Container
from webpanda.files.forms import NewContainerForm, RunForm
from webpanda.files.common import getScope
from webpanda.services import conts_
from webpanda. async .scripts import async_uploadContainer
from werkzeug.utils import redirect
bp = Blueprint('conts', __name__, url_prefix="/cont")
_logger = NrckiLogger().getLogger("dashboard.conts")
@route_s(bp, "/", methods=['GET'])
def containers():
"""
List of user's containers view
:return: Response obj
"""
hours_limit = request.args.get('hours',
current_app.config['HOURS_LIMIT'],
type=int)
display_limit = request.args.get('display_limit',
current_app.config['DISPLAY_LIMIT'],
type=int)
session['hours_limit'] = hours_limit
session['display_limit'] = display_limit
return render_template("dashboard/cont/cont_list.html")
import os
import ftplib
from webpanda.common.NrckiLogger import NrckiLogger
from webpanda.ddm.scripts import ddm_getlocalabspath
_logger = NrckiLogger().getLogger("DDM")
class FtpSEPlugin():
def __init__(self, params=None):
self.anonymode = True
self.login = '******'
self.password = ''
if 'login' in params.keys() and 'password' in params.keys():
self.anonymode = False
self.login = params['login']
self.password = params['password']
def get(self, url, dest):
dest = ddm_getlocalabspath(dest)
_logger.debug('FTP: Try to get file from %s to %s' % (url, dest))
try:
url = url.split('ftp://')[-1]
host = url.split('/')[0]
path = '/'.join(url.split('/')[1:-1])
fname = url.split('/')[-1]
destfile = os.path.join(dest, fname)
ftp = self.connect(host, self.login, self.password)
ftp.cwd(path)
from webpanda.core import WebpandaError
from webpanda.files.forms import RunForm
from werkzeug.utils import redirect
from webpanda.common.NrckiLogger import NrckiLogger
from webpanda.dashboard import route_s
from webpanda.files import Container
from webpanda.services import conts_, pipeline_types_, files_, pipelines_, tasks_
from webpanda.tasks import Pipeline
from webpanda.tasks.forms import NewPipelineForm
from webpanda.fc import client as fc
from webpanda.pipelines import client as pclient
bp = Blueprint('pipelines', __name__, url_prefix="/pipelines")
_logger = NrckiLogger().getLogger("dashboard.pipelines")
@route_s(bp, "/", methods=['GET'])
def list_all():
hours_limit = request.args.get('hours', current_app.config['HOURS_LIMIT'], type=int)
display_limit = request.args.get('display_limit', current_app.config['DISPLAY_LIMIT'], type=int)
session['hours_limit'] = hours_limit
session['display_limit'] = display_limit
return render_template("dashboard/pp/list.html")
@route_s(bp, "/<id>", methods=['GET'])
def pipeline_info(id):
"""