class Manager(Thread):
def __init__(self, app):
Thread.__init__(self)
self.must_run = False
self.app = app
self.build_list = {}
self.carrier = Carrier(pecan.conf.rabbit_server,
pecan.conf.rabbit_port, pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
self.supported_build_type = pecan.conf.supported_build_type
for build_type in self.supported_build_type:
self.carrier.declare_queue('%s.queue' % build_type)
self.logger = None
def shutdown(self):
logging.debug("Stopping Manager")
self.carrier.closing = True
self.must_run = False
def run(self):
self.must_run = True
logging.debug("Starting Manager")
while self.must_run:
time.sleep(0.1)
for build_type in self.supported_build_type:
new_build = self.carrier.get_message('%s.queue' % build_type)
build = None
if new_build is not None:
distro_name = new_build['distro_name']
build_conf = new_build['build_conf']
root_folder = new_build['root_folder']
build_path = new_build['build_path']
build = Build(new_build['build'])
job_id = new_build['job_id']
path = os.path.dirname(get_logger_path(build))
try:
os.makedirs(path)
except Exception:
pass
self.logger = get_logger(build, distro_name)
self.logger.debug(build.dumps())
builder_class = globals().get(build_type.title() +
'Builder')
builder = builder_class(distro_name, build_conf,
root_folder, self.logger, build,
build_path, job_id)
builder.run()
"""
build = Build(new_build)
build.user = User.fetch(new_build['username'],
sub_objects=False)
build.project = Project.fetch(build.user,
new_build['project_name'],
sub_objects=False)
"""
"""if build:
def post(self, send_build):
""" launch build """
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
project = Project.fetch(user.username, project_name)
if project is None:
# The project doesn't exist
# We have to create it
# TODO Maybe it's better to force users to create project before
# they can create builds
sent_project = {"name": project_name, "username": user.username}
project = Project(sent_project, sub_objects=False)
if not project.create():
# Handle error
return {"result": "Error creating %s project" % project_name}
build = Build(send_build)
build.username = user.username
build.project_name = project.name
build.create()
carrier = Carrier(
pecan.conf.rabbit_server,
pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost,
pecan.conf.rabbit_db
)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return {"result": {"build": int(build.id_)}}
class Manager(Thread):
def __init__(self, app):
Thread.__init__(self)
self.must_run = False
self.app = app
self.build_list = {}
self.carrier = Carrier(
pecan.conf.rabbit_server,
pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost,
pecan.conf.rabbit_db
)
self.supported_build_type = pecan.conf.supported_build_type
for build_type in self.supported_build_type:
self.carrier.declare_queue('%s.queue' % build_type)
self.logger = None
def shutdown(self):
logging.debug("Stopping Manager")
self.carrier.closing = True
self.must_run = False
def run(self):
self.must_run = True
logging.debug("Starting Manager")
while self.must_run:
time.sleep(0.1)
for build_type in self.supported_build_type:
new_build = self.carrier.get_message('%s.queue' % build_type)
build = None
if new_build is not None:
distro_name = new_build['distro_name']
build_conf = new_build['build_conf']
root_folder = new_build['root_folder']
build_path = new_build['build_path']
build = Build(new_build['build'])
path = os.path.dirname(get_logger_path(build))
try:
os.makedirs(path)
except Exception:
pass
self.logger = get_logger(build, distro_name)
self.logger.debug(build.dumps())
builder_class = globals().get(build_type.title() + 'Builder')
builder = builder_class(distro_name, build_conf, root_folder, self.logger, build, build_path)
builder.run()
"""
build = Build(new_build)
build.user = User.fetch(new_build['username'],
sub_objects=False)
build.project = Project.fetch(build.user,
new_build['project_name'],
sub_objects=False)
"""
"""if build:
def post(self, send_build):
""" launch build """
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
project = Project.fetch(user.username, project_name)
if project is None:
# The project doesn't exist
# We have to create it
# TODO Maybe it's better to force users to create project before
# they can create builds
sent_project = {"name": project_name, "username": user.username}
project = Project(sent_project, sub_objects=False)
if not project.create():
# Handle error
return {"result": "Error creating %s project" % project_name}
build = Build(send_build)
build.username = user.username
build.project_name = project.name
build.create()
carrier = Carrier(pecan.conf.rabbit_server, pecan.conf.rabbit_port,
pecan.conf.rabbit_user, pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return {"result": {"build": int(build.id_)}}
def __init__(self, app):
self.must_run = False
self.app = app
self.build_list = {}
self.carrier = Carrier(pecan.conf.rabbit_server,
pecan.conf.rabbit_port, pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
self.carrier.declare_queue('builds.queue')
def dispatch(self, packer_conf, root_folder, global_conf=None):
carrier = Carrier(pecan.conf.rabbit_server,
pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost,
pecan.conf.rabbit_db
)
# carrier.declare_queue('docker.queue')
for distro_name, build_conf in packer_conf.items():
if not hasattr(self.build, "forced_distro") or (
self.build.forced_distro == distro_name or
not self.build.forced_distro):
if 'type' not in build_conf:
# Default type is docker to keep backward compat
build_conf['type'] = 'docker'
# if not build_conf['type'] == 'docker':
queue = "%s.queue" % build_conf['type']
carrier.declare_queue(queue)
# Add global external repos
if not build_conf.get('repos'):
build_conf['repos'] = {}
if not build_conf.get('repos').get('rpm'):
build_conf['repos']['rpm'] = []
if global_conf:
build_conf['repos']['rpm'] += global_conf['repos']['rpm']
#
build_conf['distro'] = distro_name
build_conf['branch'] = self.branch
build_conf['root_folder'] = root_folder
job_data = {
'distro': distro_name,
'username': self.build.username,
'project_name': self.build.project_name,
'build_id': self.build.id_,
}
job = Job(job_data)
job.create()
message = {
'distro_name': distro_name,
'build_conf': build_conf,
'root_folder': root_folder,
'log_path': get_logger_path(self.build),
'id_': self.id_,
'job_id': job.id_,
'build': self.build.dumps(),
'build_path': self.build.get_folder_path()
}
if not carrier.send_message(message, queue):
self.logger.error("Can't post message to rabbitmq")
else:
self.logger.info("Posted build to %s" % queue)
self.build.inc_job_count()
def dispatch(self, packer_conf, root_folder, global_conf=None):
carrier = Carrier(pecan.conf.rabbit_server, pecan.conf.rabbit_port,
pecan.conf.rabbit_user, pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
# carrier.declare_queue('docker.queue')
for distro_name, build_conf in packer_conf.items():
if not hasattr(self.build, "forced_distro") or (
self.build.forced_distro == distro_name
or not self.build.forced_distro):
if 'type' not in build_conf:
# Default type is docker to keep backward compat
build_conf['type'] = 'docker'
# if not build_conf['type'] == 'docker':
queue = "%s.queue" % build_conf['type']
carrier.declare_queue(queue)
# Add global external repos
if not build_conf.get('repos'):
build_conf['repos'] = {}
if not build_conf.get('repos').get('rpm'):
build_conf['repos']['rpm'] = []
if global_conf:
build_conf['repos']['rpm'] += global_conf['repos']['rpm']
#
build_conf['distro'] = distro_name
build_conf['branch'] = self.branch
build_conf['root_folder'] = root_folder
job_data = {
'distro': distro_name,
'username': self.build.username,
'project_name': self.build.project_name,
'build_id': self.build.id_,
}
job = Job(job_data)
job.create()
message = {
'distro_name': distro_name,
'build_conf': build_conf,
'root_folder': root_folder,
'log_path': get_logger_path(self.build),
'id_': self.id_,
'job_id': job.id_,
'build': self.build.dumps(),
'build_path': self.build.get_folder_path()
}
if not carrier.send_message(message, queue):
self.logger.error("Can't post message to rabbitmq")
else:
self.logger.info("Posted build to %s" % queue)
self.build.inc_job_count()
def __init__(self, app):
Thread.__init__(self)
self.must_run = False
self.app = app
self.build_list = {}
self.carrier = Carrier(pecan.conf.rabbit_server,
pecan.conf.rabbit_port, pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
self.supported_build_type = pecan.conf.supported_build_type
for build_type in self.supported_build_type:
self.carrier.declare_queue('%s.queue' % build_type)
self.logger = None
def __init__(self, app):
Thread.__init__(self)
self.must_run = False
self.app = app
self.build_list = {}
self.carrier = Carrier(
pecan.conf.rabbit_server,
pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost,
pecan.conf.rabbit_db
)
self.supported_build_type = pecan.conf.supported_build_type
for build_type in self.supported_build_type:
self.carrier.declare_queue('%s.queue' % build_type)
self.logger = None
def post(self):
""" launch build from gitlab webhook"""
body = pecan.request.json
# Get use
if not body.get('user_id'):
abort(403)
# Get token
token = pecan.request.GET.get('token')
if token is None:
abort(403)
# Get project
project = Project.fetch_from_token(token, False)
if project is None:
abort(403)
if body.get('project_id') != project.gitlab_project_id:
abort(403)
if body.get('object_kind') not in ['push', 'tag']:
abort(403)
else:
# If it's a TAG event we DON'T make snaphot
snapshot = False
if body.get('object_kind') == 'push':
# If it's a PUSH event we make snapshot
snapshot = True
if not body.get('repository'):
abort(403)
repository = body.get('repository')
project_name = repository.get('name')
if project_name != project.name:
abort(403)
new_build = {
"source_url": repository.get('git_http_url'),
#"source_type": "gitlab",
"source_type": "git",
"commit": body.get('after'),
# TODO Find how decide if is a snapshot or not
"snapshot": snapshot,
# TODO Check if branch ~= ref
"branch": body.get('ref'),
}
build = Build(new_build)
build.username = project.username
build.project_name = project.name
build.create()
carrier = Carrier(pecan.conf.rabbit_server, pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return json.dumps({"result": True, "build": int(build.id_)})
abort(403)
def post(self):
"""Launch build from github webhook"""
body = pecan.request.json
# Get user
if not body.get('sender'):
abort(403)
# Get username
username = body.get('repository').get('owner').get('name')
if username is None:
username = body.get('repository').get('owner').get('login')
user = User.fetch(username)
if user is None:
abort(403)
# Check signature
signature = pecan.request.headers.get('X-Hub-Signature')
sha_name, signature = signature.split("=")
if sha_name != 'sha1':
abort(403)
mac = hmac.new(user.token.encode("utf-8"), pecan.request.text, digestmod=hashlib.sha1)
if not hmac.compare_digest(mac.hexdigest(), signature):
abort(403)
# Ping event
if pecan.request.headers.get('X-Github-Event') == 'ping':
return json.dumps({"result": True, "event": "ping"})
# TODO handle tag event
# Push Event
if pecan.request.headers.get('X-Github-Event') == 'push':
if not body.get('repository'):
abort(403)
repository = body.get('repository')
project_name = repository.get('name')
if not project_name:
abort(403)
project = Project.fetch(user.username, project_name)
if project is None:
# Error project doesn't exits
# Maybe We should create it
return json.dumps({"result": False , "error": "project not found"})
new_build = {"source_url": repository.get('clone_url'),
#"source_type": "github",
"source_type": "git",
"commit": repository.get('commit'),
# TODO Find how decide if is a snapshot or not
# Answer: on tag event => NOT
"snapshot": True,
# TODO Check if branch ~= ref
"branch": repository.get('ref'),
}
build = Build(new_build)
build.username = user.username
build.project_name = project.name
build.create()
carrier = Carrier(
pecan.conf.rabbit_server,
pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost,
pecan.conf.rabbit_db
)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return json.dumps({"result": True, "build": int(build.id_)})
abort(403)
class Manager(object):
def __init__(self, app):
self.must_run = False
self.app = app
self.build_list = {}
self.carrier = Carrier(pecan.conf.rabbit_server,
pecan.conf.rabbit_port, pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
self.carrier.declare_queue('builds.queue')
def shutdown(self):
logging.debug("Stopping Manager")
self.carrier.closing = True
self.must_run = False
def run(self):
self.must_run = True
logging.debug("Starting Manager")
while self.must_run:
time.sleep(0.1)
new_build = self.carrier.get_message('builds.queue')
build = None
if new_build is not None:
build = Build(new_build)
if build:
build.user = User.fetch(new_build['username'],
sub_objects=False)
build.project = Project.fetch(build.username,
new_build['project_name'],
sub_objects=False)
logging.debug("Task received")
build.set_status("dispatching")
dispatcher = Dispatcher(build)
self.build_list[dispatcher.uuid2] = dispatcher
dispatcher.start()
self.check_builds_status()
def check_builds_status(self):
builds = mongo.builds.find(
{"status": {
"$nin": ["succeeded", "failed"]
}})
for b in builds:
finished = 0
build = Build(b)
jobs = build.get_jobs()
if len(jobs) == build.job_count and build.job_count > 0:
for job in jobs:
if job.status in ['succeeded', 'failed']:
finished += 1
if finished == len(jobs):
if all([
True if j.status == 'succeeded' else False
for j in jobs
]):
build.set_status('succeeded')
else:
build.set_status('failed')
build.finishing()
# collect old jammed build
time_limit = time.time() - pecan.conf.build_lifetime
builds = mongo.builds.find({
"status": {
"$nin": ["succeeded", "failed"]
},
"created": {
"$lte": time_limit
}
})
for b in builds:
build = Build(b)
build.set_status('failed')
build.finishing()
jobs = build.get_jobs()
for job in jobs:
if job.status not in ['succeeded', 'failed']:
job.set_status('failed')
def post(self):
""" launch build from gitlab webhook"""
body = pecan.request.json
# Get use
if not body.get('user_id'):
abort(403)
# Get token
token = pecan.request.GET.get('token')
if token is None:
abort(403)
# Get project
project = Project.fetch_from_token(token, False)
if project is None:
abort(403)
if body.get('project_id') != project.gitlab_project_id:
abort(403)
if body.get('object_kind') not in ['push', 'tag']:
abort(403)
else:
# If it's a TAG event we DON'T make snaphot
snapshot = False
if body.get('object_kind') == 'push':
# If it's a PUSH event we make snapshot
snapshot = True
if not body.get('repository'):
abort(403)
repository = body.get('repository')
project_name = repository.get('name')
if project_name != project.name:
abort(403)
new_build = {"source_url": repository.get('git_http_url'),
#"source_type": "gitlab",
"source_type": "git",
"commit": body.get('after'),
# TODO Find how decide if is a snapshot or not
"snapshot": snapshot,
# TODO Check if branch ~= ref
"branch": body.get('ref'),
}
build = Build(new_build)
build.username = project.username
build.project_name = project.name
build.create()
carrier = Carrier(
pecan.conf.rabbit_server,
pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost,
pecan.conf.rabbit_db
)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return json.dumps({"result": True, "build": int(build.id_)})
abort(403)