def run(self):
self.must_run = True
logging.debug("Starting Manager")
while self.must_run:
time.sleep(0.1)
for build_type in self.supported_build_type:
new_build = self.carrier.get_message('%s.queue' % build_type)
build = None
if new_build is not None:
distro_name = new_build['distro_name']
build_conf = new_build['build_conf']
root_folder = new_build['root_folder']
build_path = new_build['build_path']
build = Build(new_build['build'])
path = os.path.dirname(get_logger_path(build))
try:
os.makedirs(path)
except Exception:
pass
self.logger = get_logger(build, distro_name)
self.logger.debug(build.dumps())
builder_class = globals().get(build_type.title() + 'Builder')
builder = builder_class(distro_name, build_conf, root_folder, self.logger, build, build_path)
builder.run()
"""
build = Build(new_build)
build.user = User.fetch(new_build['username'],
sub_objects=False)
build.project = Project.fetch(build.user,
new_build['project_name'],
sub_objects=False)
"""
"""if build:
def get_build(self):
build_data = self.get_message('builds.queue')
if build_data is not None:
build = Build(build_data)
build.user = User.fetch(build_data['username'], sub_objects=False)
if build.user is None:
return None
build.project = Project.fetch(build.user.username,
build_data['project_name'],
sub_objects=False)
return build
def get_build(self):
build_data = self.get_message('builds.queue')
if build_data is not None:
build = Build(build_data)
build.user = User.fetch(build_data['username'], sub_objects=False)
if build.user is None:
return None
build.project = Project.fetch(build.user.username,
build_data['project_name'],
sub_objects=False)
return build
def get(self):
"""Returns output content"""
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project_name = pecan.request.context['project_name']
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=False)
if build is None:
return
# Get output folder
output_folder = build.get_output_folder_path()
# Test if output folder exists
if not os.path.isdir(output_folder):
return
output = {}
for path, folders, files in os.walk(output_folder):
current_path = path.replace(output_folder, "").strip("/")
for folder in folders:
output[folder.strip("/")] = {}
if current_path != '':
output[current_path] = files
return output
def get(self):
"""Download all files in one archive"""
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project_name = pecan.request.context['project_name']
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=False)
if build is None:
return
# Get options
archive = pecan.request.GET.get('type', 'tgz')
distro = pecan.request.GET.get('distro', None)
if distro not in supported_distros:
distro = None
# Get output folder
output_folder = build.get_output_folder_path(distro)
# Test if output folder exists
if not os.path.isdir(output_folder):
return None
# Set headers
headers = pecan.response.headers
# Prepare content type
content_type = archive_types.get(archive, 'application/x-gzip')
pecan.core.override_template(None, content_type)
# Prepare archive
f = BytesIO()
if archive == 'zip':
# Zip
zip_archive = zipfile.ZipFile(f, "w" )
for file_ in glob.glob(output_folder + "/*"):
zip_archive.write(file_, os.path.basename(file_))
zip_archive.close()
extension = "zip"
else:
# Tarball
tar_archive = tarfile.open(fileobj=f, mode="w:gz")
for file_ in glob.glob(output_folder + "/*"):
tar_archive.add(file_, os.path.basename(file_))
tar_archive.close()
extension = "tar.gz"
if build.package_name is None:
return
filename = project_name + "_%(package_version)s-%(package_release)s" % build.as_dict()
if distro is not None:
filename = filename + "-" + distro
else:
filename = filename + "-all_distros"
filename = ".".join((filename, extension))
headers.add("Content-Disposition", str("attachment;filename=%s" % filename))
# returns
return f.getvalue()
def get_builds(self):
""" return all build ids """
builds = mongo.builds.find({
"username": self.username,
"project_name": self.name
}).sort("id_", DESCENDING)
return [Build(x) for x in builds]
def post(self, send_build):
""" launch build """
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
project = Project.fetch(user.username, project_name)
if project is None:
# The project doesn't exist
# We have to create it
# TODO Maybe it's better to force users to create project before
# they can create builds
sent_project = {"name": project_name, "username": user.username}
project = Project(sent_project, sub_objects=False)
if not project.create():
# Handle error
return {"result": "Error creating %s project" % project_name}
build = Build(send_build)
build.username = user.username
build.project_name = project.name
build.create()
carrier = Carrier(pecan.conf.rabbit_server, pecan.conf.rabbit_port,
pecan.conf.rabbit_user, pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return {"result": {"build": int(build.id_)}}
def get_latest_build(self):
build = mongo.builds.find_one(
{
"username": self.username,
"project_name": self.name
},
sort=[("id_", -1)])
if build:
return Build(build)
return None
def post(self, send_build):
""" launch build """
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
project = Project.fetch(user.username, project_name)
if project is None:
# The project doesn't exist
# We have to create it
# TODO Maybe it's better to force users to create project before
# they can create builds
sent_project = {"name": project_name, "username": user.username}
project = Project(sent_project, sub_objects=False)
if not project.create():
# Handle error
return {"result": "Error creating %s project" % project_name}
build = Build(send_build)
build.username = user.username
build.project_name = project.name
build.create()
carrier = Carrier(
pecan.conf.rabbit_server,
pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost,
pecan.conf.rabbit_db
)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return {"result": {"build": int(build.id_)}}
def run(self):
self.must_run = True
logging.debug("Starting Manager")
while self.must_run:
time.sleep(0.1)
for build_type in self.supported_build_type:
new_build = self.carrier.get_message('%s.queue' % build_type)
build = None
if new_build is not None:
distro_name = new_build['distro_name']
build_conf = new_build['build_conf']
root_folder = new_build['root_folder']
build_path = new_build['build_path']
build = Build(new_build['build'])
job_id = new_build['job_id']
path = os.path.dirname(get_logger_path(build))
try:
os.makedirs(path)
except Exception:
pass
self.logger = get_logger(build, distro_name)
self.logger.debug(build.dumps())
builder_class = globals().get(build_type.title() +
'Builder')
builder = builder_class(distro_name, build_conf,
root_folder, self.logger, build,
build_path, job_id)
builder.run()
"""
build = Build(new_build)
build.user = User.fetch(new_build['username'],
sub_objects=False)
build.project = Project.fetch(build.user,
new_build['project_name'],
sub_objects=False)
"""
"""if build:
def get(self):
"""Returns log of a specific distro."""
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
if user is None:
return None
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=True)
return build.jobs
def get(self):
"""Returns log of a specific distro."""
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
if user is None:
return None
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=True)
return build.jobs
def get(self):
"""Returns build status"""
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project_name = pecan.request.context['project_name']
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = self.id_
if self.id_ in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=True)
if build:
return build
return None
def get(self):
"""Returns build status"""
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project_name = pecan.request.context['project_name']
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = self.id_
if self.id_ in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=True)
if build:
return build
return None
def get(self):
"""Download one output file"""
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project_name = pecan.request.context['project_name']
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=False)
if build is None:
return
# Get options
distro = pecan.request.GET.get('distro', None)
filename = pecan.request.GET.get('filename', None)
if distro not in supported_distros:
distro = None
# Get output folder
output_folder = build.get_output_folder_path(distro)
# Test if output folder exists
if not os.path.isdir(output_folder):
return None
# Set headers
headers = pecan.response.headers
# Get one file
if filename is not None and distro is not None:
file_path = os.path.join(output_folder, filename)
if not os.path.isfile(file_path):
return None
# TODO clean the following lines
mime = MimeTypes()
contenttype, _ = mime.guess_type(file_path)
headers.add("Content-Disposition", str("attachment;filename=%s" % filename))
fhandler = open(file_path, 'r')
wsme_file = wtypes.File(filename=filename,
file=fhandler,
contenttype=contenttype)
return wsme_file.content
return None
def get(self):
"""Returns log of a specific job."""
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
if user is None:
return None
project = Project.fetch(user.username, project_name)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, False)
job_id = pecan.request.context['job_id']
job = Job.fetch(build, job_id)
log = job.get_log().strip().decode('unicode-escape')
html = pecan.request.GET.get('html', False)
print html
if html:
log = "<p><a></a><span>" + log.replace("\n", "</span></p><p><a></a>") + "</p>"
return log
def get(self):
"""Returns log of a specific job."""
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
if user is None:
return None
project = Project.fetch(user.username, project_name)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, False)
job_id = pecan.request.context['job_id']
job = Job.fetch(build, job_id)
log = job.get_log().strip().decode('unicode-escape')
html = pecan.request.GET.get('html', False)
print html
if html:
log = "<p><a></a><span>" + log.replace(
"\n", "</span></p><p><a></a>") + "</p>"
return log
def run(self):
self.must_run = True
logging.debug("Starting Manager")
while self.must_run:
time.sleep(0.1)
new_build = self.carrier.get_message('builds.queue')
build = None
if new_build is not None:
build = Build(new_build)
if build:
build.user = User.fetch(new_build['username'],
sub_objects=False)
build.project = Project.fetch(build.username,
new_build['project_name'],
sub_objects=False)
logging.debug("Task received")
build.set_status("dispatching")
dispatcher = Dispatcher(build)
self.build_list[dispatcher.uuid2] = dispatcher
dispatcher.start()
self.check_builds_status()
def post(self):
""" launch build from gitlab webhook"""
body = pecan.request.json
# Get use
if not body.get('user_id'):
abort(403)
# Get token
token = pecan.request.GET.get('token')
if token is None:
abort(403)
# Get project
project = Project.fetch_from_token(token, False)
if project is None:
abort(403)
if body.get('project_id') != project.gitlab_project_id:
abort(403)
if body.get('object_kind') not in ['push', 'tag']:
abort(403)
else:
# If it's a TAG event we DON'T make snaphot
snapshot = False
if body.get('object_kind') == 'push':
# If it's a PUSH event we make snapshot
snapshot = True
if not body.get('repository'):
abort(403)
repository = body.get('repository')
project_name = repository.get('name')
if project_name != project.name:
abort(403)
new_build = {
"source_url": repository.get('git_http_url'),
#"source_type": "gitlab",
"source_type": "git",
"commit": body.get('after'),
# TODO Find how decide if is a snapshot or not
"snapshot": snapshot,
# TODO Check if branch ~= ref
"branch": body.get('ref'),
}
build = Build(new_build)
build.username = project.username
build.project_name = project.name
build.create()
carrier = Carrier(pecan.conf.rabbit_server, pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return json.dumps({"result": True, "build": int(build.id_)})
abort(403)
def post(self):
"""Launch build from github webhook"""
body = pecan.request.json
# Get user
if not body.get('sender'):
abort(403)
# Get username
username = body.get('repository').get('owner').get('name')
if username is None:
username = body.get('repository').get('owner').get('login')
user = User.fetch(username)
if user is None:
abort(403)
# Check signature
signature = pecan.request.headers.get('X-Hub-Signature')
sha_name, signature = signature.split("=")
if sha_name != 'sha1':
abort(403)
mac = hmac.new(user.token.encode("utf-8"), pecan.request.text, digestmod=hashlib.sha1)
if not hmac.compare_digest(mac.hexdigest(), signature):
abort(403)
# Ping event
if pecan.request.headers.get('X-Github-Event') == 'ping':
return json.dumps({"result": True, "event": "ping"})
# TODO handle tag event
# Push Event
if pecan.request.headers.get('X-Github-Event') == 'push':
if not body.get('repository'):
abort(403)
repository = body.get('repository')
project_name = repository.get('name')
if not project_name:
abort(403)
project = Project.fetch(user.username, project_name)
if project is None:
# Error project doesn't exits
# Maybe We should create it
return json.dumps({"result": False , "error": "project not found"})
new_build = {"source_url": repository.get('clone_url'),
#"source_type": "github",
"source_type": "git",
"commit": repository.get('commit'),
# TODO Find how decide if is a snapshot or not
# Answer: on tag event => NOT
"snapshot": True,
# TODO Check if branch ~= ref
"branch": repository.get('ref'),
}
build = Build(new_build)
build.username = user.username
build.project_name = project.name
build.create()
carrier = Carrier(
pecan.conf.rabbit_server,
pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost,
pecan.conf.rabbit_db
)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return json.dumps({"result": True, "build": int(build.id_)})
abort(403)
def post(self):
""" launch build from gitlab webhook"""
body = pecan.request.json
# Get use
if not body.get('user_id'):
abort(403)
# Get token
token = pecan.request.GET.get('token')
if token is None:
abort(403)
# Get project
project = Project.fetch_from_token(token, False)
if project is None:
abort(403)
if body.get('project_id') != project.gitlab_project_id:
abort(403)
if body.get('object_kind') not in ['push', 'tag']:
abort(403)
else:
# If it's a TAG event we DON'T make snaphot
snapshot = False
if body.get('object_kind') == 'push':
# If it's a PUSH event we make snapshot
snapshot = True
if not body.get('repository'):
abort(403)
repository = body.get('repository')
project_name = repository.get('name')
if project_name != project.name:
abort(403)
new_build = {"source_url": repository.get('git_http_url'),
#"source_type": "gitlab",
"source_type": "git",
"commit": body.get('after'),
# TODO Find how decide if is a snapshot or not
"snapshot": snapshot,
# TODO Check if branch ~= ref
"branch": body.get('ref'),
}
build = Build(new_build)
build.username = project.username
build.project_name = project.name
build.create()
carrier = Carrier(
pecan.conf.rabbit_server,
pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost,
pecan.conf.rabbit_db
)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return json.dumps({"result": True, "build": int(build.id_)})
abort(403)
def check_builds_status(self):
builds = mongo.builds.find(
{"status": {
"$nin": ["succeeded", "failed"]
}})
for b in builds:
finished = 0
build = Build(b)
jobs = build.get_jobs()
if len(jobs) == build.job_count and build.job_count > 0:
for job in jobs:
if job.status in ['succeeded', 'failed']:
finished += 1
if finished == len(jobs):
if all([
True if j.status == 'succeeded' else False
for j in jobs
]):
build.set_status('succeeded')
else:
build.set_status('failed')
build.finishing()
# collect old jammed build
time_limit = time.time() - pecan.conf.build_lifetime
builds = mongo.builds.find({
"status": {
"$nin": ["succeeded", "failed"]
},
"created": {
"$lte": time_limit
}
})
for b in builds:
build = Build(b)
build.set_status('failed')
build.finishing()
jobs = build.get_jobs()
for job in jobs:
if job.status not in ['succeeded', 'failed']:
job.set_status('failed')
