def post(self, send_build):
""" launch build """
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
project = Project.fetch(user.username, project_name)
if project is None:
# The project doesn't exist
# We have to create it
# TODO Maybe it's better to force users to create project before
# they can create builds
sent_project = {"name": project_name, "username": user.username}
project = Project(sent_project, sub_objects=False)
if not project.create():
# Handle error
return {"result": "Error creating %s project" % project_name}
build = Build(send_build)
build.username = user.username
build.project_name = project.name
build.create()
carrier = Carrier(pecan.conf.rabbit_server, pecan.conf.rabbit_port,
pecan.conf.rabbit_user, pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return {"result": {"build": int(build.id_)}}
def post(self, send_build):
""" launch build """
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
project = Project.fetch(user.username, project_name)
if project is None:
# The project doesn't exist
# We have to create it
# TODO Maybe it's better to force users to create project before
# they can create builds
sent_project = {"name": project_name, "username": user.username}
project = Project(sent_project, sub_objects=False)
if not project.create():
# Handle error
return {"result": "Error creating %s project" % project_name}
build = Build(send_build)
build.username = user.username
build.project_name = project.name
build.create()
carrier = Carrier(
pecan.conf.rabbit_server,
pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost,
pecan.conf.rabbit_db
)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return {"result": {"build": int(build.id_)}}
def post(self, sent_project):
"""Create project"""
projects = self.user.get_projects()
if self.project_name not in [p.name for p in projects]:
sent_project.name = self.project_name
new_project = Project(sent_project.as_dict())
new_project.username = self.user.username
if not new_project.create():
# Handle error
return {"result": "Error creating %s with data %s" % (self.project_name, sent_project)}
return {"result": "Project %s created" % self.project_name}
else:
return {"result": "Project %s exists" % self.project_name}
def get(self):
"""Returns output content"""
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project_name = pecan.request.context['project_name']
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=False)
if build is None:
return
# Get output folder
output_folder = build.get_output_folder_path()
# Test if output folder exists
if not os.path.isdir(output_folder):
return
output = {}
for path, folders, files in os.walk(output_folder):
current_path = path.replace(output_folder, "").strip("/")
for folder in folders:
output[folder.strip("/")] = {}
if current_path != '':
output[current_path] = files
return output
def get(self):
"""Download all files in one archive"""
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project_name = pecan.request.context['project_name']
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=False)
if build is None:
return
# Get options
archive = pecan.request.GET.get('type', 'tgz')
distro = pecan.request.GET.get('distro', None)
if distro not in supported_distros:
distro = None
# Get output folder
output_folder = build.get_output_folder_path(distro)
# Test if output folder exists
if not os.path.isdir(output_folder):
return None
# Set headers
headers = pecan.response.headers
# Prepare content type
content_type = archive_types.get(archive, 'application/x-gzip')
pecan.core.override_template(None, content_type)
# Prepare archive
f = BytesIO()
if archive == 'zip':
# Zip
zip_archive = zipfile.ZipFile(f, "w" )
for file_ in glob.glob(output_folder + "/*"):
zip_archive.write(file_, os.path.basename(file_))
zip_archive.close()
extension = "zip"
else:
# Tarball
tar_archive = tarfile.open(fileobj=f, mode="w:gz")
for file_ in glob.glob(output_folder + "/*"):
tar_archive.add(file_, os.path.basename(file_))
tar_archive.close()
extension = "tar.gz"
if build.package_name is None:
return
filename = project_name + "_%(package_version)s-%(package_release)s" % build.as_dict()
if distro is not None:
filename = filename + "-" + distro
else:
filename = filename + "-all_distros"
filename = ".".join((filename, extension))
headers.add("Content-Disposition", str("attachment;filename=%s" % filename))
# returns
return f.getvalue()
def delete(self):
"""Delete project"""
project = Project.fetch(self.user.username, self.project_name)
if project is not None:
if project.delete():
return APIResult(result="Project %s deleted" % self.project_name)
return APIResult(result="Project doesn't exist")
def post(self, sent_project):
"""Create project"""
projects = self.user.get_projects()
if self.project_name not in [p.name for p in projects]:
sent_project.name = self.project_name
new_project = Project(sent_project.as_dict())
new_project.username = self.user.username
if not new_project.create():
# Handle error
return {
"result":
"Error creating %s with data %s" %
(self.project_name, sent_project)
}
return {"result": "Project %s created" % self.project_name}
else:
return {"result": "Project %s exists" % self.project_name}
def delete(self):
"""Delete project"""
project = Project.fetch(self.user.username, self.project_name)
if project is not None:
if project.delete():
return APIResult(result="Project %s deleted" %
self.project_name)
return APIResult(result="Project doesn't exist")
def update_user_info_from_gitlab(username, access_token):
user = User.fetch(username, with_password=False)
# TODO handle better access
# user = User.fetch_from_github_token(access_token)
# if user is None or user.username != username:
# return False
# Get data from github
gl_user = get_user(user.id_gitlab, access_token)
gl_repos = get_user_repos(user.id_gitlab, access_token)
gl_orgs = get_user_orgs(username, access_token)
# Update user data
user.email = gl_user.get('email')
#user.github_url = gh_user['html_url']
user.name = gl_user['name']
user.orgs = [org['name'] for org in gl_orgs]
# Update repos and create it if not exists
for repo in gl_repos:
project = Project.fetch(user.username, repo['name'])
if project is None:
project = Project({
"name": repo['name'],
"description": repo['description'],
"username": user.username,
"enabled": False,
"gitlab_project_id": repo['id'],
"url": repo['web_url'],
})
project.create()
# TODO get Project status (webhook) (enable) from gitlab
# Save project
project.update()
# Update user orgs
user._save()
return True
def update_group_info_from_gitlab(group, access_token):
# TODO handle better access
# user = User.fetch_from_github_token(access_token)
# if user is None or user.username != username:
# return False
# Get data from gitlab
gl_repos = get_group_repos(group.id_gitlab, access_token)
# Update repos and create it if not exists
for repo in gl_repos:
project = Project.fetch(group.username, repo['name'])
if project is None:
project = Project({
"name": repo['name'],
"description": repo['description'],
"username": group.username,
"enabled": False,
"gitlab_project_id": repo['id'],
"url": repo['web_url'],
})
project.create()
# TODO get Project status (enable) from gitlab
# Save project
project.update()
return True
def get(self, name=None, username=None, limit=30, offset=0, get_last_build=False, pattern=''):
"""Returns project"""
projects = Project.search(name=name,
username=username,
limit=limit,
offset=offset,
get_last_build=get_last_build,
pattern=pattern)
return projects
def get_all(self):
"""Returns all builds."""
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
return project.get_builds()
def get_all(self):
"""Returns all builds."""
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
return project.get_builds()
def get_build(self):
build_data = self.get_message('builds.queue')
if build_data is not None:
build = Build(build_data)
build.user = User.fetch(build_data['username'], sub_objects=False)
if build.user is None:
return None
build.project = Project.fetch(build.user.username,
build_data['project_name'],
sub_objects=False)
return build
def get_build(self):
build_data = self.get_message('builds.queue')
if build_data is not None:
build = Build(build_data)
build.user = User.fetch(build_data['username'], sub_objects=False)
if build.user is None:
return None
build.project = Project.fetch(build.user.username,
build_data['project_name'],
sub_objects=False)
return build
def update_user_info_from_gitlab(username, access_token):
user = User.fetch(username, with_password=False)
# TODO handle better access
# user = User.fetch_from_github_token(access_token)
# if user is None or user.username != username:
# return False
# Get data from github
gl_user = get_user(user.id_gitlab, access_token)
gl_repos = get_user_repos(user.id_gitlab, access_token)
gl_orgs = get_user_orgs(username, access_token)
# Update user data
user.email = gl_user.get('email')
#user.github_url = gh_user['html_url']
user.name = gl_user['name']
user.orgs = [org['name'] for org in gl_orgs]
# Update repos and create it if not exists
for repo in gl_repos:
project = Project.fetch(user.username, repo['name'])
if project is None:
project = Project({"name": repo['name'],
"description": repo['description'],
"username": user.username,
"enabled": False,
"gitlab_project_id": repo['id'],
"url": repo['web_url'],
})
project.create()
# TODO get Project status (webhook) (enable) from gitlab
# Save project
project.update()
# Update user orgs
user._save()
return True
def get(self, access_token):
"""toogle project"""
user = User.fetch(pecan.request.context['username'])
project = Project.fetch(user.username, pecan.request.context['project_name'])
# Github or Gitlab
new_state = None
if pecan.conf.auth == 'gitlab':
new_state = gitlab.toggle_project_webhook(user, project, access_token)
elif pecan.conf.auth == 'github':
new_state = github.toggle_project_webhook(user, project, access_token)
if new_state is None:
return {"result": "error"}
return {"result": new_state}
def get(self):
"""Returns log of a specific distro."""
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
if user is None:
return None
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=True)
return build.jobs
def get(self):
"""Returns log of a specific distro."""
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
if user is None:
return None
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=True)
return build.jobs
def get(self,
name=None,
username=None,
limit=30,
offset=0,
get_last_build=False,
pattern=''):
"""Returns project"""
projects = Project.search(name=name,
username=username,
limit=limit,
offset=offset,
get_last_build=get_last_build,
pattern=pattern)
return projects
def get(self, access_token):
"""toogle project"""
user = User.fetch(pecan.request.context['username'])
project = Project.fetch(user.username,
pecan.request.context['project_name'])
# Github or Gitlab
new_state = None
if pecan.conf.auth == 'gitlab':
new_state = gitlab.toggle_project_webhook(user, project,
access_token)
elif pecan.conf.auth == 'github':
new_state = github.toggle_project_webhook(user, project,
access_token)
if new_state is None:
return {"result": "error"}
return {"result": new_state}
def get(self):
"""Returns build status"""
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project_name = pecan.request.context['project_name']
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = self.id_
if self.id_ in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=True)
if build:
return build
return None
def get(self):
"""Returns build status"""
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project_name = pecan.request.context['project_name']
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = self.id_
if self.id_ in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=True)
if build:
return build
return None
def get(self):
"""Download one output file"""
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project_name = pecan.request.context['project_name']
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=False)
if build is None:
return
# Get options
distro = pecan.request.GET.get('distro', None)
filename = pecan.request.GET.get('filename', None)
if distro not in supported_distros:
distro = None
# Get output folder
output_folder = build.get_output_folder_path(distro)
# Test if output folder exists
if not os.path.isdir(output_folder):
return None
# Set headers
headers = pecan.response.headers
# Get one file
if filename is not None and distro is not None:
file_path = os.path.join(output_folder, filename)
if not os.path.isfile(file_path):
return None
# TODO clean the following lines
mime = MimeTypes()
contenttype, _ = mime.guess_type(file_path)
headers.add("Content-Disposition", str("attachment;filename=%s" % filename))
fhandler = open(file_path, 'r')
wsme_file = wtypes.File(filename=filename,
file=fhandler,
contenttype=contenttype)
return wsme_file.content
return None
def get(self):
"""Returns log of a specific job."""
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
if user is None:
return None
project = Project.fetch(user.username, project_name)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, False)
job_id = pecan.request.context['job_id']
job = Job.fetch(build, job_id)
log = job.get_log().strip().decode('unicode-escape')
html = pecan.request.GET.get('html', False)
print html
if html:
log = "<p><a></a><span>" + log.replace("\n", "</span></p><p><a></a>") + "</p>"
return log
def get(self):
"""Returns log of a specific job."""
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
if user is None:
return None
project = Project.fetch(user.username, project_name)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, False)
job_id = pecan.request.context['job_id']
job = Job.fetch(build, job_id)
log = job.get_log().strip().decode('unicode-escape')
html = pecan.request.GET.get('html', False)
print html
if html:
log = "<p><a></a><span>" + log.replace(
"\n", "</span></p><p><a></a>") + "</p>"
return log
def run(self):
self.must_run = True
logging.debug("Starting Manager")
while self.must_run:
time.sleep(0.1)
new_build = self.carrier.get_message('builds.queue')
build = None
if new_build is not None:
build = Build(new_build)
if build:
build.user = User.fetch(new_build['username'],
sub_objects=False)
build.project = Project.fetch(build.username,
new_build['project_name'],
sub_objects=False)
logging.debug("Task received")
build.set_status("dispatching")
dispatcher = Dispatcher(build)
self.build_list[dispatcher.uuid2] = dispatcher
dispatcher.start()
self.check_builds_status()
def update_group_info_from_gitlab(group, access_token):
# TODO handle better access
# user = User.fetch_from_github_token(access_token)
# if user is None or user.username != username:
# return False
# Get data from gitlab
gl_repos = get_group_repos(group.id_gitlab, access_token)
# Update repos and create it if not exists
for repo in gl_repos:
project = Project.fetch(group.username, repo['name'])
if project is None:
project = Project({"name": repo['name'],
"description": repo['description'],
"username": group.username,
"enabled": False,
"gitlab_project_id": repo['id'],
"url": repo['web_url'],
})
project.create()
# TODO get Project status (enable) from gitlab
# Save project
project.update()
return True
def post(self):
""" launch build from gitlab webhook"""
body = pecan.request.json
# Get use
if not body.get('user_id'):
abort(403)
# Get token
token = pecan.request.GET.get('token')
if token is None:
abort(403)
# Get project
project = Project.fetch_from_token(token, False)
if project is None:
abort(403)
if body.get('project_id') != project.gitlab_project_id:
abort(403)
if body.get('object_kind') not in ['push', 'tag']:
abort(403)
else:
# If it's a TAG event we DON'T make snaphot
snapshot = False
if body.get('object_kind') == 'push':
# If it's a PUSH event we make snapshot
snapshot = True
if not body.get('repository'):
abort(403)
repository = body.get('repository')
project_name = repository.get('name')
if project_name != project.name:
abort(403)
new_build = {
"source_url": repository.get('git_http_url'),
#"source_type": "gitlab",
"source_type": "git",
"commit": body.get('after'),
# TODO Find how decide if is a snapshot or not
"snapshot": snapshot,
# TODO Check if branch ~= ref
"branch": body.get('ref'),
}
build = Build(new_build)
build.username = project.username
build.project_name = project.name
build.create()
carrier = Carrier(pecan.conf.rabbit_server, pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return json.dumps({"result": True, "build": int(build.id_)})
abort(403)
def post(self):
"""Launch build from github webhook"""
body = pecan.request.json
# Get user
if not body.get('sender'):
abort(403)
# Get username
username = body.get('repository').get('owner').get('name')
if username is None:
username = body.get('repository').get('owner').get('login')
user = User.fetch(username)
if user is None:
abort(403)
# Check signature
signature = pecan.request.headers.get('X-Hub-Signature')
sha_name, signature = signature.split("=")
if sha_name != 'sha1':
abort(403)
mac = hmac.new(user.token.encode("utf-8"), pecan.request.text, digestmod=hashlib.sha1)
if not hmac.compare_digest(mac.hexdigest(), signature):
abort(403)
# Ping event
if pecan.request.headers.get('X-Github-Event') == 'ping':
return json.dumps({"result": True, "event": "ping"})
# TODO handle tag event
# Push Event
if pecan.request.headers.get('X-Github-Event') == 'push':
if not body.get('repository'):
abort(403)
repository = body.get('repository')
project_name = repository.get('name')
if not project_name:
abort(403)
project = Project.fetch(user.username, project_name)
if project is None:
# Error project doesn't exits
# Maybe We should create it
return json.dumps({"result": False , "error": "project not found"})
new_build = {"source_url": repository.get('clone_url'),
#"source_type": "github",
"source_type": "git",
"commit": repository.get('commit'),
# TODO Find how decide if is a snapshot or not
# Answer: on tag event => NOT
"snapshot": True,
# TODO Check if branch ~= ref
"branch": repository.get('ref'),
}
build = Build(new_build)
build.username = user.username
build.project_name = project.name
build.create()
carrier = Carrier(
pecan.conf.rabbit_server,
pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost,
pecan.conf.rabbit_db
)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return json.dumps({"result": True, "build": int(build.id_)})
abort(403)
def get(self, get_last_build=False):
"""Returns project"""
project = Project.fetch(self.user.username,
self.project_name,
get_last_build=get_last_build)
return project
def post(self):
""" launch build from gitlab webhook"""
body = pecan.request.json
# Get use
if not body.get('user_id'):
abort(403)
# Get token
token = pecan.request.GET.get('token')
if token is None:
abort(403)
# Get project
project = Project.fetch_from_token(token, False)
if project is None:
abort(403)
if body.get('project_id') != project.gitlab_project_id:
abort(403)
if body.get('object_kind') not in ['push', 'tag']:
abort(403)
else:
# If it's a TAG event we DON'T make snaphot
snapshot = False
if body.get('object_kind') == 'push':
# If it's a PUSH event we make snapshot
snapshot = True
if not body.get('repository'):
abort(403)
repository = body.get('repository')
project_name = repository.get('name')
if project_name != project.name:
abort(403)
new_build = {"source_url": repository.get('git_http_url'),
#"source_type": "gitlab",
"source_type": "git",
"commit": body.get('after'),
# TODO Find how decide if is a snapshot or not
"snapshot": snapshot,
# TODO Check if branch ~= ref
"branch": body.get('ref'),
}
build = Build(new_build)
build.username = project.username
build.project_name = project.name
build.create()
carrier = Carrier(
pecan.conf.rabbit_server,
pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost,
pecan.conf.rabbit_db
)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return json.dumps({"result": True, "build": int(build.id_)})
abort(403)
def get_projects(self):
projects = mongo.projects.find({"username": self.username})
return [Project(x) for x in projects]
def get(self, get_last_build=False):
"""Returns project"""
project = Project.fetch(self.user.username, self.project_name, get_last_build=get_last_build)
return project