def login(code):
""" Get username and access token from github """
if code is None:
return None
access_token = get_access_token(code)
if access_token:
# Check if this user exists in DB
# if not we need to create it
user = User.fetch_from_github_token(access_token)
if user is None:
# Get data from github
data = get_user_from_token(access_token)
if data:
# Save this new user
user = User({"username": data['login'],
"name": data['name'],
"github_url": data['html_url'],
"email": data['email'],
"token_github": access_token
})
if not user.create():
return None
update_user_info_from_github(user.username, user.token_github)
else:
return None
return {"access_token": access_token,
"username": user.username}
return None
def login(username, password):
""" Get username and access token from github """
gitlab_userdata = get_access_token(username, password)
if gitlab_userdata is None:
return None
access_token = gitlab_userdata.get('private_token')
if access_token:
# Check if this user exists in DB
# if not we need to create it
user = User.fetch_from_gitlab_token(access_token)
# Save this new user
if user is None:
user = User({"username": gitlab_userdata['username'],
"name": gitlab_userdata['name'],
# "gitlab_url": data['html_url'],
"email": gitlab_userdata['email'],
"token_gitlab": access_token,
"id_gitlab": gitlab_userdata['id'],
})
if not user.create():
return None
update_user_info_from_gitlab(user.username, user.token_gitlab)
return {"access_token": access_token,
"username": user.username}
return None
def post(self, sent_user):
"""Create/Edit user"""
user = User.fetch(self.username)
if user is None:
sent_user.username = self.username
new_user = User(sent_user.as_dict())
if not new_user.create():
# Handle error
return {"result": "Error creating user %s with data %s" % (sent_user.username, sent_user)}
return {"result": "User %s created" % sent_user.username}
else:
if not user.update(sent_user):
# Handle error
return {"result": "Error editing"}
return {"result": "User %s edited" % self.username}
def get(self):
"""Returns output content"""
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project_name = pecan.request.context['project_name']
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=False)
if build is None:
return
# Get output folder
output_folder = build.get_output_folder_path()
# Test if output folder exists
if not os.path.isdir(output_folder):
return
output = {}
for path, folders, files in os.walk(output_folder):
current_path = path.replace(output_folder, "").strip("/")
for folder in folders:
output[folder.strip("/")] = {}
if current_path != '':
output[current_path] = files
return output
def get(self):
"""Download all files in one archive"""
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project_name = pecan.request.context['project_name']
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=False)
if build is None:
return
# Get options
archive = pecan.request.GET.get('type', 'tgz')
distro = pecan.request.GET.get('distro', None)
if distro not in supported_distros:
distro = None
# Get output folder
output_folder = build.get_output_folder_path(distro)
# Test if output folder exists
if not os.path.isdir(output_folder):
return None
# Set headers
headers = pecan.response.headers
# Prepare content type
content_type = archive_types.get(archive, 'application/x-gzip')
pecan.core.override_template(None, content_type)
# Prepare archive
f = BytesIO()
if archive == 'zip':
# Zip
zip_archive = zipfile.ZipFile(f, "w" )
for file_ in glob.glob(output_folder + "/*"):
zip_archive.write(file_, os.path.basename(file_))
zip_archive.close()
extension = "zip"
else:
# Tarball
tar_archive = tarfile.open(fileobj=f, mode="w:gz")
for file_ in glob.glob(output_folder + "/*"):
tar_archive.add(file_, os.path.basename(file_))
tar_archive.close()
extension = "tar.gz"
if build.package_name is None:
return
filename = project_name + "_%(package_version)s-%(package_release)s" % build.as_dict()
if distro is not None:
filename = filename + "-" + distro
else:
filename = filename + "-all_distros"
filename = ".".join((filename, extension))
headers.add("Content-Disposition", str("attachment;filename=%s" % filename))
# returns
return f.getvalue()
def update_user_info_from_gitlab(username, access_token):
user = User.fetch(username, with_password=False)
# TODO handle better access
# user = User.fetch_from_github_token(access_token)
# if user is None or user.username != username:
# return False
# Get data from github
gl_user = get_user(user.id_gitlab, access_token)
gl_repos = get_user_repos(user.id_gitlab, access_token)
gl_orgs = get_user_orgs(username, access_token)
# Update user data
user.email = gl_user.get('email')
#user.github_url = gh_user['html_url']
user.name = gl_user['name']
user.orgs = [org['name'] for org in gl_orgs]
# Update repos and create it if not exists
for repo in gl_repos:
project = Project.fetch(user.username, repo['name'])
if project is None:
project = Project({"name": repo['name'],
"description": repo['description'],
"username": user.username,
"enabled": False,
"gitlab_project_id": repo['id'],
"url": repo['web_url'],
})
project.create()
# TODO get Project status (webhook) (enable) from gitlab
# Save project
project.update()
# Update user orgs
user._save()
return True
def post(self, send_build):
""" launch build """
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
project = Project.fetch(user.username, project_name)
if project is None:
# The project doesn't exist
# We have to create it
# TODO Maybe it's better to force users to create project before
# they can create builds
sent_project = {"name": project_name, "username": user.username}
project = Project(sent_project, sub_objects=False)
if not project.create():
# Handle error
return {"result": "Error creating %s project" % project_name}
build = Build(send_build)
build.username = user.username
build.project_name = project.name
build.create()
carrier = Carrier(
pecan.conf.rabbit_server,
pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost,
pecan.conf.rabbit_db
)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return {"result": {"build": int(build.id_)}}
def delete(self, sent_user):
"""Delete user and user folder"""
user = User.fetch(self.username)
if user is not None:
if user.delete(sent_user.password):
return {"result": "User %s deleted" % self.username}
return {"result": "User Doesn't exist"}
def post(self, send_build):
""" launch build """
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
project = Project.fetch(user.username, project_name)
if project is None:
# The project doesn't exist
# We have to create it
# TODO Maybe it's better to force users to create project before
# they can create builds
sent_project = {"name": project_name, "username": user.username}
project = Project(sent_project, sub_objects=False)
if not project.create():
# Handle error
return {"result": "Error creating %s project" % project_name}
build = Build(send_build)
build.username = user.username
build.project_name = project.name
build.create()
carrier = Carrier(pecan.conf.rabbit_server, pecan.conf.rabbit_port,
pecan.conf.rabbit_user, pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return {"result": {"build": int(build.id_)}}
def update_user_info_from_gitlab(username, access_token):
user = User.fetch(username, with_password=False)
# TODO handle better access
# user = User.fetch_from_github_token(access_token)
# if user is None or user.username != username:
# return False
# Get data from github
gl_user = get_user(user.id_gitlab, access_token)
gl_repos = get_user_repos(user.id_gitlab, access_token)
gl_orgs = get_user_orgs(username, access_token)
# Update user data
user.email = gl_user.get('email')
#user.github_url = gh_user['html_url']
user.name = gl_user['name']
user.orgs = [org['name'] for org in gl_orgs]
# Update repos and create it if not exists
for repo in gl_repos:
project = Project.fetch(user.username, repo['name'])
if project is None:
project = Project({
"name": repo['name'],
"description": repo['description'],
"username": user.username,
"enabled": False,
"gitlab_project_id": repo['id'],
"url": repo['web_url'],
})
project.create()
# TODO get Project status (webhook) (enable) from gitlab
# Save project
project.update()
# Update user orgs
user._save()
return True
def __init__(self, project_name):
pecan.request.context['project_name'] = project_name
self.project_name = project_name
self.user = User.fetch(pecan.request.context['username'],
sub_objects=False)
if self.user is None:
raise Exception("User not found")
def get(self, username):
""" launch build from gitlab webhook"""
access_token = pecan.request.GET.get('access_token')
if access_token:
# Check if this user exists in DB
user = User.fetch(username)
if user is None:
# This user maybe a group
data = gitlab.get_group(username, access_token)
if data is not None:
url = "%s/groups/%s" % (pecan.conf.gitlab_url,
data['name'])
if pecan.conf.gitlab_secure:
url = "https://"
else:
url = "http://"
# Save this new group
user = User({"username": data['name'],
"name": data['name'],
"gitlab_url": url,
"gitlab_group": True,
"id_gitlab": data['id'],
})
if not user.create():
return None
else:
# if user doesn't exists and this is not a group
# This request is strange so 403
abort(403)
else:
# if not we need to create it
gitlab_user = gitlab.get_user(user.id_gitlab, access_token)
# Update user info
# TODO Disabled for now
# if gitlab_user:
# user.name = gitlab_user.get('name')
# user.email = gitlab_user.get('email')
# if not user.update():
# return None
if user.gitlab_group:
return gitlab.update_group_info_from_gitlab(user, access_token)
else:
return gitlab.update_user_info_from_gitlab(username, access_token)
return None
def get_all(self):
"""Returns all builds."""
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
return project.get_builds()
def get_all(self):
"""Returns all builds."""
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
return project.get_builds()
def get(self, username):
""" launch build from github webhook"""
access_token = pecan.request.GET.get('access_token')
if access_token:
# Check if this user exists in DB
# if not we need to create it
data = github.get_user(username, access_token)
if data:
# Save this new user
user = User({"username": data['login'],
"name": data['name'],
"github_url": data['html_url'],
"email": data['email'],
"token_github": access_token
})
if not user.create():
return None
return github.update_user_info_from_github(username, access_token)
return None
def get_build(self):
build_data = self.get_message('builds.queue')
if build_data is not None:
build = Build(build_data)
build.user = User.fetch(build_data['username'], sub_objects=False)
if build.user is None:
return None
build.project = Project.fetch(build.user.username,
build_data['project_name'],
sub_objects=False)
return build
def get_build(self):
build_data = self.get_message('builds.queue')
if build_data is not None:
build = Build(build_data)
build.user = User.fetch(build_data['username'], sub_objects=False)
if build.user is None:
return None
build.project = Project.fetch(build.user.username,
build_data['project_name'],
sub_objects=False)
return build
def get(self, access_token):
"""toogle project"""
user = User.fetch(pecan.request.context['username'])
project = Project.fetch(user.username, pecan.request.context['project_name'])
# Github or Gitlab
new_state = None
if pecan.conf.auth == 'gitlab':
new_state = gitlab.toggle_project_webhook(user, project, access_token)
elif pecan.conf.auth == 'github':
new_state = github.toggle_project_webhook(user, project, access_token)
if new_state is None:
return {"result": "error"}
return {"result": new_state}
def get(self):
"""Returns log of a specific distro."""
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
if user is None:
return None
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=True)
return build.jobs
def get(self):
"""Returns log of a specific distro."""
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
if user is None:
return None
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=True)
return build.jobs
def get(self, access_token):
"""toogle project"""
user = User.fetch(pecan.request.context['username'])
project = Project.fetch(user.username,
pecan.request.context['project_name'])
# Github or Gitlab
new_state = None
if pecan.conf.auth == 'gitlab':
new_state = gitlab.toggle_project_webhook(user, project,
access_token)
elif pecan.conf.auth == 'github':
new_state = github.toggle_project_webhook(user, project,
access_token)
if new_state is None:
return {"result": "error"}
return {"result": new_state}
def get(self):
"""Returns build status"""
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project_name = pecan.request.context['project_name']
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = self.id_
if self.id_ in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=True)
if build:
return build
return None
def get(self):
"""Returns build status"""
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project_name = pecan.request.context['project_name']
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = self.id_
if self.id_ in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=True)
if build:
return build
return None
def get(self):
"""Download one output file"""
user = User.fetch(pecan.request.context['username'], sub_objects=False)
if user is None:
return None
project_name = pecan.request.context['project_name']
project = Project.fetch(user.username, project_name, sub_objects=False)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, sub_objects=False)
if build is None:
return
# Get options
distro = pecan.request.GET.get('distro', None)
filename = pecan.request.GET.get('filename', None)
if distro not in supported_distros:
distro = None
# Get output folder
output_folder = build.get_output_folder_path(distro)
# Test if output folder exists
if not os.path.isdir(output_folder):
return None
# Set headers
headers = pecan.response.headers
# Get one file
if filename is not None and distro is not None:
file_path = os.path.join(output_folder, filename)
if not os.path.isfile(file_path):
return None
# TODO clean the following lines
mime = MimeTypes()
contenttype, _ = mime.guess_type(file_path)
headers.add("Content-Disposition", str("attachment;filename=%s" % filename))
fhandler = open(file_path, 'r')
wsme_file = wtypes.File(filename=filename,
file=fhandler,
contenttype=contenttype)
return wsme_file.content
return None
def get(self):
"""Returns log of a specific job."""
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
if user is None:
return None
project = Project.fetch(user.username, project_name)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, False)
job_id = pecan.request.context['job_id']
job = Job.fetch(build, job_id)
log = job.get_log().strip().decode('unicode-escape')
html = pecan.request.GET.get('html', False)
print html
if html:
log = "<p><a></a><span>" + log.replace("\n", "</span></p><p><a></a>") + "</p>"
return log
def get(self):
"""Returns log of a specific job."""
project_name = pecan.request.context['project_name']
user = User.fetch(pecan.request.context['username'])
if user is None:
return None
project = Project.fetch(user.username, project_name)
if project is None:
return None
build_id = pecan.request.context['build_id']
if build_id in ["latest"]:
build_id = project.get_latest_build_id()
build = Build.fetch(project, build_id, False)
job_id = pecan.request.context['job_id']
job = Job.fetch(build, job_id)
log = job.get_log().strip().decode('unicode-escape')
html = pecan.request.GET.get('html', False)
print html
if html:
log = "<p><a></a><span>" + log.replace(
"\n", "</span></p><p><a></a>") + "</p>"
return log
def run(self):
self.must_run = True
logging.debug("Starting Manager")
while self.must_run:
time.sleep(0.1)
new_build = self.carrier.get_message('builds.queue')
build = None
if new_build is not None:
build = Build(new_build)
if build:
build.user = User.fetch(new_build['username'],
sub_objects=False)
build.project = Project.fetch(build.username,
new_build['project_name'],
sub_objects=False)
logging.debug("Task received")
build.set_status("dispatching")
dispatcher = Dispatcher(build)
self.build_list[dispatcher.uuid2] = dispatcher
dispatcher.start()
self.check_builds_status()
def __init__(self, project_name):
pecan.request.context['project_name'] = project_name
self.project_name = project_name
self.user = User.fetch(pecan.request.context['username'], sub_objects=False)
if self.user is None:
raise Exception("User not found")
def get(self, access_token=None):
"""Returns user"""
user = User.fetch(self.username, with_password=False)
return user
def post(self):
"""Launch build from github webhook"""
body = pecan.request.json
# Get user
if not body.get('sender'):
abort(403)
# Get username
username = body.get('repository').get('owner').get('name')
if username is None:
username = body.get('repository').get('owner').get('login')
user = User.fetch(username)
if user is None:
abort(403)
# Check signature
signature = pecan.request.headers.get('X-Hub-Signature')
sha_name, signature = signature.split("=")
if sha_name != 'sha1':
abort(403)
mac = hmac.new(user.token.encode("utf-8"), pecan.request.text, digestmod=hashlib.sha1)
if not hmac.compare_digest(mac.hexdigest(), signature):
abort(403)
# Ping event
if pecan.request.headers.get('X-Github-Event') == 'ping':
return json.dumps({"result": True, "event": "ping"})
# TODO handle tag event
# Push Event
if pecan.request.headers.get('X-Github-Event') == 'push':
if not body.get('repository'):
abort(403)
repository = body.get('repository')
project_name = repository.get('name')
if not project_name:
abort(403)
project = Project.fetch(user.username, project_name)
if project is None:
# Error project doesn't exits
# Maybe We should create it
return json.dumps({"result": False , "error": "project not found"})
new_build = {"source_url": repository.get('clone_url'),
#"source_type": "github",
"source_type": "git",
"commit": repository.get('commit'),
# TODO Find how decide if is a snapshot or not
# Answer: on tag event => NOT
"snapshot": True,
# TODO Check if branch ~= ref
"branch": repository.get('ref'),
}
build = Build(new_build)
build.username = user.username
build.project_name = project.name
build.create()
carrier = Carrier(
pecan.conf.rabbit_server,
pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost,
pecan.conf.rabbit_db
)
carrier.declare_queue('builds.queue')
# carrier.declare_builds()
if not carrier.send_message(build.dumps(), 'builds.queue'):
return None
return json.dumps({"result": True, "build": int(build.id_)})
abort(403)
def put(self, access_token):
""" Update user from github """
user = User.fetch(self.username, with_password=False)
if user is None:
return {"result": "bad user"}