def api_cross_symlink(username, repository, sourcename, linkname):
flist = HubPublicFlist(config, repository, sourcename)
linkflist = HubPublicFlist(config, username, linkname)
if not flist.user_exists:
return api_response("source repository not found", 404)
if not flist.file_exists:
return api_response("source not found", 404)
# remove previous symlink if existing
if os.path.islink(linkflist.target):
os.unlink(linkflist.target)
# if it was not a link but a regular file, we don't overwrite
# existing flist, we only allows updating links
if os.path.isfile(linkflist.target):
return api_response("link destination is already a file", 401)
cwd = os.getcwd()
os.chdir(linkflist.user_path)
os.symlink("../" + flist.username + "/" + flist.filename,
linkflist.filename)
os.chdir(cwd)
return api_response()
def api_promote(username, sourcerepo, sourcefile, targetname):
flist = HubPublicFlist(config, sourcerepo, sourcefile)
destination = HubPublicFlist(config, username, targetname)
if not flist.user_exists:
return api_response("user not found", 404)
if not flist.file_exists:
return api_response("source not found", 404)
# ensure target exists
if not destination.user_exists:
destination.user_create()
# remove previous file if existing
if os.path.exists(destination.target):
os.unlink(destination.target)
print("[+] promote: %s -> %s" % (flist.target, destination.target))
shutil.copy(flist.target, destination.target)
status = {
'source': {
'username': flist.username,
'filename': flist.filename,
},
'destination': {
'username': destination.username,
'filename': destination.filename,
}
}
return api_response(extra=status)
def api_flist_upload(request, username, validate=False):
# check if the post request has the file part
if 'file' not in request.files:
return {'status': 'error', 'message': 'no file found'}
file = request.files['file']
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '':
return {'status': 'error', 'message': 'no file selected'}
if not allowed_file(file.filename, validate):
return {'status': 'error', 'message': 'this file is not allowed'}
#
# processing the file
#
filename = secure_filename(file.filename)
print("[+] saving file")
source = os.path.join(config['upload-directory'], filename)
file.save(source)
cleanfilename = file_from_flist(filename)
flist = HubPublicFlist(config, username, cleanfilename)
flist.user_create()
# it's a new flist, let's do the normal flow
if not validate:
workspace = flist.raw.workspace()
flist.raw.unpack(source, workspace.name)
stats = flist.raw.create(workspace.name, flist.target)
# we have an existing flist and checking contents
# we don't need to create the flist, we just ensure the
# contents is on the backend
else:
flist.loads(source)
stats = flist.validate()
if stats['response']['failure'] > 0:
return {
'status':
'error',
'message':
'unauthorized upload, contents is not fully present on backend'
}
flist.commit()
# removing uploaded source file
os.unlink(source)
return {
'status': 'success',
'flist': flist.filename,
'home': username,
'stats': stats,
'timing': {}
}
def api_my_rename(source, destination):
username = session['username']
flist = HubPublicFlist(config, username, source)
destflist = HubPublicFlist(config, username, destination)
if not flist.user_exists:
return api_response("user not found", 404)
if not flist.file_exists:
return api_response("source not found", 404)
os.rename(flist.target, destflist.target)
return api_response()
def api_my_merge(target):
username = session['username']
sources = request.get_json(silent=True, force=True)
data = flist_merge_data(sources, target)
if data['error'] != None:
return api_response(data['error'], 500)
flist = HubPublicFlist(config, username, data['target'])
status = flist.merge(data['sources'])
if not status == True:
return api_response(status, 500)
return api_response()
def api_flist_upload_prepare(request, username, validate=False):
# check if the post request has the file part
if 'file' not in request.files:
return {'status': 'error', 'message': 'no file found'}
file = request.files['file']
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '':
return {'status': 'error', 'message': 'no file selected'}
if not allowed_file(file.filename, validate):
return {'status': 'error', 'message': 'this file is not allowed'}
#
# processing the file
#
filename = secure_filename(file.filename)
print("[+] saving file")
source = os.path.join(config['upload-directory'], filename)
file.save(source)
cleanfilename = file_from_flist(filename)
flist = HubPublicFlist(config, username, cleanfilename, announcer)
flist.raw.newtask()
print("[+] flist creation id: %s" % flist.raw.jobid)
job = threading.Thread(target=flist.create, args=(source, ))
job.start()
return {'status': 'success', 'jobid': flist.raw.jobid}
"""
def api_user_contents(username):
flist = HubPublicFlist(config, username, "unknown")
if not flist.user_exists:
abort(404)
contents = api_user_contents(username, flist.user_path)
response = make_response(json.dumps(contents) + "\n")
response.headers["Content-Type"] = "application/json"
return response
def api_fileslist():
repositories = api_repositories()
fileslist = {}
for repository in repositories:
flist = HubPublicFlist(config, repository['name'], "unknown")
contents = api_user_contents(flist.username, flist.user_path)
fileslist[repository['name']] = contents
return fileslist
def checksum_flist(username, flist):
flist = HubPublicFlist(config, username, flist)
hash = flist.checksum
if not hash:
abort(404)
response = make_response(hash + "\n")
response.headers["Content-Type"] = "text/plain"
return response
def flist_merge():
username = session['username']
if request.method == 'POST':
data = flist_merge_post()
print(data)
if data['error']:
return internalRedirect("merge.html", data['error'])
flist = HubPublicFlist(config, username, data['target'])
status = flist.merge(data['sources'])
if not status == True:
variables = {'error': status}
return globalTemplate("merge.html", variables)
return uploadSuccess(data['target'], 0, data['target'])
# Merge page
return internalRedirect("merge.html")
def api_delete(username, source):
flist = HubPublicFlist(config, username, source)
if not flist.user_exists:
return api_response("user not found", 404)
if not flist.file_exists:
return api_response("source not found", 404)
os.unlink(flist.target)
return api_response()
def api_readme(username, flist):
flist = HubPublicFlist(config, username, flist)
if not flist.user_exists:
return api_response("user not found", 404)
if not flist.file_exists:
return api_response("source not found", 404)
readme = api_flist_md(flist)
response = make_response(json.dumps(readme) + "\n")
response.headers["Content-Type"] = "application/json"
return response
def flist_merge_data(sources, target):
data = {}
data['error'] = None
data['sources'] = sources
data['target'] = target
if not isinstance(sources, list):
data['error'] = 'malformed json request'
return data
if len(data['sources']) == 0:
data['error'] = "no source found"
return data
# ensure .flist extension to each sources
fsources = []
for source in data['sources']:
# missing username/filename
if "/" not in source:
data['error'] = "malformed source filename"
return data
cleaned = source if source.endswith(".flist") else source + ".flist"
fsources.append(cleaned)
data['sources'] = fsources
# ensure each sources exists
for source in data['sources']:
temp = source.split("/")
item = HubPublicFlist(config, temp[0], temp[1])
if not item.file_exists:
data['error'] = "%s does not exists" % source
return data
if not data['target']:
data['error'] = "missing build (target) name"
return data
if "/" in data['target']:
data['error'] = "build name not allowed"
return data
if not data['target'].endswith('.flist'):
data['target'] += '.flist'
return data
def show_flist_txt(username, flist):
flist = HubPublicFlist(config, username, flist)
if not flist.file_exists:
abort(404)
text = "File: %s\n" % flist.filename
text += "Uploader: %s\n" % username
text += "Source: %s/%s/%s\n" % (config['public-website'], username,
flist.filename)
text += "Storage: zdb://%s:%d\n" % (config['backend-public-host'],
config['backend-public-port'])
text += "Checksum: %s\n" % flist.checksum
response = make_response(text)
response.headers["Content-Type"] = "text/plain"
return response
def api_inspect(username, flist):
flist = HubPublicFlist(config, username, flist)
if not flist.user_exists:
return api_response("user not found", 404)
if not flist.file_exists:
return api_response("source not found", 404)
if request.method == 'GET':
contents = api_contents(flist)
if request.method == 'INFO':
contents = api_flist_info(flist)
response = make_response(json.dumps(contents) + "\n")
response.headers["Content-Type"] = "application/json"
return response
def show_flist_md(username, flist):
flist = HubPublicFlist(config, username, flist)
if not flist.file_exists:
abort(404)
variables = {
'targetuser':
username,
'flistname':
flist.filename,
'flisturl':
"%s/%s/%s" % (config['public-website'], username, flist.filename),
'ardbhost':
'zdb://%s:%d' %
(config['backend-public-host'], config['backend-public-port']),
'checksum':
flist.checksum
}
return globalTemplate("preview.html", variables)
def show_flist_json(username, flist):
flist = HubPublicFlist(config, username, flist)
if not flist.file_exists:
abort(404)
data = {
'flist':
flist,
'uploader':
username,
'source':
"%s/%s/%s" % (config['public-website'], username, flist),
'storage':
"zdb://%s:%d" %
(config['backend-public-host'], config['backend-public-port']),
'checksum':
flist.checksum
}
response = make_response(json.dumps(data) + "\n")
response.headers["Content-Type"] = "application/json"
return response
def converter(self, dockerimage, username="******"):
dockername = uuid.uuid4().hex
print("[+] docker-convert: temporary docker name: %s" % dockername)
if ":" not in dockerimage:
dockerimage = "%s:latest" % dockerimage
#
# loading image from docker-hub
#
print("[+] docker-convert: pulling image: %s" % dockerimage)
self.progress("Pulling docker image: %s" % dockerimage, 10)
try:
# progress pull
self.pull(dockerimage)
# fetch real image name
image = self.dockerclient.images.pull(dockerimage)
except docker.errors.ImageNotFound:
return {'status': 'error', 'message': 'docker image not found'}
except docker.errors.APIError:
return {'status': 'error', 'message': 'could not pull this image'}
#
# building init-command line
#
command = None
if not image.attrs['Config']['Cmd'] and not image.attrs['Config'][
'Entrypoint']:
command = "/bin/sh"
print("[+] docker-convert: starting temporary container")
self.progress("Initializing temporary container", 52)
cn = self.dockerclient.containers.create(dockerimage,
name=dockername,
hostname=dockername,
command=command)
#
# exporting docker
#
print("[+] docker-convert: creating target directory")
tmpdir = tempfile.TemporaryDirectory(
prefix=dockername, dir=self.config['docker-work-directory'])
os.chmod(tmpdir.name, 0o755)
print("[+] docker-convert: dumping files to: %s" % tmpdir.name)
self.progress("Extracting container root filesystem", 54)
subprocess.call([
'sh', '-c',
'docker export %s | tar -xpf - -C %s' % (dockername, tmpdir.name)
])
#
# docker init command to container startup command
#
print("[+] docker-convert: creating container entrypoint")
self.progress("Creating container metadata", 55)
command, args, env, cwd = self.container_boot(cn)
boot = {
'startup': {
'entry': {
'name': "core.system",
'args': {
'name': command,
'args': args,
'env': env,
'dir': cwd,
}
}
}
}
with open(os.path.join(tmpdir.name, '.startup.toml'), 'w') as f:
f.write(toml.dumps(boot))
#
# bundle the image
#
print("[+] docker-convert: parsing the flist")
self.progress("Preparing filesystem flist", 56)
flistname = dockerimage.replace(":", "-").replace('/', '-')
flist = HubPublicFlist(self.config, username, flistname,
self.announcer)
flist.raw.jobid = self.jobid
flist.user_create()
info = flist.raw.create(tmpdir.name, flist.target)
print("[+] docker-convert: cleaning temporary files")
self.progress("Cleaning up temporary files", 95)
tmpdir.cleanup()
print("[+] docker-convert: destroying the container")
self.progress("Cleaning up container", 97)
cn.remove(force=True)
print("[+] docker-convert: cleaning up the docker image")
self.progress("Cleaning up docker image", 99)
self.dockerclient.images.remove(dockerimage, force=True)
if info['success'] == False:
return {'status': 'error', 'message': info['error']['message']}
self.progress("Image ready !", 100)
return {
'status': 'success',
'file': flist.filename,
'flist': info['response'],
'timing': {}
}
def show_user(username):
flist = HubPublicFlist(config, username, "unknown")
if not flist.user_exists:
abort(404)
return globalTemplate("user.html", {'targetuser': username})
def download_flist(username, flist):
flist = HubPublicFlist(config, username, flist)
return send_from_directory(directory=flist.user_path,
filename=flist.filename)