def test_get_file_hash():
print("Testing utils.get_file_hash")
from sregistry.utils import get_file_hash
here = os.path.dirname(os.path.abspath(__file__))
testdata = os.path.join(here, "testdata", "hashtest.txt")
assert (get_file_hash(testdata) ==
"6bb92117bded3da774363713657a629a9f38eac2e57cd47e1dcda21d3445c67d")
assert get_file_hash(testdata, "md5") == "e5d376ca96081dd561ff303c3a631fd5"
def create_build_package(package_files, working_dir=None):
"""given a list of files, copy them to a temporary folder,
compress into a .tar.gz, and rename based on the file hash.
Return the full path to the .tar.gz in the temporary folder.
Parameters
==========
package_files: a list of files to include in the tar.gz
working_dir: If set, the path derived for the recipe and
files is relative to this.
"""
# Ensure package files all exist
for package_file in package_files:
if not os.path.exists(package_file):
bot.exit("Cannot find %s." % package_file)
bot.log("Generating build package for %s files..." % len(package_files))
build_dir = get_tmpdir(prefix="sregistry-build")
build_tar = "%s/build.tar.gz" % build_dir
tar = tarfile.open(build_tar, "w:gz")
# Create the tar.gz, making sure relative to working_dir
for package_file in package_files:
# Get a relative path
relative_path = get_relative_path(package_file, working_dir)
tar.add(package_file, arcname=relative_path)
tar.close()
# Get hash (sha256), and rename file
sha256 = get_file_hash(build_tar)
hash_tar = "%s/%s.tar.gz" % (build_dir, sha256)
shutil.move(build_tar, hash_tar)
return hash_tar
def push(self, path, name, tag=None):
"""push an image to Google Cloud Storage, meaning uploading it
Parameters
==========
path: should correspond to an absolte image path (or derive it)
name: should be the complete uri that the user has requested to push.
tag: should correspond with an image tag. This is provided to mirror Docker
"""
path = os.path.abspath(path)
bot.debug("PUSH %s" % path)
if not os.path.exists(path):
bot.exit("%s does not exist." % path)
# This returns a data structure with collection, container, based on uri
names = parse_image_name(remove_uri(name), tag=tag)
if names["version"] is None:
version = get_file_hash(path, "sha256")
names = parse_image_name(remove_uri(name), tag=tag, version=version)
# Update metadata with names
metadata = self.get_metadata(path, names=names)
manifest = self._upload(source=path,
destination=names["storage"],
metadata=metadata)
print(manifest["mediaLink"])
def create_build_package(package_files):
'''given a list of files, copy them to a temporary folder,
compress into a .tar.gz, and rename based on the file hash.
Return the full path to the .tar.gz in the temporary folder.
Parameters
==========
package_files: a list of files to include in the tar.gz
'''
# Ensure package files all exist
for package_file in package_files:
if not os.path.exists(package_file):
bot.exit('Cannot find %s.' % package_file)
bot.log('Generating build package for %s files...' % len(package_files))
build_dir = get_tmpdir(prefix="sregistry-build")
build_tar = '%s/build.tar.gz' % build_dir
tar = tarfile.open(build_tar, "w:gz")
# Create the tar.gz
for package_file in package_files:
tar.add(package_file)
tar.close()
# Get hash (sha256), and rename file
sha256 = get_file_hash(build_tar)
hash_tar = "%s/%s.tar.gz" % (build_dir, sha256)
shutil.move(build_tar, hash_tar)
return hash_tar
def calculate_version(cid):
'''calculate version is run as a separate task after a container upload.
Instead of using md5 provided by nginx we calculate sha256 sum and
then include as the version variable.
'''
from shub.apps.main.views import get_container
from sregistry.utils import get_file_hash
print("Calculating version for upload.")
container = get_container(cid)
version = "sha256.%s" % get_file_hash(container.image.datafile.path,
"sha256")
container.version = version
container.save()
def push(self, path, name, tag=None):
"""push an image to Google Cloud Drive, meaning uploading it
path: should correspond to an absolte image path (or derive it)
name: should be the complete uri that the user has requested to push.
tag: should correspond with an image tag. This is provided to mirror Docker
"""
# The root of the drive for containers (the parent folder)
parent = self._get_or_create_folder(self._base)
image = None
path = os.path.abspath(path)
bot.debug("PUSH %s" % path)
if not os.path.exists(path):
bot.exit("%s does not exist." % path)
names = parse_image_name(remove_uri(name), tag=tag)
if names["version"] is None:
version = get_file_hash(path, "sha256")
names = parse_image_name(remove_uri(name), tag=tag, version=version)
# Update metadata with names, flatten to only include labels
metadata = self.get_metadata(path, names=names)
file_metadata = {
"name": names["storage"],
"mimeType": "application/octet-stream",
"parents": [parent["id"]],
"properties": metadata,
}
media = MediaFileUpload(path, resumable=True)
try:
bot.spinner.start()
image = (self._service.files().create(body=file_metadata,
media_body=media,
fields="id").execute())
# Add a thumbnail!
thumbnail = get_thumbnail()
with open(thumbnail, "rb") as f:
body = {
"contentHints": {
"thumbnail": {
"image":
base64.urlsafe_b64encode(f.read()).decode("utf8"),
"mimeType": "image/png",
}
}
}
image = (self._service.files().update(fileId=image["id"],
body=body).execute())
bot.spinner.stop()
print(image["name"])
except HttpError:
bot.error("Error uploading %s" % path)
return image
def add(self,
image_path=None,
image_uri=None,
image_name=None,
url=None,
metadata=None,
save=True,
copy=False):
'''get or create a container, including the collection to add it to.
This function can be used from a file on the local system, or via a URL
that has been downloaded. Either way, if one of url, version, or image_file
is not provided, the model is created without it. If a version is not
provided but a file path is, then the file hash is used.
Parameters
==========
image_path: full path to image file
image_name: if defined, the user wants a custom name (and not based on uri)
metadata: any extra metadata to keep for the image (dict)
save: if True, move the image to the cache if it's not there
copy: If True, copy the image instead of moving it.
image_name: a uri that gets parsed into a names object that looks like:
{'collection': 'vsoch',
'image': 'hello-world',
'storage': 'vsoch/hello-world-latest.img',
'tag': 'latest',
'version': '12345'
'uri': 'vsoch/hello-world:latest@12345'}
After running add, the user will take some image in a working
directory, add it to the database, and have it available for search
and use under SREGISTRY_STORAGE/<collection>/<container>
If the container was retrieved from a webby place, it should have version
If no version is found, the file hash is used.
'''
from sregistry.database.models import Container
# We can only save if the image is provided
if image_path is not None:
if not os.path.exists(image_path) and save is True:
bot.exit('Cannot find %s' % image_path)
# An image uri is required for version, tag, etc.
if image_uri is None:
bot.exit(
'You must provide an image uri --name <collection>/<namespace>')
names = parse_image_name(remove_uri(image_uri))
bot.debug('Adding %s to registry' % names['uri'])
# If Singularity is installed, inspect image for metadata
metadata = self.get_metadata(image_path, names=names)
collection = self.get_or_create_collection(names['collection'])
# Get a hash of the file for the version, or use provided
version = names.get('version')
if not version:
if image_path:
version = get_file_hash(image_path, "sha256")
else:
version = '' # we can't determine a version, not in API/no file
names = parse_image_name(remove_uri(image_uri), version=version)
# If save, move to registry storage first
if save and image_path:
# If the user hasn't defined a custom name
if image_name is None:
image_name = self._get_storage_name(names)
if copy:
copyfile(image_path, image_name)
else:
shutil.move(image_path, image_name)
image_path = image_name
# Just in case the client didn't provide it, see if we have in metadata
if url is None and "url" in metadata:
url = metadata['url']
# First check that we don't have one already!
container = self.get_container(name=names['image'],
collection_id=collection.id,
tag=names['tag'],
version=version)
# The container did not exist, create it
if not container:
action = "new"
container = Container(metrics=json.dumps(metadata),
name=names['image'],
image=image_path,
client=self.client_name,
tag=names['tag'],
version=version,
url=url,
uri=names['uri'],
collection_id=collection.id)
self.session.add(container)
collection.containers.append(container)
# The container existed, update it.
else:
action = "update"
metrics = json.loads(container.metrics)
metrics.update(metadata)
container.url = url
container.client = self.client_name
if image_path is not None:
container.image = image_path
container.metrics = json.dumps(metrics)
self.session.commit()
bot.info("[container][%s] %s" % (action, names['uri']))
return container