def docker_pull(self, namespace, repos):
# Test pull
# Docker -> Index
resp = requests.get('{0}/v1/repositories/{1}/{2}/images'.format(
self.index_endpoint, namespace, repos),)
self.assertEqual(resp.status_code, 200)
resp = requests.get('{0}/v1/repositories/{1}/{2}/images'.format(
self.index_endpoint, namespace, repos),
auth=tuple(self.user_credentials),
headers={'X-Docker-Token': 'true'})
self.assertEqual(resp.status_code, 200)
self.token = resp.headers.get('x-docker-token')
# Here we should use the 'X-Endpoints' returned in a real environment
# Docker -> Registry
resp = requests.get('{0}/v1/repositories/{1}/{2}/tags/latest'.format(
self.registry_endpoint, namespace, repos),
headers={'Authorization': 'Token ' + self.token})
self.assertEqual(resp.status_code, 200, resp.text)
resp = requests.get('{0}/v1/repositories/{1}/{2}/tags/latest'.format(
self.registry_endpoint, namespace, repos),
)
self.assertEqual(resp.status_code, 200, resp.text)
# Docker -> Registry
# Note(dmp): unicode patch XXX not applied assume requests does the job
image_id = json.loads(resp.text)
resp = requests.get('{0}/v1/images/{1}/ancestry'.format(
self.registry_endpoint, image_id),
)
self.assertEqual(resp.status_code, 200, resp.text)
# Note(dmp): unicode patch XXX not applied assume requests does the job
ancestry = json.loads(resp.text)
# We got the ancestry, let's fetch all the images there
for image_id in ancestry:
json_data, checksum, blob = self.fetch_image(image_id)
# check queried checksum and local computed checksum from the image
# are the same
tmpfile = StringIO()
tmpfile.write(blob)
tmpfile.seek(0)
computed_checksum = checksums.compute_simple(tmpfile, json_data)
tmpfile.close()
self.assertEqual(checksum, computed_checksum)
# Remove the repository
resp = requests.delete('{0}/v1/repositories/{1}/{2}/images'.format(
self.registry_endpoint, namespace, repos), )
self.assertEqual(resp.status_code, 204, resp.text)
# Remove image_id, then parent_id
store = storage.load()
try:
store.remove(os.path.join(store.images, self.image_id))
except Exception:
pass
try:
store.remove(os.path.join(store.images, self.parent_id))
except Exception:
pass
def _handle_repository_updated(self, sender, namespace, repository, value):
"""
Triggered after a docker push operation has completed via Signals
@type sender: List
@param sender: Flask object with request details
@type namespace: String
@param namespace: Docker namespace under which the image layers are stored
@type repository: String
@param repository: Docker repository under which the image layers are stored
@type value: List
@param value: List of Docker image layer JSON metadata objects
"""
#Get the list of checksums for all the image layers which will be included in the parent layer
checkSums = self.get_checksums(value)
#Loop over the image layers in this repository and collect the metadata
store = storage.load()
documentList = []
for item in value:
#Load the image layer data from the data store
data = store.get_content(store.image_json_path(item['id']))
data = json.loads(data)
#Generate the index document for this layer
appUrl, document = self.create_index_document(
data, checkSums, namespace, repository)
documentList.append(document)
#If this document has a maintainer URL then retrieve Github metadata
if appUrl:
jsonMetadata = self.get_app_metadata(appUrl)
#Add the JSON metadata to the (single) zeroth parent layer document only
for item in documentList:
if (item.get("head", "") == "true"):
item['github_sha'] = jsonMetadata.get("sha", "")
item['github_url'] = jsonMetadata.get("url", "")
item['github_html_url'] = jsonMetadata.get(
"html_url", "")
#Store the document list in the search index
self.set_in_index(documentList)
def _handle_repository_updated(self, sender, namespace, repository, value):
"""
Triggered after a docker push operation has completed via Signals
@type sender: List
@param sender: Flask object with request details
@type namespace: String
@param namespace: Docker namespace under which the image layers are stored
@type repository: String
@param repository: Docker repository under which the image layers are stored
@type value: List
@param value: List of Docker image layer JSON metadata objects
"""
#Get the list of checksums for all the image layers which will be included in the parent layer
checkSums = self.get_checksums(value)
#Loop over the image layers in this repository and collect the metadata
store = storage.load()
documentList = []
for item in value:
#Load the image layer data from the data store
data = store.get_content(store.image_json_path(item['id']))
data = json.loads(data)
#Generate the index document for this layer
appUrl, document = self.create_index_document(data, checkSums, namespace, repository)
documentList.append(document)
#If this document has a maintainer URL then retrieve Github metadata
if appUrl:
jsonMetadata = self.get_app_metadata(appUrl)
#Add the JSON metadata to the (single) zeroth parent layer document only
for item in documentList:
if (item.get("head","") == "true"):
item['github_sha'] = jsonMetadata.get("sha", "")
item['github_url'] = jsonMetadata.get("url", "")
item['github_html_url'] = jsonMetadata.get("html_url", "")
#Store the document list in the search index
self.set_in_index(documentList)
def setUp(self):
self.store = storage.load(kind='file')
self.filenames = list(comp(5, rndstr))
def setUp(self):
self._storage = storage.load("local")
#!/usr/bin/env python
import hashlib
import sys
import simplejson as json
from docker_registry import storage
store = storage.load()
images_cache = {}
ancestry_cache = {}
dry_run = True
def warning(msg):
print >>sys.stderr, '# Warning: ' + msg
def get_image_parent(image_id):
if image_id in images_cache:
return images_cache[image_id]
image_json = store.image_json_path(image_id)
parent_id = None
try:
info = json.loads(store.get_content(image_json))
if info['id'] != image_id:
warning('image_id != json image_id for image_id: ' + image_id)
parent_id = info.get('parent')
except IOError:
#!/usr/bin/env python
import sys
import simplejson as json
from docker_registry import storage
store = storage.load()
def walk_all_tags():
for namespace_path in store.list_directory(store.repositories):
for repos_path in store.list_directory(namespace_path):
try:
for tag in store.list_directory(repos_path):
fname = tag.split('/').pop()
if not fname.startswith('tag_'):
continue
(namespace, repos) = repos_path.split('/')[-2:]
yield (namespace, repos, store.get_content(tag))
except OSError:
pass
def walk_ancestry(image_id):
try:
ancestry_data = store.get_content(store.image_ancestry_path(image_id))
ancestry = json.loads(ancestry_data)
return iter(ancestry)
except IOError:
def setUp(self):
self._storage = storage.load('s3')
def setUp(self):
self._storage = storage.load('swift')
self._storage._swift_connection.put_container(
self._storage._swift_container
)
def _handle_repository_updated(self, sender, namespace, repository, value):
#Extract the repository version
appUrl = ""
appName,appVersion = repository.rsplit("-", 1)
logger.debug("######SOLR APP VERSION :: " + appVersion)
#Get the list of checksums and set the unique id per document
checkSums = []
for item in value:
checkSums.append(item['id'])
#Loop over the image layers in this repository and collect the metadata in a single list which will be bulk-updated in
store = storage.load()
documentList = []
for item in value:
#Load the image layer data from the data store
data = store.get_content(store.image_json_path(item['id']))
document={}
data = json.loads(data)
#Concatenate the <namespace>_<repository>_imageid to generate a unique primary key id
document['id'] = namespace + "_" + repository + "_" + data['id']
document['namespace'] = namespace
document['repository'] = repository
#If this is a parent container image layer then create a new field with all its child layers
if (data.get('parent') == None):
document['isParent'] = "true"
document['imageLayers'] = checkSums
else:
document['isParent'] = "false"
document['parent'] = data.get('parent',"")
document['created'] = data.get('created',"")
document['container'] = data.get('container',"")
document['author'] = data.get('author',"")
document['architecture'] = data.get('architecture',"")
document['os'] = data.get('os',"")
document['size'] = data.get('Size',"")
document['comment'] = data.get('comment',"")
document['hostname'] = data.get('container_config').get('Hostname',"")
#Parse the cmd field JSON (if any) from the image layer.
#If the image version matches the repository version then we have the correct version number for this image
field = data.get('container_config').get('Cmd',"")
appUrlCheck = self.getAppUrl(field, appVersion)
if appUrlCheck is not None:
appUrl = appUrlCheck
document['cmd'] = data.get('container_config').get('Cmd',"")
document['domainName'] = data.get('container_config').get('Domainname',"")
document['user'] = data.get('container_config').get('User',"")
document['memory'] = data.get('container_config').get('Memory',"")
document['memorySwap'] = data.get('container_config').get('MemorySwap',"")
document['cpuShares'] = data.get('container_config').get('CpuShares',"")
document['attachStdin'] = data.get('container_config').get('AttachStdin',"")
document['attachStdout'] = data.get('container_config').get('AttachStdout',"")
document['attachStderr'] = data.get('container_config').get('AttachStderr',"")
document['portSpecs'] = data.get('container_config').get('PortSpecs',"")
#document['exposedPorts'] = data.get('container_config').get('ExposedPorts',"")
document['tty'] = data.get('container_config').get('Tty',"")
document['openStdin'] = data.get('container_config').get('OpenStdin',"")
document['stdinOnce'] = data.get('container_config').get('StdinOnce',"")
document['dns'] = data.get('container_config').get('Dns',"")
document['image'] = data.get('container_config').get('Image',"")
document['volumes'] = data.get('container_config').get('Volumes',"")
document['volumesFrom'] = data.get('container_config').get('VolumesFrom',"")
document['workingDir'] = data.get('container_config').get('WorkingDir',"")
document['entrypoint'] = data.get('container_config').get('Entrypoint',"")
document['networkDisabled'] = data.get('container_config').get('NetworkDisabled',"")
#document['onBuild'] = data.get('container_config').get('OnBuild',"")
document['domainName'] = data.get('container_config').get('Domainname',"")
document['dockerVersion'] = data.get('docker_version',"")
documentList.append(document)
#Append the metadata to the parent image layer
logger.debug("######APP URL FINAL :: " + appUrl)
response = urllib2.urlopen(appUrl)
try:
#Parse the response JSON AND APPEND
responseMetadata = response.read()
jsonMetadata = json.loads(responseMetadata)
#Add the JSON metadata to the
for item in documentList:
if (item.get("isParent","") == "true"):
item['github_sha'] = jsonMetadata.get("sha", "")
item['github_url'] = jsonMetadata.get("url", "")
item['github_html_url'] = jsonMetadata.get("html_url", "")
except ValueError, e:
logger.debug("Error parsing JSON: " + responseMetadata)
def setUp(self):
self._storage = storage.load('gcs')
def setUp(self):
self._storage = storage.load('gcs')
def setUp(self):
self._storage = storage.load('s3')
def _generate_index(self):
store = storage.load()
with self._repositoryTable.batch_write() as batch:
for repository in self._walk_storage(store=store):
logger.info('Populating repository: {0}'.format(repository['name']))
batch.put_item(data=repository)
def setUp(self):
self._storage = storage.load('local')
