def call_main_on_test_file(self):
self.main_output = StringIO()
with self.make_test_file() as testfile:
path = testfile.name
arv_put.main(['--stream', '--no-progress', path], self.main_output)
self.assertTrue(
os.path.exists(os.path.join(os.environ['KEEP_LOCAL_STORE'],
'098f6bcd4621d373cade4e832627b4f6')),
"did not find file stream in Keep store")
def call_main_with_args(self, args):
self.main_stdout = StringIO()
self.main_stderr = StringIO()
return arv_put.main(args, self.main_stdout, self.main_stderr)
def call_main_with_args(self, args):
self.main_stdout = tutil.StringIO()
self.main_stderr = tutil.StringIO()
return arv_put.main(args, self.main_stdout, self.main_stderr)
def main(arguments=None,
stdout=sys.stdout,
install_sig_handlers=True,
api=None):
args = arg_parser.parse_args(arguments)
if api is None:
api = arvados.api('v1')
if args.image is None or args.image == 'images':
fmt = "{:30} {:10} {:12} {:29} {:20}\n"
stdout.write(
fmt.format("REPOSITORY", "TAG", "IMAGE ID", "COLLECTION",
"CREATED"))
try:
for i, j in list_images_in_arv(api, args.retries):
stdout.write(
fmt.format(j["repo"], j["tag"], j["dockerhash"][0:12], i,
j["timestamp"].strftime("%c")))
except IOError as e:
if e.errno == errno.EPIPE:
pass
else:
raise
sys.exit(0)
if re.search(r':\w[-.\w]{0,127}$', args.image):
# image ends with :valid-tag
if args.tag is not None:
logger.error(
"image %r already includes a tag, cannot add tag argument %r",
args.image, args.tag)
sys.exit(1)
# rsplit() accommodates "myrepo.example:8888/repo/image:tag"
args.image, args.tag = args.image.rsplit(':', 1)
elif args.tag is None:
args.tag = 'latest'
# Pull the image if requested, unless the image is specified as a hash
# that we already have.
if args.pull and not find_image_hashes(args.image):
pull_image(args.image, args.tag)
try:
image_hash = find_one_image_hash(args.image, args.tag)
except DockerError as error:
logger.error(error.message)
sys.exit(1)
if not docker_image_compatible(api, image_hash):
if args.force_image_format:
logger.warning("forcing incompatible image")
else:
logger.error("refusing to store " \
"incompatible format (use --force-image-format to override)")
sys.exit(1)
image_repo_tag = '{}:{}'.format(
args.image,
args.tag) if not image_hash.startswith(args.image.lower()) else None
if args.name is None:
if image_repo_tag:
collection_name = 'Docker image {} {}'.format(
image_repo_tag, image_hash[0:12])
else:
collection_name = 'Docker image {}'.format(image_hash[0:12])
else:
collection_name = args.name
# Acquire a lock so that only one arv-keepdocker process will
# dump/upload a particular docker image at a time. Do this before
# checking if the image already exists in Arvados so that if there
# is an upload already underway, when that upload completes and
# this process gets a turn, it will discover the Docker image is
# already available and exit quickly.
outfile_name = '{}.tar'.format(image_hash)
lockfile_name = '{}.lock'.format(outfile_name)
lockfile = None
cache_dir = get_cache_dir()
if cache_dir:
lockfile = open(os.path.join(cache_dir, lockfile_name), 'w+')
fcntl.flock(lockfile, fcntl.LOCK_EX)
try:
if not args.force:
# Check if this image is already in Arvados.
# Project where everything should be owned
parent_project_uuid = args.project_uuid or api.users().current(
).execute(num_retries=args.retries)['uuid']
# Find image hash tags
existing_links = _get_docker_links(
api,
args.retries,
filters=[['link_class', '=', 'docker_image_hash'],
['name', '=', image_hash]])
if existing_links:
# get readable collections
collections = api.collections().list(
filters=[[
'uuid', 'in',
[link['head_uuid'] for link in existing_links]
]],
select=["uuid", "owner_uuid", "name", "manifest_text"
]).execute(num_retries=args.retries)['items']
if collections:
# check for repo+tag links on these collections
if image_repo_tag:
existing_repo_tag = _get_docker_links(
api,
args.retries,
filters=[[
'link_class', '=', 'docker_image_repo+tag'
], ['name', '=', image_repo_tag],
[
'head_uuid', 'in',
[c["uuid"] for c in collections]
]])
else:
existing_repo_tag = []
try:
coll_uuid = next(
items_owned_by(parent_project_uuid,
collections))['uuid']
except StopIteration:
# create new collection owned by the project
coll_uuid = api.collections().create(
body={
"manifest_text":
collections[0]['manifest_text'],
"name": collection_name,
"owner_uuid": parent_project_uuid
},
ensure_unique_name=True).execute(
num_retries=args.retries)['uuid']
link_base = {
'owner_uuid': parent_project_uuid,
'head_uuid': coll_uuid,
'properties': existing_links[0]['properties']
}
if not any(
items_owned_by(parent_project_uuid,
existing_links)):
# create image link owned by the project
make_link(api, args.retries, 'docker_image_hash',
image_hash, **link_base)
if image_repo_tag and not any(
items_owned_by(parent_project_uuid,
existing_repo_tag)):
# create repo+tag link owned by the project
make_link(api, args.retries, 'docker_image_repo+tag',
image_repo_tag, **link_base)
stdout.write(coll_uuid + "\n")
sys.exit(0)
# Open a file for the saved image, and write it if needed.
image_file, need_save = prep_image_file(outfile_name)
if need_save:
save_image(image_hash, image_file)
# Call arv-put with switches we inherited from it
# (a.k.a., switches that aren't our own).
put_args = keepdocker_parser.parse_known_args(arguments)[1]
if args.name is None:
put_args += ['--name', collection_name]
coll_uuid = arv_put.main(
put_args + ['--filename', outfile_name, image_file.name],
stdout=stdout,
install_sig_handlers=install_sig_handlers).strip()
# Read the image metadata and make Arvados links from it.
image_file.seek(0)
image_tar = tarfile.open(fileobj=image_file)
image_hash_type, _, raw_image_hash = image_hash.rpartition(':')
if image_hash_type:
json_filename = raw_image_hash + '.json'
else:
json_filename = raw_image_hash + '/json'
json_file = image_tar.extractfile(image_tar.getmember(json_filename))
image_metadata = json.load(json_file)
json_file.close()
image_tar.close()
link_base = {'head_uuid': coll_uuid, 'properties': {}}
if 'created' in image_metadata:
link_base['properties']['image_timestamp'] = image_metadata[
'created']
if args.project_uuid is not None:
link_base['owner_uuid'] = args.project_uuid
make_link(api, args.retries, 'docker_image_hash', image_hash,
**link_base)
if image_repo_tag:
make_link(api, args.retries, 'docker_image_repo+tag',
image_repo_tag, **link_base)
# Clean up.
image_file.close()
for filename in [stat_cache_name(image_file), image_file.name]:
try:
os.unlink(filename)
except OSError as error:
if error.errno != errno.ENOENT:
raise
finally:
if lockfile is not None:
# Closing the lockfile unlocks it.
lockfile.close()
def main(arguments=None, stdout=sys.stdout):
args = arg_parser.parse_args(arguments)
api = arvados.api('v1')
if args.image is None or args.image == 'images':
fmt = "{:30} {:10} {:12} {:29} {:20}\n"
stdout.write(
fmt.format("REPOSITORY", "TAG", "IMAGE ID", "COLLECTION",
"CREATED"))
for i, j in list_images_in_arv(api, args.retries):
stdout.write(
fmt.format(j["repo"], j["tag"], j["dockerhash"][0:12], i,
j["timestamp"].strftime("%c")))
sys.exit(0)
# Pull the image if requested, unless the image is specified as a hash
# that we already have.
if args.pull and not find_image_hashes(args.image):
pull_image(args.image, args.tag)
try:
image_hash = find_one_image_hash(args.image, args.tag)
except DockerError as error:
print >> sys.stderr, "arv-keepdocker:", error.message
sys.exit(1)
if not docker_image_compatible(api, image_hash):
if args.force_image_format:
print >> sys.stderr, "arv-keepdocker: forcing incompatible image"
else:
print >>sys.stderr, "arv-keepdocker: refusing to store " \
"incompatible format (use --force-image-format to override)"
sys.exit(1)
image_repo_tag = '{}:{}'.format(
args.image,
args.tag) if not image_hash.startswith(args.image.lower()) else None
if args.name is None:
if image_repo_tag:
collection_name = 'Docker image {} {}'.format(
image_repo_tag, image_hash[0:12])
else:
collection_name = 'Docker image {}'.format(image_hash[0:12])
else:
collection_name = args.name
if not args.force:
# Check if this image is already in Arvados.
# Project where everything should be owned
if args.project_uuid:
parent_project_uuid = args.project_uuid
else:
parent_project_uuid = api.users().current().execute(
num_retries=args.retries)['uuid']
# Find image hash tags
existing_links = _get_docker_links(
api,
args.retries,
filters=[['link_class', '=', 'docker_image_hash'],
['name', '=', image_hash]])
if existing_links:
# get readable collections
collections = api.collections().list(
filters=[[
'uuid', 'in',
[link['head_uuid'] for link in existing_links]
]],
select=["uuid", "owner_uuid", "name", "manifest_text"
]).execute(num_retries=args.retries)['items']
if collections:
# check for repo+tag links on these collections
if image_repo_tag:
existing_repo_tag = _get_docker_links(
api,
args.retries,
filters=[['link_class', '=', 'docker_image_repo+tag'],
['name', '=', image_repo_tag],
['head_uuid', 'in', collections]])
else:
existing_repo_tag = []
try:
coll_uuid = next(
items_owned_by(parent_project_uuid,
collections))['uuid']
except StopIteration:
# create new collection owned by the project
coll_uuid = api.collections().create(
body={
"manifest_text": collections[0]['manifest_text'],
"name": collection_name,
"owner_uuid": parent_project_uuid
},
ensure_unique_name=True).execute(
num_retries=args.retries)['uuid']
link_base = {
'owner_uuid': parent_project_uuid,
'head_uuid': coll_uuid,
'properties': existing_links[0]['properties']
}
if not any(items_owned_by(parent_project_uuid,
existing_links)):
# create image link owned by the project
make_link(api, args.retries, 'docker_image_hash',
image_hash, **link_base)
if image_repo_tag and not any(
items_owned_by(parent_project_uuid,
existing_repo_tag)):
# create repo+tag link owned by the project
make_link(api, args.retries, 'docker_image_repo+tag',
image_repo_tag, **link_base)
stdout.write(coll_uuid + "\n")
sys.exit(0)
# Open a file for the saved image, and write it if needed.
outfile_name = '{}.tar'.format(image_hash)
image_file, need_save = prep_image_file(outfile_name)
if need_save:
save_image(image_hash, image_file)
# Call arv-put with switches we inherited from it
# (a.k.a., switches that aren't our own).
put_args = keepdocker_parser.parse_known_args(arguments)[1]
if args.name is None:
put_args += ['--name', collection_name]
coll_uuid = arv_put.main(put_args +
['--filename', outfile_name, image_file.name],
stdout=stdout).strip()
# Read the image metadata and make Arvados links from it.
image_file.seek(0)
image_tar = tarfile.open(fileobj=image_file)
image_hash_type, _, raw_image_hash = image_hash.rpartition(':')
if image_hash_type:
json_filename = raw_image_hash + '.json'
else:
json_filename = raw_image_hash + '/json'
json_file = image_tar.extractfile(image_tar.getmember(json_filename))
image_metadata = json.load(json_file)
json_file.close()
image_tar.close()
link_base = {'head_uuid': coll_uuid, 'properties': {}}
if 'created' in image_metadata:
link_base['properties']['image_timestamp'] = image_metadata['created']
if args.project_uuid is not None:
link_base['owner_uuid'] = args.project_uuid
make_link(api, args.retries, 'docker_image_hash', image_hash, **link_base)
if image_repo_tag:
make_link(api, args.retries, 'docker_image_repo+tag', image_repo_tag,
**link_base)
# Clean up.
image_file.close()
for filename in [stat_cache_name(image_file), image_file.name]:
try:
os.unlink(filename)
except OSError as error:
if error.errno != errno.ENOENT:
raise
def main(arguments=None):
args = arg_parser.parse_args(arguments)
api = arvados.api('v1')
if args.image is None or args.image == 'images':
fmt = "{:30} {:10} {:12} {:29} {:20}"
print fmt.format("REPOSITORY", "TAG", "IMAGE ID", "COLLECTION", "CREATED")
for i, j in list_images_in_arv(api, args.retries):
print(fmt.format(j["repo"], j["tag"], j["dockerhash"][0:12], i, j["timestamp"].strftime("%c")))
sys.exit(0)
# Pull the image if requested, unless the image is specified as a hash
# that we already have.
if args.pull and not find_image_hashes(args.image):
pull_image(args.image, args.tag)
try:
image_hash = find_one_image_hash(args.image, args.tag)
except DockerError as error:
print >>sys.stderr, "arv-keepdocker:", error.message
sys.exit(1)
image_repo_tag = '{}:{}'.format(args.image, args.tag) if not image_hash.startswith(args.image.lower()) else None
if args.name is None:
if image_repo_tag:
collection_name = 'Docker image {} {}'.format(image_repo_tag, image_hash[0:12])
else:
collection_name = 'Docker image {}'.format(image_hash[0:12])
else:
collection_name = args.name
if not args.force:
# Check if this image is already in Arvados.
# Project where everything should be owned
if args.project_uuid:
parent_project_uuid = args.project_uuid
else:
parent_project_uuid = api.users().current().execute(
num_retries=args.retries)['uuid']
# Find image hash tags
existing_links = api.links().list(
filters=[['link_class', '=', 'docker_image_hash'],
['name', '=', image_hash]]
).execute(num_retries=args.retries)['items']
if existing_links:
# get readable collections
collections = api.collections().list(
filters=[['uuid', 'in', [link['head_uuid'] for link in existing_links]]],
select=["uuid", "owner_uuid", "name", "manifest_text"]
).execute(num_retries=args.retries)['items']
if collections:
# check for repo+tag links on these collections
existing_repo_tag = (api.links().list(
filters=[['link_class', '=', 'docker_image_repo+tag'],
['name', '=', image_repo_tag],
['head_uuid', 'in', collections]]
).execute(num_retries=args.retries)['items']) if image_repo_tag else []
# Filter on elements owned by the parent project
owned_col = [c for c in collections if c['owner_uuid'] == parent_project_uuid]
owned_img = [c for c in existing_links if c['owner_uuid'] == parent_project_uuid]
owned_rep = [c for c in existing_repo_tag if c['owner_uuid'] == parent_project_uuid]
if owned_col:
# already have a collection owned by this project
coll_uuid = owned_col[0]['uuid']
else:
# create new collection owned by the project
coll_uuid = api.collections().create(
body={"manifest_text": collections[0]['manifest_text'],
"name": collection_name,
"owner_uuid": parent_project_uuid},
ensure_unique_name=True
).execute(num_retries=args.retries)['uuid']
link_base = {'owner_uuid': parent_project_uuid,
'head_uuid': coll_uuid }
if not owned_img:
# create image link owned by the project
make_link(api, args.retries,
'docker_image_hash', image_hash, **link_base)
if not owned_rep and image_repo_tag:
# create repo+tag link owned by the project
make_link(api, args.retries, 'docker_image_repo+tag',
image_repo_tag, **link_base)
print(coll_uuid)
sys.exit(0)
# Open a file for the saved image, and write it if needed.
outfile_name = '{}.tar'.format(image_hash)
image_file, need_save = prep_image_file(outfile_name)
if need_save:
save_image(image_hash, image_file)
# Call arv-put with switches we inherited from it
# (a.k.a., switches that aren't our own).
put_args = keepdocker_parser.parse_known_args(arguments)[1]
if args.name is None:
put_args += ['--name', collection_name]
coll_uuid = arv_put.main(
put_args + ['--filename', outfile_name, image_file.name]).strip()
# Read the image metadata and make Arvados links from it.
image_file.seek(0)
image_tar = tarfile.open(fileobj=image_file)
json_file = image_tar.extractfile(image_tar.getmember(image_hash + '/json'))
image_metadata = json.load(json_file)
json_file.close()
image_tar.close()
link_base = {'head_uuid': coll_uuid, 'properties': {}}
if 'created' in image_metadata:
link_base['properties']['image_timestamp'] = image_metadata['created']
if args.project_uuid is not None:
link_base['owner_uuid'] = args.project_uuid
make_link(api, args.retries, 'docker_image_hash', image_hash, **link_base)
if image_repo_tag:
make_link(api, args.retries,
'docker_image_repo+tag', image_repo_tag, **link_base)
# Clean up.
image_file.close()
for filename in [stat_cache_name(image_file), image_file.name]:
try:
os.unlink(filename)
except OSError as error:
if error.errno != errno.ENOENT:
raise
def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None):
args = arg_parser.parse_args(arguments)
if api is None:
api = arvados.api('v1')
if args.image is None or args.image == 'images':
fmt = "{:30} {:10} {:12} {:29} {:20}\n"
stdout.write(fmt.format("REPOSITORY", "TAG", "IMAGE ID", "COLLECTION", "CREATED"))
try:
for i, j in list_images_in_arv(api, args.retries):
stdout.write(fmt.format(j["repo"], j["tag"], j["dockerhash"][0:12], i, j["timestamp"].strftime("%c")))
except IOError as e:
if e.errno == errno.EPIPE:
pass
else:
raise
sys.exit(0)
if re.search(r':\w[-.\w]{0,127}$', args.image):
# image ends with :valid-tag
if args.tag is not None:
logger.error(
"image %r already includes a tag, cannot add tag argument %r",
args.image, args.tag)
sys.exit(1)
# rsplit() accommodates "myrepo.example:8888/repo/image:tag"
args.image, args.tag = args.image.rsplit(':', 1)
elif args.tag is None:
args.tag = 'latest'
# Pull the image if requested, unless the image is specified as a hash
# that we already have.
if args.pull and not find_image_hashes(args.image):
pull_image(args.image, args.tag)
try:
image_hash = find_one_image_hash(args.image, args.tag)
except DockerError as error:
logger.error(error.message)
sys.exit(1)
if not docker_image_compatible(api, image_hash):
if args.force_image_format:
logger.warning("forcing incompatible image")
else:
logger.error("refusing to store " \
"incompatible format (use --force-image-format to override)")
sys.exit(1)
image_repo_tag = '{}:{}'.format(args.image, args.tag) if not image_hash.startswith(args.image.lower()) else None
if args.name is None:
if image_repo_tag:
collection_name = 'Docker image {} {}'.format(image_repo_tag, image_hash[0:12])
else:
collection_name = 'Docker image {}'.format(image_hash[0:12])
else:
collection_name = args.name
# Acquire a lock so that only one arv-keepdocker process will
# dump/upload a particular docker image at a time. Do this before
# checking if the image already exists in Arvados so that if there
# is an upload already underway, when that upload completes and
# this process gets a turn, it will discover the Docker image is
# already available and exit quickly.
outfile_name = '{}.tar'.format(image_hash)
lockfile_name = '{}.lock'.format(outfile_name)
lockfile = None
cache_dir = get_cache_dir()
if cache_dir:
lockfile = open(os.path.join(cache_dir, lockfile_name), 'w+')
fcntl.flock(lockfile, fcntl.LOCK_EX)
try:
if not args.force:
# Check if this image is already in Arvados.
# Project where everything should be owned
parent_project_uuid = args.project_uuid or api.users().current().execute(
num_retries=args.retries)['uuid']
# Find image hash tags
existing_links = _get_docker_links(
api, args.retries,
filters=[['link_class', '=', 'docker_image_hash'],
['name', '=', image_hash]])
if existing_links:
# get readable collections
collections = api.collections().list(
filters=[['uuid', 'in', [link['head_uuid'] for link in existing_links]]],
select=["uuid", "owner_uuid", "name", "manifest_text"]
).execute(num_retries=args.retries)['items']
if collections:
# check for repo+tag links on these collections
if image_repo_tag:
existing_repo_tag = _get_docker_links(
api, args.retries,
filters=[['link_class', '=', 'docker_image_repo+tag'],
['name', '=', image_repo_tag],
['head_uuid', 'in', [c["uuid"] for c in collections]]])
else:
existing_repo_tag = []
try:
coll_uuid = next(items_owned_by(parent_project_uuid, collections))['uuid']
except StopIteration:
# create new collection owned by the project
coll_uuid = api.collections().create(
body={"manifest_text": collections[0]['manifest_text'],
"name": collection_name,
"owner_uuid": parent_project_uuid},
ensure_unique_name=True
).execute(num_retries=args.retries)['uuid']
link_base = {'owner_uuid': parent_project_uuid,
'head_uuid': coll_uuid,
'properties': existing_links[0]['properties']}
if not any(items_owned_by(parent_project_uuid, existing_links)):
# create image link owned by the project
make_link(api, args.retries,
'docker_image_hash', image_hash, **link_base)
if image_repo_tag and not any(items_owned_by(parent_project_uuid, existing_repo_tag)):
# create repo+tag link owned by the project
make_link(api, args.retries, 'docker_image_repo+tag',
image_repo_tag, **link_base)
stdout.write(coll_uuid + "\n")
sys.exit(0)
# Open a file for the saved image, and write it if needed.
image_file, need_save = prep_image_file(outfile_name)
if need_save:
save_image(image_hash, image_file)
# Call arv-put with switches we inherited from it
# (a.k.a., switches that aren't our own).
put_args = keepdocker_parser.parse_known_args(arguments)[1]
if args.name is None:
put_args += ['--name', collection_name]
coll_uuid = arv_put.main(
put_args + ['--filename', outfile_name, image_file.name], stdout=stdout,
install_sig_handlers=install_sig_handlers).strip()
# Read the image metadata and make Arvados links from it.
image_file.seek(0)
image_tar = tarfile.open(fileobj=image_file)
image_hash_type, _, raw_image_hash = image_hash.rpartition(':')
if image_hash_type:
json_filename = raw_image_hash + '.json'
else:
json_filename = raw_image_hash + '/json'
json_file = image_tar.extractfile(image_tar.getmember(json_filename))
image_metadata = json.loads(json_file.read().decode())
json_file.close()
image_tar.close()
link_base = {'head_uuid': coll_uuid, 'properties': {}}
if 'created' in image_metadata:
link_base['properties']['image_timestamp'] = image_metadata['created']
if args.project_uuid is not None:
link_base['owner_uuid'] = args.project_uuid
make_link(api, args.retries, 'docker_image_hash', image_hash, **link_base)
if image_repo_tag:
make_link(api, args.retries,
'docker_image_repo+tag', image_repo_tag, **link_base)
# Clean up.
image_file.close()
for filename in [stat_cache_name(image_file), image_file.name]:
try:
os.unlink(filename)
except OSError as error:
if error.errno != errno.ENOENT:
raise
finally:
if lockfile is not None:
# Closing the lockfile unlocks it.
lockfile.close()
def main(arguments=None, stdout=sys.stdout):
args = arg_parser.parse_args(arguments)
api = arvados.api('v1')
if args.image is None or args.image == 'images':
fmt = "{:30} {:10} {:12} {:29} {:20}\n"
stdout.write(fmt.format("REPOSITORY", "TAG", "IMAGE ID", "COLLECTION", "CREATED"))
try:
for i, j in list_images_in_arv(api, args.retries):
stdout.write(fmt.format(j["repo"], j["tag"], j["dockerhash"][0:12], i, j["timestamp"].strftime("%c")))
except IOError as e:
if e.errno == errno.EPIPE:
pass
else:
raise
sys.exit(0)
# Pull the image if requested, unless the image is specified as a hash
# that we already have.
if args.pull and not find_image_hashes(args.image):
pull_image(args.image, args.tag)
try:
image_hash = find_one_image_hash(args.image, args.tag)
except DockerError as error:
logger.error(error.message)
sys.exit(1)
if not docker_image_compatible(api, image_hash):
if args.force_image_format:
logger.warning("forcing incompatible image")
else:
logger.error("refusing to store " \
"incompatible format (use --force-image-format to override)")
sys.exit(1)
image_repo_tag = '{}:{}'.format(args.image, args.tag) if not image_hash.startswith(args.image.lower()) else None
if args.name is None:
if image_repo_tag:
collection_name = 'Docker image {} {}'.format(image_repo_tag, image_hash[0:12])
else:
collection_name = 'Docker image {}'.format(image_hash[0:12])
else:
collection_name = args.name
if not args.force:
# Check if this image is already in Arvados.
# Project where everything should be owned
if args.project_uuid:
parent_project_uuid = args.project_uuid
else:
parent_project_uuid = api.users().current().execute(
num_retries=args.retries)['uuid']
# Find image hash tags
existing_links = _get_docker_links(
api, args.retries,
filters=[['link_class', '=', 'docker_image_hash'],
['name', '=', image_hash]])
if existing_links:
# get readable collections
collections = api.collections().list(
filters=[['uuid', 'in', [link['head_uuid'] for link in existing_links]]],
select=["uuid", "owner_uuid", "name", "manifest_text"]
).execute(num_retries=args.retries)['items']
if collections:
# check for repo+tag links on these collections
if image_repo_tag:
existing_repo_tag = _get_docker_links(
api, args.retries,
filters=[['link_class', '=', 'docker_image_repo+tag'],
['name', '=', image_repo_tag],
['head_uuid', 'in', [c["uuid"] for c in collections]]])
else:
existing_repo_tag = []
try:
coll_uuid = next(items_owned_by(parent_project_uuid, collections))['uuid']
except StopIteration:
# create new collection owned by the project
coll_uuid = api.collections().create(
body={"manifest_text": collections[0]['manifest_text'],
"name": collection_name,
"owner_uuid": parent_project_uuid},
ensure_unique_name=True
).execute(num_retries=args.retries)['uuid']
link_base = {'owner_uuid': parent_project_uuid,
'head_uuid': coll_uuid,
'properties': existing_links[0]['properties']}
if not any(items_owned_by(parent_project_uuid, existing_links)):
# create image link owned by the project
make_link(api, args.retries,
'docker_image_hash', image_hash, **link_base)
if image_repo_tag and not any(items_owned_by(parent_project_uuid, existing_repo_tag)):
# create repo+tag link owned by the project
make_link(api, args.retries, 'docker_image_repo+tag',
image_repo_tag, **link_base)
stdout.write(coll_uuid + "\n")
sys.exit(0)
# Open a file for the saved image, and write it if needed.
outfile_name = '{}.tar'.format(image_hash)
image_file, need_save = prep_image_file(outfile_name)
if need_save:
save_image(image_hash, image_file)
# Call arv-put with switches we inherited from it
# (a.k.a., switches that aren't our own).
put_args = keepdocker_parser.parse_known_args(arguments)[1]
if args.name is None:
put_args += ['--name', collection_name]
coll_uuid = arv_put.main(
put_args + ['--filename', outfile_name, image_file.name], stdout=stdout).strip()
# Read the image metadata and make Arvados links from it.
image_file.seek(0)
image_tar = tarfile.open(fileobj=image_file)
image_hash_type, _, raw_image_hash = image_hash.rpartition(':')
if image_hash_type:
json_filename = raw_image_hash + '.json'
else:
json_filename = raw_image_hash + '/json'
json_file = image_tar.extractfile(image_tar.getmember(json_filename))
image_metadata = json.load(json_file)
json_file.close()
image_tar.close()
link_base = {'head_uuid': coll_uuid, 'properties': {}}
if 'created' in image_metadata:
link_base['properties']['image_timestamp'] = image_metadata['created']
if args.project_uuid is not None:
link_base['owner_uuid'] = args.project_uuid
make_link(api, args.retries, 'docker_image_hash', image_hash, **link_base)
if image_repo_tag:
make_link(api, args.retries,
'docker_image_repo+tag', image_repo_tag, **link_base)
# Clean up.
image_file.close()
for filename in [stat_cache_name(image_file), image_file.name]:
try:
os.unlink(filename)
except OSError as error:
if error.errno != errno.ENOENT:
raise
def main(arguments=None):
args = arg_parser.parse_args(arguments)
api = arvados.api('v1')
if args.image is None or args.image == 'images':
list_images_in_arv(api, args.retries)
sys.exit(0)
# Pull the image if requested, unless the image is specified as a hash
# that we already have.
if args.pull and not find_image_hashes(args.image):
pull_image(args.image, args.tag)
try:
image_hash = find_one_image_hash(args.image, args.tag)
except DockerError as error:
print >> sys.stderr, "arv-keepdocker:", error.message
sys.exit(1)
image_repo_tag = '{}:{}'.format(
args.image,
args.tag) if not image_hash.startswith(args.image.lower()) else None
if args.name is None:
if image_repo_tag:
collection_name = 'Docker image {} {}'.format(
image_repo_tag, image_hash[0:12])
else:
collection_name = 'Docker image {}'.format(image_hash[0:12])
else:
collection_name = args.name
if not args.force:
# Check if this image is already in Arvados.
# Project where everything should be owned
if args.project_uuid:
parent_project_uuid = args.project_uuid
else:
parent_project_uuid = api.users().current().execute(
num_retries=args.retries)['uuid']
# Find image hash tags
existing_links = api.links().list(
filters=[['link_class', '=', 'docker_image_hash'],
['name', '=', image_hash]]).execute(
num_retries=args.retries)['items']
if existing_links:
# get readable collections
collections = api.collections().list(
filters=[[
'uuid', 'in',
[link['head_uuid'] for link in existing_links]
]],
select=["uuid", "owner_uuid", "name", "manifest_text"
]).execute(num_retries=args.retries)['items']
if collections:
# check for repo+tag links on these collections
existing_repo_tag = (api.links().list(
filters=[['link_class', '=', 'docker_image_repo+tag'],
['name', '=', image_repo_tag],
['head_uuid', 'in', collections]]).execute(
num_retries=args.retries)['items']
) if image_repo_tag else []
# Filter on elements owned by the parent project
owned_col = [
c for c in collections
if c['owner_uuid'] == parent_project_uuid
]
owned_img = [
c for c in existing_links
if c['owner_uuid'] == parent_project_uuid
]
owned_rep = [
c for c in existing_repo_tag
if c['owner_uuid'] == parent_project_uuid
]
if owned_col:
# already have a collection owned by this project
coll_uuid = owned_col[0]['uuid']
else:
# create new collection owned by the project
coll_uuid = api.collections().create(
body={
"manifest_text": collections[0]['manifest_text'],
"name": collection_name,
"owner_uuid": parent_project_uuid
},
ensure_unique_name=True).execute(
num_retries=args.retries)['uuid']
link_base = {
'owner_uuid': parent_project_uuid,
'head_uuid': coll_uuid
}
if not owned_img:
# create image link owned by the project
make_link(api, args.retries, 'docker_image_hash',
image_hash, **link_base)
if not owned_rep and image_repo_tag:
# create repo+tag link owned by the project
make_link(api, args.retries, 'docker_image_repo+tag',
image_repo_tag, **link_base)
print(coll_uuid)
sys.exit(0)
# Open a file for the saved image, and write it if needed.
outfile_name = '{}.tar'.format(image_hash)
image_file, need_save = prep_image_file(outfile_name)
if need_save:
save_image(image_hash, image_file)
# Call arv-put with switches we inherited from it
# (a.k.a., switches that aren't our own).
put_args = keepdocker_parser.parse_known_args(arguments)[1]
if args.name is None:
put_args += ['--name', collection_name]
coll_uuid = arv_put.main(
put_args + ['--filename', outfile_name, image_file.name]).strip()
# Read the image metadata and make Arvados links from it.
image_file.seek(0)
image_tar = tarfile.open(fileobj=image_file)
json_file = image_tar.extractfile(image_tar.getmember(image_hash +
'/json'))
image_metadata = json.load(json_file)
json_file.close()
image_tar.close()
link_base = {'head_uuid': coll_uuid, 'properties': {}}
if 'created' in image_metadata:
link_base['properties']['image_timestamp'] = image_metadata['created']
if args.project_uuid is not None:
link_base['owner_uuid'] = args.project_uuid
make_link(api, args.retries, 'docker_image_hash', image_hash, **link_base)
if image_repo_tag:
make_link(api, args.retries, 'docker_image_repo+tag', image_repo_tag,
**link_base)
# Clean up.
image_file.close()
for filename in [stat_cache_name(image_file), image_file.name]:
try:
os.unlink(filename)
except OSError as error:
if error.errno != errno.ENOENT:
raise
def main(arguments=None, stdout=sys.stdout):
args = arg_parser.parse_args(arguments)
api = arvados.api("v1")
if args.image is None or args.image == "images":
fmt = "{:30} {:10} {:12} {:29} {:20}\n"
stdout.write(fmt.format("REPOSITORY", "TAG", "IMAGE ID", "COLLECTION", "CREATED"))
for i, j in list_images_in_arv(api, args.retries):
stdout.write(fmt.format(j["repo"], j["tag"], j["dockerhash"][0:12], i, j["timestamp"].strftime("%c")))
sys.exit(0)
# Pull the image if requested, unless the image is specified as a hash
# that we already have.
if args.pull and not find_image_hashes(args.image):
pull_image(args.image, args.tag)
try:
image_hash = find_one_image_hash(args.image, args.tag)
except DockerError as error:
print >>sys.stderr, "arv-keepdocker:", error.message
sys.exit(1)
image_repo_tag = "{}:{}".format(args.image, args.tag) if not image_hash.startswith(args.image.lower()) else None
if args.name is None:
if image_repo_tag:
collection_name = "Docker image {} {}".format(image_repo_tag, image_hash[0:12])
else:
collection_name = "Docker image {}".format(image_hash[0:12])
else:
collection_name = args.name
if not args.force:
# Check if this image is already in Arvados.
# Project where everything should be owned
if args.project_uuid:
parent_project_uuid = args.project_uuid
else:
parent_project_uuid = api.users().current().execute(num_retries=args.retries)["uuid"]
# Find image hash tags
existing_links = _get_docker_links(
api, args.retries, filters=[["link_class", "=", "docker_image_hash"], ["name", "=", image_hash]]
)
if existing_links:
# get readable collections
collections = (
api.collections()
.list(
filters=[["uuid", "in", [link["head_uuid"] for link in existing_links]]],
select=["uuid", "owner_uuid", "name", "manifest_text"],
)
.execute(num_retries=args.retries)["items"]
)
if collections:
# check for repo+tag links on these collections
if image_repo_tag:
existing_repo_tag = _get_docker_links(
api,
args.retries,
filters=[
["link_class", "=", "docker_image_repo+tag"],
["name", "=", image_repo_tag],
["head_uuid", "in", collections],
],
)
else:
existing_repo_tag = []
try:
coll_uuid = next(items_owned_by(parent_project_uuid, collections))["uuid"]
except StopIteration:
# create new collection owned by the project
coll_uuid = (
api.collections()
.create(
body={
"manifest_text": collections[0]["manifest_text"],
"name": collection_name,
"owner_uuid": parent_project_uuid,
},
ensure_unique_name=True,
)
.execute(num_retries=args.retries)["uuid"]
)
link_base = {
"owner_uuid": parent_project_uuid,
"head_uuid": coll_uuid,
"properties": existing_links[0]["properties"],
}
if not any(items_owned_by(parent_project_uuid, existing_links)):
# create image link owned by the project
make_link(api, args.retries, "docker_image_hash", image_hash, **link_base)
if image_repo_tag and not any(items_owned_by(parent_project_uuid, existing_repo_tag)):
# create repo+tag link owned by the project
make_link(api, args.retries, "docker_image_repo+tag", image_repo_tag, **link_base)
stdout.write(coll_uuid + "\n")
sys.exit(0)
# Open a file for the saved image, and write it if needed.
outfile_name = "{}.tar".format(image_hash)
image_file, need_save = prep_image_file(outfile_name)
if need_save:
save_image(image_hash, image_file)
# Call arv-put with switches we inherited from it
# (a.k.a., switches that aren't our own).
put_args = keepdocker_parser.parse_known_args(arguments)[1]
if args.name is None:
put_args += ["--name", collection_name]
coll_uuid = arv_put.main(put_args + ["--filename", outfile_name, image_file.name], stdout=stdout).strip()
# Read the image metadata and make Arvados links from it.
image_file.seek(0)
image_tar = tarfile.open(fileobj=image_file)
json_file = image_tar.extractfile(image_tar.getmember(image_hash + "/json"))
image_metadata = json.load(json_file)
json_file.close()
image_tar.close()
link_base = {"head_uuid": coll_uuid, "properties": {}}
if "created" in image_metadata:
link_base["properties"]["image_timestamp"] = image_metadata["created"]
if args.project_uuid is not None:
link_base["owner_uuid"] = args.project_uuid
make_link(api, args.retries, "docker_image_hash", image_hash, **link_base)
if image_repo_tag:
make_link(api, args.retries, "docker_image_repo+tag", image_repo_tag, **link_base)
# Clean up.
image_file.close()
for filename in [stat_cache_name(image_file), image_file.name]:
try:
os.unlink(filename)
except OSError as error:
if error.errno != errno.ENOENT:
raise
def call_main_with_args(self, args):
self.main_stdout.seek(0, 0)
self.main_stdout.truncate(0)
self.main_stderr.seek(0, 0)
self.main_stderr.truncate(0)
return arv_put.main(args, self.main_stdout, self.main_stderr)
def main(arguments=None):
args = arg_parser.parse_args(arguments)
# Pull the image if requested, unless the image is specified as a hash
# that we already have.
if args.pull and not find_image_hashes(args.image):
pull_image(args.image, args.tag)
try:
image_hash = find_one_image_hash(args.image, args.tag)
except DockerError as error:
print >>sys.stderr, "arv-keepdocker:", error.message
sys.exit(1)
if not args.force:
# Abort if this image is already in Arvados.
existing_links = arvados.api('v1').links().list(
filters=[['link_class', '=', 'docker_image_hash'],
['name', '=', image_hash]]).execute()['items']
if existing_links:
message = [
"arv-keepdocker: Image {} already stored in collection(s):".
format(image_hash)]
message.extend(link['head_uuid'] for link in existing_links)
print >>sys.stderr, "\n".join(message)
sys.exit(0)
# Open a file for the saved image, and write it if needed.
outfile_name = '{}.tar'.format(image_hash)
image_file, need_save = prep_image_file(outfile_name)
if need_save:
save_image(image_hash, image_file)
# Call arv-put with switches we inherited from it
# (a.k.a., switches that aren't our own).
put_args = opt_parser.parse_known_args(arguments)[1]
coll_uuid = arv_put.main(
put_args + ['--filename', outfile_name, image_file.name]).strip()
# Read the image metadata and make Arvados links from it.
image_file.seek(0)
image_tar = tarfile.open(fileobj=image_file)
json_file = image_tar.extractfile(image_tar.getmember(image_hash + '/json'))
image_metadata = json.load(json_file)
json_file.close()
image_tar.close()
link_base = {'head_uuid': coll_uuid, 'properties': {}}
if 'created' in image_metadata:
link_base['properties']['image_timestamp'] = image_metadata['created']
make_link('docker_image_hash', image_hash, **link_base)
if not image_hash.startswith(args.image.lower()):
make_link('docker_image_repository', args.image, **link_base)
make_link('docker_image_repo+tag', '{}:{}'.format(args.image, args.tag),
**link_base)
# Clean up.
image_file.close()
for filename in [stat_cache_name(image_file), image_file.name]:
try:
os.unlink(filename)
except OSError as error:
if error.errno != errno.ENOENT:
raise
