def copy_image(session, headers, source_image_content):
glance = chi.glance(session=session)
extra = {
k.lower().replace(f"{helpers.SWIFT_META_HEADER_PREFIX}", ""): v
for k, v in headers.items()
if k.lower().startswith(f"{helpers.SWIFT_META_HEADER_PREFIX}build")
}
tmp_image_name = f"img-cc-prod-{ulid.ulid()}"
new_image = glance.images.create(
name=tmp_image_name,
visibility="private",
disk_format=headers[f"{helpers.SWIFT_META_HEADER_PREFIX}disk-format"],
container_format='bare',
**extra)
try:
glance.images.upload(
new_image['id'],
io.BytesIO(source_image_content),
)
except Exception as e:
# will raise exception if deleting fails; in this case, please
# manually delete the empty image!
glance.images.delete(new_image['id'])
raise e
return new_image
def image(request, keystone):
image_arg = request.config.getoption('--image')
if not image_arg:
pytest.exit('--image argument is required.')
glance = chi.glance(session=keystone)
image = list(glance.images.list(filters={'name': image_arg}))
if len(image) != 1:
image = list(glance.images.list(filters={'id': image_arg}))
if len(image) != 1:
pytest.exit('No single image found with name or ID: "{}"'.format(
image_arg))
image = image[0]
image_id = image['id']
image_distro = image.get('build-distro',
request.config.getoption("--distro"))
image_release = image.get('build-release',
request.config.getoption("--release"))
image_variant = image.get('build-variant',
request.config.getoption("--variant"))
if image_distro is None or image_release is None or image_variant is None:
pytest.exit(
'Image does not contain distro/release/variant in metadata. Cannot '
'automatically infer test parameter; they must be '
'manually specified.')
return {
'id': image_id,
'distro': image_distro,
'release': image_release,
'variant': image_variant,
}
def archive_image(auth_session, image, image_production_name):
glance = chi.glance(session=auth_session)
new_name = helpers.archival_name(image_production_name, image=image)
logging.info(
f"renaming image {image['name']} ({image['id']}) to {new_name}")
glance.images.update(image['id'], name=new_name)
def do_upload(ip, rc, metadata, disk_format, **build_results):
session = helpers.get_auth_session_from_rc(rc)
glance = chi.glance(session=session)
if disk_format != 'qcow2':
converted_image = None
if build_results['image_loc'].endswith('.qcow2'):
converted_image = build_results['image_loc'][:-6] + '.img'
else:
converted_image = build_results['image_loc'] + '.img'
out = helpers.remote_run(
ip=ip,
command='qemu-img convert -f qcow2 -O {} {} {}'.format(
disk_format, build_results['image_loc'], converted_image))
if out.failed:
raise RuntimeError('converting image failed')
build_results['image_loc'] = converted_image
build_results['checksum'] = helpers.remote_run(
ip=ip,
command='md5sum {}'.format(converted_image)).split()[0].strip()
image = glance.images.create(name='image-{}-{}-{}'.format(
metadata['build-distro'], metadata['build-release'],
metadata['build-tag']),
disk_format=disk_format,
container_format='bare',
**metadata)
upload_command = textwrap.dedent('''\
curl -i -X PUT -H "X-Auth-Token: {token}" \
-H "Content-Type: application/octet-stream" \
-H "Connection: keep-alive" \
-T "{filepath}" \
{url}'''.format(
token=session.get_token(),
filepath=build_results['image_loc'],
url=session.get_endpoint(service_type="image") +
f"/v2/images/{image['id']}/file",
))
out = helpers.remote_run(ip=ip, command=upload_command)
image = glance.images.get(image["id"])
if build_results['checksum'] != image['checksum']:
raise RuntimeError('checksum mismatch! build: {} vs glance: {}'.format(
repr(build_results['checksum']),
repr(image['checksum']),
))
return image
def get_image_by_name(auth_data, name):
glance = chi.glance(session=helpers.get_auth_session_from_rc(auth_data))
images = list(
glance.images.list(filters={
'name': name,
'visibility': 'public'
}))
if len(images) > 1:
raise ValueError(f"More than one {name} images found!")
elif len(images) == 0:
# first time deployment
return None
else:
return images[0]
def archive_image(auth_session, owner, image):
'''
auth : Auth object
Authentication/authorization object indicating site to target.
image : str or dict
ID of image to archive or dictionary with contents. New name autogenerated based on metadata.
'''
glance = chi.glance(session=auth_session)
if isinstance(image, str):
image = glance.images.get(image)
new_name = helpers.archival_name(production_name(image=image), image=image)
print('renaming image {} ({}) to {}'.format(image['name'], image['id'],
new_name))
glance.images.update(image['id'], name=new_name)
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument("--site-yaml",
type=str,
required=True,
help="A yaml file with site credentials.")
parser.add_argument(
'--latest',
type=str,
nargs=3,
metavar=("distro", "release", "variant"),
help=
'Publish latest tested image given 3 args:<distro> <release> <variant>'
)
parser.add_argument('--image', type=str, help='Image id to publish')
args = parser.parse_args(argv[1:])
with open("/etc/chameleon_image_tools/supports.yaml", 'r') as f:
supports = yaml.safe_load(f)
auth_session = helpers.get_auth_session_from_yaml(args.site_yaml)
release_images = []
if args.image:
headers, content = get_image_obj_by_id(args.image)
if not headers or not content:
raise RuntimeError(f"Image {args.image} found")
release_images.append((headers, content))
elif args.latest:
distro, release, variant = args.latest
release_images = get_latest_image_objs([(distro, release, variant)])
else:
# release all images
identifiers = []
for distro, dv in supports["supported_distros"].items():
if "releases" not in dv:
continue
releases = dv["releases"]
for release, rv in releases.items():
if "variants" not in rv:
continue
for variant in rv["variants"]:
identifiers.append((distro, release, variant))
release_images = get_latest_image_objs(identifiers)
glance = chi.glance(session=auth_session)
for img in release_images:
resp_headers = img[0]
source_image_content = img[1]
image_production_name = production_name(resp_headers, supports)
# check if the latest image has been published
latest_image = find_latest_published_image(glance, resp_headers,
image_production_name)
timestamp_header = f"{helpers.SWIFT_META_HEADER_PREFIX}build-timestamp"
revision_header = f"{helpers.SWIFT_META_HEADER_PREFIX}build-os-base-image-revision"
if (latest_image and latest_image.get(
"build-timestamp", None) == resp_headers[timestamp_header]
and latest_image.get("build-os-base-image-revision",
None) == resp_headers[revision_header]):
d, r, v = get_identifiers(resp_headers)
logging.info(
f"The latest image {d}-{r}-{v} has been released. Nothing to do."
)
continue
# publish image
new_image = copy_image(auth_session, resp_headers,
source_image_content)
# rename old image
with open(args.site_yaml, 'r') as f:
site = yaml.safe_load(f)
named_images = list(
glance.images.list(
filters={
'name': image_production_name,
'owner': site["admin_project"],
'visibility': 'public'
}))
if len(named_images) == 1:
archive_image(auth_session, named_images[0], image_production_name)
elif len(named_images) > 1:
raise RuntimeError('multiple images with the name "{}"'.format(
image_production_name))
elif len(named_images) < 1:
# do nothing
logging.info(
f"no public production images {image_production_name} found on site"
)
# rename new image
glance.images.update(
new_image["id"],
name=image_production_name,
visibility="public",
)
logging.info(
f"{image_production_name} has been published successfully!")
def copy_image(source_session, target_session, source_image_id):
glance = chi.glance(session=source_session)
source_image = glance.images.get(source_image_id)
extra = extract_extra_properties(source_image)
with tempfile.TemporaryDirectory() as tempdir:
img_file = os.path.join(tempdir, 'image')
glance_source = chi.glance(session=source_session)
data = glance_source.images.data(source_image['id'])
with open(img_file, "wb") as f:
for chunk in data:
f.write(chunk)
glance_target = chi.glance(session=target_session)
disk_format = source_image['disk_format']
new_image = glance_target.images.create(
name=source_image['name'],
visibility=source_image['visibility'],
disk_format=disk_format,
container_format='bare',
**extra)
try:
glance_target.images.upload(
new_image['id'],
open(img_file, "rb"),
backend=helpers.CENTRALIZED_STORE,
)
except Exception as e:
# will raise exception if deleting fails; in this case, please
# manually delete the empty image!
glance_target.images.delete(new_image['id'])
raise e
new_image_full = glance_target.images.get(new_image['id'])
if new_image_full['checksum'] != source_image['checksum']:
# skip checksum check for kvm site
raise RuntimeError('checksum mismatch')
# add metadata to swift object
swift_conn = helpers.connect_to_swift_with_admin(
target_session, helpers.CENTRALIZED_STORE_REGION_NAME)
try:
meta_headers = {
f"{helpers.SWIFT_META_HEADER_PREFIX}{k}": f"{new_image[k]}"
for k in new_image.keys() if k.startswith("build")
}
meta_headers[
f"{helpers.SWIFT_META_HEADER_PREFIX}disk-format"] = new_image[
"disk_format"]
existing_headers = swift_conn.head_object(
helpers.CENTRALIZED_CONTAINER_NAME, new_image['id'])
for header in KEEP_SWIFT_HEADERS:
meta_headers[header] = existing_headers.get(header, None)
swift_conn.put_object(
container=helpers.CENTRALIZED_CONTAINER_NAME,
obj=new_image['id'],
headers=meta_headers,
contents=None,
)
except Exception as e:
glance_target.images.delete(new_image['id'])
raise e
return new_image_full
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
'auth_jsons',
type=str,
help='File with auth info in JSON format for core sites.')
parser.add_argument('--from-site',
type=str,
default="tacc",
help='Staging image located site')
parser.add_argument(
'--latest',
type=str,
nargs=3,
metavar=('distro', 'release', 'variant'),
help=
'Publish latest tested image given 3 args:<distro> <release> <variant>'
)
parser.add_argument('--image', type=str, help='Image id to publish')
parser.add_argument('--notify',
type=str,
nargs=2,
help="Send notifications to emails (comma-separated)"
" using relay: <relay> <emails>")
args = parser.parse_args(argv[1:])
with open(args.auth_jsons) as f:
auth_data = json.load(f)
auth_sessions = {}
for site, auth_info in auth_data['auths'].items():
if site in helpers.CHAMELEON_CORE_SITES:
auth_sessions[site] = helpers.get_auth_session_from_rc(auth_info)
centralized_auth_session = auth_sessions[helpers.CENTRALIZED_STORE_SITE]
glance_source = chi.glance(session=auth_sessions[args.from_site])
if args.image:
source_image = glance_source.get(args.image)
print('found specified image {} ({}) to publish'.format(
source_image['name'], source_image['id']))
elif args.latest:
distro, release, variant = args.latest
query = {
'build-distro': distro,
'build-release': release,
'status': 'active',
'build-variant': variant,
}
matching_images = list(glance_source.images.list(filters=query))
matching_images.sort(reverse=True,
key=operator.itemgetter('created_at'))
latest_image = next(iter(matching_images), None)
if not latest_image:
print(f"No latest image found with query {query}")
return 0
if latest_image.get("stores", None) == helpers.CENTRALIZED_STORE:
print(
f"The latest {distro}-{release} {variant} image has been released.",
file=sys.stderr)
return 0
source_image = latest_image
print('found latest image {} ({}) to publish'.format(
source_image['name'], source_image['id']))
else:
print('must provide --latest or --image', file=sys.stderr)
return 1
image_production_name = production_name(image=source_image)
# publish image
new_image = copy_image(auth_sessions[args.from_site],
centralized_auth_session, source_image['id'])
# rename old image at centralized object store
glance = chi.glance(session=centralized_auth_session)
named_images = list(
glance.images.list(
filters={
'name': image_production_name,
'owner': auth_data['auths'][args.from_site]['OS_PROJECT_ID'],
'visibility': 'public',
'stores': helpers.CENTRALIZED_STORE
}))
if len(named_images) == 1:
archive_image(centralized_auth_session,
auth_data['auths'][args.from_site]['OS_PROJECT_ID'],
named_images[0]['id'])
elif len(named_images) > 1:
raise RuntimeError(
'multiple images with the name "{}"'.format(image_production_name))
elif len(named_images) < 1:
# do nothing
print(
f"no public production images {image_production_name} found on site {helpers.CENTRALIZED_STORE_SITE}"
)
# rename new image at centralized object store
glance = chi.glance(session=centralized_auth_session)
glance.images.update(
new_image['id'],
name=image_production_name,
visibility='public',
)
# delete tmp image
print('delete tmp image {} from site {}'.format(source_image['id'],
args.from_site))
glance = chi.glance(session=auth_sessions[args.from_site])
glance.images.delete(source_image['id'])
if args.notify:
relay, to_emails = args.notify
print(f"sending emails to {to_emails}")
helpers.send_notification_mail(relay, "*****@*****.**",
to_emails.split(","), new_image)
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument("--site-yaml",
type=str,
required=True,
help="A yaml file with site credentials.")
parser.add_argument("--dry-run",
action='store_true',
help="Dry run; no actual hiding and deleting")
args = parser.parse_args(argv[1:])
with open("/etc/chameleon_image_tools/supports.yaml", 'r') as f:
supports = yaml.safe_load(f)
with open(args.site_yaml, 'r') as f:
site_specs = yaml.safe_load(f)
hide_image_age_in_month = site_specs["hide_image_age_in_month"]
delete_image_age_in_month = site_specs["delete_image_age_in_month"]
if hide_image_age_in_month >= delete_image_age_in_month:
raise ValueError(
"Parameter hide_image_age_in_month must be smaller than delete_image_age_in_month"
)
today = date.today()
hide_datetime_cutoff = today + relativedelta(
months=-hide_image_age_in_month)
delete_datetime_cuttoff = today + relativedelta(
months=-delete_image_age_in_month)
auth_session = helpers.get_auth_session_from_yaml(args.site_yaml)
glance = chi.glance(session=auth_session)
nova = chi.nova(session=auth_session)
ready_to_delete_images = []
for distro, dv in supports["supported_distros"].items():
if "releases" not in dv:
continue
releases = dv["releases"]
for release, rv in releases.items():
if "variants" not in rv:
continue
for variant in rv["variants"]:
prod_name = production_name(supports, distro, release, variant)
ready_to_delete_images.extend(
find_images(glance, nova, distro, release, variant,
prod_name))
skip_images = []
if args.dry_run:
logging.info("It's dry-run. Print messages only.")
if "skip_images" in site_specs:
skip_images = site_specs["skip_images"]
for img in ready_to_delete_images:
if img["id"] in skip_images:
logging.info(f"Skip image {img['name']} (id: {img['id']}).")
continue
create_date = datetime.strptime(img["created_at"],
"%Y-%m-%dT%H:%M:%SZ").date()
if create_date <= delete_datetime_cuttoff:
try:
if not args.dry_run:
glance.images.delete(img['id'])
logging.info(
f"Image {img['name']} (id: {img['id']}) has been deleted.")
except Exception:
logging.exception(
f"Failed to delete {img['name']} (id: {img['id']}).")
elif create_date <= hide_datetime_cutoff:
try:
if not args.dry_run:
glance.images.update(
img['id'],
visibility='private',
)
logging.info(
f"Image {img['name']} (id: {img['id']}) has been hided.")
except Exception:
logging.exception(
f"Failed to hide {img['name']} (id: {img['id']}).")
def glance(self):
return chi.glance(session=self.session)