def prepare_two_workdirs(deployment_id: str,
blueprint_id: str,
version_id: str,
inputs: dict,
location: Path = None):
location_old = InvocationService.deployment_location(
str(uuid.uuid4()), str(uuid.uuid4()))
location_new = location or InvocationService.deployment_location(
str(uuid.uuid4()), str(uuid.uuid4()))
# old Deployed instance
# TODO Next line should use PostgreSQL.get_deployment_status(deployment_id), had to be changed since
# old blueprint_id is part of second to last invocation, last is already current
inv_old = PostgreSQL.get_last_completed_invocation(deployment_id)
CSAR_db.get_revision(inv_old.blueprint_id, location_old,
inv_old.version_id)
InvocationService.get_dot_opera_from_db(deployment_id, location_old)
storage_old = Storage.create(str(location_old / '.opera'))
# new blueprint
CSAR_db.get_revision(blueprint_id, location_new, version_id)
storage_new = Storage.create(str(location_new / '.opera'))
storage_new.write_json(inputs or {}, "inputs")
storage_new.write(str(entry_definitions(location_new)), "root_file")
return storage_old, location_old, storage_new, location_new
def _parser_callback(args):
if args.instance_path and not path.isdir(args.instance_path):
raise argparse.ArgumentTypeError(
"Directory {} is not a valid path!".format(args.instance_path))
if args.workers < 1:
print("{} is not a positive number!".format(args.workers))
return 1
storage_old = Storage.create(args.instance_path)
comparer = TemplateComparer()
instance_comparer = InstanceComparer()
if args.template:
service_template_new = args.template.name
else:
print("Template file for update was not supplied.")
return 1
try:
if args.inputs:
inputs_new = yaml.safe_load(args.inputs)
else:
inputs_new = {}
except YAMLError as e:
print("Invalid inputs: {}".format(e))
return 1
workdir_old = get_workdir(storage_old)
try:
with tempfile.TemporaryDirectory() as temp_path:
storage_new = Storage.create(temp_path)
storage_new.write_json(inputs_new, "inputs")
storage_new.write(service_template_new, "root_file")
workdir_new = get_workdir(storage_new)
instance_diff = diff_instances(storage_old, workdir_old,
storage_new, workdir_new, comparer,
instance_comparer, args.verbose)
update(storage_old,
workdir_old,
storage_new,
workdir_new,
instance_comparer,
instance_diff,
args.verbose,
args.workers,
overwrite=True)
except ParseError as e:
print("{}: {}".format(e.loc, e))
return 1
except DataError as e:
print(str(e))
return 1
return 0
def save_invocation(cls, inv: Invocation):
# TODO database
# SQL_database.update_deployment_log(invocation_id, inv)
storage = Storage.create(".opera-api")
filename = "invocation-{}.json".format(inv.invocation_id)
dump = json.dumps(inv.to_dict(), cls=image_builder_util.UUIDEncoder)
storage.write(dump, filename)
def _deploy_continue(location: Path, inv: ExtendedInvocation):
# get blueprint
CSAR_db.get_revision(inv.blueprint_id, location, inv.version_id)
# get session data (.opera)
InvocationService.get_dot_opera_from_db(inv.deployment_id, location)
with xopera_util.cwd(location):
try:
if inv.user_id and Settings.secure_workdir:
xopera_util.setup_user([location], inv.user_id,
inv.access_token)
opera_storage = Storage.create(".opera")
service_template = entry_definitions(location)
opera_deploy(service_template,
inv.inputs,
opera_storage,
verbose_mode=False,
num_workers=inv.workers,
delete_existing_state=inv.clean_state)
outputs = opera_outputs(opera_storage)
return outputs
finally:
if inv.user_id and Settings.secure_workdir:
xopera_util.cleanup_user()
def _undeploy(location: Path, inv: ExtendedInvocation):
# get blueprint
CSAR_db.get_revision(inv.blueprint_id, location, inv.version_id)
# get session data (.opera)
if not InvocationService.get_dot_opera_from_db(inv.deployment_id,
location):
raise MissingDeploymentDataError(
'Could not get .opera data from previous job, aborting...')
with xopera_util.cwd(location):
try:
if inv.user_id and Settings.secure_workdir:
xopera_util.setup_user([location], inv.user_id,
inv.access_token)
opera_storage = Storage.create(".opera")
if inv.inputs:
opera_storage.write_json(inv.inputs, "inputs")
opera_undeploy(opera_storage,
verbose_mode=False,
num_workers=inv.workers)
# Outputs in undeployment are not returned
return None
finally:
if inv.user_id and Settings.secure_workdir:
xopera_util.cleanup_user()
def _parser_callback(args):
if args.instance_path and not path.isdir(args.instance_path):
raise argparse.ArgumentTypeError(
"Directory {0} is not a valid path!".format(args.instance_path))
storage = Storage.create(args.instance_path)
try:
inputs = yaml.safe_load(args.inputs) if args.inputs else {}
except Exception as e:
print("Invalid inputs: {}".format(e))
return 1
try:
if is_zipfile(args.csar.name):
initialize_compressed_csar(args.csar.name, inputs, storage)
print("CSAR was initialized")
else:
initialize_service_template(args.csar.name, inputs, storage)
print("Service template was initialized")
except ParseError as e:
print("{}: {}".format(e.loc, e))
return 1
except DataError as e:
print(str(e))
return 1
except Exception as e:
print("Invalid CSAR: {}".format(e))
return 1
return 0
def _parser_callback(args):
try:
inputs = yaml.safe_load(args.inputs) if args.inputs else {}
except yaml.YAMLError as e:
print(f"Invalid inputs: {e}")
return 1
storage = Storage.create(args.instance_path)
if args.csar_or_service_template is None:
csar_or_st_path = PurePath(".")
else:
csar_or_st_path = PurePath(args.csar_or_service_template)
try:
if is_zipfile(csar_or_st_path) or Path(csar_or_st_path).is_dir():
print("Validating CSAR...")
validate_csar(csar_or_st_path, inputs, storage, args.verbose,
args.executors)
else:
print("Validating service template...")
validate_service_template(csar_or_st_path, inputs, storage,
args.verbose, args.executors)
print("Done.")
except ParseError as e:
print(f"{e.loc}: {e}")
return 1
except OperaError as e:
print(str(e))
return 1
return 0
def _parser_callback(args):
print("Warning: 'opera init' command is deprecated and will probably be removed within one of the next releases. "
"Please use 'opera deploy' to initialize and deploy service templates or compressed CSARs.")
if args.instance_path and not path.isdir(args.instance_path):
raise argparse.ArgumentTypeError("Directory {} is not a valid path!".format(args.instance_path))
storage = Storage.create(args.instance_path)
try:
inputs = yaml.safe_load(args.inputs) if args.inputs else {}
except YAMLError as e:
print("Invalid inputs: {}".format(e))
return 1
try:
if is_zipfile(args.csar.name):
init_compressed_csar(args.csar.name, inputs, storage, args.clean)
print("CSAR was initialized")
else:
init_service_template(args.csar.name, inputs, storage, args.clean)
print("Service template was initialized")
except ParseError as e:
print("{}: {}".format(e.loc, e))
return 1
except DataError as e:
print(str(e))
return 1
except OperaError as e:
print("Invalid CSAR: {}".format(e))
return 1
return 0
def _parser_callback(args):
if args.instance_path and not path.isdir(args.instance_path):
raise argparse.ArgumentTypeError(
"Directory {} is not a valid path!".format(args.instance_path))
storage = Storage.create(args.instance_path)
status = info(None, storage)["status"]
if not args.force and storage.exists("instances"):
if status == "initialized":
print(
"Running notify without previously running deploy might have unexpected consequences."
)
question = prompt_yes_no_question()
if not question:
return 0
elif status in ("deploying", "undeploying"):
print(
"The project is in the middle of some other operation. Please try again after some time."
)
return 0
elif status == "undeployed":
print(
"Running notify in an undeployed project might have unexpected consequences."
)
question = prompt_yes_no_question()
if not question:
return 0
elif status == "error":
print(
"Running notify after a deployment with an error might have unexpected consequences."
)
question = prompt_yes_no_question()
if not question:
return 0
if not args.force and not args.trigger:
print(
"You have not specified which policy trigger to use (with --trigger/-t or --event/-e) "
"and in this case all the triggers will be invoked which might not be what you want."
)
question = prompt_yes_no_question()
if not question:
return 0
# read the notification file and the pass its contents to the library function
notification_file_contents = Path(
args.notification.name).read_text() if args.notification else None
try:
notify(storage, args.verbose, args.trigger, notification_file_contents)
except ParseError as e:
print("{}: {}".format(e.loc, e))
return 1
except DataError as e:
print(str(e))
return 1
return 0
def _deploy(service_template: str, inputs: typing.Optional[dict],
num_workers: int, clean_state: bool):
opera_storage = Storage.create()
opera_deploy(service_template,
inputs,
opera_storage,
verbose_mode=True,
num_workers=num_workers,
delete_existing_state=clean_state)
def outputs(deployment_id: str):
with tempfile.TemporaryDirectory() as location:
InvocationService.prepare_location(deployment_id, Path(location))
try:
with xopera_util.cwd(location):
opera_storage = Storage.create(".opera")
return opera_outputs(opera_storage), None
except Exception as e:
return None, (e.__class__.__name__,
xopera_util.mask_workdir(location, str(e)))
def info():
logger.debug("Entry: info")
try:
opera_storage = Storage.create()
result = opera_info(PurePath("."), opera_storage)
except Exception as e:
return {"message": "General error: {}".format(str(e))}, 500
serialized = Info(**result)
return serialized, 200
def _update(service_template: str, inputs: typing.Optional[dict],
num_workers: int):
original_storage = Storage.create()
new_storage = Storage.create(instance_path=".opera-api-update")
new_storage.write_json(inputs, "inputs")
new_storage.write(service_template, "root_file")
instance_diff = opera_diff_instances(original_storage, ".",
new_storage, ".",
TemplateComparer(),
InstanceComparer(), True)
opera_update(original_storage,
".",
new_storage,
".",
InstanceComparer(),
instance_diff,
True,
num_workers,
overwrite=True)
def outputs():
logger.debug("Entry: outputs")
try:
opera_storage = Storage.create()
result = opera_outputs(opera_storage)
except Exception as e:
logger.error("Error getting outputs.", e)
return {"message": str(e)}, 500
if not result:
return {"message": "No outputs exist for this deployment."}, 404
return result, 200
def diff(body: dict = None):
logger.debug("Entry: diff")
diff_request = DiffRequest.from_dict(body)
try:
original_storage = Storage.create()
original_service_template = original_storage.read("root_file")
original_inputs = original_storage.read_json("inputs")
with tempfile.TemporaryDirectory(prefix=".opera-api-diff",
dir=".") as new_storage_root:
new_storage = Storage.create(instance_path=new_storage_root)
with tempfile.NamedTemporaryFile(prefix="diff-service-template",
dir=".") as new_service_template:
new_service_template.write(
diff_request.new_service_template_contents)
new_service_template.flush()
if diff_request.template_only:
diff_result = opera_diff_templates(
original_service_template, ".", original_inputs,
new_service_template.name, ".", diff_request.inputs,
TemplateComparer(), True)
else:
diff_result = opera_diff_instances(original_storage, ".",
new_storage, ".",
TemplateComparer(),
InstanceComparer(),
True)
result = Diff(added=diff_result.added,
changed=diff_result.changed,
deleted=diff_result.deleted)
except Exception as e:
logger.error("Error performing diff.", e)
return {"message": str(e)}, 500
return result, 200
def init(csar_initialization_input: CsarInitializationInput):
logger.debug("Entry: init")
try:
opera_storage = Storage.create()
opera_init_compressed_csar(".", csar_initialization_input.inputs,
opera_storage,
csar_initialization_input.clean)
return {"success": True, "message": ""}, 200
except Exception as e:
return {
"success": False,
"message": "General error: {}".format(str(e))
}, 500
def _parser_callback(args):
if args.instance_path and not path.isdir(args.instance_path):
raise argparse.ArgumentTypeError(
"Directory {} is not a valid path!".format(args.instance_path))
if args.workers < 1:
print("{} is not a positive number!".format(args.workers))
return 1
storage = Storage.create(args.instance_path)
status = info(None, storage)["status"]
if storage.exists("instances"):
if args.resume and status == "error":
if not args.force:
print(
"The resume undeploy option might have unexpected consequences on the already "
"undeployed blueprint.")
question = prompt_yes_no_question()
if not question:
return 0
elif status == "initialized":
print(
"The project is initialized. You have to deploy it first to be able to run undeploy."
)
return 0
elif status == "deploying":
print(
"The project is currently deploying. Please try again after the deployment."
)
return 0
elif status == "undeployed":
print("All instances have already been undeployed.")
return 0
elif status == "error":
print(
"The instance model already exists. Use --resume/-r option to continue current undeployment process."
)
return 0
try:
undeploy(storage, args.verbose, args.workers)
except ParseError as e:
print("{}: {}".format(e.loc, e))
return 1
except DataError as e:
print(str(e))
return 1
return 0
def build_image(inv: Invocation):
with tempfile.TemporaryDirectory() as workdir:
dir_util.copy_tree(Settings.TOSCA_path, workdir)
with image_builder_util.cwd(workdir):
opera_storage = Storage.create(".opera")
service_template = "docker_image_definition.yaml"
build_params = transform_build_params(inv.build_params)
logger.info(json.dumps(build_params))
opera_deploy(service_template,
build_params,
opera_storage,
verbose_mode=False,
num_workers=1,
delete_existing_state=True)
return opera_outputs(opera_storage)
def _parser_callback(args):
if args.instance_path and not path.isdir(args.instance_path):
raise argparse.ArgumentTypeError(
"Directory {0} is not a valid path!".format(args.instance_path))
storage = Storage.create(args.instance_path)
try:
outs = info(storage)
print(format_outputs(outs, args.format))
except ParseError as e:
print("{}: {}".format(e.loc, e))
return 1
except DataError as e:
print(str(e))
return 1
return 0
def run_test(registry_ip: str, test_name: str):
"""
Runs yaml test and returns exit_code
"""
test_path = test_build_params / f'{test_name}.json'
inputs = json_to_yaml(test_path, registry_ip)
with cwd(tosca_path):
opera_storage = Storage.create('.opera')
try:
opera_deploy('docker_image_definition.yaml', inputs, opera_storage,
verbose_mode=False, num_workers=1, delete_existing_state=True)
except OperationError:
return 1
finally:
shutil.rmtree((tosca_path / ".opera"), ignore_errors=True)
return 0
def validate(blueprint_id: str, version_tag: str, inputs: dict):
with tempfile.TemporaryDirectory() as location:
CSAR_db.get_revision(blueprint_id, location, version_tag)
try:
with xopera_util.cwd(location):
opera_storage = Storage.create(".opera")
service_template = Path(location) / entry_definitions(
location)
opera_validate(service_template,
inputs,
opera_storage,
verbose=False,
executors=False)
return None
except Exception as e:
return "{}: {}".format(
e.__class__.__name__,
xopera_util.mask_workdir(location, str(e)))
def load_invocation(cls, job_id: uuid) -> Optional[Invocation]:
# TODO database
# try:
# inv = SQL_database.get_deployment_status(deployment_id)
# # if inv.state == InvocationState.IN_PROGRESS:
# # inv.stdout = InvocationWorkerProcess.read_file(cls.stdout_file(inv.deployment_id))
# # inv.stderr = InvocationWorkerProcess.read_file(cls.stderr_file(inv.deployment_id))
# return inv
#
# except Exception in (FileNotFoundError, AttributeError):
# return None
storage = Storage.create(".opera-api")
filename = "invocation-{}.json".format(job_id)
try:
dump = storage.read_json(filename)
except FileNotFoundError:
return None
return Invocation.from_dict(dump)
def _parser_callback(args):
if args.instance_path and not path.isdir(args.instance_path):
raise argparse.ArgumentTypeError(
f"Directory {args.instance_path} is not a valid path!")
storage = Storage.create(args.instance_path)
try:
outs = outputs(storage)
if args.output:
save_outputs(outs, args.format, args.output)
else:
print(format_outputs(outs, args.format).strip())
except ParseError as e:
print(f"{e.loc}: {e}")
return 1
except DataError as e:
print(str(e))
return 1
return 0
def _parser_callback(args):
if args.instance_path and not path.isdir(args.instance_path):
raise argparse.ArgumentTypeError(
"Directory {0} is not a valid path!".format(args.instance_path))
if args.workers < 1:
print("{0} is not a positive number!".format(args.workers))
return 1
storage = Storage.create(args.instance_path)
try:
undeploy(storage, args.verbose, args.workers)
except ParseError as e:
print("{}: {}".format(e.loc, e))
return 1
except DataError as e:
print(str(e))
return 1
return 0
def validate_new(CSAR: FileStorage, inputs: dict):
try:
with tempfile.TemporaryDirectory() as location:
with tempfile.TemporaryDirectory() as csar_workdir:
csar_path = Path(csar_workdir) / Path(CSAR.filename)
CSAR.save(Path(csar_path).open('wb'))
csar_to_blueprint(csar=csar_path, dst=location)
with xopera_util.cwd(location):
opera_storage = Storage.create(".opera")
service_template = Path(location) / entry_definitions(
location)
opera_validate(service_template,
inputs,
opera_storage,
verbose=False,
executors=False)
return None
except Exception as e:
return "{}: {}".format(
e.__class__.__name__,
xopera_util.mask_workdirs([location, csar_workdir], str(e)),
)
def _parser_callback(args):
if args.instance_path and not path.isdir(args.instance_path):
raise argparse.ArgumentTypeError(
"Directory {0} is not a valid path!".format(args.instance_path))
storage = Storage.create(args.instance_path)
try:
if args.csar_or_rootdir is None:
csar_path = "."
else:
csar_path = args.csar_or_rootdir
outs = info(PurePath(csar_path), storage)
if args.output:
save_outputs(outs, args.format, args.output)
else:
print(format_outputs(outs, args.format))
except ParseError as e:
print("{}: {}".format(e.loc, e))
return 1
except DataError as e:
print(str(e))
return 1
return 0
def _parser_callback(args):
if args.instance_path and not path.isdir(args.instance_path):
raise argparse.ArgumentTypeError(
f"Directory {args.instance_path} is not a valid path!")
storage_old = Storage.create(args.instance_path)
comparer = TemplateComparer()
if args.template:
service_template_new_path = Path(args.template.name)
else:
print("Template file for comparison was not supplied.")
return 1
if storage_old.exists("root_file"):
service_template_old_path = Path(storage_old.read("root_file"))
else:
print("There is no root_file in storage.")
return 1
if storage_old.exists("inputs"):
inputs_old = storage_old.read_json("inputs")
else:
inputs_old = {}
try:
if args.inputs:
inputs_new = yaml.safe_load(args.inputs)
else:
inputs_new = {}
except yaml.YAMLError as e:
print(f"Invalid inputs: {e}")
return 1
workdir_old = get_workdir(storage_old)
workdir_new = Path(service_template_new_path.parent)
try:
if args.template_only:
template_diff = diff_templates(service_template_old_path,
workdir_old, inputs_old,
service_template_new_path,
workdir_new, inputs_new, comparer,
args.verbose)
else:
instance_comparer = InstanceComparer()
with tempfile.TemporaryDirectory() as temp_path:
storage_new = Storage.create(temp_path)
storage_new.write_json(inputs_new, "inputs")
storage_new.write(str(service_template_new_path), "root_file")
template_diff = diff_instances(storage_old, workdir_old,
storage_new, workdir_new,
comparer, instance_comparer,
args.verbose)
outputs = template_diff.outputs()
if args.output:
save_outputs(outputs, args.format, args.output)
else:
print(format_outputs(outputs, args.format))
except ParseError as e:
print(f"{e.loc}: {e}")
return 1
except DataError as e:
print(str(e))
return 1
return 0
def _parser_callback(args):
if args.instance_path and not path.isdir(args.instance_path):
raise argparse.ArgumentTypeError(
"Directory {0} is not a valid path!".format(args.instance_path))
storage = Storage.create(args.instance_path)
if args.workers < 1:
print("{0} is not a positive number!".format(args.workers))
return 1
delete_existing_state = False
if storage.exists("instances"):
if args.resume:
if not args.force:
print("The resume deploy option might have unexpected "
"consequences on the already deployed blueprint.")
question = prompt_yes_no_question()
if not question:
return 0
elif args.clean_state:
if args.force:
delete_existing_state = True
else:
print("The clean state deploy option might have unexpected "
"consequences on the already deployed blueprint.")
question = prompt_yes_no_question()
if question:
delete_existing_state = True
else:
return 0
else:
print("The instance model already exists. Use --resume to "
"continue or --clean-state to delete current deployment "
"state and start over the deployment.")
return 0
if args.template:
service_template = args.template.name
else:
if storage.exists("root_file"):
service_template = storage.read("root_file")
else:
print("CSAR or template root file does not exist. "
"Maybe you have forgotten to initialize it.")
return 1
try:
if args.inputs:
inputs = yaml.safe_load(args.inputs)
else:
inputs = None
except Exception as e:
print("Invalid inputs: {}".format(e))
return 1
try:
deploy(service_template, inputs, storage, args.verbose, args.workers,
delete_existing_state)
except ParseError as e:
print("{}: {}".format(e.loc, e))
return 1
except DataError as e:
print(str(e))
return 1
return 0
def _parser_callback(args): # pylint: disable=too-many-statements
if args.instance_path and not path.isdir(args.instance_path):
raise argparse.ArgumentTypeError(
f"Directory {args.instance_path} is not a valid path!")
if args.workers < 1:
print(f"{args.workers} is not a positive number!")
return 1
storage = Storage.create(args.instance_path)
status = info(None, storage)["status"]
delete_existing_state = False
if storage.exists("instances"):
if args.resume and status == "error":
if not args.force:
print(
"The resume deploy option might have unexpected consequences on the already deployed blueprint."
)
question = prompt_yes_no_question()
if not question:
return 0
elif args.clean_state:
if args.force:
delete_existing_state = True
else:
print("The clean state deploy option might have unexpected "
"consequences on the already deployed blueprint.")
question = prompt_yes_no_question()
if question:
delete_existing_state = True
else:
return 0
elif status == "initialized":
print(
"The project is initialized. You have to deploy it first to be able to run undeploy."
)
return 0
elif status == "undeploying":
print(
"The project is currently undeploying. Please try again after the undeployment."
)
return 0
elif status == "deployed":
print("All instances have already been deployed.")
return 0
elif status == "error":
print(
"The instance model already exists. Use --resume/-r to continue or --clean-state/-c to delete "
"current deployment state and start over the deployment.")
return 0
if args.template:
csar_or_st_path = PurePath(args.template.name)
else:
if storage.exists("root_file"):
csar_or_st_path = PurePath(storage.read("root_file"))
else:
print(
"CSAR or template root file does not exist. Maybe you have forgotten to initialize it."
)
return 1
try:
if args.inputs:
inputs = yaml.safe_load(args.inputs)
else:
inputs = None
except yaml.YAMLError as e:
print(f"Invalid inputs: {e}")
return 1
try:
if is_zipfile(csar_or_st_path):
deploy_compressed_csar(csar_or_st_path, inputs, storage,
args.verbose, args.workers,
delete_existing_state)
else:
deploy_service_template(csar_or_st_path, inputs, storage,
args.verbose, args.workers,
delete_existing_state)
except ParseError as e:
print(f"{e.loc}: {e}")
return 1
except DataError as e:
print(str(e))
return 1
return 0
def write_invocation(cls, inv: Invocation):
storage = Storage.create(".opera-api")
filename = "invocation-{}.json".format(inv.id)
dump = json.dumps(inv.to_dict())
storage.write(dump, filename)