def prepare_two_workdirs(deployment_id: str, blueprint_id: str, version_id: str, inputs: dict, location: Path = None): location_old = InvocationService.deployment_location( str(uuid.uuid4()), str(uuid.uuid4())) location_new = location or InvocationService.deployment_location( str(uuid.uuid4()), str(uuid.uuid4())) # old Deployed instance # TODO Next line should use PostgreSQL.get_deployment_status(deployment_id), had to be changed since # old blueprint_id is part of second to last invocation, last is already current inv_old = PostgreSQL.get_last_completed_invocation(deployment_id) CSAR_db.get_revision(inv_old.blueprint_id, location_old, inv_old.version_id) InvocationService.get_dot_opera_from_db(deployment_id, location_old) storage_old = Storage.create(str(location_old / '.opera')) # new blueprint CSAR_db.get_revision(blueprint_id, location_new, version_id) storage_new = Storage.create(str(location_new / '.opera')) storage_new.write_json(inputs or {}, "inputs") storage_new.write(str(entry_definitions(location_new)), "root_file") return storage_old, location_old, storage_new, location_new
def _parser_callback(args): if args.instance_path and not path.isdir(args.instance_path): raise argparse.ArgumentTypeError( "Directory {} is not a valid path!".format(args.instance_path)) if args.workers < 1: print("{} is not a positive number!".format(args.workers)) return 1 storage_old = Storage.create(args.instance_path) comparer = TemplateComparer() instance_comparer = InstanceComparer() if args.template: service_template_new = args.template.name else: print("Template file for update was not supplied.") return 1 try: if args.inputs: inputs_new = yaml.safe_load(args.inputs) else: inputs_new = {} except YAMLError as e: print("Invalid inputs: {}".format(e)) return 1 workdir_old = get_workdir(storage_old) try: with tempfile.TemporaryDirectory() as temp_path: storage_new = Storage.create(temp_path) storage_new.write_json(inputs_new, "inputs") storage_new.write(service_template_new, "root_file") workdir_new = get_workdir(storage_new) instance_diff = diff_instances(storage_old, workdir_old, storage_new, workdir_new, comparer, instance_comparer, args.verbose) update(storage_old, workdir_old, storage_new, workdir_new, instance_comparer, instance_diff, args.verbose, args.workers, overwrite=True) except ParseError as e: print("{}: {}".format(e.loc, e)) return 1 except DataError as e: print(str(e)) return 1 return 0
def save_invocation(cls, inv: Invocation): # TODO database # SQL_database.update_deployment_log(invocation_id, inv) storage = Storage.create(".opera-api") filename = "invocation-{}.json".format(inv.invocation_id) dump = json.dumps(inv.to_dict(), cls=image_builder_util.UUIDEncoder) storage.write(dump, filename)
def _deploy_continue(location: Path, inv: ExtendedInvocation): # get blueprint CSAR_db.get_revision(inv.blueprint_id, location, inv.version_id) # get session data (.opera) InvocationService.get_dot_opera_from_db(inv.deployment_id, location) with xopera_util.cwd(location): try: if inv.user_id and Settings.secure_workdir: xopera_util.setup_user([location], inv.user_id, inv.access_token) opera_storage = Storage.create(".opera") service_template = entry_definitions(location) opera_deploy(service_template, inv.inputs, opera_storage, verbose_mode=False, num_workers=inv.workers, delete_existing_state=inv.clean_state) outputs = opera_outputs(opera_storage) return outputs finally: if inv.user_id and Settings.secure_workdir: xopera_util.cleanup_user()
def _undeploy(location: Path, inv: ExtendedInvocation): # get blueprint CSAR_db.get_revision(inv.blueprint_id, location, inv.version_id) # get session data (.opera) if not InvocationService.get_dot_opera_from_db(inv.deployment_id, location): raise MissingDeploymentDataError( 'Could not get .opera data from previous job, aborting...') with xopera_util.cwd(location): try: if inv.user_id and Settings.secure_workdir: xopera_util.setup_user([location], inv.user_id, inv.access_token) opera_storage = Storage.create(".opera") if inv.inputs: opera_storage.write_json(inv.inputs, "inputs") opera_undeploy(opera_storage, verbose_mode=False, num_workers=inv.workers) # Outputs in undeployment are not returned return None finally: if inv.user_id and Settings.secure_workdir: xopera_util.cleanup_user()
def _parser_callback(args): if args.instance_path and not path.isdir(args.instance_path): raise argparse.ArgumentTypeError( "Directory {0} is not a valid path!".format(args.instance_path)) storage = Storage.create(args.instance_path) try: inputs = yaml.safe_load(args.inputs) if args.inputs else {} except Exception as e: print("Invalid inputs: {}".format(e)) return 1 try: if is_zipfile(args.csar.name): initialize_compressed_csar(args.csar.name, inputs, storage) print("CSAR was initialized") else: initialize_service_template(args.csar.name, inputs, storage) print("Service template was initialized") except ParseError as e: print("{}: {}".format(e.loc, e)) return 1 except DataError as e: print(str(e)) return 1 except Exception as e: print("Invalid CSAR: {}".format(e)) return 1 return 0
def _parser_callback(args): try: inputs = yaml.safe_load(args.inputs) if args.inputs else {} except yaml.YAMLError as e: print(f"Invalid inputs: {e}") return 1 storage = Storage.create(args.instance_path) if args.csar_or_service_template is None: csar_or_st_path = PurePath(".") else: csar_or_st_path = PurePath(args.csar_or_service_template) try: if is_zipfile(csar_or_st_path) or Path(csar_or_st_path).is_dir(): print("Validating CSAR...") validate_csar(csar_or_st_path, inputs, storage, args.verbose, args.executors) else: print("Validating service template...") validate_service_template(csar_or_st_path, inputs, storage, args.verbose, args.executors) print("Done.") except ParseError as e: print(f"{e.loc}: {e}") return 1 except OperaError as e: print(str(e)) return 1 return 0
def _parser_callback(args): print("Warning: 'opera init' command is deprecated and will probably be removed within one of the next releases. " "Please use 'opera deploy' to initialize and deploy service templates or compressed CSARs.") if args.instance_path and not path.isdir(args.instance_path): raise argparse.ArgumentTypeError("Directory {} is not a valid path!".format(args.instance_path)) storage = Storage.create(args.instance_path) try: inputs = yaml.safe_load(args.inputs) if args.inputs else {} except YAMLError as e: print("Invalid inputs: {}".format(e)) return 1 try: if is_zipfile(args.csar.name): init_compressed_csar(args.csar.name, inputs, storage, args.clean) print("CSAR was initialized") else: init_service_template(args.csar.name, inputs, storage, args.clean) print("Service template was initialized") except ParseError as e: print("{}: {}".format(e.loc, e)) return 1 except DataError as e: print(str(e)) return 1 except OperaError as e: print("Invalid CSAR: {}".format(e)) return 1 return 0
def _parser_callback(args): if args.instance_path and not path.isdir(args.instance_path): raise argparse.ArgumentTypeError( "Directory {} is not a valid path!".format(args.instance_path)) storage = Storage.create(args.instance_path) status = info(None, storage)["status"] if not args.force and storage.exists("instances"): if status == "initialized": print( "Running notify without previously running deploy might have unexpected consequences." ) question = prompt_yes_no_question() if not question: return 0 elif status in ("deploying", "undeploying"): print( "The project is in the middle of some other operation. Please try again after some time." ) return 0 elif status == "undeployed": print( "Running notify in an undeployed project might have unexpected consequences." ) question = prompt_yes_no_question() if not question: return 0 elif status == "error": print( "Running notify after a deployment with an error might have unexpected consequences." ) question = prompt_yes_no_question() if not question: return 0 if not args.force and not args.trigger: print( "You have not specified which policy trigger to use (with --trigger/-t or --event/-e) " "and in this case all the triggers will be invoked which might not be what you want." ) question = prompt_yes_no_question() if not question: return 0 # read the notification file and the pass its contents to the library function notification_file_contents = Path( args.notification.name).read_text() if args.notification else None try: notify(storage, args.verbose, args.trigger, notification_file_contents) except ParseError as e: print("{}: {}".format(e.loc, e)) return 1 except DataError as e: print(str(e)) return 1 return 0
def _deploy(service_template: str, inputs: typing.Optional[dict], num_workers: int, clean_state: bool): opera_storage = Storage.create() opera_deploy(service_template, inputs, opera_storage, verbose_mode=True, num_workers=num_workers, delete_existing_state=clean_state)
def outputs(deployment_id: str): with tempfile.TemporaryDirectory() as location: InvocationService.prepare_location(deployment_id, Path(location)) try: with xopera_util.cwd(location): opera_storage = Storage.create(".opera") return opera_outputs(opera_storage), None except Exception as e: return None, (e.__class__.__name__, xopera_util.mask_workdir(location, str(e)))
def info(): logger.debug("Entry: info") try: opera_storage = Storage.create() result = opera_info(PurePath("."), opera_storage) except Exception as e: return {"message": "General error: {}".format(str(e))}, 500 serialized = Info(**result) return serialized, 200
def _update(service_template: str, inputs: typing.Optional[dict], num_workers: int): original_storage = Storage.create() new_storage = Storage.create(instance_path=".opera-api-update") new_storage.write_json(inputs, "inputs") new_storage.write(service_template, "root_file") instance_diff = opera_diff_instances(original_storage, ".", new_storage, ".", TemplateComparer(), InstanceComparer(), True) opera_update(original_storage, ".", new_storage, ".", InstanceComparer(), instance_diff, True, num_workers, overwrite=True)
def outputs(): logger.debug("Entry: outputs") try: opera_storage = Storage.create() result = opera_outputs(opera_storage) except Exception as e: logger.error("Error getting outputs.", e) return {"message": str(e)}, 500 if not result: return {"message": "No outputs exist for this deployment."}, 404 return result, 200
def diff(body: dict = None): logger.debug("Entry: diff") diff_request = DiffRequest.from_dict(body) try: original_storage = Storage.create() original_service_template = original_storage.read("root_file") original_inputs = original_storage.read_json("inputs") with tempfile.TemporaryDirectory(prefix=".opera-api-diff", dir=".") as new_storage_root: new_storage = Storage.create(instance_path=new_storage_root) with tempfile.NamedTemporaryFile(prefix="diff-service-template", dir=".") as new_service_template: new_service_template.write( diff_request.new_service_template_contents) new_service_template.flush() if diff_request.template_only: diff_result = opera_diff_templates( original_service_template, ".", original_inputs, new_service_template.name, ".", diff_request.inputs, TemplateComparer(), True) else: diff_result = opera_diff_instances(original_storage, ".", new_storage, ".", TemplateComparer(), InstanceComparer(), True) result = Diff(added=diff_result.added, changed=diff_result.changed, deleted=diff_result.deleted) except Exception as e: logger.error("Error performing diff.", e) return {"message": str(e)}, 500 return result, 200
def init(csar_initialization_input: CsarInitializationInput): logger.debug("Entry: init") try: opera_storage = Storage.create() opera_init_compressed_csar(".", csar_initialization_input.inputs, opera_storage, csar_initialization_input.clean) return {"success": True, "message": ""}, 200 except Exception as e: return { "success": False, "message": "General error: {}".format(str(e)) }, 500
def _parser_callback(args): if args.instance_path and not path.isdir(args.instance_path): raise argparse.ArgumentTypeError( "Directory {} is not a valid path!".format(args.instance_path)) if args.workers < 1: print("{} is not a positive number!".format(args.workers)) return 1 storage = Storage.create(args.instance_path) status = info(None, storage)["status"] if storage.exists("instances"): if args.resume and status == "error": if not args.force: print( "The resume undeploy option might have unexpected consequences on the already " "undeployed blueprint.") question = prompt_yes_no_question() if not question: return 0 elif status == "initialized": print( "The project is initialized. You have to deploy it first to be able to run undeploy." ) return 0 elif status == "deploying": print( "The project is currently deploying. Please try again after the deployment." ) return 0 elif status == "undeployed": print("All instances have already been undeployed.") return 0 elif status == "error": print( "The instance model already exists. Use --resume/-r option to continue current undeployment process." ) return 0 try: undeploy(storage, args.verbose, args.workers) except ParseError as e: print("{}: {}".format(e.loc, e)) return 1 except DataError as e: print(str(e)) return 1 return 0
def build_image(inv: Invocation): with tempfile.TemporaryDirectory() as workdir: dir_util.copy_tree(Settings.TOSCA_path, workdir) with image_builder_util.cwd(workdir): opera_storage = Storage.create(".opera") service_template = "docker_image_definition.yaml" build_params = transform_build_params(inv.build_params) logger.info(json.dumps(build_params)) opera_deploy(service_template, build_params, opera_storage, verbose_mode=False, num_workers=1, delete_existing_state=True) return opera_outputs(opera_storage)
def _parser_callback(args): if args.instance_path and not path.isdir(args.instance_path): raise argparse.ArgumentTypeError( "Directory {0} is not a valid path!".format(args.instance_path)) storage = Storage.create(args.instance_path) try: outs = info(storage) print(format_outputs(outs, args.format)) except ParseError as e: print("{}: {}".format(e.loc, e)) return 1 except DataError as e: print(str(e)) return 1 return 0
def run_test(registry_ip: str, test_name: str): """ Runs yaml test and returns exit_code """ test_path = test_build_params / f'{test_name}.json' inputs = json_to_yaml(test_path, registry_ip) with cwd(tosca_path): opera_storage = Storage.create('.opera') try: opera_deploy('docker_image_definition.yaml', inputs, opera_storage, verbose_mode=False, num_workers=1, delete_existing_state=True) except OperationError: return 1 finally: shutil.rmtree((tosca_path / ".opera"), ignore_errors=True) return 0
def validate(blueprint_id: str, version_tag: str, inputs: dict): with tempfile.TemporaryDirectory() as location: CSAR_db.get_revision(blueprint_id, location, version_tag) try: with xopera_util.cwd(location): opera_storage = Storage.create(".opera") service_template = Path(location) / entry_definitions( location) opera_validate(service_template, inputs, opera_storage, verbose=False, executors=False) return None except Exception as e: return "{}: {}".format( e.__class__.__name__, xopera_util.mask_workdir(location, str(e)))
def load_invocation(cls, job_id: uuid) -> Optional[Invocation]: # TODO database # try: # inv = SQL_database.get_deployment_status(deployment_id) # # if inv.state == InvocationState.IN_PROGRESS: # # inv.stdout = InvocationWorkerProcess.read_file(cls.stdout_file(inv.deployment_id)) # # inv.stderr = InvocationWorkerProcess.read_file(cls.stderr_file(inv.deployment_id)) # return inv # # except Exception in (FileNotFoundError, AttributeError): # return None storage = Storage.create(".opera-api") filename = "invocation-{}.json".format(job_id) try: dump = storage.read_json(filename) except FileNotFoundError: return None return Invocation.from_dict(dump)
def _parser_callback(args): if args.instance_path and not path.isdir(args.instance_path): raise argparse.ArgumentTypeError( f"Directory {args.instance_path} is not a valid path!") storage = Storage.create(args.instance_path) try: outs = outputs(storage) if args.output: save_outputs(outs, args.format, args.output) else: print(format_outputs(outs, args.format).strip()) except ParseError as e: print(f"{e.loc}: {e}") return 1 except DataError as e: print(str(e)) return 1 return 0
def _parser_callback(args): if args.instance_path and not path.isdir(args.instance_path): raise argparse.ArgumentTypeError( "Directory {0} is not a valid path!".format(args.instance_path)) if args.workers < 1: print("{0} is not a positive number!".format(args.workers)) return 1 storage = Storage.create(args.instance_path) try: undeploy(storage, args.verbose, args.workers) except ParseError as e: print("{}: {}".format(e.loc, e)) return 1 except DataError as e: print(str(e)) return 1 return 0
def validate_new(CSAR: FileStorage, inputs: dict): try: with tempfile.TemporaryDirectory() as location: with tempfile.TemporaryDirectory() as csar_workdir: csar_path = Path(csar_workdir) / Path(CSAR.filename) CSAR.save(Path(csar_path).open('wb')) csar_to_blueprint(csar=csar_path, dst=location) with xopera_util.cwd(location): opera_storage = Storage.create(".opera") service_template = Path(location) / entry_definitions( location) opera_validate(service_template, inputs, opera_storage, verbose=False, executors=False) return None except Exception as e: return "{}: {}".format( e.__class__.__name__, xopera_util.mask_workdirs([location, csar_workdir], str(e)), )
def _parser_callback(args): if args.instance_path and not path.isdir(args.instance_path): raise argparse.ArgumentTypeError( "Directory {0} is not a valid path!".format(args.instance_path)) storage = Storage.create(args.instance_path) try: if args.csar_or_rootdir is None: csar_path = "." else: csar_path = args.csar_or_rootdir outs = info(PurePath(csar_path), storage) if args.output: save_outputs(outs, args.format, args.output) else: print(format_outputs(outs, args.format)) except ParseError as e: print("{}: {}".format(e.loc, e)) return 1 except DataError as e: print(str(e)) return 1 return 0
def _parser_callback(args): if args.instance_path and not path.isdir(args.instance_path): raise argparse.ArgumentTypeError( f"Directory {args.instance_path} is not a valid path!") storage_old = Storage.create(args.instance_path) comparer = TemplateComparer() if args.template: service_template_new_path = Path(args.template.name) else: print("Template file for comparison was not supplied.") return 1 if storage_old.exists("root_file"): service_template_old_path = Path(storage_old.read("root_file")) else: print("There is no root_file in storage.") return 1 if storage_old.exists("inputs"): inputs_old = storage_old.read_json("inputs") else: inputs_old = {} try: if args.inputs: inputs_new = yaml.safe_load(args.inputs) else: inputs_new = {} except yaml.YAMLError as e: print(f"Invalid inputs: {e}") return 1 workdir_old = get_workdir(storage_old) workdir_new = Path(service_template_new_path.parent) try: if args.template_only: template_diff = diff_templates(service_template_old_path, workdir_old, inputs_old, service_template_new_path, workdir_new, inputs_new, comparer, args.verbose) else: instance_comparer = InstanceComparer() with tempfile.TemporaryDirectory() as temp_path: storage_new = Storage.create(temp_path) storage_new.write_json(inputs_new, "inputs") storage_new.write(str(service_template_new_path), "root_file") template_diff = diff_instances(storage_old, workdir_old, storage_new, workdir_new, comparer, instance_comparer, args.verbose) outputs = template_diff.outputs() if args.output: save_outputs(outputs, args.format, args.output) else: print(format_outputs(outputs, args.format)) except ParseError as e: print(f"{e.loc}: {e}") return 1 except DataError as e: print(str(e)) return 1 return 0
def _parser_callback(args): if args.instance_path and not path.isdir(args.instance_path): raise argparse.ArgumentTypeError( "Directory {0} is not a valid path!".format(args.instance_path)) storage = Storage.create(args.instance_path) if args.workers < 1: print("{0} is not a positive number!".format(args.workers)) return 1 delete_existing_state = False if storage.exists("instances"): if args.resume: if not args.force: print("The resume deploy option might have unexpected " "consequences on the already deployed blueprint.") question = prompt_yes_no_question() if not question: return 0 elif args.clean_state: if args.force: delete_existing_state = True else: print("The clean state deploy option might have unexpected " "consequences on the already deployed blueprint.") question = prompt_yes_no_question() if question: delete_existing_state = True else: return 0 else: print("The instance model already exists. Use --resume to " "continue or --clean-state to delete current deployment " "state and start over the deployment.") return 0 if args.template: service_template = args.template.name else: if storage.exists("root_file"): service_template = storage.read("root_file") else: print("CSAR or template root file does not exist. " "Maybe you have forgotten to initialize it.") return 1 try: if args.inputs: inputs = yaml.safe_load(args.inputs) else: inputs = None except Exception as e: print("Invalid inputs: {}".format(e)) return 1 try: deploy(service_template, inputs, storage, args.verbose, args.workers, delete_existing_state) except ParseError as e: print("{}: {}".format(e.loc, e)) return 1 except DataError as e: print(str(e)) return 1 return 0
def _parser_callback(args): # pylint: disable=too-many-statements if args.instance_path and not path.isdir(args.instance_path): raise argparse.ArgumentTypeError( f"Directory {args.instance_path} is not a valid path!") if args.workers < 1: print(f"{args.workers} is not a positive number!") return 1 storage = Storage.create(args.instance_path) status = info(None, storage)["status"] delete_existing_state = False if storage.exists("instances"): if args.resume and status == "error": if not args.force: print( "The resume deploy option might have unexpected consequences on the already deployed blueprint." ) question = prompt_yes_no_question() if not question: return 0 elif args.clean_state: if args.force: delete_existing_state = True else: print("The clean state deploy option might have unexpected " "consequences on the already deployed blueprint.") question = prompt_yes_no_question() if question: delete_existing_state = True else: return 0 elif status == "initialized": print( "The project is initialized. You have to deploy it first to be able to run undeploy." ) return 0 elif status == "undeploying": print( "The project is currently undeploying. Please try again after the undeployment." ) return 0 elif status == "deployed": print("All instances have already been deployed.") return 0 elif status == "error": print( "The instance model already exists. Use --resume/-r to continue or --clean-state/-c to delete " "current deployment state and start over the deployment.") return 0 if args.template: csar_or_st_path = PurePath(args.template.name) else: if storage.exists("root_file"): csar_or_st_path = PurePath(storage.read("root_file")) else: print( "CSAR or template root file does not exist. Maybe you have forgotten to initialize it." ) return 1 try: if args.inputs: inputs = yaml.safe_load(args.inputs) else: inputs = None except yaml.YAMLError as e: print(f"Invalid inputs: {e}") return 1 try: if is_zipfile(csar_or_st_path): deploy_compressed_csar(csar_or_st_path, inputs, storage, args.verbose, args.workers, delete_existing_state) else: deploy_service_template(csar_or_st_path, inputs, storage, args.verbose, args.workers, delete_existing_state) except ParseError as e: print(f"{e.loc}: {e}") return 1 except DataError as e: print(str(e)) return 1 return 0
def write_invocation(cls, inv: Invocation): storage = Storage.create(".opera-api") filename = "invocation-{}.json".format(inv.id) dump = json.dumps(inv.to_dict()) storage.write(dump, filename)