def _prepare_and_submit_blueprint(cls, file_server_root, app_dir, blueprint_id, visibility): args = get_args_and_verify_arguments([ Argument('private_resource', type=boolean), Argument('visibility'), Argument('application_file_name', default='') ]) app_file_name = cls._extract_application_file( file_server_root, app_dir, args.application_file_name) # add to blueprints manager (will also dsl_parse it) try: blueprint = get_resource_manager().publish_blueprint( app_dir, app_file_name, file_server_root, blueprint_id, args.private_resource, visibility) # moving the app directory in the file server to be under a # directory named after the blueprint id tenant_dir = os.path.join(file_server_root, FILE_SERVER_BLUEPRINTS_FOLDER, current_tenant.name) mkdirs(tenant_dir) shutil.move(os.path.join(file_server_root, app_dir), os.path.join(tenant_dir, blueprint.id)) cls._process_plugins(file_server_root, blueprint.id) return blueprint except manager_exceptions.DslParseException, ex: shutil.rmtree(os.path.join(file_server_root, app_dir)) raise manager_exceptions.InvalidBlueprintError( 'Invalid blueprint - {0}'.format(ex.message))
def post(self, maintenance_action, **_): maintenance_file_path = get_maintenance_file_path() if maintenance_action == 'activate': if os.path.isfile(maintenance_file_path): state = utils.read_json_file(maintenance_file_path) return state, 304 now = utils.get_formatted_timestamp() try: user = current_user.username except AttributeError: user = '' remaining_executions = get_running_executions() status = MAINTENANCE_MODE_ACTIVATING \ if remaining_executions else MAINTENANCE_MODE_ACTIVATED activated_at = '' if remaining_executions else now utils.mkdirs(config.instance.maintenance_folder) new_state = prepare_maintenance_dict( status=status, activation_requested_at=now, activated_at=activated_at, remaining_executions=remaining_executions, requested_by=user) utils.write_dict_to_json_file(maintenance_file_path, new_state) return new_state if maintenance_action == 'deactivate': if not os.path.isfile(maintenance_file_path): return prepare_maintenance_dict( MAINTENANCE_MODE_DEACTIVATED), 304 os.remove(maintenance_file_path) return prepare_maintenance_dict(MAINTENANCE_MODE_DEACTIVATED) valid_actions = ['activate', 'deactivate'] raise BadParametersError('Invalid action: {0}, Valid action ' 'values are: {1}'.format( maintenance_action, valid_actions))
def _save_manager_logs_after_test(self, purge=True): self.logger.debug('_save_manager_logs_after_test started') logs_dir = os.environ.get('CFY_LOGS_PATH') test_path = self.id().split('.')[-2:] if not logs_dir: self.logger.debug('not saving manager logs') return if os.environ.get('SKIP_LOG_SAVE_ON_SUCCESS') \ and sys.exc_info() == (None, None, None): self.logger.info('Not saving manager logs for successful test: ' '{0}'.format(test_path[-1])) return self.logger.info('Saving manager logs for test: {0}'.format( test_path[-1])) logs_dir = os.path.join(os.path.expanduser(logs_dir), *test_path) mkdirs(logs_dir) target = os.path.join(logs_dir, 'logs.tar.gz') self.cfy.logs.download(output_path=target) if purge: self.cfy.logs.purge(force=True) self.logger.debug('opening tar.gz: {0}'.format(target)) with tarfile.open(target) as tar: tar.extractall(path=logs_dir) self.logger.debug('removing {0}'.format(target)) os.remove(target) self.logger.debug('_save_manager_logs_after_test completed')
def _prepare_and_submit_blueprint(cls, file_server_root, app_dir, blueprint_id): args = cls._get_args() app_dir, app_file_name = cls._extract_application_file( file_server_root, app_dir, args.application_file_name) # add to blueprints manager (will also dsl_parse it) try: blueprint = get_resource_manager().publish_blueprint( app_dir, app_file_name, file_server_root, blueprint_id, private_resource=args.private_resource) # moving the app directory in the file server to be under a # directory named after the blueprint id tenant_dir = os.path.join( file_server_root, FILE_SERVER_BLUEPRINTS_FOLDER, current_app.config[CURRENT_TENANT_CONFIG].name) mkdirs(tenant_dir) shutil.move(os.path.join(file_server_root, app_dir), os.path.join(tenant_dir, blueprint.id)) cls._process_plugins(file_server_root, blueprint.id) return blueprint except manager_exceptions.DslParseException, ex: shutil.rmtree(os.path.join(file_server_root, app_dir)) raise manager_exceptions.InvalidBlueprintError( 'Invalid blueprint - {0}'.format(ex.message))
def _save_manager_logs_after_test(self, purge=True): self.logger.debug('_save_manager_logs_after_test started') logs_dir = os.environ.get('CFY_LOGS_PATH_REMOTE') self.logger.info( "Cloudify remote log saving path found: [{}]. If " "you're running via itest-runner, make sure to set a " "local path as well with CFY_LOGS_PATH_LOCAL.".format(logs_dir)) test_path = self.id().split('.')[-2:] if not logs_dir: self.logger.debug('not saving manager logs') return if os.environ.get('SKIP_LOG_SAVE_ON_SUCCESS') \ and sys.exc_info() == (None, None, None): self.logger.info('Not saving manager logs for successful test: ' '{0}'.format(test_path[-1])) return self.logger.info('Saving manager logs for test: {0}'.format( test_path[-1])) logs_dir = os.path.join(os.path.expanduser(logs_dir), *test_path) mkdirs(logs_dir) target = os.path.join(logs_dir, 'logs.tar.gz') self.cfy.logs.download(output_path=target) if purge: self.cfy.logs.purge(force=True) if not bool(os.environ.get('SKIP_LOGS_EXTRACTION')): with tarfile.open(target) as tar: self.logger.debug('Extracting tar.gz: {0}'.format(target)) tar.extractall(path=logs_dir) self.logger.debug('Removing {0}'.format(target)) os.remove(target) self.logger.debug('_save_manager_logs_after_test completed')
def post(self, maintenance_action, **_): maintenance_file_path = get_maintenance_file_path() if maintenance_action == 'activate': if os.path.isfile(maintenance_file_path): state = utils.read_json_file(maintenance_file_path) return state, 304 now = utils.get_formatted_timestamp() try: user = current_user.username except AttributeError: user = '' remaining_executions = get_running_executions() status = MAINTENANCE_MODE_ACTIVATING \ if remaining_executions else MAINTENANCE_MODE_ACTIVATED activated_at = '' if remaining_executions else now utils.mkdirs(config.instance.maintenance_folder) new_state = prepare_maintenance_dict( status=status, activation_requested_at=now, activated_at=activated_at, remaining_executions=remaining_executions, requested_by=user) utils.write_dict_to_json_file(maintenance_file_path, new_state) return new_state if maintenance_action == 'deactivate': if not os.path.isfile(maintenance_file_path): return prepare_maintenance_dict( MAINTENANCE_MODE_DEACTIVATED), 304 os.remove(maintenance_file_path) return prepare_maintenance_dict(MAINTENANCE_MODE_DEACTIVATED) valid_actions = ['activate', 'deactivate'] raise BadParametersError( 'Invalid action: {0}, Valid action ' 'values are: {1}'.format(maintenance_action, valid_actions))
def _save_logs(self, purge=True): self.logger.debug('save_logs started') logs_dir = os.environ.get('CFY_LOGS_PATH') test_path = self.id().split('.')[1:] if not logs_dir: self.logger.info('Saving manager logs is disabled by configuration' ' for test: {0}'.format(test_path[-1])) self.logger.info('To enable logs keeping, define "CFY_LOGS_PATH"') return self.logger.info('Saving manager logs for test: {0}'.format( test_path[-1])) logs_dir = os.path.join(os.path.expanduser(logs_dir), *test_path) mkdirs(logs_dir) target = os.path.join(logs_dir, 'logs.tar.gz') self.cfy.logs.download(output_path=target) if purge: self.cfy.logs.purge(force=True) self.logger.debug('opening tar.gz: {0}'.format(target)) with tarfile.open(target) as tar: tar.extractall(path=logs_dir) self.logger.debug('removing {0}'.format(target)) os.remove(target) self.logger.debug('save_logs completed')
def _save_logs(self, purge=True): logs_dir = os.environ.get('CFY_LOGS_PATH') if not logs_dir: return logs_dir = os.path.join(logs_dir, *self.id().split('.')) mkdirs(logs_dir) self.cfy.logs.download(output_path=logs_dir) if purge: self.cfy.logs.purge(force=True)
def _prepare_and_submit_blueprint(cls, file_server_root, app_dir, deployment_id, additional_inputs=None, runtime_only_evaluation=False): app_file_name = cls._extract_application_file(file_server_root, app_dir) # add to deployment update manager (will also dsl_parse it) try: cls._process_plugins(file_server_root, app_dir) update = get_deployment_updates_manager().stage_deployment_update( deployment_id, app_dir, app_file_name, additional_inputs=additional_inputs or {}, runtime_only_evaluation=runtime_only_evaluation ) # Moving the contents of the app dir to the dest dir, while # overwriting any file encountered # create the destination root dir file_server_deployment_root = \ os.path.join(file_server_root, FILE_SERVER_DEPLOYMENTS_FOLDER, current_tenant.name, deployment_id) app_root_dir = os.path.join(file_server_root, app_dir) for root, dirs, files in os.walk(app_root_dir): # Creates a corresponding dir structure in the deployment dir dest_rel_dir = os.path.relpath(root, app_root_dir) dest_dir = os.path.abspath( os.path.join(file_server_deployment_root, dest_rel_dir)) mkdirs(dest_dir) # Calculate source dir source_dir = os.path.join(file_server_root, app_dir, root) for file_name in files: source_file = os.path.join(source_dir, file_name) relative_dest_path = os.path.relpath(source_file, app_root_dir) dest_file = os.path.join(file_server_deployment_root, relative_dest_path) shutil.copy(source_file, dest_file) return update finally: shutil.rmtree(os.path.join(file_server_root, app_dir))
def _prepare_and_submit_blueprint(cls, file_server_root, app_dir, deployment_id, additional_inputs=None): app_dir, app_file_name = \ cls._extract_application_file(file_server_root, app_dir) # add to deployment update manager (will also dsl_parse it) try: cls._process_plugins(file_server_root, app_dir, deployment_id) update = get_deployment_updates_manager().stage_deployment_update( deployment_id, app_dir, app_file_name, additional_inputs=additional_inputs or {} ) # Moving the contents of the app dir to the dest dir, while # overwriting any file encountered # create the destination root dir file_server_deployment_root = \ os.path.join(file_server_root, config.instance().file_server_deployments_folder, deployment_id) app_root_dir = os.path.join(file_server_root, app_dir) for root, dirs, files in os.walk(app_root_dir): # Creates a corresponding dir structure in the deployment dir dest_rel_dir = os.path.relpath(root, app_root_dir) dest_dir = os.path.abspath( os.path.join(file_server_deployment_root, dest_rel_dir)) utils.mkdirs(dest_dir) # Calculate source dir source_dir = os.path.join(file_server_root, app_dir, root) for file_name in files: source_file = os.path.join(source_dir, file_name) relative_dest_path = os.path.relpath(source_file, app_root_dir) dest_file = os.path.join(file_server_deployment_root, relative_dest_path) shutil.copy(source_file, dest_file) return update except Exception: shutil.rmtree(os.path.join(file_server_root, app_dir)) raise
def _prepare_and_submit_blueprint(cls, file_server_root, app_dir, deployment_id, additional_inputs=None): app_dir, app_file_name = \ cls._extract_application_file(file_server_root, app_dir) # add to deployment update manager (will also dsl_parse it) try: cls._process_plugins(file_server_root, app_dir, deployment_id) update = get_deployment_updates_manager().stage_deployment_update( deployment_id, app_dir, app_file_name, additional_inputs=additional_inputs or {}) # Moving the contents of the app dir to the dest dir, while # overwriting any file encountered # create the destination root dir file_server_deployment_root = \ os.path.join(file_server_root, config.instance().file_server_deployments_folder, deployment_id) app_root_dir = os.path.join(file_server_root, app_dir) for root, dirs, files in os.walk(app_root_dir): # Creates a corresponding dir structure in the deployment dir dest_rel_dir = os.path.relpath(root, app_root_dir) dest_dir = os.path.abspath( os.path.join(file_server_deployment_root, dest_rel_dir)) utils.mkdirs(dest_dir) # Calculate source dir source_dir = os.path.join(file_server_root, app_dir, root) for file_name in files: source_file = os.path.join(source_dir, file_name) relative_dest_path = os.path.relpath( source_file, app_root_dir) dest_file = os.path.join(file_server_deployment_root, relative_dest_path) shutil.copy(source_file, dest_file) return update except Exception: shutil.rmtree(os.path.join(file_server_root, app_dir)) raise
def _set_local_dir(target_dir, logs_dir, build, edition): links = {} for root, dirs, files in os.walk(logs_dir): for log_file in files: if log_file in SKIP_FILES: continue abs_path = os.path.join(root, log_file) rel_path = abs_path.replace(logs_dir, '').strip('/').split('/') test_dir = '{0}-{1}-{2}'.format(build, rel_path[0], rel_path[1]) rel_target = os.path.join(edition, build, test_dir, *rel_path[2:]) abs_target = os.path.join(target_dir, rel_target) if abs_target.endswith('.log'): abs_target += '.txt' mkdirs(os.path.join('/', *abs_target.split('/')[:-1])) shutil.copy(abs_path, abs_target) links.setdefault(test_dir, []).append(rel_target) return links
def post(self, maintenance_action, **kwargs): maintenance_file_path = get_maintenance_file_path() if maintenance_action == 'activate': if os.path.isfile(maintenance_file_path): return {'status': MAINTENANCE_MODE_ACTIVE}, 304 utils.mkdirs(config.instance().maintenance_folder) write_maintenance_state(ACTIVATING_MAINTENANCE_MODE) return {'status': ACTIVATING_MAINTENANCE_MODE} if maintenance_action == 'deactivate': if not os.path.isfile(maintenance_file_path): return {'status': NOT_IN_MAINTENANCE_MODE}, 304 os.remove(maintenance_file_path) return {'status': NOT_IN_MAINTENANCE_MODE}
def _prepare_and_submit_blueprint(cls, file_server_root, app_dir, blueprint_id, visibility): args = get_args_and_verify_arguments([ Argument('private_resource', type=boolean), Argument('visibility'), Argument('application_file_name', default='')]) app_file_name = cls._extract_application_file( file_server_root, app_dir, args.application_file_name) # add to blueprints manager (will also dsl_parse it) try: blueprint = get_resource_manager().publish_blueprint( app_dir, app_file_name, file_server_root, blueprint_id, args.private_resource, visibility ) # moving the app directory in the file server to be under a # directory named after the blueprint id tenant_dir = os.path.join( file_server_root, FILE_SERVER_BLUEPRINTS_FOLDER, current_tenant.name) mkdirs(tenant_dir) shutil.move(os.path.join(file_server_root, app_dir), os.path.join(tenant_dir, blueprint.id)) cls._process_plugins(file_server_root, blueprint.id) return blueprint except manager_exceptions.DslParseException, ex: shutil.rmtree(os.path.join(file_server_root, app_dir)) raise manager_exceptions.InvalidBlueprintError( 'Invalid blueprint - {0}'.format(ex.message))
def extract_blueprint_archive_to_file_server(self, blueprint_id, tenant): sm = get_resource_manager().sm file_server_root = config.instance.file_server_root local_path = os.path.join(config.instance.file_server_root, FILE_SERVER_UPLOADED_BLUEPRINTS_FOLDER, tenant, blueprint_id) for arc_type in SUPPORTED_ARCHIVE_TYPES: # attempting to find the archive file on the file system local_file_path = os.path.join( local_path, '{0}.{1}'.format(blueprint_id, arc_type)) if os.path.isfile(local_file_path): break else: error_msg = "Could not find blueprint's archive; " \ "Blueprint ID: {0}".format(blueprint_id) blueprint = sm.get(Blueprint, blueprint_id) blueprint.state = \ BlueprintUploadState.FAILED_EXTRACTING_TO_FILE_SERVER blueprint.error = error_msg sm.update(blueprint) raise manager_exceptions.NotFoundError(error_msg) try: app_dir = self._extract_file_to_file_server( local_file_path, file_server_root) except Exception as e: blueprint = sm.get(Blueprint, blueprint_id) blueprint.state = \ BlueprintUploadState.FAILED_EXTRACTING_TO_FILE_SERVER blueprint.error = str(e) sm.update(blueprint) remove(local_path) raise e tenant_dir = os.path.join(file_server_root, FILE_SERVER_BLUEPRINTS_FOLDER, tenant) mkdirs(tenant_dir) shutil.move(os.path.join(file_server_root, app_dir), os.path.join(tenant_dir, blueprint_id)) self._process_plugins(file_server_root, blueprint_id)