def _prepare_and_submit_blueprint(cls, file_server_root, app_dir,
blueprint_id, visibility):
args = get_args_and_verify_arguments([
Argument('private_resource', type=boolean),
Argument('visibility'),
Argument('application_file_name', default='')
])
app_file_name = cls._extract_application_file(
file_server_root, app_dir, args.application_file_name)
# add to blueprints manager (will also dsl_parse it)
try:
blueprint = get_resource_manager().publish_blueprint(
app_dir, app_file_name, file_server_root, blueprint_id,
args.private_resource, visibility)
# moving the app directory in the file server to be under a
# directory named after the blueprint id
tenant_dir = os.path.join(file_server_root,
FILE_SERVER_BLUEPRINTS_FOLDER,
current_tenant.name)
mkdirs(tenant_dir)
shutil.move(os.path.join(file_server_root, app_dir),
os.path.join(tenant_dir, blueprint.id))
cls._process_plugins(file_server_root, blueprint.id)
return blueprint
except manager_exceptions.DslParseException, ex:
shutil.rmtree(os.path.join(file_server_root, app_dir))
raise manager_exceptions.InvalidBlueprintError(
'Invalid blueprint - {0}'.format(ex.message))
def post(self, maintenance_action, **_):
maintenance_file_path = get_maintenance_file_path()
if maintenance_action == 'activate':
if os.path.isfile(maintenance_file_path):
state = utils.read_json_file(maintenance_file_path)
return state, 304
now = utils.get_formatted_timestamp()
try:
user = current_user.username
except AttributeError:
user = ''
remaining_executions = get_running_executions()
status = MAINTENANCE_MODE_ACTIVATING \
if remaining_executions else MAINTENANCE_MODE_ACTIVATED
activated_at = '' if remaining_executions else now
utils.mkdirs(config.instance.maintenance_folder)
new_state = prepare_maintenance_dict(
status=status,
activation_requested_at=now,
activated_at=activated_at,
remaining_executions=remaining_executions,
requested_by=user)
utils.write_dict_to_json_file(maintenance_file_path, new_state)
return new_state
if maintenance_action == 'deactivate':
if not os.path.isfile(maintenance_file_path):
return prepare_maintenance_dict(
MAINTENANCE_MODE_DEACTIVATED), 304
os.remove(maintenance_file_path)
return prepare_maintenance_dict(MAINTENANCE_MODE_DEACTIVATED)
valid_actions = ['activate', 'deactivate']
raise BadParametersError('Invalid action: {0}, Valid action '
'values are: {1}'.format(
maintenance_action, valid_actions))
def _save_manager_logs_after_test(self, purge=True):
self.logger.debug('_save_manager_logs_after_test started')
logs_dir = os.environ.get('CFY_LOGS_PATH')
test_path = self.id().split('.')[-2:]
if not logs_dir:
self.logger.debug('not saving manager logs')
return
if os.environ.get('SKIP_LOG_SAVE_ON_SUCCESS') \
and sys.exc_info() == (None, None, None):
self.logger.info('Not saving manager logs for successful test: '
'{0}'.format(test_path[-1]))
return
self.logger.info('Saving manager logs for test: {0}'.format(
test_path[-1]))
logs_dir = os.path.join(os.path.expanduser(logs_dir), *test_path)
mkdirs(logs_dir)
target = os.path.join(logs_dir, 'logs.tar.gz')
self.cfy.logs.download(output_path=target)
if purge:
self.cfy.logs.purge(force=True)
self.logger.debug('opening tar.gz: {0}'.format(target))
with tarfile.open(target) as tar:
tar.extractall(path=logs_dir)
self.logger.debug('removing {0}'.format(target))
os.remove(target)
self.logger.debug('_save_manager_logs_after_test completed')
def _prepare_and_submit_blueprint(cls, file_server_root, app_dir,
blueprint_id):
args = cls._get_args()
app_dir, app_file_name = cls._extract_application_file(
file_server_root, app_dir, args.application_file_name)
# add to blueprints manager (will also dsl_parse it)
try:
blueprint = get_resource_manager().publish_blueprint(
app_dir,
app_file_name,
file_server_root,
blueprint_id,
private_resource=args.private_resource)
# moving the app directory in the file server to be under a
# directory named after the blueprint id
tenant_dir = os.path.join(
file_server_root, FILE_SERVER_BLUEPRINTS_FOLDER,
current_app.config[CURRENT_TENANT_CONFIG].name)
mkdirs(tenant_dir)
shutil.move(os.path.join(file_server_root, app_dir),
os.path.join(tenant_dir, blueprint.id))
cls._process_plugins(file_server_root, blueprint.id)
return blueprint
except manager_exceptions.DslParseException, ex:
shutil.rmtree(os.path.join(file_server_root, app_dir))
raise manager_exceptions.InvalidBlueprintError(
'Invalid blueprint - {0}'.format(ex.message))
def _save_manager_logs_after_test(self, purge=True):
self.logger.debug('_save_manager_logs_after_test started')
logs_dir = os.environ.get('CFY_LOGS_PATH_REMOTE')
self.logger.info(
"Cloudify remote log saving path found: [{}]. If "
"you're running via itest-runner, make sure to set a "
"local path as well with CFY_LOGS_PATH_LOCAL.".format(logs_dir))
test_path = self.id().split('.')[-2:]
if not logs_dir:
self.logger.debug('not saving manager logs')
return
if os.environ.get('SKIP_LOG_SAVE_ON_SUCCESS') \
and sys.exc_info() == (None, None, None):
self.logger.info('Not saving manager logs for successful test: '
'{0}'.format(test_path[-1]))
return
self.logger.info('Saving manager logs for test: {0}'.format(
test_path[-1]))
logs_dir = os.path.join(os.path.expanduser(logs_dir), *test_path)
mkdirs(logs_dir)
target = os.path.join(logs_dir, 'logs.tar.gz')
self.cfy.logs.download(output_path=target)
if purge:
self.cfy.logs.purge(force=True)
if not bool(os.environ.get('SKIP_LOGS_EXTRACTION')):
with tarfile.open(target) as tar:
self.logger.debug('Extracting tar.gz: {0}'.format(target))
tar.extractall(path=logs_dir)
self.logger.debug('Removing {0}'.format(target))
os.remove(target)
self.logger.debug('_save_manager_logs_after_test completed')
def post(self, maintenance_action, **_):
maintenance_file_path = get_maintenance_file_path()
if maintenance_action == 'activate':
if os.path.isfile(maintenance_file_path):
state = utils.read_json_file(maintenance_file_path)
return state, 304
now = utils.get_formatted_timestamp()
try:
user = current_user.username
except AttributeError:
user = ''
remaining_executions = get_running_executions()
status = MAINTENANCE_MODE_ACTIVATING \
if remaining_executions else MAINTENANCE_MODE_ACTIVATED
activated_at = '' if remaining_executions else now
utils.mkdirs(config.instance.maintenance_folder)
new_state = prepare_maintenance_dict(
status=status,
activation_requested_at=now,
activated_at=activated_at,
remaining_executions=remaining_executions,
requested_by=user)
utils.write_dict_to_json_file(maintenance_file_path, new_state)
return new_state
if maintenance_action == 'deactivate':
if not os.path.isfile(maintenance_file_path):
return prepare_maintenance_dict(
MAINTENANCE_MODE_DEACTIVATED), 304
os.remove(maintenance_file_path)
return prepare_maintenance_dict(MAINTENANCE_MODE_DEACTIVATED)
valid_actions = ['activate', 'deactivate']
raise BadParametersError(
'Invalid action: {0}, Valid action '
'values are: {1}'.format(maintenance_action, valid_actions))
def _save_logs(self, purge=True):
self.logger.debug('save_logs started')
logs_dir = os.environ.get('CFY_LOGS_PATH')
test_path = self.id().split('.')[1:]
if not logs_dir:
self.logger.info('Saving manager logs is disabled by configuration'
' for test: {0}'.format(test_path[-1]))
self.logger.info('To enable logs keeping, define "CFY_LOGS_PATH"')
return
self.logger.info('Saving manager logs for test: {0}'.format(
test_path[-1]))
logs_dir = os.path.join(os.path.expanduser(logs_dir), *test_path)
mkdirs(logs_dir)
target = os.path.join(logs_dir, 'logs.tar.gz')
self.cfy.logs.download(output_path=target)
if purge:
self.cfy.logs.purge(force=True)
self.logger.debug('opening tar.gz: {0}'.format(target))
with tarfile.open(target) as tar:
tar.extractall(path=logs_dir)
self.logger.debug('removing {0}'.format(target))
os.remove(target)
self.logger.debug('save_logs completed')
def _save_logs(self, purge=True):
logs_dir = os.environ.get('CFY_LOGS_PATH')
if not logs_dir:
return
logs_dir = os.path.join(logs_dir, *self.id().split('.'))
mkdirs(logs_dir)
self.cfy.logs.download(output_path=logs_dir)
if purge:
self.cfy.logs.purge(force=True)
def _prepare_and_submit_blueprint(cls,
file_server_root,
app_dir,
deployment_id,
additional_inputs=None,
runtime_only_evaluation=False):
app_file_name = cls._extract_application_file(file_server_root,
app_dir)
# add to deployment update manager (will also dsl_parse it)
try:
cls._process_plugins(file_server_root, app_dir)
update = get_deployment_updates_manager().stage_deployment_update(
deployment_id,
app_dir,
app_file_name,
additional_inputs=additional_inputs or {},
runtime_only_evaluation=runtime_only_evaluation
)
# Moving the contents of the app dir to the dest dir, while
# overwriting any file encountered
# create the destination root dir
file_server_deployment_root = \
os.path.join(file_server_root,
FILE_SERVER_DEPLOYMENTS_FOLDER,
current_tenant.name,
deployment_id)
app_root_dir = os.path.join(file_server_root, app_dir)
for root, dirs, files in os.walk(app_root_dir):
# Creates a corresponding dir structure in the deployment dir
dest_rel_dir = os.path.relpath(root, app_root_dir)
dest_dir = os.path.abspath(
os.path.join(file_server_deployment_root,
dest_rel_dir))
mkdirs(dest_dir)
# Calculate source dir
source_dir = os.path.join(file_server_root, app_dir, root)
for file_name in files:
source_file = os.path.join(source_dir, file_name)
relative_dest_path = os.path.relpath(source_file,
app_root_dir)
dest_file = os.path.join(file_server_deployment_root,
relative_dest_path)
shutil.copy(source_file, dest_file)
return update
finally:
shutil.rmtree(os.path.join(file_server_root, app_dir))
def _prepare_and_submit_blueprint(cls,
file_server_root,
app_dir,
deployment_id,
additional_inputs=None):
app_dir, app_file_name = \
cls._extract_application_file(file_server_root, app_dir)
# add to deployment update manager (will also dsl_parse it)
try:
cls._process_plugins(file_server_root, app_dir, deployment_id)
update = get_deployment_updates_manager().stage_deployment_update(
deployment_id,
app_dir,
app_file_name,
additional_inputs=additional_inputs or {}
)
# Moving the contents of the app dir to the dest dir, while
# overwriting any file encountered
# create the destination root dir
file_server_deployment_root = \
os.path.join(file_server_root,
config.instance().file_server_deployments_folder,
deployment_id)
app_root_dir = os.path.join(file_server_root, app_dir)
for root, dirs, files in os.walk(app_root_dir):
# Creates a corresponding dir structure in the deployment dir
dest_rel_dir = os.path.relpath(root, app_root_dir)
dest_dir = os.path.abspath(
os.path.join(file_server_deployment_root,
dest_rel_dir))
utils.mkdirs(dest_dir)
# Calculate source dir
source_dir = os.path.join(file_server_root, app_dir, root)
for file_name in files:
source_file = os.path.join(source_dir, file_name)
relative_dest_path = os.path.relpath(source_file,
app_root_dir)
dest_file = os.path.join(file_server_deployment_root,
relative_dest_path)
shutil.copy(source_file, dest_file)
return update
except Exception:
shutil.rmtree(os.path.join(file_server_root, app_dir))
raise
def _prepare_and_submit_blueprint(cls,
file_server_root,
app_dir,
deployment_id,
additional_inputs=None):
app_dir, app_file_name = \
cls._extract_application_file(file_server_root, app_dir)
# add to deployment update manager (will also dsl_parse it)
try:
cls._process_plugins(file_server_root, app_dir, deployment_id)
update = get_deployment_updates_manager().stage_deployment_update(
deployment_id,
app_dir,
app_file_name,
additional_inputs=additional_inputs or {})
# Moving the contents of the app dir to the dest dir, while
# overwriting any file encountered
# create the destination root dir
file_server_deployment_root = \
os.path.join(file_server_root,
config.instance().file_server_deployments_folder,
deployment_id)
app_root_dir = os.path.join(file_server_root, app_dir)
for root, dirs, files in os.walk(app_root_dir):
# Creates a corresponding dir structure in the deployment dir
dest_rel_dir = os.path.relpath(root, app_root_dir)
dest_dir = os.path.abspath(
os.path.join(file_server_deployment_root, dest_rel_dir))
utils.mkdirs(dest_dir)
# Calculate source dir
source_dir = os.path.join(file_server_root, app_dir, root)
for file_name in files:
source_file = os.path.join(source_dir, file_name)
relative_dest_path = os.path.relpath(
source_file, app_root_dir)
dest_file = os.path.join(file_server_deployment_root,
relative_dest_path)
shutil.copy(source_file, dest_file)
return update
except Exception:
shutil.rmtree(os.path.join(file_server_root, app_dir))
raise
def _set_local_dir(target_dir, logs_dir, build, edition):
links = {}
for root, dirs, files in os.walk(logs_dir):
for log_file in files:
if log_file in SKIP_FILES:
continue
abs_path = os.path.join(root, log_file)
rel_path = abs_path.replace(logs_dir, '').strip('/').split('/')
test_dir = '{0}-{1}-{2}'.format(build, rel_path[0], rel_path[1])
rel_target = os.path.join(edition, build, test_dir, *rel_path[2:])
abs_target = os.path.join(target_dir, rel_target)
if abs_target.endswith('.log'):
abs_target += '.txt'
mkdirs(os.path.join('/', *abs_target.split('/')[:-1]))
shutil.copy(abs_path, abs_target)
links.setdefault(test_dir, []).append(rel_target)
return links
def post(self, maintenance_action, **kwargs):
maintenance_file_path = get_maintenance_file_path()
if maintenance_action == 'activate':
if os.path.isfile(maintenance_file_path):
return {'status': MAINTENANCE_MODE_ACTIVE}, 304
utils.mkdirs(config.instance().maintenance_folder)
write_maintenance_state(ACTIVATING_MAINTENANCE_MODE)
return {'status': ACTIVATING_MAINTENANCE_MODE}
if maintenance_action == 'deactivate':
if not os.path.isfile(maintenance_file_path):
return {'status': NOT_IN_MAINTENANCE_MODE}, 304
os.remove(maintenance_file_path)
return {'status': NOT_IN_MAINTENANCE_MODE}
def _set_local_dir(target_dir, logs_dir, build, edition):
links = {}
for root, dirs, files in os.walk(logs_dir):
for log_file in files:
if log_file in SKIP_FILES:
continue
abs_path = os.path.join(root, log_file)
rel_path = abs_path.replace(logs_dir, '').strip('/').split('/')
test_dir = '{0}-{1}-{2}'.format(build, rel_path[0], rel_path[1])
rel_target = os.path.join(edition, build, test_dir, *rel_path[2:])
abs_target = os.path.join(target_dir, rel_target)
if abs_target.endswith('.log'):
abs_target += '.txt'
mkdirs(os.path.join('/', *abs_target.split('/')[:-1]))
shutil.copy(abs_path, abs_target)
links.setdefault(test_dir, []).append(rel_target)
return links
def _prepare_and_submit_blueprint(cls,
file_server_root,
app_dir,
blueprint_id,
visibility):
args = get_args_and_verify_arguments([
Argument('private_resource', type=boolean),
Argument('visibility'),
Argument('application_file_name',
default='')])
app_file_name = cls._extract_application_file(
file_server_root, app_dir, args.application_file_name)
# add to blueprints manager (will also dsl_parse it)
try:
blueprint = get_resource_manager().publish_blueprint(
app_dir,
app_file_name,
file_server_root,
blueprint_id,
args.private_resource,
visibility
)
# moving the app directory in the file server to be under a
# directory named after the blueprint id
tenant_dir = os.path.join(
file_server_root,
FILE_SERVER_BLUEPRINTS_FOLDER,
current_tenant.name)
mkdirs(tenant_dir)
shutil.move(os.path.join(file_server_root, app_dir),
os.path.join(tenant_dir, blueprint.id))
cls._process_plugins(file_server_root, blueprint.id)
return blueprint
except manager_exceptions.DslParseException, ex:
shutil.rmtree(os.path.join(file_server_root, app_dir))
raise manager_exceptions.InvalidBlueprintError(
'Invalid blueprint - {0}'.format(ex.message))
def extract_blueprint_archive_to_file_server(self, blueprint_id, tenant):
sm = get_resource_manager().sm
file_server_root = config.instance.file_server_root
local_path = os.path.join(config.instance.file_server_root,
FILE_SERVER_UPLOADED_BLUEPRINTS_FOLDER,
tenant, blueprint_id)
for arc_type in SUPPORTED_ARCHIVE_TYPES:
# attempting to find the archive file on the file system
local_file_path = os.path.join(
local_path, '{0}.{1}'.format(blueprint_id, arc_type))
if os.path.isfile(local_file_path):
break
else:
error_msg = "Could not find blueprint's archive; " \
"Blueprint ID: {0}".format(blueprint_id)
blueprint = sm.get(Blueprint, blueprint_id)
blueprint.state = \
BlueprintUploadState.FAILED_EXTRACTING_TO_FILE_SERVER
blueprint.error = error_msg
sm.update(blueprint)
raise manager_exceptions.NotFoundError(error_msg)
try:
app_dir = self._extract_file_to_file_server(
local_file_path, file_server_root)
except Exception as e:
blueprint = sm.get(Blueprint, blueprint_id)
blueprint.state = \
BlueprintUploadState.FAILED_EXTRACTING_TO_FILE_SERVER
blueprint.error = str(e)
sm.update(blueprint)
remove(local_path)
raise e
tenant_dir = os.path.join(file_server_root,
FILE_SERVER_BLUEPRINTS_FOLDER, tenant)
mkdirs(tenant_dir)
shutil.move(os.path.join(file_server_root, app_dir),
os.path.join(tenant_dir, blueprint_id))
self._process_plugins(file_server_root, blueprint_id)
