def start_manager_rest(self): from manager_rest.file_server import PORT as FS_PORT file_server_base_uri = 'http://localhost:{0}'.format(FS_PORT) self.manager_rest_process = ManagerRestProcess( MANAGER_REST_PORT, self.fileserver_dir, file_server_base_uri, FILE_SERVER_BLUEPRINTS_FOLDER, FILE_SERVER_UPLOADED_BLUEPRINTS_FOLDER, FILE_SERVER_RESOURCES_URI, self.test_working_dir) self.manager_rest_process.start()
def start_manager_rest(self): from manager_rest.file_server import PORT as FS_PORT file_server_base_uri = 'http://localhost:{0}'.format(FS_PORT) self.manager_rest_process = ManagerRestProcess( MANAGER_REST_PORT, self.fileserver_dir, file_server_base_uri, FILE_SERVER_BLUEPRINTS_FOLDER, FILE_SERVER_DEPLOYMENTS_FOLDER, FILE_SERVER_UPLOADED_BLUEPRINTS_FOLDER, FILE_SERVER_RESOURCES_URI, self.rest_service_log_level, self.rest_service_log_path, self.rest_service_log_file_size_MB, self.rest_service_log_files_backup_count, self.securest_log_level, self.securest_log_file, self.securest_log_file_size_MB, self.securest_log_files_backup_count, self.test_working_dir, self.amqp_username, self.amqp_password, self.maintenance_folder) self.manager_rest_process.start()
def start_manager_rest(self): from manager_rest.file_server import PORT as FS_PORT file_server_base_uri = 'http://localhost:{0}'.format(FS_PORT) self.manager_rest_process = ManagerRestProcess( MANAGER_REST_PORT, self.fileserver_dir, file_server_base_uri, FILE_SERVER_BLUEPRINTS_FOLDER, FILE_SERVER_UPLOADED_BLUEPRINTS_FOLDER, FILE_SERVER_RESOURCES_URI, self.rest_service_log_level, self.rest_service_log_path, self.rest_service_log_file_size_MB, self.rest_service_log_files_backup_count, self.securest_log_level, self.securest_log_file, self.securest_log_file_size_MB, self.securest_log_files_backup_count, self.test_working_dir, self.amqp_username, self.amqp_password) self.manager_rest_process.start()
class TestEnvironment(object): manager_rest_process = None elasticsearch_process = None riemann_process = None file_server_process = None celery_management_worker_process = None def __init__(self, test_working_dir): super(TestEnvironment, self).__init__() self.test_working_dir = test_working_dir self.plugins_storage_dir = os.path.join( self.test_working_dir, 'plugins-storage' ) os.makedirs(self.plugins_storage_dir) self.fileserver_dir = path.join(self.test_working_dir, 'fileserver') self.maintenance_folder = path.join(self.test_working_dir, 'maintenance') self.rest_service_log_level = 'DEBUG' self.rest_service_log_path = path.join( self.test_working_dir, 'cloudify-rest-service.log') self.rest_service_log_file_size_MB = 100 self.rest_service_log_files_backup_count = 20 self.securest_log_level = 'DEBUG' self.securest_log_file = path.join( self.test_working_dir, 'rest-security-audit.log') self.securest_log_file_size_MB = 100 self.securest_log_files_backup_count = 20 self.amqp_username = '******' self.amqp_password = '******' self.events_and_logs_dir = \ path.join(self.test_working_dir, 'tests-events-and-logs') os.mkdir(self.events_and_logs_dir) def create(self): try: logger.info('Setting up test environment... workdir=[{0}]' .format(self.test_working_dir)) # events/logs polling start_events_and_logs_polling( logs_handler_retriever=self._logs_handler_retriever) self.start_elasticsearch() self.start_riemann() self.start_fileserver() self.start_manager_rest() self.create_management_worker() except BaseException as error: s_traceback = StringIO.StringIO() traceback.print_exc(file=s_traceback) logger.error("Error in test environment setup: %s", error) logger.error(s_traceback.getvalue()) self.destroy() raise def create_management_worker(self): mock_plugins_path = os.path.dirname(mock_plugins.__file__) os.environ['MOCK_PLUGINS_PATH'] = mock_plugins_path self.celery_management_worker_process = CeleryWorkerProcess( queues=['cloudify.management'], test_working_dir=self.test_working_dir, # we need high concurrency since all management and # central deployment operations/workflow will be executed # by this worker concurrency=10 ) # copy plugins to worker env mock_plugins_path = os.path.dirname(mock_plugins.__file__) shutil.copytree( src=mock_plugins_path, dst=self.celery_management_worker_process.envdir, ignore=shutil.ignore_patterns('*.pyc') ) def start_riemann(self): riemann_config_path = self._get_riemann_config() libs_path = self._get_libs_path() self.riemann_process = RiemannProcess(riemann_config_path, libs_path) self.riemann_process.start() def start_manager_rest(self): from manager_rest.file_server import PORT as FS_PORT file_server_base_uri = 'http://*****:*****@timestamp'] = timestamp es_client = Elasticsearch() doc_type = event['type'] # simulate log index res = es_client.index(index=index, doc_type=doc_type, body=event) if not res['created']: raise Exception('failed to write to elasticsearch') self.handle_logs = es_log_handler def _logs_handler_retriever(self): return self.handle_logs @classmethod def _get_riemann_config(cls): manager_dir = cls._get_manager_root() plugins_dir = os.path.join(manager_dir, 'plugins') riemann_dir = os.path.join(plugins_dir, 'riemann-controller') package_dir = os.path.join(riemann_dir, 'riemann_controller') resources_dir = os.path.join(package_dir, 'resources') manager_config = os.path.join(resources_dir, 'manager.config') return manager_config @classmethod def _get_libs_path(cls): return path.join(cls._get_manager_root(), '.libs') @staticmethod def reset_elasticsearch_data(): global testenv_instance testenv_instance.elasticsearch_process.reset_data() @staticmethod def stop_celery_management_worker(): global testenv_instance testenv_instance.celery_management_worker_process.stop() @staticmethod def read_celery_management_logs(): global testenv_instance process = testenv_instance.celery_management_worker_process return process.try_read_logfile() @classmethod def stop_all_celery_processes(cls): logger.info('Shutting down all celery processes') os.system("pkill -9 -f 'celery worker'") @classmethod def stop_dispatch_processes(cls): logger.info('Shutting down all dispatch processes') os.system("pkill -9 -f 'cloudify/dispatch.py'") @staticmethod def start_celery_management_worker(): global testenv_instance testenv_instance.celery_management_worker_process.start() @staticmethod def riemann_cleanup(): global testenv_instance shutil.rmtree(TestEnvironment.riemann_workdir()) os.mkdir(TestEnvironment.riemann_workdir()) testenv_instance.riemann_process.restart() @staticmethod def riemann_workdir(): global testenv_instance return testenv_instance.\ celery_management_worker_process.\ riemann_config_dir @staticmethod def _get_manager_root(): init_file = __file__ testenv_dir = dirname(init_file) tests_dir = dirname(testenv_dir) manager_dir = dirname(tests_dir) return manager_dir @staticmethod def patch_source_urls(resources): with open(path.join(resources, 'cloudify', 'types', 'types.yaml')) as f: types_yaml = yaml.safe_load(f.read()) for policy_type in types_yaml.get('policy_types', {}).values(): in_path = '/cloudify/policies/' source = policy_type['source'] if in_path in source: source = source[source.index(in_path) + 1:] policy_type['source'] = source for policy_trigger in types_yaml.get('policy_triggers', {}).values(): in_path = '/cloudify/triggers/' source = policy_trigger['source'] if in_path in source: source = source[source.index(in_path) + 1:] policy_trigger['source'] = source with open(path.join(resources, 'cloudify', 'types', 'types.yaml'), 'w') as f: f.write(yaml.safe_dump(types_yaml))
class TestEnvironment(object): manager_rest_process = None elasticsearch_process = None riemann_process = None file_server_process = None celery_management_worker_process = None def __init__(self, test_working_dir): super(TestEnvironment, self).__init__() self.test_working_dir = test_working_dir self.plugins_storage_dir = os.path.join( self.test_working_dir, 'plugins-storage' ) os.makedirs(self.plugins_storage_dir) self.fileserver_dir = path.join(self.test_working_dir, 'fileserver') self.rest_service_log_path = path.join( self.test_working_dir, 'cloudify-rest-service.log') def create(self): try: logger.info('Setting up test environment... workdir=[{0}]' .format(self.test_working_dir)) # events/logs polling start_events_and_logs_polling() self.start_elasticsearch() self.start_riemann() self.start_fileserver() self.start_manager_rest() self.create_management_worker() except BaseException as error: s_traceback = StringIO.StringIO() traceback.print_exc(file=s_traceback) logger.error("Error in test environment setup: %s", error) logger.error(s_traceback.getvalue()) self.destroy() raise def create_management_worker(self): mock_plugins_path = os.path.dirname(mock_plugins.__file__) os.environ['MOCK_PLUGINS_PATH'] = mock_plugins_path self.celery_management_worker_process = CeleryWorkerProcess( queues=['cloudify.management'], test_working_dir=self.test_working_dir, # these plugins are already installed. # so we just need to append to the includes. # note that these are not mocks, but the actual production # code plugins. additional_includes=[ 'riemann_controller.tasks', 'cloudify_system_workflows.deployment_environment', 'cloudify.plugins.workflows', 'diamond_agent.tasks', 'script_runner.tasks' ], # we need higher concurrency since # 'deployment_environment.create' calls # 'plugin_installer.install' as a sub-task # and they are both executed inside # this worker concurrency=2 ) # copy plugins to worker env mock_plugins_path = os.path.dirname(mock_plugins.__file__) shutil.copytree( src=mock_plugins_path, dst=self.celery_management_worker_process.envdir, ignore=shutil.ignore_patterns('*.pyc') ) def start_riemann(self): riemann_config_path = self._get_riemann_config() libs_path = self._get_libs_path() self.riemann_process = RiemannProcess(riemann_config_path, libs_path) self.riemann_process.start() def start_manager_rest(self): from manager_rest.file_server import PORT as FS_PORT file_server_base_uri = 'http://localhost:{0}'.format(FS_PORT) self.manager_rest_process = ManagerRestProcess( MANAGER_REST_PORT, self.fileserver_dir, file_server_base_uri, FILE_SERVER_BLUEPRINTS_FOLDER, FILE_SERVER_UPLOADED_BLUEPRINTS_FOLDER, FILE_SERVER_RESOURCES_URI, self.rest_service_log_path, self.test_working_dir) self.manager_rest_process.start() def start_elasticsearch(self): # elasticsearch self.elasticsearch_process = ElasticSearchProcess() self.elasticsearch_process.start() def start_fileserver(self): # workaround to update path manager_rest_path = \ path.dirname(path.dirname(path.dirname(__file__))) manager_rest_path = path.join(manager_rest_path, 'rest-service') sys.path.append(manager_rest_path) os.mkdir(self.fileserver_dir) from manager_rest.file_server import FileServer from manager_rest.util import copy_resources self.file_server_process = FileServer(self.fileserver_dir) self.file_server_process.start() # copy resources (base yaml etc) resources_path = path.abspath(__file__) resources_path = path.dirname(resources_path) resources_path = path.dirname(resources_path) resources_path = path.dirname(resources_path) resources_path = path.join(resources_path, 'resources') copy_resources(self.fileserver_dir, resources_path) self.patch_source_urls(self.fileserver_dir) def destroy(self): logger.info('Destroying test environment...') if self.riemann_process: self.riemann_process.close() if self.elasticsearch_process: self.elasticsearch_process.close() if self.manager_rest_process: self.manager_rest_process.close() if self.file_server_process: self.file_server_process.stop() self.delete_working_directory() def delete_working_directory(self): if os.path.exists(self.test_working_dir): logger.info('Deleting test environment from: %s', self.test_working_dir) shutil.rmtree(self.test_working_dir, ignore_errors=True) @classmethod def _get_riemann_config(cls): manager_dir = cls._get_manager_root() plugins_dir = os.path.join(manager_dir, 'plugins') riemann_dir = os.path.join(plugins_dir, 'riemann-controller') package_dir = os.path.join(riemann_dir, 'riemann_controller') resources_dir = os.path.join(package_dir, 'resources') manager_config = os.path.join(resources_dir, 'manager.config') return manager_config @classmethod def _get_libs_path(cls): return path.join(cls._get_manager_root(), '.libs') @staticmethod def reset_elasticsearch_data(): global testenv_instance testenv_instance.elasticsearch_process.reset_data() @staticmethod def stop_celery_management_worker(): global testenv_instance testenv_instance.celery_management_worker_process.stop() @staticmethod def read_celery_management_logs(): global testenv_instance process = testenv_instance.celery_management_worker_process return process.try_read_logfile() @classmethod def stop_all_celery_processes(cls): logger.info('Shutting down all celery processes') os.system("pkill -9 -f 'celery worker'") @staticmethod def start_celery_management_worker(): global testenv_instance testenv_instance.celery_management_worker_process.start() @staticmethod def riemann_cleanup(): global testenv_instance shutil.rmtree(TestEnvironment.riemann_workdir()) os.mkdir(TestEnvironment.riemann_workdir()) testenv_instance.riemann_process.restart() @staticmethod def riemann_workdir(): global testenv_instance return testenv_instance.\ celery_management_worker_process.\ riemann_config_dir @staticmethod def _get_manager_root(): init_file = __file__ testenv_dir = dirname(init_file) tests_dir = dirname(testenv_dir) manager_dir = dirname(tests_dir) return manager_dir @staticmethod def patch_source_urls(resources): with open(path.join(resources, 'cloudify', 'types', 'types.yaml')) as f: types_yaml = yaml.safe_load(f.read()) for policy_type in types_yaml.get('policy_types', {}).values(): in_path = '/cloudify/policies/' source = policy_type['source'] if in_path in source: source = source[source.index(in_path) + 1:] policy_type['source'] = source for policy_trigger in types_yaml.get('policy_triggers', {}).values(): in_path = '/cloudify/triggers/' source = policy_trigger['source'] if in_path in source: source = source[source.index(in_path) + 1:] policy_trigger['source'] = source with open(path.join(resources, 'cloudify', 'types', 'types.yaml'), 'w') as f: f.write(yaml.safe_dump(types_yaml))
class TestEnvironment(object): manager_rest_process = None elasticsearch_process = None riemann_process = None file_server_process = None celery_management_worker_process = None def __init__(self, test_working_dir): super(TestEnvironment, self).__init__() self.test_working_dir = test_working_dir self.plugins_storage_dir = os.path.join(self.test_working_dir, 'plugins-storage') os.makedirs(self.plugins_storage_dir) self.fileserver_dir = path.join(self.test_working_dir, 'fileserver') self.rest_service_log_level = 'DEBUG' self.rest_service_log_path = path.join(self.test_working_dir, 'cloudify-rest-service.log') self.rest_service_log_file_size_MB = 100 self.rest_service_log_files_backup_count = 20 self.securest_log_level = 'DEBUG' self.securest_log_file = path.join(self.test_working_dir, 'rest-security-audit.log') self.securest_log_file_size_MB = 100 self.securest_log_files_backup_count = 20 self.amqp_username = '******' self.amqp_password = '******' self.events_and_logs_dir = \ path.join(self.test_working_dir, 'tests-events-and-logs') os.mkdir(self.events_and_logs_dir) def create(self): try: logger.info('Setting up test environment... workdir=[{0}]'.format( self.test_working_dir)) # events/logs polling start_events_and_logs_polling( logs_handler_retriever=self._logs_handler_retriever) self.start_elasticsearch() self.start_riemann() self.start_fileserver() self.start_manager_rest() self.create_management_worker() except BaseException as error: s_traceback = StringIO.StringIO() traceback.print_exc(file=s_traceback) logger.error("Error in test environment setup: %s", error) logger.error(s_traceback.getvalue()) self.destroy() raise def create_management_worker(self): mock_plugins_path = os.path.dirname(mock_plugins.__file__) os.environ['MOCK_PLUGINS_PATH'] = mock_plugins_path self.celery_management_worker_process = CeleryWorkerProcess( queues=['cloudify.management'], test_working_dir=self.test_working_dir, # these plugins are already installed. # so we just need to append to the includes. # note that these are not mocks, but the actual production # code plugins. additional_includes=[ 'riemann_controller.tasks', 'cloudify_system_workflows.deployment_environment', 'cloudify.plugins.workflows', 'diamond_agent.tasks', 'script_runner.tasks', # modules in the agent intended for backwards compatibility 'worker_installer.tasks', 'windows_agent_installer.tasks', 'plugin_installer.tasks', 'windows_plugin_installer.tasks', ], # we need higher concurrency since # 'deployment_environment.create' calls # 'plugin_installer.install' as a sub-task # and they are both executed inside # this worker concurrency=2) # copy plugins to worker env mock_plugins_path = os.path.dirname(mock_plugins.__file__) shutil.copytree(src=mock_plugins_path, dst=self.celery_management_worker_process.envdir, ignore=shutil.ignore_patterns('*.pyc')) def start_riemann(self): riemann_config_path = self._get_riemann_config() libs_path = self._get_libs_path() self.riemann_process = RiemannProcess(riemann_config_path, libs_path) self.riemann_process.start() def start_manager_rest(self): from manager_rest.file_server import PORT as FS_PORT file_server_base_uri = 'http://*****:*****@timestamp'] = timestamp es_client = Elasticsearch() doc_type = event['type'] # simulate log index res = es_client.index(index=index, doc_type=doc_type, body=event) if not res['created']: raise Exception('failed to write to elasticsearch') self.handle_logs = es_log_handler def _logs_handler_retriever(self): return self.handle_logs @classmethod def _get_riemann_config(cls): manager_dir = cls._get_manager_root() plugins_dir = os.path.join(manager_dir, 'plugins') riemann_dir = os.path.join(plugins_dir, 'riemann-controller') package_dir = os.path.join(riemann_dir, 'riemann_controller') resources_dir = os.path.join(package_dir, 'resources') manager_config = os.path.join(resources_dir, 'manager.config') return manager_config @classmethod def _get_libs_path(cls): return path.join(cls._get_manager_root(), '.libs') @staticmethod def reset_elasticsearch_data(): global testenv_instance testenv_instance.elasticsearch_process.reset_data() @staticmethod def stop_celery_management_worker(): global testenv_instance testenv_instance.celery_management_worker_process.stop() @staticmethod def read_celery_management_logs(): global testenv_instance process = testenv_instance.celery_management_worker_process return process.try_read_logfile() @classmethod def stop_all_celery_processes(cls): logger.info('Shutting down all celery processes') os.system("pkill -9 -f 'celery worker'") @staticmethod def start_celery_management_worker(): global testenv_instance testenv_instance.celery_management_worker_process.start() @staticmethod def riemann_cleanup(): global testenv_instance shutil.rmtree(TestEnvironment.riemann_workdir()) os.mkdir(TestEnvironment.riemann_workdir()) testenv_instance.riemann_process.restart() @staticmethod def riemann_workdir(): global testenv_instance return testenv_instance.\ celery_management_worker_process.\ riemann_config_dir @staticmethod def _get_manager_root(): init_file = __file__ testenv_dir = dirname(init_file) tests_dir = dirname(testenv_dir) manager_dir = dirname(tests_dir) return manager_dir @staticmethod def patch_source_urls(resources): with open(path.join(resources, 'cloudify', 'types', 'types.yaml')) as f: types_yaml = yaml.safe_load(f.read()) for policy_type in types_yaml.get('policy_types', {}).values(): in_path = '/cloudify/policies/' source = policy_type['source'] if in_path in source: source = source[source.index(in_path) + 1:] policy_type['source'] = source for policy_trigger in types_yaml.get('policy_triggers', {}).values(): in_path = '/cloudify/triggers/' source = policy_trigger['source'] if in_path in source: source = source[source.index(in_path) + 1:] policy_trigger['source'] = source with open(path.join(resources, 'cloudify', 'types', 'types.yaml'), 'w') as f: f.write(yaml.safe_dump(types_yaml))
class TestEnvironment(object): manager_rest_process = None elasticsearch_process = None riemann_process = None file_server_process = None celery_management_worker_process = None def __init__(self, test_working_dir): super(TestEnvironment, self).__init__() self.test_working_dir = test_working_dir self.fileserver_dir = path.join(self.test_working_dir, 'fileserver') def create(self): try: logger.info('Setting up test environment... workdir=[{0}]' .format(self.test_working_dir)) # events/logs polling start_events_and_logs_polling() self.start_elasticsearch() self.start_riemann() self.start_fileserver() self.start_manager_rest() self.create_management_worker() except BaseException as error: s_traceback = StringIO.StringIO() traceback.print_exc(file=s_traceback) logger.error("Error in test environment setup: %s", error) logger.error(s_traceback.getvalue()) self.destroy() raise def create_management_worker(self): self.celery_management_worker_process = CeleryWorkerProcess( queues=['cloudify.management'], test_working_dir=self.test_working_dir, # these plugins are already installed. # so we just need to append to the includes. # note that these are not mocks, but the actual production # code plugins. additional_includes=[ 'riemann_controller.tasks', 'cloudify_system_workflows.deployment_environment', 'cloudify.plugins.workflows', 'diamond_agent.tasks', 'script_runner.tasks' ], # we need higher concurrency since # 'deployment_environment.create' calls # 'plugin_installer.install' as a sub-task # and they are both executed inside # this worker concurrency=2 ) # copy plugins to worker env import mock_plugins mock_plugins_path = os.path.dirname(mock_plugins.__file__) shutil.copytree( src=mock_plugins_path, dst=self.celery_management_worker_process.envdir, ignore=shutil.ignore_patterns('*.pyc') ) def start_riemann(self): riemann_config_path = self._get_riemann_config() libs_path = self._get_libs_path() self.riemann_process = RiemannProcess(riemann_config_path, libs_path) self.riemann_process.start() def start_manager_rest(self): from manager_rest.file_server import PORT as FS_PORT file_server_base_uri = 'http://localhost:{0}'.format(FS_PORT) self.manager_rest_process = ManagerRestProcess( MANAGER_REST_PORT, self.fileserver_dir, file_server_base_uri, FILE_SERVER_BLUEPRINTS_FOLDER, FILE_SERVER_UPLOADED_BLUEPRINTS_FOLDER, FILE_SERVER_RESOURCES_URI, self.test_working_dir) self.manager_rest_process.start() def start_elasticsearch(self): # elasticsearch self.elasticsearch_process = ElasticSearchProcess() self.elasticsearch_process.start() def start_fileserver(self): # workaround to update path manager_rest_path = \ path.dirname(path.dirname(path.dirname(__file__))) manager_rest_path = path.join(manager_rest_path, 'rest-service') sys.path.append(manager_rest_path) os.mkdir(self.fileserver_dir) from manager_rest.file_server import FileServer from manager_rest.util import copy_resources self.file_server_process = FileServer(self.fileserver_dir) self.file_server_process.start() # copy resources (base yaml etc) resources_path = path.abspath(__file__) resources_path = path.dirname(resources_path) resources_path = path.dirname(resources_path) resources_path = path.dirname(resources_path) resources_path = path.join(resources_path, 'resources') copy_resources(self.fileserver_dir, resources_path) self.patch_source_urls(self.fileserver_dir) def destroy(self): logger.info('Destroying test environment...') if self.riemann_process: self.riemann_process.close() if self.elasticsearch_process: self.elasticsearch_process.close() if self.manager_rest_process: self.manager_rest_process.close() if self.file_server_process: self.file_server_process.stop() self.delete_working_directory() def delete_working_directory(self): if os.path.exists(self.test_working_dir): logger.info('Deleting test environment from: %s', self.test_working_dir) shutil.rmtree(self.test_working_dir, ignore_errors=True) @classmethod def _get_riemann_config(cls): manager_dir = cls._get_manager_root() plugins_dir = os.path.join(manager_dir, 'plugins') riemann_dir = os.path.join(plugins_dir, 'riemann-controller') package_dir = os.path.join(riemann_dir, 'riemann_controller') resources_dir = os.path.join(package_dir, 'resources') manager_config = os.path.join(resources_dir, 'manager.config') return manager_config @classmethod def _get_libs_path(cls): return path.join(cls._get_manager_root(), '.libs') @staticmethod def reset_elasticsearch_data(): global testenv_instance testenv_instance.elasticsearch_process.reset_data() @staticmethod def stop_celery_management_worker(): global testenv_instance testenv_instance.celery_management_worker_process.stop() @staticmethod def start_celery_management_worker(): global testenv_instance testenv_instance.celery_management_worker_process.start() @staticmethod def riemann_workdir(): global testenv_instance return testenv_instance.\ celery_management_worker_process.\ riemann_config_dir @staticmethod def _get_manager_root(): init_file = __file__ testenv_dir = dirname(init_file) tests_dir = dirname(testenv_dir) manager_dir = dirname(tests_dir) return manager_dir @staticmethod def patch_source_urls(resources): with open(path.join(resources, 'cloudify', 'types', 'types.yaml')) as f: types_yaml = yaml.safe_load(f.read()) for policy_type in types_yaml.get('policy_types', {}).values(): in_path = '/cloudify/policies/' source = policy_type['source'] if in_path in source: source = source[source.index(in_path) + 1:] policy_type['source'] = source for policy_trigger in types_yaml.get('policy_triggers', {}).values(): in_path = '/cloudify/triggers/' source = policy_trigger['source'] if in_path in source: source = source[source.index(in_path) + 1:] policy_trigger['source'] = source with open(path.join(resources, 'cloudify', 'types', 'types.yaml'), 'w') as f: f.write(yaml.safe_dump(types_yaml))
class TestEnvironment(object): manager_rest_process = None elasticsearch_process = None riemann_process = None file_server_process = None celery_management_worker_process = None def __init__(self, test_working_dir): super(TestEnvironment, self).__init__() self.test_working_dir = test_working_dir self.plugins_storage_dir = os.path.join(self.test_working_dir, "plugins-storage") os.makedirs(self.plugins_storage_dir) self.fileserver_dir = path.join(self.test_working_dir, "fileserver") self.rest_service_log_level = "DEBUG" self.rest_service_log_path = path.join(self.test_working_dir, "cloudify-rest-service.log") self.rest_service_log_file_size_MB = 100 self.rest_service_log_files_backup_count = 20 self.securest_log_level = "DEBUG" self.securest_log_file = path.join(self.test_working_dir, "rest-security-audit.log") self.securest_log_file_size_MB = 100 self.securest_log_files_backup_count = 20 self.amqp_username = "******" self.amqp_password = "******" self.events_and_logs_dir = path.join(self.test_working_dir, "tests-events-and-logs") os.mkdir(self.events_and_logs_dir) def create(self): try: logger.info("Setting up test environment... workdir=[{0}]".format(self.test_working_dir)) # events/logs polling start_events_and_logs_polling(logs_handler_retriever=self._logs_handler_retriever) self.start_elasticsearch() self.start_riemann() self.start_fileserver() self.start_manager_rest() self.create_management_worker() except BaseException as error: s_traceback = StringIO.StringIO() traceback.print_exc(file=s_traceback) logger.error("Error in test environment setup: %s", error) logger.error(s_traceback.getvalue()) self.destroy() raise def create_management_worker(self): mock_plugins_path = os.path.dirname(mock_plugins.__file__) os.environ["MOCK_PLUGINS_PATH"] = mock_plugins_path self.celery_management_worker_process = CeleryWorkerProcess( queues=["cloudify.management"], test_working_dir=self.test_working_dir, # these plugins are already installed. # so we just need to append to the includes. # note that these are not mocks, but the actual production # code plugins. additional_includes=[ "riemann_controller.tasks", "cloudify_system_workflows.deployment_environment", "cloudify.plugins.workflows", "diamond_agent.tasks", "script_runner.tasks", # modules in the agent intended for backwards compatibility "worker_installer.tasks", "windows_agent_installer.tasks", "plugin_installer.tasks", "windows_plugin_installer.tasks", ], # we need higher concurrency since # 'deployment_environment.create' calls # 'plugin_installer.install' as a sub-task # and they are both executed inside # this worker concurrency=2, ) # copy plugins to worker env mock_plugins_path = os.path.dirname(mock_plugins.__file__) shutil.copytree( src=mock_plugins_path, dst=self.celery_management_worker_process.envdir, ignore=shutil.ignore_patterns("*.pyc"), ) def start_riemann(self): riemann_config_path = self._get_riemann_config() libs_path = self._get_libs_path() self.riemann_process = RiemannProcess(riemann_config_path, libs_path) self.riemann_process.start() def start_manager_rest(self): from manager_rest.file_server import PORT as FS_PORT file_server_base_uri = "http://localhost:{0}".format(FS_PORT) self.manager_rest_process = ManagerRestProcess( MANAGER_REST_PORT, self.fileserver_dir, file_server_base_uri, FILE_SERVER_BLUEPRINTS_FOLDER, FILE_SERVER_UPLOADED_BLUEPRINTS_FOLDER, FILE_SERVER_RESOURCES_URI, self.rest_service_log_level, self.rest_service_log_path, self.rest_service_log_file_size_MB, self.rest_service_log_files_backup_count, self.securest_log_level, self.securest_log_file, self.securest_log_file_size_MB, self.securest_log_files_backup_count, self.test_working_dir, self.amqp_username, self.amqp_password, ) self.manager_rest_process.start() def start_elasticsearch(self): # elasticsearch self.elasticsearch_process = ElasticSearchProcess() self.elasticsearch_process.start() def start_fileserver(self): # workaround to update path manager_rest_path = path.dirname(path.dirname(path.dirname(__file__))) manager_rest_path = path.join(manager_rest_path, "rest-service") sys.path.append(manager_rest_path) os.mkdir(self.fileserver_dir) from manager_rest.file_server import FileServer from manager_rest.utils import copy_resources self.file_server_process = FileServer(self.fileserver_dir) self.file_server_process.start() # copy resources (base yaml etc) resources_path = path.abspath(__file__) resources_path = path.dirname(resources_path) resources_path = path.dirname(resources_path) resources_path = path.dirname(resources_path) resources_path = path.join(resources_path, "resources") copy_resources(self.fileserver_dir, resources_path) self.patch_source_urls(self.fileserver_dir) def destroy(self): logger.info("Destroying test environment...") if self.riemann_process: self.riemann_process.close() if self.elasticsearch_process: self.elasticsearch_process.close() if self.manager_rest_process: self.manager_rest_process.close() if self.file_server_process: self.file_server_process.stop() self.delete_working_directory() def delete_working_directory(self): if os.path.exists(self.test_working_dir): logger.info("Deleting test environment from: %s", self.test_working_dir) # shutil.rmtree(self.test_working_dir, ignore_errors=True) def handle_logs(self, output, event): pass def _logs_handler_retriever(self): return self.handle_logs @classmethod def _get_riemann_config(cls): manager_dir = cls._get_manager_root() plugins_dir = os.path.join(manager_dir, "plugins") riemann_dir = os.path.join(plugins_dir, "riemann-controller") package_dir = os.path.join(riemann_dir, "riemann_controller") resources_dir = os.path.join(package_dir, "resources") manager_config = os.path.join(resources_dir, "manager.config") return manager_config @classmethod def _get_libs_path(cls): return path.join(cls._get_manager_root(), ".libs") @staticmethod def reset_elasticsearch_data(): global testenv_instance testenv_instance.elasticsearch_process.reset_data() @staticmethod def stop_celery_management_worker(): global testenv_instance testenv_instance.celery_management_worker_process.stop() @staticmethod def read_celery_management_logs(): global testenv_instance process = testenv_instance.celery_management_worker_process return process.try_read_logfile() @classmethod def stop_all_celery_processes(cls): logger.info("Shutting down all celery processes") os.system("pkill -9 -f 'celery worker'") @staticmethod def start_celery_management_worker(): global testenv_instance testenv_instance.celery_management_worker_process.start() @staticmethod def riemann_cleanup(): global testenv_instance shutil.rmtree(TestEnvironment.riemann_workdir()) os.mkdir(TestEnvironment.riemann_workdir()) testenv_instance.riemann_process.restart() @staticmethod def riemann_workdir(): global testenv_instance return testenv_instance.celery_management_worker_process.riemann_config_dir @staticmethod def _get_manager_root(): init_file = __file__ testenv_dir = dirname(init_file) tests_dir = dirname(testenv_dir) manager_dir = dirname(tests_dir) return manager_dir @staticmethod def patch_source_urls(resources): with open(path.join(resources, "cloudify", "types", "types.yaml")) as f: types_yaml = yaml.safe_load(f.read()) for policy_type in types_yaml.get("policy_types", {}).values(): in_path = "/cloudify/policies/" source = policy_type["source"] if in_path in source: source = source[source.index(in_path) + 1 :] policy_type["source"] = source for policy_trigger in types_yaml.get("policy_triggers", {}).values(): in_path = "/cloudify/triggers/" source = policy_trigger["source"] if in_path in source: source = source[source.index(in_path) + 1 :] policy_trigger["source"] = source with open(path.join(resources, "cloudify", "types", "types.yaml"), "w") as f: f.write(yaml.safe_dump(types_yaml))