def setup_environment(): global redis_pid # Set the port to the test redis server global_config.REDIS_SERVER_SERVER = "localhost" global_config.REDIS_SERVER_PORT = 7000 # Set the path to redis WORKER_LOGFILE # global_config.WORKER_LOGFILE = "/var/log/redis/redis" # home = os.getenv("HOME") # GRASS GIS # Setup the test environment global_config.GRASS_GIS_BASE = "/usr/local/grass78/" global_config.GRASS_GIS_START_SCRIPT = "/usr/local/bin/grass78" # global_config.GRASS_DATABASE= "/usr/local/grass_test_db" # global_config.GRASS_DATABASE = "%s/actinia/grass_test_db" % home global_config.GRASS_TMP_DATABASE = "/tmp" if server_test is False and custom_actinia_cfg is False: # Start the redis server for user and logging management redis_pid = os.spawnl(os.P_NOWAIT, "/usr/bin/redis-server", "common/redis.conf", "--port %i" % global_config.REDIS_SERVER_PORT) time.sleep(1) if server_test is False and custom_actinia_cfg is not False: global_config.read(custom_actinia_cfg)
def setup_environment(): # If docker config if custom_actinia_cfg is not False: global_config.read(custom_actinia_cfg) return global redis_pid # Set the port to the test redis server global_config.REDIS_SERVER_SERVER = "localhost" global_config.REDIS_SERVER_PORT = 7000 # home = os.getenv("HOME") # GRASS # Setup the test environment global_config.GRASS_GIS_BASE = "/usr/local/grass78/" global_config.GRASS_GIS_START_SCRIPT = "/usr/local/bin/grass78" # global_config.GRASS_DATABASE= "/usr/local/grass_test_db" # global_config.GRASS_DATABASE = "%s/actinia/grass_test_db" % home # Start the redis server for user and logging management redis_pid = os.spawnl(os.P_NOWAIT, "/usr/bin/redis-server", "common/redis.conf", "--port %i" % global_config.REDIS_SERVER_PORT) time.sleep(1)
def tearDownClass(cls): # reset config if cls.save_interim_results_value is False: cls.save_config(cls.tmp_cfg_file, cls.cfg_file, 'False') os.remove(cls.tmp_cfg_file) global_config.read(cls.cfg_file) super(JobResumptionProcessingTestCase, cls).tearDownClass()
def setUpClass(cls): if custom_actinia_cfg is not False: global_config.read(custom_actinia_cfg) print(global_config) else: global_config.REDIS_SERVER_URL = "localhost" global_config.REDIS_SERVER_PORT = 7000 args = (global_config.REDIS_SERVER_URL, global_config.REDIS_SERVER_PORT) if global_config.REDIS_SERVER_PW and global_config.REDIS_SERVER_PW is not None: args = (*args, global_config.REDIS_SERVER_PW) redis_interface.connect(*args)
def __init__(self, user_id, resource_id, iteration): """Init method for InterimResult class Args: user_id (str): The unique user name/id resource_id (str): The id of the resource old_pc_step (int): The number of the successfully finished steps of the process chain in the previous iteration """ global_config.read(DEFAULT_CONFIG_PATH) self.logger = MessageLogger() self.user_resource_interim_storage_path = os.path.join( global_config.GRASS_RESOURCE_DIR, user_id, "interim") self.saving_interim_results = global_config.SAVE_INTERIM_RESULTS self.resource_id = resource_id self.iteration = iteration if iteration is not None else 1 self.old_pc_step = None
Logging interface """ import logging import logging.handlers import sys import platform from datetime import datetime from pythonjsonlogger import jsonlogger from actinia_core.core.common.config import global_config # unfortunately, the config is read twice because of the read call here but it # is needed to load the correct interface and log level at this time global_config.read() if "colored" in [ global_config.LOG_STDOUT_FORMAT, global_config.LOG_FILE_FORMAT ]: from colorlog import ColoredFormatter if global_config.LOG_INTERFACE == "fluentd": from fluent import handler __license__ = "GPLv3" __author__ = "Sören Gebbert, Carmen Tawalika" __copyright__ = "Copyright 2016-present, Sören Gebbert and mundialis GmbH & Co. KG" class BasicLogger(object):
from actinia_core.core.common.app import flask_app from actinia_core.core.common.config import global_config, DEFAULT_CONFIG_PATH from actinia_core.core.common.redis_interface import connect, create_job_queues from actinia_core.core.common.process_queue import create_process_queue __license__ = "GPLv3" __author__ = "Sören Gebbert" __copyright__ = "Copyright 2016-2018, Sören Gebbert and mundialis GmbH & Co. KG" __maintainer__ = "Sören Gebbert" __email__ = "*****@*****.**" # if os.environ.get('DEFAULT_CONFIG_PATH'): # DEFAULT_CONFIG_PATH = os.environ['DEFAULT_CONFIG_PATH'] if os.path.exists(DEFAULT_CONFIG_PATH) is True and os.path.isfile( DEFAULT_CONFIG_PATH): global_config.read(DEFAULT_CONFIG_PATH) # Create the endpoints based on the global config create_endpoints() init_versions() # TODO: Implement a better error handler # @flask_app.errorhandler(InvalidUsage) # def handle_invalid_usage(error): # response = error.to_json() # response.status_code = error.status_code # return response # Connect the redis interfaces redis_args = (global_config.REDIS_SERVER_URL, global_config.REDIS_SERVER_PORT) if global_config.REDIS_SERVER_PW and global_config.REDIS_SERVER_PW is not None:
def put(self, user_id, resource_id): """Updates/Resumes the status of a resource.""" global_config.read(DEFAULT_CONFIG_PATH) if global_config.SAVE_INTERIM_RESULTS is not True: return make_response( jsonify( SimpleResponseModel( status="error", message= "Interim results are not set in the configureation")), 404) ret = self.check_permissions(user_id=user_id) if ret: return ret # check if latest iteration is found old_iteration, response_data = self.resource_logger.get_latest_iteration( user_id, resource_id) old_iteration = 1 if old_iteration is None else old_iteration if response_data is None: return make_response( jsonify( SimpleResponseModel(status="error", message="Resource does not exist")), 400) # check if a new iteration is possible _, response_model = pickle.loads(response_data) err_msg = self._check_possibility_of_new_iteration( response_model, user_id, resource_id) if err_msg is not None: return make_response( jsonify(SimpleResponseModel(status="error", message=err_msg)), 404) # get step of the process chain pc_step = response_model['progress']['step'] - 1 for iter in range(old_iteration - 1, 0, -1): if iter == 1: old_response_data = self.resource_logger.get( user_id, resource_id) else: old_response_data = self.resource_logger.get( user_id, resource_id, iter) if old_response_data is None: return None _, old_response_model = pickle.loads(old_response_data) pc_step += old_response_model['progress']['step'] - 1 # start new iteration iteration = old_iteration + 1 # use post_url if iteration > 1 if old_iteration and old_iteration == 1: post_url = response_model['api_info']['request_url'] elif old_iteration and old_iteration > 1: post_url = response_model['api_info']['post_url'] else: post_url = None rdc, processing_resource, start_job = \ self._create_ResourceDataContainer_for_resumption( post_url, pc_step, user_id, resource_id, iteration) # enqueue job if rdc: enqueue_job(processing_resource.job_timeout, start_job, rdc) html_code, response_model = pickle.loads( processing_resource.response_data) return make_response(jsonify(response_model), html_code)
def setUpClass(cls): if cls.server_test is False and cls.custom_actinia_cfg is False: global_config.REDIS_SERVER_SERVER = "localhost" global_config.REDIS_SERVER_PORT = 7000 global_config.GRASS_RESOURCE_DIR = "/tmp" global_config.DOWNLOAD_CACHE = "/tmp/download_cache" global_config.REDIS_QUEUE_SERVER_URL = "localhost" global_config.REDIS_QUEUE_SERVER_PORT = 6379 global_config.NUMBER_OF_WORKERS = 3 # Create the job queue # redis_interface.create_job_queues( # global_config.REDIS_QUEUE_SERVER_URL, # global_config.REDIS_QUEUE_SERVER_PORT, # global_config.NUMBER_OF_WORKERS) # If the custom_actinia_cfg variable is set, then the actinia config # file will be read to configure Redis queue if cls.server_test is False and cls.custom_actinia_cfg is not False: global_config.read(cls.custom_actinia_cfg) # Create the job queue # redis_interface.create_job_queues(global_config.REDIS_QUEUE_SERVER_URL, # global_config.REDIS_QUEUE_SERVER_PORT, # global_config.NUMBER_OF_WORKERS) # Start the redis interface redis_args = (global_config.REDIS_SERVER_URL, global_config.REDIS_SERVER_PORT) if global_config.REDIS_SERVER_PW and global_config.REDIS_SERVER_PW is not None: redis_args = (*redis_args, global_config.REDIS_SERVER_PW) redis_interface.connect(*redis_args) # Process queue create_process_queue(config=global_config) # We create 4 user for all roles: guest, user, admin, root accessible_datasets = { "nc_spm_08": ["PERMANENT", "user1", "landsat", "modis_lst", "test_mapset"], "ECAD": ["PERMANENT"], "latlong_wgs84": ["PERMANENT"] } # Create users cls.guest_id, cls.guest_group, cls.guest_auth_header = cls.create_user( name="guest", role="guest", process_num_limit=3, process_time_limit=2, accessible_datasets=accessible_datasets) cls.user_id, cls.user_group, cls.user_auth_header = cls.create_user( name="user", role="user", process_num_limit=3, process_time_limit=4, accessible_datasets=accessible_datasets) cls.admin_id, cls.admin_group, cls.admin_auth_header = cls.create_user( name="admin", role="admin", accessible_datasets=accessible_datasets) cls.root_id, cls.root_group, cls.root_auth_header = cls.create_user( name="superadmin", role="superadmin", accessible_datasets=accessible_datasets)