def __init__(self, config, with_cache=False):
"""
Initialize Mapchete processing endpoint.
Parameters
----------
config : MapcheteConfig
Mapchete process configuration
with_cache : bool
cache processed output data in memory (default: False)
"""
LOGGER.info("preparing process ...")
if not isinstance(config, MapcheteConfig):
raise TypeError("config must be MapcheteConfig object")
self.config = config
try:
py_compile.compile(self.config.process_file, doraise=True)
except py_compile.PyCompileError as e:
raise MapcheteProcessSyntaxError(e)
self.process_name = os.path.splitext(
os.path.basename(self.config.process_file))[0]
if self.config.mode == "memory":
self.with_cache = True
else:
self.with_cache = with_cache
if self.with_cache:
self.process_tile_cache = LRUCache(maxsize=32)
self.current_processes = {}
self.process_lock = threading.Lock()
def _load_process_module(process_path=None,
config_dir=None,
run_compile=False):
if process_path.endswith(".py"):
module_path = os.path.join(config_dir, process_path)
if not os.path.isfile(module_path):
raise MapcheteConfigError(f"{module_path} is not available")
try:
if run_compile:
py_compile.compile(module_path, doraise=True)
module_name = os.path.splitext(os.path.basename(module_path))[0]
# load module
spec = importlib.util.spec_from_file_location(
module_name, module_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
# required to make imported module available using multiprocessing
sys.modules[module_name] = module
# configure process file logger
add_module_logger(module.__name__)
except py_compile.PyCompileError as e:
raise MapcheteProcessSyntaxError(e)
except ImportError as e:
raise MapcheteProcessImportError(e)
else:
try:
module = importlib.import_module(process_path)
except ImportError as e:
raise MapcheteProcessImportError(e)
return module
def _validate_process_file(config):
abs_path = os.path.join(config["config_dir"], config["process_file"])
if not os.path.isfile(abs_path):
raise MapcheteConfigError("%s is not available" % abs_path)
try:
py_compile.compile(abs_path, doraise=True)
except py_compile.PyCompileError as e:
raise MapcheteProcessSyntaxError(e)
return abs_path
def _validate_process_file(config):
abs_path = os.path.join(config["config_dir"], config["process_file"])
if not os.path.isfile(abs_path):
raise MapcheteConfigError("%s is not available" % abs_path)
try:
py_compile.compile(abs_path, doraise=True)
imp.load_source(os.path.splitext(os.path.basename(abs_path))[0], abs_path)
except py_compile.PyCompileError as e:
raise MapcheteProcessSyntaxError(e)
except ImportError as e:
raise MapcheteProcessImportError(e)
return abs_path
def _load_process_module(config):
if config["process"].endswith(".py"):
abs_path = os.path.join(config["config_dir"], config["process"])
if not os.path.isfile(abs_path):
raise MapcheteConfigError("%s is not available" % abs_path)
try:
py_compile.compile(abs_path, doraise=True)
module = imp.load_source(
os.path.splitext(os.path.basename(abs_path))[0], abs_path)
except py_compile.PyCompileError as e:
raise MapcheteProcessSyntaxError(e)
except ImportError as e:
raise MapcheteProcessImportError(e)
else:
try:
module = importlib.import_module(config["process"])
except ImportError as e:
raise MapcheteProcessImportError(e)
return module
def _load_process_module(process_path=None,
config_dir=None,
run_compile=False):
if process_path.endswith(".py"):
abs_path = os.path.join(config_dir, process_path)
if not os.path.isfile(abs_path):
raise MapcheteConfigError("%s is not available" % abs_path)
try:
if run_compile:
py_compile.compile(abs_path, doraise=True)
module = imp.load_source(
os.path.splitext(os.path.basename(abs_path))[0], abs_path)
# configure process file logger
add_module_logger(module.__name__)
except py_compile.PyCompileError as e:
raise MapcheteProcessSyntaxError(e)
except ImportError as e:
raise MapcheteProcessImportError(e)
else:
try:
module = importlib.import_module(process_path)
except ImportError as e:
raise MapcheteProcessImportError(e)
return module