def _load_process_module(process_path=None,
config_dir=None,
run_compile=False):
if process_path.endswith(".py"):
module_path = os.path.join(config_dir, process_path)
if not os.path.isfile(module_path):
raise MapcheteConfigError(f"{module_path} is not available")
try:
if run_compile:
py_compile.compile(module_path, doraise=True)
module_name = os.path.splitext(os.path.basename(module_path))[0]
# load module
spec = importlib.util.spec_from_file_location(
module_name, module_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
# required to make imported module available using multiprocessing
sys.modules[module_name] = module
# configure process file logger
add_module_logger(module.__name__)
except py_compile.PyCompileError as e:
raise MapcheteProcessSyntaxError(e)
except ImportError as e:
raise MapcheteProcessImportError(e)
else:
try:
module = importlib.import_module(process_path)
except ImportError as e:
raise MapcheteProcessImportError(e)
return module
def _load_process_module(config):
if config["process"].endswith(".py"):
abs_path = os.path.join(config["config_dir"], config["process"])
if not os.path.isfile(abs_path):
raise MapcheteConfigError("%s is not available" % abs_path)
try:
py_compile.compile(abs_path, doraise=True)
module = imp.load_source(
os.path.splitext(os.path.basename(abs_path))[0], abs_path)
except py_compile.PyCompileError as e:
raise MapcheteProcessSyntaxError(e)
except ImportError as e:
raise MapcheteProcessImportError(e)
else:
try:
module = importlib.import_module(config["process"])
except ImportError as e:
raise MapcheteProcessImportError(e)
return module
def _validate_process_file(config):
abs_path = os.path.join(config["config_dir"], config["process_file"])
if not os.path.isfile(abs_path):
raise MapcheteConfigError("%s is not available" % abs_path)
try:
py_compile.compile(abs_path, doraise=True)
imp.load_source(os.path.splitext(os.path.basename(abs_path))[0], abs_path)
except py_compile.PyCompileError as e:
raise MapcheteProcessSyntaxError(e)
except ImportError as e:
raise MapcheteProcessImportError(e)
return abs_path
def _load_process_module(process_path=None,
config_dir=None,
run_compile=False):
if process_path.endswith(".py"):
abs_path = os.path.join(config_dir, process_path)
if not os.path.isfile(abs_path):
raise MapcheteConfigError("%s is not available" % abs_path)
try:
if run_compile:
py_compile.compile(abs_path, doraise=True)
module = imp.load_source(
os.path.splitext(os.path.basename(abs_path))[0], abs_path)
# configure process file logger
add_module_logger(module.__name__)
except py_compile.PyCompileError as e:
raise MapcheteProcessSyntaxError(e)
except ImportError as e:
raise MapcheteProcessImportError(e)
else:
try:
module = importlib.import_module(process_path)
except ImportError as e:
raise MapcheteProcessImportError(e)
return module
def get_process_func(process_path=None, config_dir=None, run_compile=False):
logger.debug("get process function from %s", process_path)
process_module = _load_process_module(process_path=process_path,
config_dir=config_dir,
run_compile=run_compile)
try:
if hasattr(process_module, "Process"):
logger.error("""instanciating MapcheteProcess is deprecated, """
"""provide execute() function instead""")
if hasattr(process_module, "execute"):
return process_module.execute
else:
raise ImportError("No execute() function found in %s" %
process_path)
except ImportError as e:
raise MapcheteProcessImportError(e)
def process_func(self):
process_module = _load_process_module(self._raw)
try:
if hasattr(process_module, "Process"):
logger.error(
"""instanciating MapcheteProcess is deprecated, """
"""provide execute() function instead""")
if hasattr(process_module, "execute"):
user_execute = process_module.execute
if len(inspect.signature(user_execute).parameters) == 0:
raise ImportError(
"execute() function has to accept at least one argument"
)
return user_execute
else:
raise ImportError("No execute() function found in %s" %
self._raw["process"])
except ImportError as e:
raise MapcheteProcessImportError(e)
def process_func(self):
try:
user_process_py = imp.load_source(
os.path.splitext(os.path.basename(self.process_file))[0],
self.process_file
)
if hasattr(user_process_py, "Process"):
logger.error(
"""instanciating MapcheteProcess is deprecated, """
"""provide execute() function instead""")
if hasattr(user_process_py, "execute"):
user_execute = user_process_py.execute
if len(inspect.getargspec(user_execute).args) == 0:
raise ImportError(
"execute() function has to accept at least one argument"
)
return user_execute
else:
raise ImportError(
"No execute() function found in %s" % self.process_file
)
except ImportError as e:
raise MapcheteProcessImportError(e)
def _execute(self, process_tile):
# If baselevel is active and zoom is outside of baselevel,
# interpolate from other zoom levels.
if self.config.baselevels:
if process_tile.zoom < min(self.config.baselevels["zooms"]):
process_data = self._interpolate_from_baselevel(
process_tile, "lower")
# Analyze proess output.
return self._streamline_output(process_data, process_tile)
elif process_tile.zoom > max(self.config.baselevels["zooms"]):
process_data = self._interpolate_from_baselevel(
process_tile, "higher")
# Analyze proess output.
return self._streamline_output(process_data, process_tile)
# Otherwise, load process source and execute.
try:
user_process_py = imp.load_source(
self.process_name + "process_file", self.config.process_file)
if hasattr(user_process_py, "execute"):
process_is_function = True
tile_process = MapcheteProcess(config=self.config,
tile=process_tile,
params=self.config.at_zoom(
process_tile.zoom))
user_execute = user_process_py.execute
elif hasattr(user_process_py, "Process"):
warnings.warn(
"""instanciating MapcheteProcess will be deprecated, """
"""provide execute() function instead""")
process_is_function = False
tile_process = user_process_py.Process(
config=self.config,
tile=process_tile,
params=self.config.at_zoom(process_tile.zoom))
else:
raise ImportError(
"No execute() function or Process object found in %s" %
(self.config.process_file))
except ImportError as e:
raise MapcheteProcessImportError(e)
try:
if process_is_function:
if len(inspect.getargspec(user_execute).args) != 1:
raise MapcheteProcessImportError(
"execute() function has to accept exactly one argument"
)
starttime = time.time()
# Actually run process.
process_data = user_execute(tile_process)
else:
starttime = time.time()
# Actually run process.
process_data = tile_process.execute()
except Exception as e:
# Log process time
elapsed = "%ss" % (round((time.time() - starttime), 3))
LOGGER.error(
(process_tile.id, "exception in user process", e, elapsed))
for line in format_exc().split("\n"):
LOGGER.error(line)
raise MapcheteProcessException(format_exc())
finally:
del tile_process
elapsed = "%ss" % (round((time.time() - starttime), 3))
LOGGER.debug((process_tile.id, "processed", elapsed))
# Analyze proess output.
return self._streamline_output(process_data, process_tile)