def _execute_plugin_():
fn = CrawlinoModulesStore.find_module(module_name, plugin_type)
log.debug(f"Executing plugin "
f"'{plugin_type}' - Parameters: {plugin_config}")
#
# Sources module runs the first, then the don't have any previous
# result from other step
#
return fn(previous_step_result, **plugin_config)
def __init__(self, type: str, config: Dict or None, name: str = None):
self.type = type
self.name = name or ""
self.config = config or {}
if CrawlinoModulesStore.find_module("hooks", self.type) is None:
raise CrawlinoValueError("Invalid 'type' property value",
exc_info=True,
extra={
"given_source_type": self.type
})
if self.config is None:
raise CrawlinoValueError("Input must has a 'config' property")
def __init__(self, type: str, config: Dict, name: str = None):
self.type = type
self.name = name or ""
self.config = config
if not self.type:
raise CrawlinoValueError("Source must has the 'type' property")
if self.config is None:
raise CrawlinoValueError("Source must has a 'config' property")
if CrawlinoModulesStore.find_module(STEP_SOURCES, self.type) is None:
raise CrawlinoValueError("Invalid 'type' property value",
exc_info=True,
extra={"given_source_type": self.type})
def __init__(self, type: str, config: Dict or None, name: str = None):
self.type = type
self.name = name or ""
self.config = config or {}
if not self.type:
raise CrawlinoValueError("Config must has the 'type' property")
if self.config is None:
raise CrawlinoValueError("Source must has a 'config' property")
if CrawlinoModulesStore.find_module(STEP_EXTRACTOR, self.type) is None:
raise CrawlinoValueError(
f"Invalid 'type' property value: "
f"'{self.type}'",
exc_info=True,
extra={"input_type": self.type})
def convert(item, current: str = None):
if isinstance(item, dict):
return {
k: convert(
v,
f"{current}.{k}" if current else k
)
for k, v in item.items()
}
if isinstance(item, list):
#
# Currently List are not permitted
#
return [
convert(v, f"{current}.{k}") if current else k
for k, v in enumerate(item)
]
else:
# _item = str(item)
_item = item
if "$generator" in _item:
for i, (action, action_params) in \
enumerate(detect_actions(_item)):
# Set generator name
generator_name = action_params[0]
generator_params = action_params[1:]
gen[current].append(
(
generator_name,
generator_params
)
)
# Map current property with their generators
map_keys_and_generators[f"{current}_{i}"] = \
CrawlinoModulesStore.find_module(
"generators", generator_name
)(*generator_params)
def test_load_input_inline_check_valid_types(crawler_import_definitions: File):
m = CrawlinoModel(crawler_import_definitions)
assert CrawlinoModulesStore.find_module("input", m.input.type) is not None
def test_get_all_plugin_paths_oks():
assert isinstance(CrawlinoModulesStore.get_all_python_files(), dict)
def test_load_plugins_oks():
CrawlinoModulesStore.load_modules()
assert CrawlinoModulesStore.modules.get("input") is not None
def test_get_all_plugin_paths_not_contains__init__():
assert all("__init__.py" not in x
for x in CrawlinoModulesStore.get_all_python_files())