def make_action(
self,
esi_job: EsiJob,
operation_manifest: OperationManifest,
callback_manifest: Optional[CallbackManifest] = None,
observers: Optional[List[ActionObserver]] = None,
):
callback_manifest = optional_object(callback_manifest, new_manifest)
observers = optional_object(observers, list)
op_info = operation_manifest.op_info(esi_job.op_id)
op_info.check_params(esi_job.parameters)
request_params = op_info.request_params_to_locations(esi_job.parameters)
url_template = Template(operation_manifest.url_template(esi_job.op_id))
context = {"esi_job": esi_job}
aiohttp_args = AiohttpRequest(
method=op_info.method,
url=url_template.substitute(request_params.path),
params=request_params.query,
json=request_params.body,
headers=request_params.header,
)
action = AiohttpAction(
aiohttp_args=aiohttp_args,
name=esi_job.name,
id_=esi_job.id_,
max_attempts=5,
callbacks=callback_manifest.build_action_callbacks(esi_job.callbacks),
context=context,
)
action.observers.extend(observers)
return action
def do_jobs(
esi_jobs: List[EsiJob],
operation_manifest: OperationManifest,
callback_manifest: Optional[CallbackManifest] = None,
jobs_to_actions: Optional[JobsToActions] = None,
workorder_attributes: Optional[Dict[str, Any]] = None,
observers: Optional[List[ActionObserver]] = None,
worker_count: Optional[int] = None,
max_workers: int = 100,
):
"""
Do a list of jobs.
The typical workflow has a workorder containing a list of jobs, but this function
can be used when there is no workorder. If any workorder attributes are used in the
job's callbacks, they must be passed in as workorder_attributes.
Args:
esi_jobs: The jobs to do.
operation_manifest: The operation manifest.
callback_manifest: The :class:`CallbackManifest` to use if other than default.
Defaults to None.
jobs_to_actions: The `JobToAction` to use if other than default. Defaults to None.
workorder_attributes: Any additional values that might be used by the job callbacks.
Defaults to None.
observers: Observers to register with the
:class:`pfmsoft.aiohttp_queue.aiohttp.AiohttpAction` s. Defaults to None.
worker_count: The number of workers to use. If not given, a resonable number of
workers will be calculated. Defaults to None.
max_workers: The maximum number of workers. Be kind, dont DOS. Defaults to 100.
"""
callback_manifest = optional_object(callback_manifest,
CallbackManifest.manifest_factory)
jobs_to_actions = optional_object(jobs_to_actions, JobsToActions)
workorder_attributes = optional_object(workorder_attributes, dict)
observers = optional_object(observers, list)
if worker_count is None:
worker_count = get_worker_count(len(esi_jobs), max_workers)
workers = []
for _ in range(worker_count):
workers.append(AiohttpQueueWorker())
job_preprocessor = JobPreprocessor()
for esi_job in esi_jobs:
esi_job.update_attributes(workorder_attributes)
job_preprocessor.pre_process_job(esi_job)
actions = JobsToActions().make_actions(
esi_jobs=esi_jobs,
operation_manifest=operation_manifest,
callback_manifest=callback_manifest,
observers=observers,
)
asyncio.run(queue_runner(actions, workers))
def __init__(
self,
manifest_entries: Optional[Dict[str, CallbackManifestEntry]] = None
) -> None:
self.manifest_entries: Dict[str,
CallbackManifestEntry] = optional_object(
manifest_entries, dict)
def make_actions(
self,
esi_jobs: List[EsiJob],
operation_manifest: OperationManifest,
callback_manifest: Optional[CallbackManifest] = None,
observers: Optional[List[ActionObserver]] = None,
):
callback_manifest = optional_object(callback_manifest, CallbackManifest)
observers = optional_object(observers, list)
actions = []
for esi_job in esi_jobs:
action = self.make_action(
esi_job,
operation_manifest=operation_manifest,
callback_manifest=callback_manifest,
observers=observers,
)
actions.append(action)
return actions
def collect_resource_paths(
resource_path: str,
exclude_suffixes: Optional[List[str]] = None,
) -> List[Path]:
"""Returns a list of Paths in a resource directory, excluding the __init__.py"""
exclude_suffixes = optional_object(exclude_suffixes, list)
result = []
with resources.path(resource_path, "__init__.py") as data_path:
for file in data_path.parent.glob("*.*"):
if file.name != "__init__.py" and file.suffix not in exclude_suffixes:
result.append(file)
return result
def get_data_filename(route_name: str, params: Optional[Dict] = None) -> str:
route_template = ROUTE[route_name]["file_name"]
params = optional_object(params, dict)
file_name = Template(route_template).substitute(params)
return file_name
def get_data_version(route_name: str, params: Optional[Dict] = None) -> Path:
version_template = ROUTE[route_name]["version"]
params = optional_object(params, dict)
version = Template(version_template).substitute(params)
return Path(version)
def get_data_subpath(route_name: str, params: Optional[Dict] = None) -> Path:
sub_path_template = ROUTE[route_name]["sub_path"]
params = optional_object(params, dict)
sub_path = Template(sub_path_template).substitute(params)
return Path(sub_path)