def func(
coroutine: Union[str, Function, 'WorkerCoroutine'],
*,
name: Optional[str] = None,
keep_result: Optional['SecondsTimedelta'] = None,
timeout: Optional['SecondsTimedelta'] = None,
keep_result_forever: Optional[bool] = None,
max_tries: Optional[int] = None,
) -> Function:
"""
Wrapper for a job function which lets you configure more settings.
:param coroutine: coroutine function to call, can be a string to import
:param name: name for function, if None, ``coroutine.__qualname__`` is used
:param keep_result: duration to keep the result for, if 0 the result is not kept
:param keep_result_forever: whether to keep results forever, if None use Worker default, wins over ``keep_result``
:param timeout: maximum time the job should take
:param max_tries: maximum number of tries allowed for the function, use 1 to prevent retrying
"""
if isinstance(coroutine, Function):
return coroutine
if isinstance(coroutine, str):
name = name or coroutine
coroutine_: 'WorkerCoroutine' = import_string(coroutine)
else:
coroutine_ = coroutine
assert asyncio.iscoroutinefunction(
coroutine_), f'{coroutine_} is not a coroutine function'
timeout = to_seconds(timeout)
keep_result = to_seconds(keep_result)
return Function(name or coroutine_.__qualname__, coroutine_, timeout,
keep_result, keep_result_forever, max_tries)
def cli(*, worker_settings, burst, check, watch, verbose, wid):
"""
Job queues in python with asyncio and redis.
CLI to run the arq worker.
"""
sys.path.append(os.getcwd())
worker_settings = import_string(worker_settings)
prepend = f"worker.{wid:02d} | "
old_factory = logging.getLogRecordFactory()
def record_factory(*args, **kwargs):
record = old_factory(*args, **kwargs)
record.prepend = prepend
return record
logging.setLogRecordFactory(record_factory)
logging.config.dictConfig(default_log_config(verbose, prepend))
if check:
exit(check_health(worker_settings))
else:
kwargs = {} if burst is None else {'burst': burst}
kwargs['prepend'] = prepend
if watch:
with suppress(asyncio.CancelledError):
loop = asyncio.get_event_loop()
while True:
loop.run_until_complete(
watch_reload(watch, worker_settings, loop, **kwargs))
else:
run_worker(worker_settings, **kwargs)
def func(
coroutine: Union[str, Function, Callable],
*,
name: Optional[str] = None,
keep_result: Optional[SecondsTimedelta] = None,
timeout: Optional[SecondsTimedelta] = None,
max_tries: Optional[int] = None,
) -> Function:
"""
Wrapper for a job function which lets you configure more settings.
:param coroutine: coroutine function to call, can be a string to import
:param name: name for function, if None, ``coroutine.__qualname__`` is used
:param keep_result: duration to keep the result for, if 0 the result is not kept
:param timeout: maximum time the job should take
:param max_tries: maximum number of tries allowed for the function, use 1 to prevent retrying
"""
if isinstance(coroutine, Function):
return coroutine
if isinstance(coroutine, str):
name = name or coroutine
coroutine = import_string(coroutine)
assert asyncio.iscoroutinefunction(coroutine), f'{coroutine} is not a coroutine function'
timeout = to_seconds(timeout)
keep_result = to_seconds(keep_result)
return Function(name or coroutine.__qualname__, coroutine, timeout, keep_result, max_tries)
def worker(args, settings: BaseSettings):
if settings.worker_func:
logger.info('running worker...')
worker_func: Callable[[BaseSettings],
None] = import_string(settings.worker_func)
wait_for_services(settings)
worker_func(settings=settings)
else:
raise CliError("settings.worker_func not set, can't run the worker")
def get_function(self, type: str) -> Optional[Callable]:
"""Lazy load parser functions.
This requires the consumer to have install all parser dependencies.
Returns:
Optional[Callable]: parser function
"""
function_str = getattr(self, type)
if function_str:
return import_string(f"electricitymap.contrib.parsers.{function_str}")
def web(args, settings: BaseSettings):
logger.info('running web server at %s...', settings.port)
create_app: Callable[[BaseSettings],
Application] = import_string(settings.create_app)
wait_for_services(settings)
app = create_app(settings=settings)
kwargs = dict(port=settings.port,
shutdown_timeout=8,
print=lambda *args: None) # pragma: no branch
if args.access_log:
kwargs.update(access_log_class=ColouredAccessLogger,
access_log=logging.getLogger('atoolbox.access'))
else:
kwargs['access_log'] = None
run_app(app, **kwargs)
def cli(ctx: click.Context, *, app: str, verbose: bool) -> None:
"""
Job queues in python with Asyncio and Redis.
CLI to run the Darq worker.
app - path to Darq app instance.
For example: someproject.darq.darq_app
"""
sys.path.append(os.getcwd())
darq = import_string(app)
if not isinstance(darq, Darq):
raise click.BadArgumentUsage(
f'"APP" argument error. {darq!r} is not instance of {Darq!r}', )
ctx.ensure_object(dict)
ctx.obj['darq'] = darq
logging.config.dictConfig(default_log_config(verbose))
def cli(*, worker_settings, burst, check, watch, verbose):
"""
Job queues in python with asyncio and redis.
CLI to run the arq worker.
"""
sys.path.append(os.getcwd())
worker_settings = import_string(worker_settings)
logging.config.dictConfig(default_log_config(verbose))
if check:
exit(check_health(worker_settings))
else:
kwargs = {} if burst is None else {'burst': burst}
if watch:
loop = asyncio.get_event_loop()
loop.run_until_complete(watch_reload(watch, worker_settings, loop))
else:
run_worker(worker_settings, **kwargs)
def cli(*, worker_settings: str, burst: bool, check: bool, watch: str,
verbose: bool) -> None:
"""
Job queues in python with asyncio and redis.
CLI to run the arq worker.
"""
sys.path.append(os.getcwd())
worker_settings_ = cast('WorkerSettingsType',
import_string(worker_settings))
logging.config.dictConfig(default_log_config(verbose))
if check:
exit(check_health(worker_settings_))
else:
kwargs = {} if burst is None else {'burst': burst}
if watch:
asyncio.run(watch_reload(watch, worker_settings_))
else:
run_worker(worker_settings_, **kwargs)
def on_config(self, config):
self.context = {}
config["version"] = self.config["version"]
self.session = Session(db.engine)
db.create_db_and_tables()
self.pages = []
self.env = config["theme"].get_env()
self.env.filters["b64encode"] = escapeb64
self.env.filters["bom_json"] = self.bom_json
self.env.filters["md_table"] = self.md_table
self.env.filters["yes_no"] = self.yes_no
self.env.filters["slugify"] = slugify
try:
config["meta_model_class"] = import_string(self.config["meta_model_class"])
except ImportError as exc:
raise PluginError(
f"Meta Model Class {self.config['meta_model_class']} could not be imported. {exc}"
) from exc
return config
def test_import_module():
assert import_string('os.path') == os.path
:param hour: hour(s) to run the job on, 0 - 23
:param minute: minute(s) to run the job on, 0 - 59
:param second: second(s) to run the job on, 0 - 59
:param microsecond: microsecond(s) to run the job on,
defaults to 123456 as the world is busier at the top of a second, 0 - 1e6
:param run_at_startup: whether to run as worker starts
:param unique: whether the job should be only be executed once at each time
:param timeout: job timeout
:param keep_result: how long to keep the result for
:param keep_result_forever: whether to keep results forever
:param max_tries: maximum number of tries for the job
"""
if isinstance(coroutine, str):
name = name or 'cron:' + coroutine
coroutine_: WorkerCoroutine = import_string(coroutine)
else:
coroutine_ = coroutine
assert asyncio.iscoroutinefunction(coroutine_), f'{coroutine_} is not a coroutine function'
timeout = to_seconds(timeout)
keep_result = to_seconds(keep_result)
return CronJob(
name or 'cron:' + coroutine_.__qualname__,
coroutine_,
month,
day,
weekday,
hour,
minute,
:param weekday: week day(s) to run the job on, 0 - 6 or mon - sun
:param hour: hour(s) to run the job on, 0 - 23
:param minute: minute(s) to run the job on, 0 - 59
:param second: second(s) to run the job on, 0 - 59
:param microsecond: microsecond(s) to run the job on,
defaults to 123456 as the world is busier at the top of a second, 0 - 1e6
:param run_at_startup: whether to run as worker starts
:param unique: whether the job should be only be executed once at each time
:param timeout: job timeout
:param keep_result: how long to keep the result for
:param max_tries: maximum number of tries for the job
"""
if isinstance(coroutine, str):
name = name or 'cron:' + coroutine
coroutine = import_string(coroutine)
assert asyncio.iscoroutinefunction(coroutine), f'{coroutine} is not a coroutine function'
timeout = to_seconds(timeout)
keep_result = to_seconds(keep_result)
return CronJob(
name or 'cron:' + coroutine.__qualname__,
coroutine,
month,
day,
weekday,
hour,
minute,
second,
microsecond,
:param weekday: week day(s) to run the job on, 0 - 6 or mon - sun
:param hour: hour(s) to run the job on, 0 - 23
:param minute: minute(s) to run the job on, 0 - 59
:param second: second(s) to run the job on, 0 - 59
:param microsecond: microsecond(s) to run the job on,
defaults to 123456 as the world is busier at the top of a second, 0 - 1e6
:param run_at_startup: whether to run as worker starts
:param unique: whether the job should be only be executed once at each time
:param timeout: job timeout
:param keep_result: how long to keep the result for
:param max_tries: maximum number of tries for the job
"""
if isinstance(coroutine, str):
name = name or 'cron:' + coroutine
coroutine = import_string(coroutine)
assert iscoroutine_or_partial(
coroutine), f'{coroutine} is not a coroutine function or a partial'
timeout = to_seconds(timeout)
keep_result = to_seconds(keep_result)
return CronJob(
name or 'cron:' + coroutine.__qualname__,
coroutine,
month,
day,
weekday,
hour,
minute,
second,
# isort:skip_file
import sys
sys.path.extend(["./"])
from pydantic.utils import import_string
from .application import db
from .settings.arq import settings
from .settings.globals import ARQ_BACKGROUND_FUNCTIONS, DATABASE_CONFIG
FUNCTIONS: list = [
import_string(background_function)
for background_function in list(ARQ_BACKGROUND_FUNCTIONS)
] if ARQ_BACKGROUND_FUNCTIONS is not None else list()
async def startup(ctx):
"""
Binds a connection set to the db object.
"""
await db.set_bind(DATABASE_CONFIG.url)
async def shutdown(ctx):
"""
Pops the bind on the db object.
"""
await db.pop_bind().close()
def test_import_module_invalid():
with pytest.raises(ImportError) as exc_info:
import_string('xx')
assert exc_info.value.args[0] == '"xx" doesn\'t look like a module path'
def main(*args) -> int:
parser = ArgumentParser(
description=f'aiohttp-toolbox command line interface v{VERSION}')
parser.add_argument(
'command',
type=str,
choices=list(commands.keys()),
help=
('The command to run, use "auto" to infer the command from environment variables, '
'ATOOLBOX_COMMAND or DYNO (heroku) or PORT.'),
)
parser.add_argument(
'-r',
'--root',
dest='root',
default=os.getenv('ATOOLBOX_ROOT_DIR', '.'),
help=
('root directory to run the command from, defaults to to the environment variable '
'"ATOOLBOX_ROOT_DIR" or "."'),
)
parser.add_argument(
'-s',
'--settings-path',
dest='settings_path',
default=os.getenv('ATOOLBOX_SETTINGS', 'settings.Settings'),
help=
('settings path (dotted, relative to the root directory), defaults to to the environment variable '
'"ATOOLBOX_SETTINGS" or "settings.Settings"'),
)
parser.add_argument('--verbose',
action='store_true',
help='whether to print debug logs')
parser.add_argument(
'--log',
default=os.getenv('ATOOLBOX_LOG_NAME', 'app'),
help=
'Root name of logs for the app, defaults to to the environment variable "ATOOLBOX_LOG_NAME" or "app"',
)
parser.add_argument(
'--live',
action='store_true',
help=
'whether to run patches as live, default false, only applies to the "patch" command.',
)
parser.add_argument(
'--access-log',
dest='access_log',
action='store_true',
help=
'whether run the access logger on web, default false, only applies to the "web" command.',
)
parser.add_argument(
'--patches-path',
help=
'patch to import before running patches, only applies to the "patch" command.'
)
parser.add_argument('extra',
nargs='*',
default=[],
help='Extra arguments to pass to the command.')
try:
ns, extra = parser.parse_known_args(args)
except SystemExit:
return 1
ns.extra.extend(extra)
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
setup_logging(debug=ns.verbose, main_logger_name=ns.log)
try:
sys.path.append(os.getcwd())
ns.root = Path(ns.root).resolve()
sys.path.append(str(ns.root))
os.chdir(str(ns.root))
try:
settings_cls = import_string(ns.settings_path)
except (ModuleNotFoundError, ImportError) as exc:
raise CliError(
f'unable to import "{ns.settings_path}", {exc.__class__.__name__}: {exc}'
)
if not isinstance(settings_cls, type) or not issubclass(
settings_cls, PydanticBaseSettings):
raise CliError(
f'settings "{settings_cls}" (from "{ns.settings_path}"), is not a valid Settings class'
)
settings = settings_cls()
locale.setlocale(locale.LC_ALL,
getattr(settings, 'locale', 'en_US.utf8'))
func = commands[ns.command] or get_auto_command()
return func(ns, settings) or 0
except CliError as exc:
logger.error('%s', exc)
return 1
def test_import_no_attr():
with pytest.raises(ImportError) as exc_info:
import_string('os.foobar')
assert exc_info.value.args[
0] == 'Module "os" does not define a "foobar" attribute'
:param weekday: week day(s) to run the job on, 0 - 6 or mon - sun
:param hour: hour(s) to run the job on, 0 - 23
:param minute: minute(s) to run the job on, 0 - 59
:param second: second(s) to run the job on, 0 - 59
:param microsecond: microsecond(s) to run the job on, defaults to 123456
as the world is busier at the top of a second, 0 - 1e6
:param run_at_startup: whether to run as worker starts
:param unique: whether the job should be only be executed once at each time
:param timeout: job timeout
:param keep_result: how long to keep the result for
:param max_tries: maximum number of tries for the job
"""
if isinstance(task, str):
name = name or 'cron:' + task
task = import_string(task)
task = t.cast(Coro, task)
assert asyncio.iscoroutinefunction(task), \
f'{task} is not a coroutine function'
timeout = to_seconds(timeout)
keep_result = to_seconds(keep_result)
return CronJob(
name or 'cron:' + task.__qualname__,
task,
month,
day,
weekday,
hour,
