def func( coroutine: Union[str, Function, 'WorkerCoroutine'], *, name: Optional[str] = None, keep_result: Optional['SecondsTimedelta'] = None, timeout: Optional['SecondsTimedelta'] = None, keep_result_forever: Optional[bool] = None, max_tries: Optional[int] = None, ) -> Function: """ Wrapper for a job function which lets you configure more settings. :param coroutine: coroutine function to call, can be a string to import :param name: name for function, if None, ``coroutine.__qualname__`` is used :param keep_result: duration to keep the result for, if 0 the result is not kept :param keep_result_forever: whether to keep results forever, if None use Worker default, wins over ``keep_result`` :param timeout: maximum time the job should take :param max_tries: maximum number of tries allowed for the function, use 1 to prevent retrying """ if isinstance(coroutine, Function): return coroutine if isinstance(coroutine, str): name = name or coroutine coroutine_: 'WorkerCoroutine' = import_string(coroutine) else: coroutine_ = coroutine assert asyncio.iscoroutinefunction( coroutine_), f'{coroutine_} is not a coroutine function' timeout = to_seconds(timeout) keep_result = to_seconds(keep_result) return Function(name or coroutine_.__qualname__, coroutine_, timeout, keep_result, keep_result_forever, max_tries)
def cli(*, worker_settings, burst, check, watch, verbose, wid): """ Job queues in python with asyncio and redis. CLI to run the arq worker. """ sys.path.append(os.getcwd()) worker_settings = import_string(worker_settings) prepend = f"worker.{wid:02d} | " old_factory = logging.getLogRecordFactory() def record_factory(*args, **kwargs): record = old_factory(*args, **kwargs) record.prepend = prepend return record logging.setLogRecordFactory(record_factory) logging.config.dictConfig(default_log_config(verbose, prepend)) if check: exit(check_health(worker_settings)) else: kwargs = {} if burst is None else {'burst': burst} kwargs['prepend'] = prepend if watch: with suppress(asyncio.CancelledError): loop = asyncio.get_event_loop() while True: loop.run_until_complete( watch_reload(watch, worker_settings, loop, **kwargs)) else: run_worker(worker_settings, **kwargs)
def func( coroutine: Union[str, Function, Callable], *, name: Optional[str] = None, keep_result: Optional[SecondsTimedelta] = None, timeout: Optional[SecondsTimedelta] = None, max_tries: Optional[int] = None, ) -> Function: """ Wrapper for a job function which lets you configure more settings. :param coroutine: coroutine function to call, can be a string to import :param name: name for function, if None, ``coroutine.__qualname__`` is used :param keep_result: duration to keep the result for, if 0 the result is not kept :param timeout: maximum time the job should take :param max_tries: maximum number of tries allowed for the function, use 1 to prevent retrying """ if isinstance(coroutine, Function): return coroutine if isinstance(coroutine, str): name = name or coroutine coroutine = import_string(coroutine) assert asyncio.iscoroutinefunction(coroutine), f'{coroutine} is not a coroutine function' timeout = to_seconds(timeout) keep_result = to_seconds(keep_result) return Function(name or coroutine.__qualname__, coroutine, timeout, keep_result, max_tries)
def worker(args, settings: BaseSettings): if settings.worker_func: logger.info('running worker...') worker_func: Callable[[BaseSettings], None] = import_string(settings.worker_func) wait_for_services(settings) worker_func(settings=settings) else: raise CliError("settings.worker_func not set, can't run the worker")
def get_function(self, type: str) -> Optional[Callable]: """Lazy load parser functions. This requires the consumer to have install all parser dependencies. Returns: Optional[Callable]: parser function """ function_str = getattr(self, type) if function_str: return import_string(f"electricitymap.contrib.parsers.{function_str}")
def web(args, settings: BaseSettings): logger.info('running web server at %s...', settings.port) create_app: Callable[[BaseSettings], Application] = import_string(settings.create_app) wait_for_services(settings) app = create_app(settings=settings) kwargs = dict(port=settings.port, shutdown_timeout=8, print=lambda *args: None) # pragma: no branch if args.access_log: kwargs.update(access_log_class=ColouredAccessLogger, access_log=logging.getLogger('atoolbox.access')) else: kwargs['access_log'] = None run_app(app, **kwargs)
def cli(ctx: click.Context, *, app: str, verbose: bool) -> None: """ Job queues in python with Asyncio and Redis. CLI to run the Darq worker. app - path to Darq app instance. For example: someproject.darq.darq_app """ sys.path.append(os.getcwd()) darq = import_string(app) if not isinstance(darq, Darq): raise click.BadArgumentUsage( f'"APP" argument error. {darq!r} is not instance of {Darq!r}', ) ctx.ensure_object(dict) ctx.obj['darq'] = darq logging.config.dictConfig(default_log_config(verbose))
def cli(*, worker_settings, burst, check, watch, verbose): """ Job queues in python with asyncio and redis. CLI to run the arq worker. """ sys.path.append(os.getcwd()) worker_settings = import_string(worker_settings) logging.config.dictConfig(default_log_config(verbose)) if check: exit(check_health(worker_settings)) else: kwargs = {} if burst is None else {'burst': burst} if watch: loop = asyncio.get_event_loop() loop.run_until_complete(watch_reload(watch, worker_settings, loop)) else: run_worker(worker_settings, **kwargs)
def cli(*, worker_settings: str, burst: bool, check: bool, watch: str, verbose: bool) -> None: """ Job queues in python with asyncio and redis. CLI to run the arq worker. """ sys.path.append(os.getcwd()) worker_settings_ = cast('WorkerSettingsType', import_string(worker_settings)) logging.config.dictConfig(default_log_config(verbose)) if check: exit(check_health(worker_settings_)) else: kwargs = {} if burst is None else {'burst': burst} if watch: asyncio.run(watch_reload(watch, worker_settings_)) else: run_worker(worker_settings_, **kwargs)
def on_config(self, config): self.context = {} config["version"] = self.config["version"] self.session = Session(db.engine) db.create_db_and_tables() self.pages = [] self.env = config["theme"].get_env() self.env.filters["b64encode"] = escapeb64 self.env.filters["bom_json"] = self.bom_json self.env.filters["md_table"] = self.md_table self.env.filters["yes_no"] = self.yes_no self.env.filters["slugify"] = slugify try: config["meta_model_class"] = import_string(self.config["meta_model_class"]) except ImportError as exc: raise PluginError( f"Meta Model Class {self.config['meta_model_class']} could not be imported. {exc}" ) from exc return config
def test_import_module(): assert import_string('os.path') == os.path
:param hour: hour(s) to run the job on, 0 - 23 :param minute: minute(s) to run the job on, 0 - 59 :param second: second(s) to run the job on, 0 - 59 :param microsecond: microsecond(s) to run the job on, defaults to 123456 as the world is busier at the top of a second, 0 - 1e6 :param run_at_startup: whether to run as worker starts :param unique: whether the job should be only be executed once at each time :param timeout: job timeout :param keep_result: how long to keep the result for :param keep_result_forever: whether to keep results forever :param max_tries: maximum number of tries for the job """ if isinstance(coroutine, str): name = name or 'cron:' + coroutine coroutine_: WorkerCoroutine = import_string(coroutine) else: coroutine_ = coroutine assert asyncio.iscoroutinefunction(coroutine_), f'{coroutine_} is not a coroutine function' timeout = to_seconds(timeout) keep_result = to_seconds(keep_result) return CronJob( name or 'cron:' + coroutine_.__qualname__, coroutine_, month, day, weekday, hour, minute,
:param weekday: week day(s) to run the job on, 0 - 6 or mon - sun :param hour: hour(s) to run the job on, 0 - 23 :param minute: minute(s) to run the job on, 0 - 59 :param second: second(s) to run the job on, 0 - 59 :param microsecond: microsecond(s) to run the job on, defaults to 123456 as the world is busier at the top of a second, 0 - 1e6 :param run_at_startup: whether to run as worker starts :param unique: whether the job should be only be executed once at each time :param timeout: job timeout :param keep_result: how long to keep the result for :param max_tries: maximum number of tries for the job """ if isinstance(coroutine, str): name = name or 'cron:' + coroutine coroutine = import_string(coroutine) assert asyncio.iscoroutinefunction(coroutine), f'{coroutine} is not a coroutine function' timeout = to_seconds(timeout) keep_result = to_seconds(keep_result) return CronJob( name or 'cron:' + coroutine.__qualname__, coroutine, month, day, weekday, hour, minute, second, microsecond,
:param weekday: week day(s) to run the job on, 0 - 6 or mon - sun :param hour: hour(s) to run the job on, 0 - 23 :param minute: minute(s) to run the job on, 0 - 59 :param second: second(s) to run the job on, 0 - 59 :param microsecond: microsecond(s) to run the job on, defaults to 123456 as the world is busier at the top of a second, 0 - 1e6 :param run_at_startup: whether to run as worker starts :param unique: whether the job should be only be executed once at each time :param timeout: job timeout :param keep_result: how long to keep the result for :param max_tries: maximum number of tries for the job """ if isinstance(coroutine, str): name = name or 'cron:' + coroutine coroutine = import_string(coroutine) assert iscoroutine_or_partial( coroutine), f'{coroutine} is not a coroutine function or a partial' timeout = to_seconds(timeout) keep_result = to_seconds(keep_result) return CronJob( name or 'cron:' + coroutine.__qualname__, coroutine, month, day, weekday, hour, minute, second,
# isort:skip_file import sys sys.path.extend(["./"]) from pydantic.utils import import_string from .application import db from .settings.arq import settings from .settings.globals import ARQ_BACKGROUND_FUNCTIONS, DATABASE_CONFIG FUNCTIONS: list = [ import_string(background_function) for background_function in list(ARQ_BACKGROUND_FUNCTIONS) ] if ARQ_BACKGROUND_FUNCTIONS is not None else list() async def startup(ctx): """ Binds a connection set to the db object. """ await db.set_bind(DATABASE_CONFIG.url) async def shutdown(ctx): """ Pops the bind on the db object. """ await db.pop_bind().close()
def test_import_module_invalid(): with pytest.raises(ImportError) as exc_info: import_string('xx') assert exc_info.value.args[0] == '"xx" doesn\'t look like a module path'
def main(*args) -> int: parser = ArgumentParser( description=f'aiohttp-toolbox command line interface v{VERSION}') parser.add_argument( 'command', type=str, choices=list(commands.keys()), help= ('The command to run, use "auto" to infer the command from environment variables, ' 'ATOOLBOX_COMMAND or DYNO (heroku) or PORT.'), ) parser.add_argument( '-r', '--root', dest='root', default=os.getenv('ATOOLBOX_ROOT_DIR', '.'), help= ('root directory to run the command from, defaults to to the environment variable ' '"ATOOLBOX_ROOT_DIR" or "."'), ) parser.add_argument( '-s', '--settings-path', dest='settings_path', default=os.getenv('ATOOLBOX_SETTINGS', 'settings.Settings'), help= ('settings path (dotted, relative to the root directory), defaults to to the environment variable ' '"ATOOLBOX_SETTINGS" or "settings.Settings"'), ) parser.add_argument('--verbose', action='store_true', help='whether to print debug logs') parser.add_argument( '--log', default=os.getenv('ATOOLBOX_LOG_NAME', 'app'), help= 'Root name of logs for the app, defaults to to the environment variable "ATOOLBOX_LOG_NAME" or "app"', ) parser.add_argument( '--live', action='store_true', help= 'whether to run patches as live, default false, only applies to the "patch" command.', ) parser.add_argument( '--access-log', dest='access_log', action='store_true', help= 'whether run the access logger on web, default false, only applies to the "web" command.', ) parser.add_argument( '--patches-path', help= 'patch to import before running patches, only applies to the "patch" command.' ) parser.add_argument('extra', nargs='*', default=[], help='Extra arguments to pass to the command.') try: ns, extra = parser.parse_known_args(args) except SystemExit: return 1 ns.extra.extend(extra) asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) setup_logging(debug=ns.verbose, main_logger_name=ns.log) try: sys.path.append(os.getcwd()) ns.root = Path(ns.root).resolve() sys.path.append(str(ns.root)) os.chdir(str(ns.root)) try: settings_cls = import_string(ns.settings_path) except (ModuleNotFoundError, ImportError) as exc: raise CliError( f'unable to import "{ns.settings_path}", {exc.__class__.__name__}: {exc}' ) if not isinstance(settings_cls, type) or not issubclass( settings_cls, PydanticBaseSettings): raise CliError( f'settings "{settings_cls}" (from "{ns.settings_path}"), is not a valid Settings class' ) settings = settings_cls() locale.setlocale(locale.LC_ALL, getattr(settings, 'locale', 'en_US.utf8')) func = commands[ns.command] or get_auto_command() return func(ns, settings) or 0 except CliError as exc: logger.error('%s', exc) return 1
def test_import_no_attr(): with pytest.raises(ImportError) as exc_info: import_string('os.foobar') assert exc_info.value.args[ 0] == 'Module "os" does not define a "foobar" attribute'
:param weekday: week day(s) to run the job on, 0 - 6 or mon - sun :param hour: hour(s) to run the job on, 0 - 23 :param minute: minute(s) to run the job on, 0 - 59 :param second: second(s) to run the job on, 0 - 59 :param microsecond: microsecond(s) to run the job on, defaults to 123456 as the world is busier at the top of a second, 0 - 1e6 :param run_at_startup: whether to run as worker starts :param unique: whether the job should be only be executed once at each time :param timeout: job timeout :param keep_result: how long to keep the result for :param max_tries: maximum number of tries for the job """ if isinstance(task, str): name = name or 'cron:' + task task = import_string(task) task = t.cast(Coro, task) assert asyncio.iscoroutinefunction(task), \ f'{task} is not a coroutine function' timeout = to_seconds(timeout) keep_result = to_seconds(keep_result) return CronJob( name or 'cron:' + task.__qualname__, task, month, day, weekday, hour,