def main() -> None: if not lock.acquire(blocking=False): print("Couldn't acquire lock file at %s, exiting." % lock.path) sys.exit(1) log.info('Deck Chores %s started.' % __version__) try: generate_config() log_handler.setFormatter(logging.Formatter(cfg.logformat, style='{')) log.debug('Config: %s' % cfg.__dict__) if there_is_another_deck_chores_container(): log.error( "There's another container running deck-chores, maybe paused or restarting." ) raise SystemExit(1) jobs.start_scheduler() inspection_time = inspect_running_containers() listen(since=inspection_time) except SystemExit as e: exit_code = e.code except ConfigurationError as e: log.error(str(e)) exit_code = 1 except Exception as e: log.error('Caught unhandled exception:') log.exception(e) # type: ignore exit_code = 3 else: exit_code = 0 finally: shutdown() lock.release() sys.exit(exit_code)
def handle_unpause(event: dict) -> None: log.debug('Handling unpause.') container_id = event['Actor']['ID'] for job in jobs.get_jobs_for_container(container_id): log.info('Resuming job %s for %s' % (job.kwargs['job_name'], job.kwargs['container_name'])) job.resume()
def parse_labels( container_id: str) -> Tuple[Tuple[str, ...], str, Dict[str, Dict]]: labels = cfg.client.containers.get(container_id).labels log.debug(f'Parsing labels: {labels}') service_id = parse_service_id(labels) filtered_labels = { k: v for k, v in labels.items() if k.startswith(cfg.label_ns) } flags, user = parse_options(filtered_labels) jobs_labels: Mapping[str, str] if 'image' in flags: image_labels = image_definition_labels_of_container(container_id) _, image_options_user = parse_options(image_labels) user = user or image_options_user jobs_labels = ChainMap(filtered_labels, image_labels) else: jobs_labels = filtered_labels job_definitions = parse_job_definitions(jobs_labels, user) if service_id: log.debug(f'Assigning service id: {service_id}') for job_definition in job_definitions.values(): job_definition['service_id'] = service_id return service_id, flags, job_definitions
def add(container_id: str, definitions: Mapping[str, Dict], paused: bool = False) -> None: log.debug(f'Adding jobs to container {container_id}.') for job_name, definition in definitions.items(): job_id = generate_id(*definition.get("service_id") or (container_id, ), job_name) definition.update({ 'job_name': job_name, 'job_id': job_id, 'container_id': container_id }) trigger_class, trigger_config = definition['trigger'] scheduler.add_job( func=exec_job, trigger=trigger_class( *trigger_config, timezone=definition['timezone'], jitter=definition['jitter'], ), kwargs=definition, id=job_id, max_instances=definition['max'], next_run_time=None if paused else undefined_runtime, replace_existing=True, ) log.info(f"{container_name(container_id)}: Added " + ("paused " if paused else "") + f"'{job_name}' ({job_id}).")
def inspect_running_containers() -> datetime: log.debug('Fetching running containers') containers = cfg.client.containers.list() inspection_time = datetime.utcnow() # FIXME get last eventtime jobs.scheduler.add_job(exec_inspection, trigger=DateTrigger(), args=(containers, ), id='container_inspection') return inspection_time
def parse_flags(options: str) -> str: result = set(cfg.default_flags) if options: for option in split_string(options): if option.startswith('no'): result.discard(option[2:]) else: result.add(option) result_string = ','.join(sorted(result)) log.debug(f'Parsed & resolved container flags: {result_string}') return result_string
def reassign_service_lock(old_container_id: str, new_container_id: str): service_id = _service_locks_by_container_id.pop(old_container_id) assert old_container_id not in service_locks_by_container_id assert new_container_id not in service_locks_by_container_id _service_locks_by_container_id[new_container_id] = service_id assert service_id in service_locks_by_service_id _service_locks_by_service_id[service_id] = new_container_id log.debug( f"Reassigned lock for service {service_id} from container {old_container_id} " f"to {new_container_id}." )
def process_running_container_labels(container_id: str) -> None: service_id, options, definitions = parse.labels(container_id) if not definitions: return if service_id and 'service' in options: if service_id in locking_container_to_services_map.values(): log.debug('Service id has a registered job: %s' % service_id) return log.info('Locking service id: %s' % service_id) locking_container_to_services_map[container_id] = service_id jobs.add(container_id, definitions)
def parse_service_id(labels: Dict[str, str]) -> Tuple[str, ...]: filtered_labels = { k: v for k, v in labels.items() if k in cfg.service_identifiers } log.debug(f'Considering labels for service id: {filtered_labels}') if not filtered_labels: return () if len(filtered_labels) != len(cfg.service_identifiers): log.critical('Missing service identity labels: {}'.format( ', '.join(set(cfg.service_identifiers) - set(filtered_labels)))) return () return tuple(f"{k}={v}" for k, v in filtered_labels.items())
def handle_die(event: dict) -> None: log.debug('Handling die.') container_id = event['Actor']['ID'] service_id, options, definitions = parse.labels(container_id) if not definitions: return if service_id and 'service' in options: if container_id in locking_container_to_services_map: log.info('Unlocking service id: %s' % service_id) del locking_container_to_services_map[container_id] else: return container_name = cfg.client.containers.get(container_id).name for job_name in definitions: log.info("Removing job '%s' for %s" % (job_name, container_name)) jobs.remove(generate_id(container_id, job_name))
def listen(since: datetime = None) -> None: if since is None: since = datetime.utcnow() log.info('Listening to events.') for event_json in cfg.client.events(since=since): if b'container' not in event_json: continue if not any((x in event_json) for x in (b'start', b'die', b'pause', b'unpause')): continue event = from_json(event_json) log.debug('Daemon event: %s' % event) if event['Type'] != 'container': continue elif event['Action'] == 'start': handle_start(event) elif event['Action'] == 'die': handle_die(event) elif event['Action'] == 'pause': handle_pause(event) elif event['Action'] == 'unpause': handle_unpause(event)
def parse_labels(container_id: str) -> Tuple[Tuple[str, ...], str, Dict[str, Dict]]: labels = cfg.client.containers.get(container_id).labels log.debug(f'Parsing labels: {labels}') service_id = parse_service_id(labels) filtered_labels = {k: v for k, v in labels.items() if k.startswith(cfg.label_ns)} flags, user = parse_options(filtered_labels) if 'image' in flags: image_labels = image_definition_labels_of_container(container_id) user = user or parse_options(image_labels)[1] else: image_labels = {} job_definitions = parse_job_definitions( image_labels | filtered_labels, user # type: ignore # TODO remove eventually ) if service_id: log.debug(f'Assigning service id: {service_id}') for job_definition in job_definitions.values(): job_definition['service_id'] = service_id return service_id, flags, job_definitions
def parse_job_definitions(labels: Mapping[str, str], user: str) -> Dict[str, Dict]: log.debug(f'Considering labels for job definitions: {dict(labels)}') name_grouped_definitions: DefaultDict[ str, Dict[str, Union[str, Dict]] ] = defaultdict(dict) for key, value in labels.items(): key = key.removeprefix(cfg.label_ns) if '.env.' in key: name, _, variable = key.split('.', 2) name_grouped_definitions[name].setdefault('environment', {}) name_grouped_definitions[name]['environment'][ # type: ignore variable ] = value else: name, attribute = key.split('.', 1) name_grouped_definitions[name][attribute] = value log.debug(f'Job definitions: {dict(name_grouped_definitions)}') result = {} for name, definition in name_grouped_definitions.items(): log.debug(f'Processing {name}') definition['name'] = name definition.setdefault("user", user) job = job_config_validator.validated(definition) if job is None: log.error(f'Misconfigured job definition: {definition}') log.error(f'Errors: {job_config_validator.errors}') continue for trigger_name in ('cron', 'date', 'interval'): trigger = job.pop(trigger_name, None) if trigger is None: continue job['trigger'] = trigger log.debug(f'Normalized definition: {job}') result[name] = job return result
def lock_service(service_id: Tuple[str, ...], container_id: str): assert service_id not in service_locks_by_service_id _service_locks_by_service_id[service_id] = container_id assert container_id not in service_locks_by_container_id _service_locks_by_container_id[container_id] = service_id log.debug(f"Added lock for service {service_id} on container {container_id}.")
def unlock_service(container_id: str): service_id = _service_locks_by_container_id.pop(container_id, None) if service_id is None: return _service_locks_by_service_id.pop(service_id) log.debug(f"Removed lock for service {service_id} on container {container_id}.")
def exec_inspection(containers: dict) -> None: # TODO handle paused containers log.info('Inspecting running containers.') for container in containers: process_running_container_labels(container.id) log.debug('Finished inspection of running containers.')
def handle_start(event: dict) -> None: log.debug('Handling start.') container_id = event['Actor']['ID'] process_running_container_labels(container_id)