async def number_of_collectors_to_process(timestamp): number_of_collectors = await db.count(CollectorState.select().where( CollectorState.last_processing_time < timestamp - CollectorState.processing_period, )) await db.create( NumberOfCollectorsToProcess, timestamp=timestamp, value=number_of_collectors, )
async def init(): global collectors _collectors_dirs = settings.COLLECTORS_DIRS if type(_collectors_dirs) is not list: _collectors_dirs = [_collectors_dirs] for collectors_dir in _collectors_dirs: if collectors_dir.startswith('/'): raise Exception("Collector's dir cannot be absolute") if collectors_dir.startswith('..'): raise Exception("Collector's dir cannot be in parent directory") for root, dirs, files in os.walk(collectors_dir): for file in files: if file.endswith('.py'): file_path = os.path.join(root, file) if file_path.startswith('./'): file_path = file_path[2:] module_name = os.path.splitext(file_path)[0].replace( '/', '.') spec = importlib.util.spec_from_file_location( module_name, file_path) collector_module = importlib.util.module_from_spec(spec) spec.loader.exec_module(collector_module) # TODO: iterate through all classes independent of their names for name, member in inspect.getmembers( collector_module, inspect.isclass): # if inspect.isclass(member): if member.__module__ == collector_module.__name__ \ and hasattr(member, '__collector__') \ and member.__collector__: collectors[module_name + '.' + member.__name__] = member() # init db for module_name, Collector in collectors.items(): try: await db.get(CollectorState.select().where( CollectorState.identifier == module_name)) except CollectorState.DoesNotExist: await db.create( CollectorState, identifier=module_name, processing_period=Collector.processing_period, last_processing_time=0, )
async def process_collectors(self): while True: await asyncio.sleep(0.1) try: # check collectors collector_states = await db.execute( CollectorState.select().where( CollectorState.last_processing_time < time.time() - CollectorState.processing_period ).order_by(peewee.fn.Random()).limit(settings.NUMBER_OF_CONCURRENT_COLLECTORS) ) await asyncio.gather(*[ self.process_collector_of_state(collector_state) for collector_state in collector_states ]) except KeyboardInterrupt as ex: raise ex except BaseException as ex: self.collectors_logger.exception(ex) if settings.DEBUG: raise ex await asyncio.sleep(settings.SLEEP_AFTER_ERROR_PERIOD)
async def get_collector_state_html(self, request): return { "collector_states": list(await db.execute(CollectorState.select())), }