def diff_percent(self, other_image): """Calculate difference percent. :param other_image: :return: difference percent. """ img1 = self.get_pil() img2 = other_image.get_pil() if img1.mode != img2.mode: Log.debug('Image diff percent: Different kinds of images.') return 100 if img1.size != img2.size: Log.debug('Image diff percent: Different sizes.') return 100 pairs = zip(img1.getdata(), img2.getdata()) if len(img1.getbands()) == 1: dif = sum(abs(p1-p2) for p1,p2 in pairs) # for gray-scale jpegs else: dif = sum(abs(c1-c2) for p1,p2 in pairs for c1,c2 in zip(p1,p2)) ncomponents = img1.size[0] * img1.size[1] * 3 return (dif / 255.0 * 100) / ncomponents
def install_module(self, name): Log.debug('install %s module' % name) if name.startswith('git:'): return self.install_git_module(name[4:]) return self.install_mola_module(name)
def __index_modules(reload=False): """Add all modules in module_list.""" global __modules_data if __modules_data and not reload: return dir_list = sorted(os.listdir(MODULES_PATH)) nb = 0 __modules_data.clear() for module_name in dir_list: if is_disabled(module_name): continue if '__pycache__' in module_name: continue __modules_data[module_name] = { 'disabled': False, 'instance': None, 'thread': None } Log.debug('index "%s" module' % module_name) nb += 1 Log.info('%d modules indexed' % nb)
def register(event_name, handler): """Register a handler.""" Log.debug('register handler event: %s for %s' % (handler, event_name)) if event_name not in __event_handlers: __event_handlers[event_name] = [] __event_handlers[event_name].append(handler)
def started(self, component): self.load_configuration() config = self.choice_config() if not config: return self.install_modules(config) Log.debug('install sucessfull') shutil.rmtree(os.path.join(MODULES_PATH, 'install')) Daemon.restart()
def __print_pending_change(self, module_name=''): path = ROOT_PATH if module_name: path = os.path.join(path, 'modules', module_name) result = subprocess.getoutput('cd "%s" && git status' % path) if not 'nothing to commit' in result: type = '"%s" module' % module_name if module_name else 'core' Log.debug('change in %s: %s' % (type, result))
def fire(event): """ send a event.""" Log.debug('event: %s' % event) if event.name not in __event_handlers: return for handler in __event_handlers[event.name]: threading.Thread( None, handler, 'Event: %s at %s' % (event, str(handler)), (event,) ).start()
def __scan_modules(self): dir_list = sorted(os.listdir(MODULES_PATH)) for module_name in dir_list: dir_path = os.path.join(MODULES_PATH, module_name) if '__pycache__' in module_name or not os.path.isdir(dir_path): continue if not os.path.isdir(os.path.join(dir_path, '.git')): Log.debug('Not git repository: %s' % dir_path) continue self.__print_pending_change(module_name)
def load_peer(self): conf_path = '%s/configs/peer/' % \ os.path.dirname(os.path.abspath(__file__)) nb = 0 if os.path.isdir(conf_path): for name in os.listdir(conf_path): if not os.path.exists(conf_path + name) \ or name[0] == '.': continue with open(conf_path + name) as config_file: config = json.load(config_file) if 'name' not in config: config['name'] = name self.add_peer(**config) nb += 1 Log.debug('%d peer loaded' % nb)
def init(module_name): """Init module.""" if is_disabled(module_name): Log.debug('module "%s" ignored: is disabled' % module_name) return False if module_name in __modules_data and not __modules_data[module_name]['instance']: module_path = os.path.join(MODULES_PATH, module_name) module_file = os.path.join(module_path, 'Module.py') if not os.path.isfile(module_file): Log.debug('module "%s" ignored: not "Module.py" file' % module_name) return False try: module = __import__( 'modules.' + module_name + '.Module', globals(), locals(), ['Module'], ).Module() if not module.is_available(): Log.debug('module "%s" ignored: not available' % module_name) del(module) return False module.name = module_name module.module_path = module_path module._internal_init() __modules_data[module_name]['instance'] = module Log.debug('init module %s' % module_name) except ImportError as e: Log.error( 'Import error, module %s (%s):' % (module_name, e), exc_info=True ) return False except AttributeError as e: Log.error( 'Module error, module %s (%s)' % (module_name, e), exc_info=True ) return False return True
class _Tasks: def __init__(self): self.log = Log(self.__class__.__name__) async def remove_tasks(self, tasks: Dict[asyncio.Task, Any]): assert isinstance(tasks, dict) finished = tuple(task for task in tasks.keys() if task.done()) for task in finished: if task.cancelled(): self.log.warning("Task was cancelled: ", task=task) else: try: result = await task if result: self.log.debug("Task say:", task=task, result=result) except RuntimeError: raise except Exception: self.log.exception("Task shout:", task=task) del tasks[task] return tasks
class BaseWork: start_time = time() MUTE_EXCEPTION = True PARALLEL = 10 INPUT_RETRIES = 0 WAIT_COEF = 1 need_stop = False work_ids: Dict[str, int] = {} def __init__(self): work_name = self.__class__.__name__ self.log = Log(work_name) self.log.debug("Register work") self.work_ids.setdefault(work_name, -1) self.work_ids[work_name] += 1 work_id = f"{work_name}_{self.work_ids[work_name]}" self.log.debug("Work registered", work_id=work_id) self.stat = Stats(work_id, work_name) self.log.debug("Run task manager") self.task_manager = TasksManager(self.PARALLEL) self.tasks: List[TaskInfo] = [] self.state = "Base class initialized" @property def state(self): return self.stat.state @state.setter def state(self, value): self.stat.state = value self.log.debug(self.state) async def warm_up(self): pass async def input(self): yield raise NotImplementedError() async def process(self, item): yield raise NotImplementedError() async def update(self, result): raise NotImplementedError() async def shutdown(self): pass async def __call__(self): self.state = "🔥 Warming up" await self.warm_up() self.stat._start_time = time() try: await self.main_cycle() except Exception: self.log.exception("MAIN CYCLE") if not self.MUTE_EXCEPTION: raise self.stat.finished_time = time() self.state = "🛑 Shutdown" await self.shutdown() self.state = "🏁 Finished" async def main_cycle(self): self.state = "⌛️ Ready to start" await asyncio.gather(self._input_cycle(), self._result_cycle()) async def _result_cycle(self): while True: try: result = await asyncio.wait_for(self.task_manager.take(), 1) except asyncio.TimeoutError: continue if isinstance(result, TasksManager.Finish): break await self.update(result) self.stat.updated_items += 1 async def _input_cycle(self): self.stat.retries = 0 while not self.need_stop: self.state = "🔎 Wait for new item" async for item in self.input(): self.stat.input_items += 1 await self.task_manager.put(self._run_task(TaskInfo(item))) self.stat.retries = None if self.INPUT_RETRIES == 0: # Need to run only one time self.need_stop = True continue if self.stat.retries is None: # Item found self.stat.retries = 0 await asyncio.sleep(0) continue if self.stat.retries >= self.INPUT_RETRIES: self.log.warning("Too many retries, i'm done", retries=self.stat.retries) self.need_stop = True continue # Retry logic self.stat.retries += 1 self.state = f"🔎 Wait items, repeat №{self.stat.retries}" await asyncio.sleep(self.stat.retries * self.WAIT_COEF) await self.task_manager.stop() async def _run_task(self, info: TaskInfo): self.tasks.append(info) info.update("🎬 Task started") info.update(f"🛠 Processing") async for result in self.process(info.item): self.stat.returned_items += 1 info.update(f"🛠 {repr(result)}") yield result info.update(f"🛠 Processing") self.stat.processed_items += 1 info.update("✅ Finish processing") if info.processed_callback: info.update("🤙 Run callback") self.log.info("Run processed callback", processed_callback=info.processed_callback) await info.processed_callback self.stat.finished_items += 1 info.update("🏁 Task finished") self.tasks.remove(info) async def take_error(self): return await self.task_manager.take_error()
def add_debug(self, text, *args, **kwargs): """Add a debug message to logger""" text = '%s %s: %s' % (self.name_prefix, self.name, text) Log.debug(text, *args, **kwargs)
class VK: def __init__(self, config_vk): self.log = Log("VK") self.config = config_vk self.additional_params = { 'access_token': self.config.token, 'lang': 'ru', 'v': "5.103" } self.person_fields = ",".join([ "first_name", "last_name", "deactivated", "verified", "sex", "bdate", "city", # TODO: https://vk.com/dev/places.getCityById "country", # TODO: https://vk.com/dev/places.getCountryById "home_town", "photo_400_orig", "online", "has_mobile", "contacts", "education", "universities", "schools", "last_seen", "occupation", "hidden", ]) self.group_info_fields = ",".join([ 'id', 'name', 'type', 'photo_200', 'city', 'description', 'place' ]) self.post_fields = ",".join([]) self.session: aiohttp.ClientSession = None self.last_call = 0 self.threshold = 1 / 3 self._update_token = None self.query_lock = asyncio.Lock() self.stats = VKStats('vk', "VK API") self.auth_lock = asyncio.Lock() async def warm_up(self): self.session = aiohttp.ClientSession() async def call_method(self, method, **params): self.log.debug(method=method, params=params) self.stats.call_methods_count += 1 try: while True: assert self.session is not None, "call `await .warm_up()` first" async with self.query_lock: if time() - self.threshold < self.last_call: self.log.deep("Sleep", threshold=self.threshold) await asyncio.sleep(self.threshold) self.last_call = time() self.stats.queries += 1 response = await self.session.get( url=f"{self.config['api_host']}{method}", params={ **params, **self.additional_params }, timeout=10) self.stats.by_type[method] += 1 result = await response.json() if 'error' in result: self.stats.errors += 1 vk_error = VKError(result['error']) if vk_error.error_code == VKError.TOO_MANY_REQUESTS: self.stats.errors_too_many += 1 self.threshold *= 1.1 if self.threshold > 1: self.threshold = 1 self.log.warning("Too many requests", threshold=self.threshold) continue if vk_error.error_code == VKError.INVALID_SESSION: self.log.warning( "Need auth. Please run <app> <config> auth") break if vk_error.error_code == VKError.PROFILE_PRIVATE: self.log.warning("Profile private", method=method, params=params) break if vk_error.error_code == VKError.DELETED_OR_BANNED: self.log.warning("Profile deleted or banned", method=method, params=params) break if vk_error.error_code == VKError.RATE_LIMIT_REACHED: self.log.important("RATE LIMIT") raise vk_error else: self.stats.success += 1 assert 'response' in result self.threshold *= 0.991 return result['response'] finally: self.stats.call_methods_count -= 1 self.stats.threshold = self.threshold async def persons_info(self, *user_ids) -> Sequence[VKPerson]: answer = await self.call_method("users.get", user_ids=",".join(map(str, user_ids)), fields=self.person_fields) users = [] for user_info in answer: users.append(VKPerson(**user_info)) return users async def me(self) -> VKPerson: return (await self.persons_info(self.config.user_id))[0] async def group_info(self, group_id) -> VKGroup: answer = await self.call_method("groups.getById", group_id=group_id, fields=self.group_info_fields) assert len(answer) == 1 group = answer[0] return VKGroup(**group) async def person_posts(self, person_id, count): return [post async for post in self._posts_count(person_id, count)] async def person_posts_iter(self, person_id, count=None): async for post in self._posts_count(person_id, count): yield post async def group_posts_iter(self, group_id, count=None): async for post in self._posts_count(-group_id, count): yield post async def comments_iter(self, owner_id, post_id, count=None): async for raw_data in self._offsetter( count, dict( method='wall.getComments', owner_id=owner_id, post_id=post_id, need_likes=1, preview_length=0, extended=0, thread_items_count=10, )): comment = VKComment(**raw_data) comment.post_id = post_id comment.owner_id = owner_id yield comment async def group_posts(self, group_id, count=None, from_ts=None): if count is not None and from_ts is not None: raise ValueError("Use one of attribute: `count` or `from_ts`") return [post async for post in self._posts_count(-group_id, count)] async def _posts_count(self, owner_id, count): async for post in self._offsetter( count, dict(method="wall.get", owner_id=owner_id, fields=self.post_fields)): yield VKPost(**post) async def _offsetter(self, count, params): # TODO: Can be optimized! Use asyncio.gather after first query, Luke! if count is None: count = float("+inf") if count < 1: raise ValueError(f"{count=} must be more than 0") offset = 0 items_count = count while offset < items_count: to_download = min(items_count - offset, 100) try: answer = await self.call_method(**params, count=to_download, offset=offset) except VKError: self.log.exception(params=params) raise if answer is None: # Good error in call_method return items_count = min(count, answer['count']) if to_download != len(answer['items']): if to_download < items_count: self.log.warning("Downloaded items count:", wanted=to_download, actual=len(answer['items'])) offset += to_download for item in answer['items']: yield item async def group_user_ids(self, group_id, count=None) -> Sequence[int]: users = [] async for user_id in self.group_participants_iter(group_id, count): users.append(user_id) return users async def group_participants_iter(self, group_id, count=None): async for user_id in self._offsetter( count, dict(method="groups.getMembers", group_id=group_id)): yield user_id async def shutdown(self): if self.session: await self.session.close()
def install_modules(self, config): installed = [self.install_module(name) for name in config['modules_require']] Log.debug('%d modules installed' % sum(installed)) return len(config['modules_require']) == sum(installed)
def __call(self, method, url, data=None, **kwargs): if 'page' in kwargs: url = '%s/%s' (url, kwargs['page']) encoded_data = urllib.parse.urlencode(data).encode('ascii') if data \ else None Log.debug('http_lib : %s %s' % (method, url)) request = urllib.request.Request(url, encoded_data) request.get_method = lambda: method.upper() # set headers headers = kwargs.get('headers', { 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)', 'Referer': url, }) for header in headers: request.add_header(header, headers[header]) start = time.time() try: timeout = kwargs.get('timeout', self.timeout) response = urllib.request.urlopen(request, timeout=timeout) end = time.time() http_code = response.status headers = dict(response.getheaders()) if 'download_path' in kwargs: html = '' dir_path = os.path.dirname(kwargs['download_path']) if not os.path.exists(dir_path): os.makedirs(dir_path) with open(kwargs['download_path'], 'wb') as f: f.write(response.read()) else: html = response.read().decode('utf8') except urllib.error.HTTPError as e: end = time.time() html = e.reason http_code = e.getcode() headers = {} except urllib.error.URLError as e: Log.debug('http_lib: URLError: %s' % str(e)) if kwargs.get('retry', 0) > 0: if 'retry_delay' not in kwargs: kwargs['retry_delay'] = 2 time.sleep(kwargs['retry_delay']) kwargs['retry_delay'] *= 2 kwargs['retry'] -= 1 return self.__call(method, url, data=data, **kwargs) raise return { 'response_time': end - start, 'html': html, 'code': http_code, 'headers': headers, }
def load_configuration(self): conf_path = '%s/config/' % os.path.dirname(os.path.abspath(__file__)) liste = os.listdir(conf_path) loaded = [self.read_config_file(conf_path + name) for name in liste] Log.debug('%d install config load' % sum(loaded))
def install_modules(self, config): installed = [ self.install_module(name) for name in config['modules_require'] ] Log.debug('%d modules installed' % sum(installed)) return len(config['modules_require']) == sum(installed)