def diff_percent(self, other_image):
"""Calculate difference percent.
:param other_image:
:return: difference percent.
"""
img1 = self.get_pil()
img2 = other_image.get_pil()
if img1.mode != img2.mode:
Log.debug('Image diff percent: Different kinds of images.')
return 100
if img1.size != img2.size:
Log.debug('Image diff percent: Different sizes.')
return 100
pairs = zip(img1.getdata(), img2.getdata())
if len(img1.getbands()) == 1:
dif = sum(abs(p1-p2) for p1,p2 in pairs) # for gray-scale jpegs
else:
dif = sum(abs(c1-c2) for p1,p2 in pairs for c1,c2 in zip(p1,p2))
ncomponents = img1.size[0] * img1.size[1] * 3
return (dif / 255.0 * 100) / ncomponents
def install_module(self, name):
Log.debug('install %s module' % name)
if name.startswith('git:'):
return self.install_git_module(name[4:])
return self.install_mola_module(name)
def install_module(self, name):
Log.debug('install %s module' % name)
if name.startswith('git:'):
return self.install_git_module(name[4:])
return self.install_mola_module(name)
def __index_modules(reload=False):
"""Add all modules in module_list."""
global __modules_data
if __modules_data and not reload:
return
dir_list = sorted(os.listdir(MODULES_PATH))
nb = 0
__modules_data.clear()
for module_name in dir_list:
if is_disabled(module_name):
continue
if '__pycache__' in module_name:
continue
__modules_data[module_name] = {
'disabled': False,
'instance': None,
'thread': None
}
Log.debug('index "%s" module' % module_name)
nb += 1
Log.info('%d modules indexed' % nb)
def register(event_name, handler):
"""Register a handler."""
Log.debug('register handler event: %s for %s' % (handler, event_name))
if event_name not in __event_handlers:
__event_handlers[event_name] = []
__event_handlers[event_name].append(handler)
def started(self, component):
self.load_configuration()
config = self.choice_config()
if not config:
return
self.install_modules(config)
Log.debug('install sucessfull')
shutil.rmtree(os.path.join(MODULES_PATH, 'install'))
Daemon.restart()
def started(self, component):
self.load_configuration()
config = self.choice_config()
if not config:
return
self.install_modules(config)
Log.debug('install sucessfull')
shutil.rmtree(os.path.join(MODULES_PATH, 'install'))
Daemon.restart()
def __print_pending_change(self, module_name=''):
path = ROOT_PATH
if module_name:
path = os.path.join(path, 'modules', module_name)
result = subprocess.getoutput('cd "%s" && git status' % path)
if not 'nothing to commit' in result:
type = '"%s" module' % module_name if module_name else 'core'
Log.debug('change in %s: %s' % (type, result))
def fire(event):
""" send a event."""
Log.debug('event: %s' % event)
if event.name not in __event_handlers:
return
for handler in __event_handlers[event.name]:
threading.Thread(
None, handler, 'Event: %s at %s' % (event, str(handler)), (event,)
).start()
def __scan_modules(self):
dir_list = sorted(os.listdir(MODULES_PATH))
for module_name in dir_list:
dir_path = os.path.join(MODULES_PATH, module_name)
if '__pycache__' in module_name or not os.path.isdir(dir_path):
continue
if not os.path.isdir(os.path.join(dir_path, '.git')):
Log.debug('Not git repository: %s' % dir_path)
continue
self.__print_pending_change(module_name)
def load_peer(self):
conf_path = '%s/configs/peer/' % \
os.path.dirname(os.path.abspath(__file__))
nb = 0
if os.path.isdir(conf_path):
for name in os.listdir(conf_path):
if not os.path.exists(conf_path + name) \
or name[0] == '.':
continue
with open(conf_path + name) as config_file:
config = json.load(config_file)
if 'name' not in config:
config['name'] = name
self.add_peer(**config)
nb += 1
Log.debug('%d peer loaded' % nb)
def init(module_name):
"""Init module."""
if is_disabled(module_name):
Log.debug('module "%s" ignored: is disabled' % module_name)
return False
if module_name in __modules_data and not __modules_data[module_name]['instance']:
module_path = os.path.join(MODULES_PATH, module_name)
module_file = os.path.join(module_path, 'Module.py')
if not os.path.isfile(module_file):
Log.debug('module "%s" ignored: not "Module.py" file' % module_name)
return False
try:
module = __import__(
'modules.' + module_name + '.Module',
globals(),
locals(),
['Module'],
).Module()
if not module.is_available():
Log.debug('module "%s" ignored: not available' % module_name)
del(module)
return False
module.name = module_name
module.module_path = module_path
module._internal_init()
__modules_data[module_name]['instance'] = module
Log.debug('init module %s' % module_name)
except ImportError as e:
Log.error(
'Import error, module %s (%s):' % (module_name, e),
exc_info=True
)
return False
except AttributeError as e:
Log.error(
'Module error, module %s (%s)' % (module_name, e),
exc_info=True
)
return False
return True
class _Tasks:
def __init__(self):
self.log = Log(self.__class__.__name__)
async def remove_tasks(self, tasks: Dict[asyncio.Task, Any]):
assert isinstance(tasks, dict)
finished = tuple(task for task in tasks.keys() if task.done())
for task in finished:
if task.cancelled():
self.log.warning("Task was cancelled: ", task=task)
else:
try:
result = await task
if result:
self.log.debug("Task say:", task=task, result=result)
except RuntimeError:
raise
except Exception:
self.log.exception("Task shout:", task=task)
del tasks[task]
return tasks
class BaseWork:
start_time = time()
MUTE_EXCEPTION = True
PARALLEL = 10
INPUT_RETRIES = 0
WAIT_COEF = 1
need_stop = False
work_ids: Dict[str, int] = {}
def __init__(self):
work_name = self.__class__.__name__
self.log = Log(work_name)
self.log.debug("Register work")
self.work_ids.setdefault(work_name, -1)
self.work_ids[work_name] += 1
work_id = f"{work_name}_{self.work_ids[work_name]}"
self.log.debug("Work registered", work_id=work_id)
self.stat = Stats(work_id, work_name)
self.log.debug("Run task manager")
self.task_manager = TasksManager(self.PARALLEL)
self.tasks: List[TaskInfo] = []
self.state = "Base class initialized"
@property
def state(self):
return self.stat.state
@state.setter
def state(self, value):
self.stat.state = value
self.log.debug(self.state)
async def warm_up(self):
pass
async def input(self):
yield
raise NotImplementedError()
async def process(self, item):
yield
raise NotImplementedError()
async def update(self, result):
raise NotImplementedError()
async def shutdown(self):
pass
async def __call__(self):
self.state = "🔥 Warming up"
await self.warm_up()
self.stat._start_time = time()
try:
await self.main_cycle()
except Exception:
self.log.exception("MAIN CYCLE")
if not self.MUTE_EXCEPTION:
raise
self.stat.finished_time = time()
self.state = "🛑 Shutdown"
await self.shutdown()
self.state = "🏁 Finished"
async def main_cycle(self):
self.state = "⌛️ Ready to start"
await asyncio.gather(self._input_cycle(), self._result_cycle())
async def _result_cycle(self):
while True:
try:
result = await asyncio.wait_for(self.task_manager.take(), 1)
except asyncio.TimeoutError:
continue
if isinstance(result, TasksManager.Finish):
break
await self.update(result)
self.stat.updated_items += 1
async def _input_cycle(self):
self.stat.retries = 0
while not self.need_stop:
self.state = "🔎 Wait for new item"
async for item in self.input():
self.stat.input_items += 1
await self.task_manager.put(self._run_task(TaskInfo(item)))
self.stat.retries = None
if self.INPUT_RETRIES == 0:
# Need to run only one time
self.need_stop = True
continue
if self.stat.retries is None:
# Item found
self.stat.retries = 0
await asyncio.sleep(0)
continue
if self.stat.retries >= self.INPUT_RETRIES:
self.log.warning("Too many retries, i'm done",
retries=self.stat.retries)
self.need_stop = True
continue
# Retry logic
self.stat.retries += 1
self.state = f"🔎 Wait items, repeat №{self.stat.retries}"
await asyncio.sleep(self.stat.retries * self.WAIT_COEF)
await self.task_manager.stop()
async def _run_task(self, info: TaskInfo):
self.tasks.append(info)
info.update("🎬 Task started")
info.update(f"🛠 Processing")
async for result in self.process(info.item):
self.stat.returned_items += 1
info.update(f"🛠 {repr(result)}")
yield result
info.update(f"🛠 Processing")
self.stat.processed_items += 1
info.update("✅ Finish processing")
if info.processed_callback:
info.update("🤙 Run callback")
self.log.info("Run processed callback",
processed_callback=info.processed_callback)
await info.processed_callback
self.stat.finished_items += 1
info.update("🏁 Task finished")
self.tasks.remove(info)
async def take_error(self):
return await self.task_manager.take_error()
def add_debug(self, text, *args, **kwargs):
"""Add a debug message to logger"""
text = '%s %s: %s' % (self.name_prefix, self.name, text)
Log.debug(text, *args, **kwargs)
class VK:
def __init__(self, config_vk):
self.log = Log("VK")
self.config = config_vk
self.additional_params = {
'access_token': self.config.token,
'lang': 'ru',
'v': "5.103"
}
self.person_fields = ",".join([
"first_name",
"last_name",
"deactivated",
"verified",
"sex",
"bdate",
"city", # TODO: https://vk.com/dev/places.getCityById
"country", # TODO: https://vk.com/dev/places.getCountryById
"home_town",
"photo_400_orig",
"online",
"has_mobile",
"contacts",
"education",
"universities",
"schools",
"last_seen",
"occupation",
"hidden",
])
self.group_info_fields = ",".join([
'id', 'name', 'type', 'photo_200', 'city', 'description', 'place'
])
self.post_fields = ",".join([])
self.session: aiohttp.ClientSession = None
self.last_call = 0
self.threshold = 1 / 3
self._update_token = None
self.query_lock = asyncio.Lock()
self.stats = VKStats('vk', "VK API")
self.auth_lock = asyncio.Lock()
async def warm_up(self):
self.session = aiohttp.ClientSession()
async def call_method(self, method, **params):
self.log.debug(method=method, params=params)
self.stats.call_methods_count += 1
try:
while True:
assert self.session is not None, "call `await .warm_up()` first"
async with self.query_lock:
if time() - self.threshold < self.last_call:
self.log.deep("Sleep", threshold=self.threshold)
await asyncio.sleep(self.threshold)
self.last_call = time()
self.stats.queries += 1
response = await self.session.get(
url=f"{self.config['api_host']}{method}",
params={
**params,
**self.additional_params
},
timeout=10)
self.stats.by_type[method] += 1
result = await response.json()
if 'error' in result:
self.stats.errors += 1
vk_error = VKError(result['error'])
if vk_error.error_code == VKError.TOO_MANY_REQUESTS:
self.stats.errors_too_many += 1
self.threshold *= 1.1
if self.threshold > 1:
self.threshold = 1
self.log.warning("Too many requests",
threshold=self.threshold)
continue
if vk_error.error_code == VKError.INVALID_SESSION:
self.log.warning(
"Need auth. Please run <app> <config> auth")
break
if vk_error.error_code == VKError.PROFILE_PRIVATE:
self.log.warning("Profile private",
method=method,
params=params)
break
if vk_error.error_code == VKError.DELETED_OR_BANNED:
self.log.warning("Profile deleted or banned",
method=method,
params=params)
break
if vk_error.error_code == VKError.RATE_LIMIT_REACHED:
self.log.important("RATE LIMIT")
raise vk_error
else:
self.stats.success += 1
assert 'response' in result
self.threshold *= 0.991
return result['response']
finally:
self.stats.call_methods_count -= 1
self.stats.threshold = self.threshold
async def persons_info(self, *user_ids) -> Sequence[VKPerson]:
answer = await self.call_method("users.get",
user_ids=",".join(map(str, user_ids)),
fields=self.person_fields)
users = []
for user_info in answer:
users.append(VKPerson(**user_info))
return users
async def me(self) -> VKPerson:
return (await self.persons_info(self.config.user_id))[0]
async def group_info(self, group_id) -> VKGroup:
answer = await self.call_method("groups.getById",
group_id=group_id,
fields=self.group_info_fields)
assert len(answer) == 1
group = answer[0]
return VKGroup(**group)
async def person_posts(self, person_id, count):
return [post async for post in self._posts_count(person_id, count)]
async def person_posts_iter(self, person_id, count=None):
async for post in self._posts_count(person_id, count):
yield post
async def group_posts_iter(self, group_id, count=None):
async for post in self._posts_count(-group_id, count):
yield post
async def comments_iter(self, owner_id, post_id, count=None):
async for raw_data in self._offsetter(
count,
dict(
method='wall.getComments',
owner_id=owner_id,
post_id=post_id,
need_likes=1,
preview_length=0,
extended=0,
thread_items_count=10,
)):
comment = VKComment(**raw_data)
comment.post_id = post_id
comment.owner_id = owner_id
yield comment
async def group_posts(self, group_id, count=None, from_ts=None):
if count is not None and from_ts is not None:
raise ValueError("Use one of attribute: `count` or `from_ts`")
return [post async for post in self._posts_count(-group_id, count)]
async def _posts_count(self, owner_id, count):
async for post in self._offsetter(
count,
dict(method="wall.get",
owner_id=owner_id,
fields=self.post_fields)):
yield VKPost(**post)
async def _offsetter(self, count, params):
# TODO: Can be optimized! Use asyncio.gather after first query, Luke!
if count is None:
count = float("+inf")
if count < 1:
raise ValueError(f"{count=} must be more than 0")
offset = 0
items_count = count
while offset < items_count:
to_download = min(items_count - offset, 100)
try:
answer = await self.call_method(**params,
count=to_download,
offset=offset)
except VKError:
self.log.exception(params=params)
raise
if answer is None:
# Good error in call_method
return
items_count = min(count, answer['count'])
if to_download != len(answer['items']):
if to_download < items_count:
self.log.warning("Downloaded items count:",
wanted=to_download,
actual=len(answer['items']))
offset += to_download
for item in answer['items']:
yield item
async def group_user_ids(self, group_id, count=None) -> Sequence[int]:
users = []
async for user_id in self.group_participants_iter(group_id, count):
users.append(user_id)
return users
async def group_participants_iter(self, group_id, count=None):
async for user_id in self._offsetter(
count, dict(method="groups.getMembers", group_id=group_id)):
yield user_id
async def shutdown(self):
if self.session:
await self.session.close()
def install_modules(self, config):
installed = [self.install_module(name)
for name in config['modules_require']]
Log.debug('%d modules installed' % sum(installed))
return len(config['modules_require']) == sum(installed)
def __call(self, method, url, data=None, **kwargs):
if 'page' in kwargs:
url = '%s/%s' (url, kwargs['page'])
encoded_data = urllib.parse.urlencode(data).encode('ascii') if data \
else None
Log.debug('http_lib : %s %s' % (method, url))
request = urllib.request.Request(url, encoded_data)
request.get_method = lambda: method.upper()
# set headers
headers = kwargs.get('headers', {
'User-Agent': 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
'Referer': url,
})
for header in headers:
request.add_header(header, headers[header])
start = time.time()
try:
timeout = kwargs.get('timeout', self.timeout)
response = urllib.request.urlopen(request, timeout=timeout)
end = time.time()
http_code = response.status
headers = dict(response.getheaders())
if 'download_path' in kwargs:
html = ''
dir_path = os.path.dirname(kwargs['download_path'])
if not os.path.exists(dir_path):
os.makedirs(dir_path)
with open(kwargs['download_path'], 'wb') as f:
f.write(response.read())
else:
html = response.read().decode('utf8')
except urllib.error.HTTPError as e:
end = time.time()
html = e.reason
http_code = e.getcode()
headers = {}
except urllib.error.URLError as e:
Log.debug('http_lib: URLError: %s' % str(e))
if kwargs.get('retry', 0) > 0:
if 'retry_delay' not in kwargs:
kwargs['retry_delay'] = 2
time.sleep(kwargs['retry_delay'])
kwargs['retry_delay'] *= 2
kwargs['retry'] -= 1
return self.__call(method, url, data=data, **kwargs)
raise
return {
'response_time': end - start,
'html': html,
'code': http_code,
'headers': headers,
}
def load_configuration(self):
conf_path = '%s/config/' % os.path.dirname(os.path.abspath(__file__))
liste = os.listdir(conf_path)
loaded = [self.read_config_file(conf_path + name) for name in liste]
Log.debug('%d install config load' % sum(loaded))
def load_configuration(self):
conf_path = '%s/config/' % os.path.dirname(os.path.abspath(__file__))
liste = os.listdir(conf_path)
loaded = [self.read_config_file(conf_path + name) for name in liste]
Log.debug('%d install config load' % sum(loaded))
def install_modules(self, config):
installed = [
self.install_module(name) for name in config['modules_require']
]
Log.debug('%d modules installed' % sum(installed))
return len(config['modules_require']) == sum(installed)
