def _find_or_create_profile(vk_post: dict, post_date: datetime,
db_profiles: List[Profile]) -> Profile:
profile_id = vk_post['from_id']
db_profile = find(db_profiles, lambda it: it.id == profile_id)
if db_profile:
return db_profile
db_profile = Profile(id=profile_id,
join_date=post_date,
first_name='Unknown',
sex=Profile.Sex.UNKNOWN)
time.sleep(GETTING_USER_INTERVAL)
if profile_id >= 0:
vk_user = vk_api_service.get_user(profile_id)
if vk_user:
db_profile.first_name = vk_user['first_name']
db_profile.last_name = vk_user['last_name']
db_profile.sex = vk_user['sex']
db_profile.photo_50 = vk_user['photo_50']
db_profile.photo_100 = vk_user['photo_100']
else:
vk_group = vk_api_service.get_group(profile_id * -1)
if vk_group:
db_profile.first_name = vk_group['name']
db_profile.photo_50 = vk_group['photo_50']
db_profile.photo_100 = vk_group['photo_100']
db_profile.photo_200 = vk_group['photo_200']
db_profile.save()
db_profiles.append(db_profile)
return db_profile
def test_find(self):
people = [{
'id': 21,
'name': 'Jack'
}, {
'id': 11,
'name': 'Bob'
}, {
'id': 300,
'name': 'Dude'
}, {
'id': 22,
'name': 'Craig'
}]
person = util.find(people, lambda it: it['id'] == 300)
self.assertEqual(person, {'id': 300, 'name': 'Dude'})
def _sync_block_posts(vk_post_count: int, download_count: int) -> int:
db_post_count = Post.objects.count()
logger.debug(
f'>> Downloaded (before sync): {db_post_count}/{vk_post_count}')
if vk_post_count - db_post_count > download_count:
offset = vk_post_count - db_post_count - download_count
else:
offset = 0
response = vk_api_service.get_wall_posts(offset, download_count)
if response['count'] != vk_post_count:
logger.debug(
f' -- Number of posts in VK changed: {vk_post_count} -> {response["count"]}'
)
return db_post_count
vk_posts = list(reversed(response['items']))
db_profiles = list(Profile.objects.all())
last_db_posts = _get_last_posts(LAST_POSTS_COUNT)
deleted_post_ids = _remove_deleted_posts(vk_posts, last_db_posts)
for vk_post in vk_posts:
post_id = vk_post['id']
post_text = remove_non_utf8_chars(vk_post['text'])
post_date = datetime.utcfromtimestamp(vk_post['date']).astimezone(
timezone.get_default_timezone())
text_hash = md5(post_text.encode()).hexdigest()
db_post = find(last_db_posts, lambda it: it.id == post_id)
last_post = _get_last_post(last_db_posts, post_id, post_date)
last_sum_distance = last_post.sum_distance if last_post else 0
last_post_number = last_post.number if last_post else 0
if db_post:
# if post exists in db and (it isn't modified or was hand modified) than continue to next post
if text_hash == db_post.text_hash and db_post.start_sum == last_sum_distance \
or db_post.last_update is not None:
continue
_analyze_post_text(post_text, text_hash, last_sum_distance,
last_post_number, db_post, EventType.UPDATE)
continue
# Searching and creating a new profile
profile = _find_or_create_profile(vk_post, post_date, db_profiles)
new_post = Post(id=post_id,
status=Post.Status.SUCCESS,
author=profile,
date=post_date)
parser_out = _analyze_post_text(post_text, text_hash,
last_sum_distance, last_post_number,
new_post, EventType.CREATE)
# Adding a new post into last posts and post sorting by time
if parser_out:
last_db_posts.append(new_post)
last_db_posts.sort(key=lambda p: p.date, reverse=True)
# Deleting posts from the client, after sync without exceptions
for post_id in deleted_post_ids:
ws_service.main_group_send(post_id, ObjectType.POST, EventType.REMOVE)
return Post.objects.count()
def _get_last_post(posts, post_id, post_date):
return find(
posts, lambda it: it.number is not None and it.id != post_id and it.
date <= post_date)
def not_find_in_vk(post):
return find(vk_posts, lambda it: it['id'] == post.id) is None