def _find_or_create_profile(vk_post: dict, post_date: datetime, db_profiles: List[Profile]) -> Profile: profile_id = vk_post['from_id'] db_profile = find(db_profiles, lambda it: it.id == profile_id) if db_profile: return db_profile db_profile = Profile(id=profile_id, join_date=post_date, first_name='Unknown', sex=Profile.Sex.UNKNOWN) time.sleep(GETTING_USER_INTERVAL) if profile_id >= 0: vk_user = vk_api_service.get_user(profile_id) if vk_user: db_profile.first_name = vk_user['first_name'] db_profile.last_name = vk_user['last_name'] db_profile.sex = vk_user['sex'] db_profile.photo_50 = vk_user['photo_50'] db_profile.photo_100 = vk_user['photo_100'] else: vk_group = vk_api_service.get_group(profile_id * -1) if vk_group: db_profile.first_name = vk_group['name'] db_profile.photo_50 = vk_group['photo_50'] db_profile.photo_100 = vk_group['photo_100'] db_profile.photo_200 = vk_group['photo_200'] db_profile.save() db_profiles.append(db_profile) return db_profile
def test_find(self): people = [{ 'id': 21, 'name': 'Jack' }, { 'id': 11, 'name': 'Bob' }, { 'id': 300, 'name': 'Dude' }, { 'id': 22, 'name': 'Craig' }] person = util.find(people, lambda it: it['id'] == 300) self.assertEqual(person, {'id': 300, 'name': 'Dude'})
def _sync_block_posts(vk_post_count: int, download_count: int) -> int: db_post_count = Post.objects.count() logger.debug( f'>> Downloaded (before sync): {db_post_count}/{vk_post_count}') if vk_post_count - db_post_count > download_count: offset = vk_post_count - db_post_count - download_count else: offset = 0 response = vk_api_service.get_wall_posts(offset, download_count) if response['count'] != vk_post_count: logger.debug( f' -- Number of posts in VK changed: {vk_post_count} -> {response["count"]}' ) return db_post_count vk_posts = list(reversed(response['items'])) db_profiles = list(Profile.objects.all()) last_db_posts = _get_last_posts(LAST_POSTS_COUNT) deleted_post_ids = _remove_deleted_posts(vk_posts, last_db_posts) for vk_post in vk_posts: post_id = vk_post['id'] post_text = remove_non_utf8_chars(vk_post['text']) post_date = datetime.utcfromtimestamp(vk_post['date']).astimezone( timezone.get_default_timezone()) text_hash = md5(post_text.encode()).hexdigest() db_post = find(last_db_posts, lambda it: it.id == post_id) last_post = _get_last_post(last_db_posts, post_id, post_date) last_sum_distance = last_post.sum_distance if last_post else 0 last_post_number = last_post.number if last_post else 0 if db_post: # if post exists in db and (it isn't modified or was hand modified) than continue to next post if text_hash == db_post.text_hash and db_post.start_sum == last_sum_distance \ or db_post.last_update is not None: continue _analyze_post_text(post_text, text_hash, last_sum_distance, last_post_number, db_post, EventType.UPDATE) continue # Searching and creating a new profile profile = _find_or_create_profile(vk_post, post_date, db_profiles) new_post = Post(id=post_id, status=Post.Status.SUCCESS, author=profile, date=post_date) parser_out = _analyze_post_text(post_text, text_hash, last_sum_distance, last_post_number, new_post, EventType.CREATE) # Adding a new post into last posts and post sorting by time if parser_out: last_db_posts.append(new_post) last_db_posts.sort(key=lambda p: p.date, reverse=True) # Deleting posts from the client, after sync without exceptions for post_id in deleted_post_ids: ws_service.main_group_send(post_id, ObjectType.POST, EventType.REMOVE) return Post.objects.count()
def _get_last_post(posts, post_id, post_date): return find( posts, lambda it: it.number is not None and it.id != post_id and it. date <= post_date)
def not_find_in_vk(post): return find(vk_posts, lambda it: it['id'] == post.id) is None