def run(): username = input("input delete username: ") try: GithubUser.objects.get(username=username) github_information_service = GithubInformationService(username=username) github_information_service.update() except GithubUser.DoesNotExist: pass
def run(): """ 업데이트가 늦은 날짜 순으로 ordering 된 유저들의 상세 깃헙 정보 업데이트 (업데이트 한지 일주일 이내 유저 제외) rate limit 를 고려하여 새벽 시간대에만 돌림 """ try: # 1스크립트를 시작하기전 rate_limit 를 체크한다. rate_limit_check_service = GithubInformationService( is_insert_queue=False) rate_limit_check_service.get_rate_remaining() except RateLimit: return github_user_qs = GithubUser.objects.filter( updated__lte=datetime.now() - timedelta(7)).order_by('updated') if not github_user_qs: return start_time = timeit.default_timer() SlackService.slack_update_older_week_user(status='시작', message='') update_user_count = 0 with concurrent.futures.ThreadPoolExecutor() as executor: for github_user in chunkator(github_user_qs, 1000): try: # 배치에서는 queue 에 넣지 않는다 (그냥 새벽에 실행하도록) github_information_service = GithubInformationService( username=github_user.username, is_insert_queue=False) executor.submit(github_information_service.update) update_user_count += 1 except RateLimit: SlackService.slack_notify_update_fail( message=f'Rate Limit 로 인해 업데이트가 실패되었습니다. ' f'{update_user_count}명만 업데이트 되었습니다.😭') break except GitHubUserDoesNotExist: continue terminate_time = timeit.default_timer() SlackService.slack_update_older_week_user( status='완료', message=f'업데이트가 {terminate_time - start_time:.2f}초 걸렸습니다. ' f'🤖 API 호출 남은 횟수 : {rate_limit_check_service.get_rate_remaining()}', update_user=update_user_count)
def run(): """ 기본적으로 업데이트 되야할 유저 기본정보 업데이트 (업데이트 한지 일주일 이내 유저 제외) """ try: # 스크립트를 시작하기전 rate_limit 를 체크한다. rate_limit_check_service = GithubInformationService( is_insert_queue=False) rate_limit_check_service.get_rate_remaining() except RateLimit: return github_user_qs = GithubUser.objects.filter(updated__lte=datetime.now() - timedelta(7)) if not github_user_qs: return start_time = timeit.default_timer() SlackService.slack_update_basic_info(status='시작', message='') update_user_count = 0 with concurrent.futures.ThreadPoolExecutor() as executor: # max_worker default = min(32, os.cpu_count() + 4) for github_user in chunkator(github_user_qs, 1000): try: executor.submit(update_github_basic_information, github_user) update_user_count += 1 except RateLimit: SlackService.slack_notify_update_fail( message=f'Rate Limit 로 인해 업데이트가 실패되었습니다. ' f'{update_user_count}명만 업데이트 되었습니다.😭') except GitHubUserDoesNotExist: continue terminate_time = timeit.default_timer() SlackService.slack_update_basic_info( status='완료', message=f'업데이트가 {terminate_time - start_time:.2f}초 걸렸습니다. ' f'🤖 API 호출 남은 횟수 : {rate_limit_check_service.get_rate_remaining()}', update_user=update_user_count)
def retrieve(self, request, *args, **kwargs): self.serializer_class = GithubUserSerializer username = self.kwargs.get(self.lookup_url_kwarg) github_user = self.get_queryset().filter(username=username).first() if not github_user: try: github_information_service = GithubInformationService(username) github_user = github_information_service.update() except GitHubUserDoesNotExist: raise NotExistsGithubUser() except RateLimit: raise RateLimitGithubAPI() serializer = self.serializer_class(github_user) return Response(serializer.data)
def update_user_ranking(): """ total score 로 전체 유저의 순위를 계산하는 함수 """ github_users = GithubUser.objects.all() for github_user in chunkator(github_users, 1000): try: github_user.previous_user_rank = github_user.user_rank github_user.user_rank = GithubInformationService.update_user_ranking( github_user.total_score) github_user.tier = GithubInformationService.get_tier_statistics( github_user.user_rank) github_user.save( update_fields=['user_rank', 'previous_user_rank', 'tier']) except GitHubUserDoesNotExist: continue except Exception as e: capture_exception(e)
def update_github_basic_information(github_user: GithubUser): github_information_service = GithubInformationService(github_user.username) user_information = github_information_service.github_adapter.get_user_info( github_user.username) for key, value in asdict(user_information).items(): if key in github_information_service.user_update_fields: if getattr(github_user, key, '') != value: setattr(github_user, key, value) # is_completed, continuous_count = get_continuous_commit_day(github_user.username) # if is_completed: # github_user.continuous_commit_day = continuous_count github_user.total_score = github_information_service.get_total_score( github_user) github_user.user_rank = github_information_service.update_user_ranking( github_user.total_score) github_user.tier = github_information_service.get_tier_statistics( github_user.user_rank) github_user.save(update_fields=['total_score', 'user_rank', 'tier'])
def run(): start_time = timeit.default_timer() # 시작 시간 체크 print('------- start update information -------') github_users = GithubUser.objects.all() for github_user in chunkator(github_users, 1000): github_service = GithubInformationService( username=github_user.username) github_user.total_score = github_service.get_total_score(github_user) github_user.save(update_fields=['total_score']) print(f'[{github_user}] - {github_user.total_score}') for github_user in chunkator(github_users, 1000): github_service = GithubInformationService( username=github_user.username) github_user.tier = github_service.get_tier_statistics( github_user.user_rank) github_user.save(update_fields=['tier']) print( f'[{github_user}] - {github_user.get_tier_display()}[{github_user.user_rank}]' ) print('------- end update information -------') terminate_time = timeit.default_timer() # 종료 시간 체크 print(f'{terminate_time - start_time:.2f}초 걸렸습니다.')
def update(self, request, *args, **kwargs): self.serializer_class = GithubUserSerializer username = self.kwargs.get(self.lookup_url_kwarg) try: github_user = GithubUser.objects.filter(username=username).get() if self.can_update(updated_date=github_user.updated) is False: response_data = self.serializer_class(github_user).data return Response(response_data) github_information_service = GithubInformationService(username) user = github_information_service.update() response_data = self.serializer_class(user).data except GithubUser.DoesNotExist: raise exceptions.NotFound except RateLimit: raise RateLimitGithubAPI() return Response(response_data)
def run(): """ 30분마다 실행되는 Github user 업데이트 : github api가 rate_limit 걸려서 더이상 호출하지 못하는 경우 """ start_time = timeit.default_timer() # 시작 시간 체크 update_user_queue_qs = UpdateUserQueue.objects.filter( status__in=[UpdateUserQueue.READY, UpdateUserQueue.FAIL]) if not update_user_queue_qs: return try: # 스크립트를 시작하기전 rate_limit 를 체크한다. rate_limit_check_service = GithubInformationService( is_insert_queue=False) rate_limit_check_service.get_rate_remaining() except RateLimit: return SlackService.slack_update_github_user(status='시작', message='') update_user_count = 0 for user_queue in chunkator(update_user_queue_qs, 1000): try: github_information_service = GithubInformationService( user_queue.username, False) github_information_service.update() user_queue.status = UpdateUserQueue.SUCCESS user_queue.save(update_fields=['status']) update_user_count += 1 except RateLimit: # rate limit 면 다른 유저들도 업데이드 못함 SlackService.slack_notify_update_fail( message=f'Rate Limit 로 인해 업데이트가 실패되었습니다. ' f'{update_user_count}명만 업데이트 되었습니다.😭') break except Exception as e: capture_exception(e) terminate_time = timeit.default_timer() # 종료 시간 체크 SlackService.slack_update_github_user( status='완료', message=f'업데이트가 {terminate_time - start_time:.2f}초 걸렸습니다.', update_user=update_user_count)
def test_get_or_create_github_user(github_context, mock_slack_notify_new_user): username = '******' user_information_dto = UserInformationDto( name='jay', email='*****@*****.**', location='Republic of Korea', company='DirtyBoyz', avatar_url='https://avatars.githubusercontent.com/u/24591259?v=4', bio='', blog='', public_repos=0, followers=0, following=0, repos_url='', organizations_url='') github_information_service = GithubInformationService(username=username) github_user = github_information_service.get_or_create_github_user( user_information_dto) assert github_user is not None for key, value in asdict(user_information_dto).items(): if key in github_information_service.user_update_fields: assert getattr(github_user, key, '') == value
async def update_continuous_commit_day(github_user: GithubUser): """ 1일 1커밋 크롤링으로 업데이트 """ if not is_exists_github_users(github_user.username): return is_completed, continuous_count = get_continuous_commit_day( github_user.username) if is_completed: github_user.continuous_commit_day = continuous_count github_user.total_score = GithubInformationService.get_total_score( github_user) github_user.save( update_fields=['continuous_commit_day', 'total_score'])