def run():
username = input("input delete username: ")
try:
GithubUser.objects.get(username=username)
github_information_service = GithubInformationService(username=username)
github_information_service.update()
except GithubUser.DoesNotExist:
pass
def run():
"""
업데이트가 늦은 날짜 순으로 ordering 된 유저들의 상세 깃헙 정보 업데이트
(업데이트 한지 일주일 이내 유저 제외)
rate limit 를 고려하여 새벽 시간대에만 돌림
"""
try:
# 1스크립트를 시작하기전 rate_limit 를 체크한다.
rate_limit_check_service = GithubInformationService(
is_insert_queue=False)
rate_limit_check_service.get_rate_remaining()
except RateLimit:
return
github_user_qs = GithubUser.objects.filter(
updated__lte=datetime.now() - timedelta(7)).order_by('updated')
if not github_user_qs:
return
start_time = timeit.default_timer()
SlackService.slack_update_older_week_user(status='시작', message='')
update_user_count = 0
with concurrent.futures.ThreadPoolExecutor() as executor:
for github_user in chunkator(github_user_qs, 1000):
try:
# 배치에서는 queue 에 넣지 않는다 (그냥 새벽에 실행하도록)
github_information_service = GithubInformationService(
username=github_user.username, is_insert_queue=False)
executor.submit(github_information_service.update)
update_user_count += 1
except RateLimit:
SlackService.slack_notify_update_fail(
message=f'Rate Limit 로 인해 업데이트가 실패되었습니다. '
f'{update_user_count}명만 업데이트 되었습니다.😭')
break
except GitHubUserDoesNotExist:
continue
terminate_time = timeit.default_timer()
SlackService.slack_update_older_week_user(
status='완료',
message=f'업데이트가 {terminate_time - start_time:.2f}초 걸렸습니다. '
f'🤖 API 호출 남은 횟수 : {rate_limit_check_service.get_rate_remaining()}',
update_user=update_user_count)
def run():
"""
기본적으로 업데이트 되야할 유저 기본정보 업데이트
(업데이트 한지 일주일 이내 유저 제외)
"""
try:
# 스크립트를 시작하기전 rate_limit 를 체크한다.
rate_limit_check_service = GithubInformationService(
is_insert_queue=False)
rate_limit_check_service.get_rate_remaining()
except RateLimit:
return
github_user_qs = GithubUser.objects.filter(updated__lte=datetime.now() -
timedelta(7))
if not github_user_qs:
return
start_time = timeit.default_timer()
SlackService.slack_update_basic_info(status='시작', message='')
update_user_count = 0
with concurrent.futures.ThreadPoolExecutor() as executor:
# max_worker default = min(32, os.cpu_count() + 4)
for github_user in chunkator(github_user_qs, 1000):
try:
executor.submit(update_github_basic_information, github_user)
update_user_count += 1
except RateLimit:
SlackService.slack_notify_update_fail(
message=f'Rate Limit 로 인해 업데이트가 실패되었습니다. '
f'{update_user_count}명만 업데이트 되었습니다.😭')
except GitHubUserDoesNotExist:
continue
terminate_time = timeit.default_timer()
SlackService.slack_update_basic_info(
status='완료',
message=f'업데이트가 {terminate_time - start_time:.2f}초 걸렸습니다. '
f'🤖 API 호출 남은 횟수 : {rate_limit_check_service.get_rate_remaining()}',
update_user=update_user_count)
def retrieve(self, request, *args, **kwargs):
self.serializer_class = GithubUserSerializer
username = self.kwargs.get(self.lookup_url_kwarg)
github_user = self.get_queryset().filter(username=username).first()
if not github_user:
try:
github_information_service = GithubInformationService(username)
github_user = github_information_service.update()
except GitHubUserDoesNotExist:
raise NotExistsGithubUser()
except RateLimit:
raise RateLimitGithubAPI()
serializer = self.serializer_class(github_user)
return Response(serializer.data)
def update_user_ranking():
"""
total score 로 전체 유저의 순위를 계산하는 함수
"""
github_users = GithubUser.objects.all()
for github_user in chunkator(github_users, 1000):
try:
github_user.previous_user_rank = github_user.user_rank
github_user.user_rank = GithubInformationService.update_user_ranking(
github_user.total_score)
github_user.tier = GithubInformationService.get_tier_statistics(
github_user.user_rank)
github_user.save(
update_fields=['user_rank', 'previous_user_rank', 'tier'])
except GitHubUserDoesNotExist:
continue
except Exception as e:
capture_exception(e)
def update_github_basic_information(github_user: GithubUser):
github_information_service = GithubInformationService(github_user.username)
user_information = github_information_service.github_adapter.get_user_info(
github_user.username)
for key, value in asdict(user_information).items():
if key in github_information_service.user_update_fields:
if getattr(github_user, key, '') != value:
setattr(github_user, key, value)
# is_completed, continuous_count = get_continuous_commit_day(github_user.username)
# if is_completed:
# github_user.continuous_commit_day = continuous_count
github_user.total_score = github_information_service.get_total_score(
github_user)
github_user.user_rank = github_information_service.update_user_ranking(
github_user.total_score)
github_user.tier = github_information_service.get_tier_statistics(
github_user.user_rank)
github_user.save(update_fields=['total_score', 'user_rank', 'tier'])
def run():
start_time = timeit.default_timer() # 시작 시간 체크
print('------- start update information -------')
github_users = GithubUser.objects.all()
for github_user in chunkator(github_users, 1000):
github_service = GithubInformationService(
username=github_user.username)
github_user.total_score = github_service.get_total_score(github_user)
github_user.save(update_fields=['total_score'])
print(f'[{github_user}] - {github_user.total_score}')
for github_user in chunkator(github_users, 1000):
github_service = GithubInformationService(
username=github_user.username)
github_user.tier = github_service.get_tier_statistics(
github_user.user_rank)
github_user.save(update_fields=['tier'])
print(
f'[{github_user}] - {github_user.get_tier_display()}[{github_user.user_rank}]'
)
print('------- end update information -------')
terminate_time = timeit.default_timer() # 종료 시간 체크
print(f'{terminate_time - start_time:.2f}초 걸렸습니다.')
def update(self, request, *args, **kwargs):
self.serializer_class = GithubUserSerializer
username = self.kwargs.get(self.lookup_url_kwarg)
try:
github_user = GithubUser.objects.filter(username=username).get()
if self.can_update(updated_date=github_user.updated) is False:
response_data = self.serializer_class(github_user).data
return Response(response_data)
github_information_service = GithubInformationService(username)
user = github_information_service.update()
response_data = self.serializer_class(user).data
except GithubUser.DoesNotExist:
raise exceptions.NotFound
except RateLimit:
raise RateLimitGithubAPI()
return Response(response_data)
def run():
"""
30분마다 실행되는 Github user 업데이트
: github api가 rate_limit 걸려서 더이상 호출하지 못하는 경우
"""
start_time = timeit.default_timer() # 시작 시간 체크
update_user_queue_qs = UpdateUserQueue.objects.filter(
status__in=[UpdateUserQueue.READY, UpdateUserQueue.FAIL])
if not update_user_queue_qs:
return
try:
# 스크립트를 시작하기전 rate_limit 를 체크한다.
rate_limit_check_service = GithubInformationService(
is_insert_queue=False)
rate_limit_check_service.get_rate_remaining()
except RateLimit:
return
SlackService.slack_update_github_user(status='시작', message='')
update_user_count = 0
for user_queue in chunkator(update_user_queue_qs, 1000):
try:
github_information_service = GithubInformationService(
user_queue.username, False)
github_information_service.update()
user_queue.status = UpdateUserQueue.SUCCESS
user_queue.save(update_fields=['status'])
update_user_count += 1
except RateLimit:
# rate limit 면 다른 유저들도 업데이드 못함
SlackService.slack_notify_update_fail(
message=f'Rate Limit 로 인해 업데이트가 실패되었습니다. '
f'{update_user_count}명만 업데이트 되었습니다.😭')
break
except Exception as e:
capture_exception(e)
terminate_time = timeit.default_timer() # 종료 시간 체크
SlackService.slack_update_github_user(
status='완료',
message=f'업데이트가 {terminate_time - start_time:.2f}초 걸렸습니다.',
update_user=update_user_count)
def test_get_or_create_github_user(github_context, mock_slack_notify_new_user):
username = '******'
user_information_dto = UserInformationDto(
name='jay',
email='*****@*****.**',
location='Republic of Korea',
company='DirtyBoyz',
avatar_url='https://avatars.githubusercontent.com/u/24591259?v=4',
bio='',
blog='',
public_repos=0,
followers=0,
following=0,
repos_url='',
organizations_url='')
github_information_service = GithubInformationService(username=username)
github_user = github_information_service.get_or_create_github_user(
user_information_dto)
assert github_user is not None
for key, value in asdict(user_information_dto).items():
if key in github_information_service.user_update_fields:
assert getattr(github_user, key, '') == value
async def update_continuous_commit_day(github_user: GithubUser):
"""
1일 1커밋 크롤링으로 업데이트
"""
if not is_exists_github_users(github_user.username):
return
is_completed, continuous_count = get_continuous_commit_day(
github_user.username)
if is_completed:
github_user.continuous_commit_day = continuous_count
github_user.total_score = GithubInformationService.get_total_score(
github_user)
github_user.save(
update_fields=['continuous_commit_day', 'total_score'])