def test_get_user_subpath(self):
"""Test the github utility get_user method with a subpath."""
url = 'https://api.github.com/users/gitcoin/test?per_page=100'
responses.add(responses.GET, url, headers=HEADERS, json={}, status=200)
get_user('@gitcoin', '/test')
assert responses.calls[0].request.url == url
def test_get_user(self):
"""Test the github utility get_user method."""
url = 'https://api.github.com/users/gitcoin'
responses.add(responses.GET, url, headers=HEADERS, json={}, status=200)
get_user('@gitcoin')
assert responses.calls[0].request.url == url
def sync_profile(handle, user=None, hide_profile=True):
handle = handle.strip().replace('@', '')
data = get_user(handle)
email = ''
is_error = 'name' not in data.keys()
if is_error:
print("- error main")
logger.warning('Failed to fetch github username', exc_info=True, extra={'handle': handle})
return None
defaults = {'last_sync_date': timezone.now(), 'data': data, 'hide_profile': hide_profile, }
if user and isinstance(user, User):
defaults['user'] = user
try:
defaults['github_access_token'] = user.social_auth.filter(provider='github').latest('pk').access_token
if user and user.email:
defaults['email'] = user.email
except UserSocialAuth.DoesNotExist:
pass
# store the org info in postgres
try:
profile, created = Profile.objects.update_or_create(handle=handle, defaults=defaults)
print("Profile:", profile, "- created" if created else "- updated")
orgs = get_user(handle, '/orgs')
profile.organizations = [ele['login'] for ele in orgs]
keywords = []
for repo in profile.repos_data_lite:
language = repo.get('language') if repo.get('language') else ''
_keywords = language.split(',')
for key in _keywords:
if key != '' and key not in keywords:
keywords.append(key)
profile.keywords = keywords
profile.save()
except Exception as e:
logger.error(e)
return None
if user and user.email:
email = user.email
elif profile and profile.email:
email = profile.email
if email and profile:
get_or_save_email_subscriber(email, 'sync_profile', profile=profile)
if profile and not profile.github_access_token:
token = profile.get_access_token(save=False)
profile.github_access_token = token
profile.save()
return profile
def get_avatar(_org_name):
avatar = None
filename = f"{_org_name}.png"
filepath = AVATAR_BASE + filename
if _org_name == 'gitcoinco':
filepath = AVATAR_BASE + '../../v2/images/helmet.png'
try:
avatar = Image.open(filepath, 'r').convert("RGBA")
except (IOError, FileNotFoundError):
remote_user = get_user(_org_name)
if not remote_user.get('avatar_url', False):
return JsonResponse({'msg': 'invalid user'}, status=422)
remote_avatar_url = remote_user['avatar_url']
r = requests.get(remote_avatar_url, stream=True)
chunk_size = 20000
with open(filepath, 'wb') as fd:
for chunk in r.iter_content(chunk_size):
fd.write(chunk)
avatar = Image.open(filepath, 'r').convert("RGBA")
# make transparent
datas = avatar.getdata()
new_data = []
for item in datas:
if item[0] == 255 and item[1] == 255 and item[2] == 255:
new_data.append((255, 255, 255, 0))
else:
new_data.append(item)
avatar.putdata(new_data)
avatar.save(filepath, "PNG")
return filepath
def github_issues():
if settings.DEBUG:
return
from git.utils import get_issues, get_user
repos = []
for org in ['bitcoin', 'gitcoinco', 'ethereum']:
for repo in get_user(org, '/repos'):
repos.append((org, repo['name']))
for org, repo in repos:
issues = []
cont = True
page = 1
while cont:
new_issues = get_issues(org, repo, page, 'all')
issues = issues + new_issues
page += 1
cont = len(new_issues)
val = len(issues)
key = f"github_issues_{org}_{repo}"
try:
Stat.objects.create(
created_on=timezone.now(),
key=key,
val=(val),
)
except Exception:
pass
if not val:
break
def sync_profile_actions(self):
# figure out what github profiles we care about
hours = 100 if settings.DEBUG else 24
start = timezone.now() - timezone.timedelta(hours=hours)
profile_ids_logged_in_last_time_period = list(
UserAction.objects.filter(created_on__gt=start,
action="Login").values_list('profile',
flat=True))
profile_ids_interest_last_time_period = list(
Interest.objects.filter(created__gt=start).values_list('profile',
flat=True))
profile_ids_fulfilled_last_time_period = list(
BountyFulfillment.objects.filter(created_on__gt=start).values_list(
'profile', flat=True))
profile_ids = profile_ids_interest_last_time_period + profile_ids_logged_in_last_time_period + profile_ids_fulfilled_last_time_period
profiles = Profile.objects.filter(pk__in=profile_ids)
# process them
for profile in profiles:
try:
events = get_user(profile.handle).get_events()
for event in events:
if self.do_we_care(event):
GithubEvent.objects.get_or_create(
profile=profile,
payload=event._rawData,
what=event.type,
repo=event.repo.name,
created_on=event.created_at,
)
except Exception as e:
logger.error(
'Error while syncing profile actions during sync_github',
e)
def sync_profile_actions(self):
# figure out what github profiles we care about
hours = 100 if settings.DEBUG else 24
start = timezone.now() - timezone.timedelta(hours=hours)
profile_ids_logged_in_last_time_period = list(UserAction.objects.filter(created_on__gt=start, action="Login").values_list('profile', flat=True))
profile_ids_interest_last_time_period = list(Interest.objects.filter(created__gt=start).values_list('profile', flat=True))
profile_ids_fulfilled_last_time_period = list(BountyFulfillment.objects.filter(created_on__gt=start).values_list('profile', flat=True))
profile_ids = profile_ids_interest_last_time_period + profile_ids_logged_in_last_time_period + profile_ids_fulfilled_last_time_period
profiles = Profile.objects.filter(pk__in=profile_ids)
# process them
for profile in profiles:
try:
events = get_user(profile.handle, '/events')
for event in events:
try:
event_time = event.get('created_at', False)
created_on = datetime.datetime.strptime(event_time, '%Y-%m-%dT%H:%M:%SZ')
except Exception:
created_on = timezone.now()
if self.do_we_care(event):
GithubEvent.objects.get_or_create(
profile=profile,
payload=event,
what=event.get('type', ''),
repo=event.get('repo', {}).get('name', ''),
created_on=created_on,
)
except Exception as e:
logger.error('Error while syncing profile actions during sync_github', e)
def get_user_github_avatar_image(handle):
remote_user = get_user(handle)
avatar_url = remote_user.get('avatar_url')
if not avatar_url:
return None
from .models import BaseAvatar
temp_avatar = get_github_avatar_image(avatar_url, BaseAvatar.ICON_SIZE)
if not temp_avatar:
return None
return temp_avatar
def sync_profile(handle, user=None, hide_profile=True):
data = get_user(handle)
email = ''
is_error = 'name' not in data.keys()
if is_error:
print("- error main")
logger.warning('Failed to fetch github username',
exc_info=True,
extra={'handle': handle})
return None
defaults = {
'last_sync_date': timezone.now(),
'data': data,
'hide_profile': hide_profile,
}
if user and isinstance(user, User):
defaults['user'] = user
try:
defaults['github_access_token'] = user.social_auth.filter(
provider='github').latest('pk').access_token
if user and user.email:
defaults['email'] = user.email
except UserSocialAuth.DoesNotExist:
pass
# store the org info in postgres
try:
profile, created = Profile.objects.update_or_create(handle=handle,
defaults=defaults)
print("Profile:", profile, "- created" if created else "- updated")
except Exception as e:
logger.error(e)
return None
if user and user.email:
email = user.email
elif profile and profile.email:
email = profile.email
if email and profile:
get_or_save_email_subscriber(email, 'sync_profile', profile=profile)
if profile and not profile.github_access_token:
token = profile.get_access_token(save=False)
profile.github_access_token = token
profile.save()
return profile
def github_stars():
from git.utils import get_user
reops = get_user('gitcoinco', '/repos')
forks_count = sum([repo['forks_count'] for repo in reops])
Stat.objects.create(
key='github_forks_count',
val=forks_count,
)
stargazers_count = sum([repo['stargazers_count'] for repo in reops])
Stat.objects.create(
key='github_stargazers_count',
val=stargazers_count,
)
def github_stars():
from git.utils import get_user
repos = get_user('gitcoinco').get_repos()
forks_count = sum([repo.forks_count for repo in repos])
Stat.objects.create(
key='github_forks_count',
val=forks_count,
)
stargazers_count = sum([repo.stargazers_count for repo in repos])
Stat.objects.create(
key='github_stargazers_count',
val=stargazers_count,
)
def get_github_avatar(handle):
"""Pull the latest avatar from Github and store in Avatar.png.
Returns:
bool: Whether or not the Github avatar was updated.
"""
remote_user = get_user(handle)
avatar_url = remote_user.get('avatar_url')
if not avatar_url:
return False
temp_avatar = get_temp_image_file(avatar_url)
if not temp_avatar:
return False
return temp_avatar
def recursive_sync(lsynced, handle):
try:
if handle not in lsynced:
print(f'Syncing User Handle: {handle}')
profile = sync_profile(handle)
print('Profile from sync')
print(profile)
access_token = profile.user.social_auth.filter(
provider='github').latest('pk').access_token
print('Removing Stale Organizations and Groups')
remove_org_groups = [
x for x in profile.profile_organizations.all()
if x.name not in profile.organizations
]
for y in remove_org_groups:
profile.profile_organizations.remove(y)
profile.user.groups.filter(
name__contains=y.name).delete()
print(
f'Removing: {profile.handle} from Organization: {y.name} '
)
try:
gh_client = github_connect(access_token)
user_access_repos = gh_client.get_user().get_repos()
if user_access_repos.totalCount:
pass # trigger error throw if any
# Question around user repo access if we can't get user repos, should we assume all repos are no longer available in the platform?
except Exception as e:
print(e)
return lsynced
current_user_repos = []
for y in user_access_repos:
current_user_repos.append(y.name)
remove_user_repos_names = [
x for x in profile.repos.all()
if x.name not in current_user_repos
]
remove_user_repos = Repo.objects.filter(
name__in=remove_user_repos_names,
profile__handle=handle)
for y in remove_user_repos:
profile.repos.remove(y)
lsynced.append(handle)
else:
return lsynced
members_to_sync = []
if profile.organizations is None:
print("no organizations to sync")
return []
for org in profile.organizations:
try:
if org in orgs_synced:
print(f'{org} has been synced already')
continue
orgs_synced.append(org)
db_org = Organization.objects.get_or_create(
name=org)[0]
print(f'Syncing Organization: {db_org.name}')
profile.profile_organizations.add(db_org)
try:
gh_client = github_connect()
org_members = gh_client.get_organization(
db_org.name).get_members()
if org_members.totalCount:
pass # trigger error throw if any
except Exception as e:
print(e)
continue
for member in org_members:
try:
membership = get_user(
handle).get_organization_membership(
db_org.name)
role = membership.role if hasattr(
membership, 'role') else "member"
db_group = Group.objects.get_or_create(
name=f'{db_org.name}-role-{role}')[0]
db_org.groups.add(db_group)
member_profile_obj = Profile.objects.get(
handle=member.login, user__is_active=True)
member_profile_obj.user.groups.add(db_group)
members_to_sync.append(member.login)
except Exception as e:
print(e)
continue
try:
gh_client = github_connect(access_token)
org_repos = gh_client.get_organization(
db_org.name).get_repos()
if org_repos.totalCount:
pass # trigger error throw if any
except Exception as e:
print(e)
continue
for repo in org_repos:
db_repo = Repo.objects.get_or_create(
name=repo.name)[0]
db_org.repos.add(db_repo)
print(f'Syncing Repo: {db_repo.name}')
try:
gh_client = github_connect(access_token)
repo_collabs = gh_client.get_repo(
repo.full_name).get_collaborators()
if repo_collabs.totalCount:
pass # trigger error throw if any
except Exception as e:
print(e)
continue
for collaborator in repo_collabs:
if collaborator.permissions:
if collaborator.permissions.admin:
permission = "admin"
elif collaborator.permissions.push:
permission = "write"
elif collaborator.permissions.pull:
permission = "pull"
else:
permission = "none"
db_group = Group.objects.get_or_create(
name=
f'{db_org.name}-repo-{repo["name"]}-{permission}'
)[0]
db_org.groups.add(db_group)
try:
member_user_profile = Profile.objects.get(
handle=collaborator.login,
user__is_active=True)
member_user_profile.user.groups.add(
db_group)
member_user_profile.repos.add(db_repo)
if collaborator.login not in members_to_sync or \
collaborator.login not in lsynced:
members_to_sync.append(
collaborator.login)
except Exception as e:
continue
for x in members_to_sync:
try:
lsynced = lsynced + recursive_sync(lsynced, x)
except Exception as e:
# print(f'An exception happened in the Members sync Loop: handle: {handle} {e}')
continue
except Exception as e:
# print(f'An exception happened in the Organization Loop: handle {handle} {e}')
continue
except Exception as exc:
print(f'Exception occurred inside recursive_sync: {exc}')
return lsynced
def sync_profile(handle, user=None, hide_profile=True):
from dashboard.models import Profile
handle = handle.strip().replace('@', '').lower()
# data = get_user(handle, scoped=True)
if user and hasattr(user, 'profile'):
try:
access_token = user.social_auth.filter(
provider='github').latest('pk').access_token
data = get_user(handle,
'',
scoped=True,
auth=(handle, access_token))
user = User.objects.get(username__iexact=handle)
if 'login' in data:
profile = user.profile
user.username = data['login']
user.save()
profile.handle = data['login']
profile.email = user.email
profile.save()
except UserSocialAuth.DoesNotExist:
pass
else:
data = get_user(handle)
email = ''
is_error = 'name' not in data.keys()
if is_error:
print("- error main")
logger.warning(f'Failed to fetch github username {handle}',
exc_info=True,
extra={'handle': handle})
return None
defaults = {'last_sync_date': timezone.now(), 'data': data}
if user and isinstance(user, User):
defaults['user'] = user
try:
defaults['github_access_token'] = user.social_auth.filter(
provider='github').latest('pk').access_token
if user and user.email:
defaults['email'] = user.email
except UserSocialAuth.DoesNotExist:
pass
# store the org info in postgres
try:
profile_exists = Profile.objects.filter(handle=handle).count()
if not profile_exists:
defaults['hide_profile'] = hide_profile
profile, created = Profile.objects.update_or_create(handle=handle,
defaults=defaults)
access_token = profile.user.social_auth.filter(
provider='github').latest('pk').access_token
orgs = get_user(handle,
'',
scope='orgs',
auth=(profile.handle, access_token))
profile.organizations = [
ele['login'] for ele in orgs if ele and type(ele) is dict
] if orgs else []
print("Profile:", profile, "- created" if created else "- updated")
keywords = []
for repo in profile.repos_data_lite:
language = repo.get('language') if repo.get('language') else ''
_keywords = language.split(',')
for key in _keywords:
if key != '' and key not in keywords:
keywords.append(key)
profile.keywords = keywords
profile.save()
except UserSocialAuth.DoesNotExist:
pass
except Exception as e:
logger.exception(e)
return None
if user and user.email:
email = user.email
elif profile and profile.email:
email = profile.email
if email and profile:
profile.email = email
profile.save()
get_or_save_email_subscriber(email, 'sync_profile', profile=profile)
if profile and not profile.github_access_token:
token = profile.get_access_token(save=False)
profile.github_access_token = token
profile.save()
if profile and not profile.avatar_baseavatar_related.last():
github_avatar_img = get_user_github_avatar_image(profile.handle)
if github_avatar_img:
try:
github_avatar = SocialAvatar.github_avatar(
profile, github_avatar_img)
github_avatar.save()
profile.activate_avatar(github_avatar.pk)
profile.save()
except Exception as e:
logger.warning(
f'Encountered ({e}) while attempting to save a user\'s github avatar'
)
return profile
def embed(request):
# default response
could_not_find = Image.new('RGBA', (1, 1), (0, 0, 0, 0))
err_response = HttpResponse(content_type="image/jpeg")
could_not_find.save(err_response, "JPEG")
# Get maxAge GET param if provided, else default on the small side
max_age = int(request.GET.get('maxAge', 3600))
# params
repo_url = request.GET.get('repo', False)
if not repo_url or 'github.com' not in repo_url:
return err_response
try:
badge = request.GET.get('badge', False)
if badge:
open_bounties = Bounty.objects.current() \
.filter(
github_url__startswith=repo_url,
network='mainnet',
idx_status__in=['open']
)
tmpl = loader.get_template('svg_badge.txt')
response = HttpResponse(
tmpl.render({'bounties_count': open_bounties.count()}),
content_type='image/svg+xml',
)
patch_response_headers(response, cache_timeout=max_age)
return response
# get avatar of repo
_org_name = org_name(repo_url)
avatar = None
filename = f"{_org_name}.png"
filepath = 'assets/other/avatars/' + filename
try:
avatar = Image.open(filepath, 'r').convert("RGBA")
except IOError:
remote_user = get_user(_org_name)
if not hasattr(remote_user, 'avatar_url'):
return JsonResponse({'msg': 'invalid user'}, status=422)
remote_avatar_url = remote_user.avatar_url
r = requests.get(remote_avatar_url, stream=True)
chunk_size = 20000
with open(filepath, 'wb') as fd:
for chunk in r.iter_content(chunk_size):
fd.write(chunk)
avatar = Image.open(filepath, 'r').convert("RGBA")
# make transparent
datas = avatar.getdata()
new_data = []
for item in datas:
if item[0] == 255 and item[1] == 255 and item[2] == 255:
new_data.append((255, 255, 255, 0))
else:
new_data.append(item)
avatar.putdata(new_data)
avatar.save(filepath, "PNG")
# get issues
length = request.GET.get('len', 10)
super_bounties = Bounty.objects.current() \
.filter(
github_url__startswith=repo_url,
network='mainnet',
idx_status__in=['open', 'started', 'submitted']
).order_by('-_val_usd_db')
bounties = super_bounties[:length]
# config
bounty_height = 200
bounty_width = 572
font = 'assets/v2/fonts/futura/FuturaStd-Medium.otf'
width = 1776
height = 576
# setup
img = Image.new("RGBA", (width, height), (255, 255, 255))
draw = ImageDraw.Draw(img)
black = (0, 0, 0)
gray = (102, 102, 102)
h1 = ImageFont.truetype(font, 36, encoding="unic")
h2_thin = ImageFont.truetype(font, 36, encoding="unic")
p = ImageFont.truetype(font, 24, encoding="unic")
# background
background_image = 'assets/v2/images/embed-widget/background.png'
back = Image.open(background_image, 'r').convert("RGBA")
offset = 0, 0
img.paste(back, offset)
# repo logo
icon_size = (184, 184)
avatar.thumbnail(icon_size, Image.ANTIALIAS)
offset = 195, 148
img.paste(avatar, offset, avatar)
img_org_name = ImageDraw.Draw(img)
img_org_name_size = img_org_name.textsize(_org_name, h1)
img_org_name.multiline_text(
align="left",
xy=(287 - img_org_name_size[0] / 2, 360),
text=_org_name,
fill=black,
font=h1,
)
draw.multiline_text(
align="left",
xy=(110, 410),
text="supports funded issues",
fill=black,
font=h1,
)
# put bounty list in there
i = 0
for bounty in bounties[:4]:
i += 1
# execute
line_size = 2
# Limit text to 28 chars
text = f"{bounty.title_or_desc}"
text = (text[:28] + '...') if len(text) > 28 else text
x = 620 + (int((i - 1) / line_size) * (bounty_width))
y = 230 + (abs(i % line_size - 1) * bounty_height)
draw.multiline_text(align="left",
xy=(x, y),
text=text,
fill=black,
font=h2_thin)
unit = 'day'
num = int(round((bounty.expires_date - timezone.now()).days, 0))
if num == 0:
unit = 'hour'
num = int(
round((bounty.expires_date - timezone.now()).seconds /
3600 / 24, 0))
unit = unit + ("s" if num != 1 else "")
draw.multiline_text(
align="left",
xy=(x, y - 40),
text=f"Expires in {num} {unit}:",
fill=gray,
font=p,
)
bounty_eth_background = Image.new("RGBA", (200, 56),
(231, 240, 250))
bounty_usd_background = Image.new("RGBA", (200, 56),
(214, 251, 235))
img.paste(bounty_eth_background, (x, y + 50))
img.paste(bounty_usd_background, (x + 210, y + 50))
tmp = ImageDraw.Draw(img)
bounty_value_size = tmp.textsize(
f"{round(bounty.value_true, 2)} {bounty.token_name}", p)
draw.multiline_text(
align="left",
xy=(x + 100 - bounty_value_size[0] / 2, y + 67),
text=f"{round(bounty.value_true, 2)} {bounty.token_name}",
fill=(44, 35, 169),
font=p,
)
bounty_value_size = tmp.textsize(
f"{round(bounty.value_in_usdt_now, 2)} USD", p)
draw.multiline_text(
align="left",
xy=(x + 310 - bounty_value_size[0] / 2, y + 67),
text=f"{round(bounty.value_in_usdt_now, 2)} USD",
fill=(45, 168, 116),
font=p,
)
# blank slate
if bounties.count() == 0:
draw.multiline_text(
align="left",
xy=(760, 320),
text=
"No active issues. Post a funded issue at: https://gitcoin.co",
fill=gray,
font=h1,
)
if bounties.count() != 0:
text = 'Browse issues at: https://gitcoin.co/explorer'
draw.multiline_text(
align="left",
xy=(64, height - 70),
text=text,
fill=gray,
font=p,
)
draw.multiline_text(
align="left",
xy=(624, 120),
text="Recently funded issues:",
fill=(62, 36, 251),
font=p,
)
_, value = summarize_bounties(super_bounties)
value_size = tmp.textsize(value, p)
draw.multiline_text(
align="left",
xy=(1725 - value_size[0], 120),
text=value,
fill=gray,
font=p,
)
line_table_header = Image.new("RGBA", (1100, 6), (62, 36, 251))
img.paste(line_table_header, (624, 155))
# Resize back to output size for better anti-alias
img = img.resize((888, 288), Image.LANCZOS)
# Return image with right content-type
response = HttpResponse(content_type="image/png")
img.save(response, "PNG")
patch_response_headers(response, cache_timeout=max_age)
return response
except IOError as e:
print(e)
return err_response