def update(self, *args, **kwargs):
try:
super().update(*args, **kwargs)
except ConnectionError as e:
sentry.log_info(e)
except Exception as e:
sentry.log_info(e)
def build_hit(self, request, data):
hit = {}
event_data = data.copy()
user_id = data.get('user_id', request.user.id)
ip, is_routable = get_client_ip(request)
hit['api_key'] = self.api_key
if user_id:
user = User.objects.get(id=user_id)
user_email = user.email
invited_by = user.invited_by
if invited_by:
invited_by_id = invited_by.id
else:
invited_by_id = None
user_properties = {
'email': user_email,
'first_name': user.first_name,
'last_name': user.last_name,
'reputation': user.reputation,
'is_suspended': user.is_suspended,
'probable_spammer': user.probable_spammer,
'invited_by_id': invited_by_id
}
user_id = f'{user_email}_{user_id}'
if len(user_id) < 5:
user_id += '_____'
event_data['user_id'] = user_id
else:
user_properties = {
'email': '',
'first_name': 'Anonymous',
'reputation': 0,
'is_suspended': False,
'probable_spammer': False
}
event_data['user_id'] = '_Anonymous_'
event_data['user_properties'] = user_properties
if ip is not None:
try:
geo_info = geo.city(ip)
event_data['ip'] = ip
event_data['country'] = geo_info['country_name']
event_data['city'] = geo_info['city']
event_data['region'] = geo_info['region']
event_data['dma'] = geo_info['dma_code']
event_data['location_lat'] = geo_info['latitude']
event_data['location_lng'] = geo_info['longitude']
except Exception as e:
log_info(e)
hit['events'] = [event_data]
self.hit = json.dumps(hit)
def sift_check_user_content(self, request):
# https://sift.com/developers/docs/python/decisions-api/decision-webhooks/authentication
# Let's check whether this webhook actually came from Sift!
# First let's grab the signature from the postback's headers
postback_signature = request.headers.get("X-Sift-Science-Signature")
# Next, let's try to assemble the signature on our side to verify
key = SIFT_WEBHOOK_SECRET_KEY.encode("utf-8")
postback_body = request.body
h = hmac.new(key, postback_body, sha1)
verification_signature = "sha1={}".format(h.hexdigest())
if verification_signature == postback_signature:
decision_id = request.data["decision"]["id"]
user_id = request.data["entity"]["id"]
user = User.objects.get(id=user_id)
if not user.moderator or user.email not in EMAIL_WHITELIST:
if "mark_as_probable_spammer_content_abuse" in decision_id:
log_info(
f"Possible Spammer - {user.id}: {user.first_name} {user.last_name} - {decision_id}"
)
user.set_probable_spammer()
elif "suspend_user_content_abuse" in decision_id:
log_info(
f"Suspending User - {user.id}: {user.first_name} {user.last_name} - {decision_id}"
)
user.set_suspended(is_manual=False)
serialized = UserSerializer(user)
return Response(serialized.data, status=200)
else:
raise Exception("Sift verification signature mismatch")
def process_response(self, request, response):
if 'api' in request.path:
try:
self.prof.disable()
out = StringIO()
old_stdout = sys.stdout
sys.stdout = out
stats = pstats.Stats(self.prof, stream=out)
stats.print_stats()
sys.stdout = old_stdout
stats_str = out.getvalue()
total_time = str(stats.total_tt)
except Exception as e:
sentry.log_error(e)
sentry.log_info(self.prof.stats, error=e)
sentry.log_info(self.prof.getstats(), error=e)
return response
log_traceback.apply_async(
(self.data, total_time, stats_str),
priority=1,
)
return response
def forward_event(self):
headers = {'Content-Type': 'application/json', 'Accept': '*/*'}
request = requests.post(self.api_url, data=self.hit, headers=headers)
res = request.json()
if request.status_code != 200:
log_info(res)
return res
def _add_references(self, paper):
try:
if not TESTING:
add_references.apply_async((paper.id, ),
priority=5,
countdown=30)
else:
add_references(paper.id)
except Exception as e:
sentry.log_info(e)
def _add_orcid_authors(self, paper):
try:
if not TESTING:
add_orcid_authors.apply_async((paper.id, ),
priority=5,
countdown=10)
else:
add_orcid_authors(paper.id)
except Exception as e:
sentry.log_info(e)
def _delete_user_account(self, user, error=None):
user_email = user.email
email_exists = email_address_exists(user_email)
if email_exists:
user = User.objects.get(email=user_email)
social_account_exists = SocialAccount.objects.filter(
user=user).exists()
if not social_account_exists:
deletion_info = user.delete()
sentry.log_info(deletion_info, error=error)
return True
return False
def email_notifications(request):
"""Handles AWS SNS email notifications."""
data = request.data
if type(request.data) is not dict:
data = json.loads(request.data)
data_type = None
try:
data_type = data['Type']
except KeyError:
raise ParseError(f'Did not find key `Type` in {data}')
if data_type == 'SubscriptionConfirmation':
url = data['SubscribeURL']
resp = http_request('GET', url)
if resp.status_code != 200:
message = 'Failed to subscribe to SNS'
log_request_error(resp, message)
elif data_type == 'Notification':
data_message = json.loads(data['Message'])
if data_message['notificationType'] == 'Bounce':
bounced_recipients = data_message['bounce']['bouncedRecipients']
for b_r in bounced_recipients:
email_address = b_r['emailAddress']
# TODO: Sanitize email address before putting it in the db
try:
recipient, created = EmailRecipient.objects.get_or_create(
email=email_address)
recipient.bounced()
except Exception as e:
message = (
f'Failed handling bounced recipient: {email_address}')
error = EmailNotificationError(e, message)
print(error)
log_error(error, base_error=e)
elif data_type == 'Complaint':
message = (f'`email_notifications` received {data_type}')
log_info(message)
else:
message = (
f'`email_notifications` received unsupported type {data_type}')
print(message)
return Response({})
def to_representation(self, value):
"""
Serialize tagged objects to a simple textual representation.
"""
if isinstance(value, Distribution):
return DistributionSerializer(value,
context={
'exclude_stats': True
}).data
elif isinstance(value, Purchase):
return PurchaseSerializer(value, context={
'exclude_stats': True
}).data
elif isinstance(value, Withdrawal):
return WithdrawalSerializer(value, context={
'exclude_stats': True
}).data
sentry.log_info('No representation for ' + value)
return None
def notify_editor_inactivity():
User = apps.get_model('user.User')
last_week = timezone.now() - timedelta(days=7)
editors = User.objects.editors()
inactive_contributors = editors.annotate(
paper_count=Count(
'id',
filter=Q(
contributions__contribution_type=Contribution.SUBMITTER,
contributions__created_date__gte=last_week
)
),
comment_count=Count(
'id',
filter=Q(
contributions__contribution_type=Contribution.COMMENTER,
contributions__created_date__gte=last_week
)
),
total_contributions=F('paper_count') + F('comment_count')
).filter(
total_contributions__lt=3
)
logging = []
for inactive_contributor in inactive_contributors.iterator():
paper_count = inactive_contributor.paper_count
comment_count = inactive_contributor.comment_count
logging.append(
(
inactive_contributor.email,
f'Paper count: {paper_count}',
f'Comment count: {comment_count}'
)
)
inactive_contributor.notify_inactivity(
paper_count,
comment_count
)
log_info(logging)
def update(self,
thing,
refresh=None,
action='index',
parallel=False,
**kwargs):
if refresh is not None:
kwargs['refresh'] = refresh
elif self.django.auto_refresh:
kwargs['refresh'] = self.django.auto_refresh
if isinstance(thing, models.Model):
object_list = [thing]
else:
object_list = thing
objects_to_remove = []
objects_to_index = []
for obj in object_list:
if self.should_remove_from_index(obj):
objects_to_remove.append(obj)
else:
objects_to_index.append(obj)
try:
self._bulk(self._get_actions(objects_to_index, action='index'),
parallel=parallel,
**kwargs)
self._bulk(self._get_actions(objects_to_remove, action='delete'),
parallel=parallel,
**kwargs)
except ConnectionError as e:
sentry.log_info(e)
except Exception as e:
# The likely scenario is the result of removing objects
# that do not exist in elastic search - 404s
pass
def execute_editor_daily_payout_task():
log_info(f"{APP_ENV}-running payout")
result = editor_daily_payout_task()
log_info(result)
def subscribed_hub_papers(self, request):
feed_type = "subscribed"
user = request.user
hubs = user.subscribed_hubs.all()
page_number = int(request.GET["page"])
start_date = datetime.datetime.fromtimestamp(
int(request.GET.get("start_date__gte", 0)), datetime.timezone.utc)
end_date = datetime.datetime.fromtimestamp(
int(request.GET.get("end_date__lte", 0)), datetime.timezone.utc)
ordering = self._set_hub_paper_ordering(request)
if ordering == "-hot_score" and page_number == 1:
papers = {}
for hub in hubs.iterator():
hub_name = hub.slug
cache_key = get_cache_key("papers", hub_name)
cache_hit = cache.get(cache_key)
if cache_hit:
for hit in cache_hit:
paper_id = hit["id"]
abstract = hit.get("abstract", None)
if paper_id not in papers and abstract:
papers[paper_id] = hit
papers = list(papers.values())
if len(papers) < 1:
qs = self.get_queryset(
include_autopull=True).order_by("-hot_score")
papers = qs.filter(hubs__in=hubs).distinct()
else:
papers = sorted(papers, key=lambda paper: -paper["hot_score"])
papers = papers[:UNIFIED_DOC_PAGE_SIZE]
next_page = request.build_absolute_uri()
if len(papers) < UNIFIED_DOC_PAGE_SIZE:
next_page = None
else:
next_page = replace_query_param(next_page, "page", 2)
res = {
"count": len(papers),
"next": next_page,
"results": {
"data": papers,
"no_results": False,
"feed_type": feed_type,
},
}
return Response(res, status=status.HTTP_200_OK)
else:
qs = self.get_queryset(
include_autopull=True).order_by("-hot_score")
papers = qs.filter(hubs__in=hubs).distinct()
if papers.count() < 1:
log_info(f"""
No hub papers found, retrieiving trending papers.
Page: {page_number}
""")
trending_pk = "0_-hot_score_today"
cache_key_hub = get_cache_key("hub", trending_pk)
cache_hit = cache.get(cache_key_hub)
if cache_hit and page_number == 1:
return Response(cache_hit)
feed_type = "all"
papers = self.get_queryset().order_by("-hot_score")
context = self.get_serializer_context()
context["user_no_balance"] = True
context["exclude_promoted_score"] = True
context["include_wallet"] = False
order_papers = self.calculate_paper_ordering(papers, ordering,
start_date, end_date)
page = self.paginate_queryset(order_papers)
serializer = HubPaperSerializer(page, many=True, context=context)
serializer_data = serializer.data
return self.get_paginated_response({
"data": serializer_data,
"no_results": False,
"feed_type": feed_type
})
def validate(self, attrs, retry=0):
view = self.context.get('view')
request = self._get_request()
if not view:
error = serializers.ValidationError(
_("View is not defined, pass it as a context variable"))
sentry.log_error(error)
raise error
adapter_class = getattr(view, 'adapter_class', None)
if not adapter_class:
error = serializers.ValidationError(
_("Define adapter_class in view"))
sentry.log_error(error)
raise error
adapter = adapter_class(request)
app = adapter.get_provider().get_app(request)
# More info on code vs access_token
# http://stackoverflow.com/questions/8666316/facebook-oauth-2-0-code-and-token
credential = attrs.get('credential')
# Case 1: We received the access_token
if attrs.get('access_token') or credential:
access_token = attrs.get('access_token')
if credential:
access_token = credential
# Case 2: We received the authorization code
elif attrs.get('code'):
self.callback_url = getattr(view, 'callback_url', None)
self.client_class = getattr(view, 'client_class', None)
if not self.callback_url:
error = serializers.ValidationError(
_("Define callback_url in view"))
sentry.log_error(error)
raise error
if not self.client_class:
error = serializers.ValidationError(
_("Define client_class in view"))
sentry.log_error(error)
raise error
code = attrs.get('code')
provider = adapter.get_provider()
scope = provider.get_scope(request)
client = self.client_class(
request,
app.client_id,
app.secret,
adapter.access_token_method,
adapter.access_token_url,
'postmessage', # This is the callback url
scope)
token = client.get_access_token(code)
access_token = token['access_token']
else:
error = serializers.ValidationError(
_("Incorrect input. access_token or code is required."))
sentry.log_error(error)
raise serializers.ValidationError(
_("Incorrect input. access_token or code is required."))
social_token = adapter.parse_token({'access_token': access_token})
social_token.app = app
login = None
try:
login = self.get_social_login(adapter, app, social_token,
access_token)
complete_social_login(request, login)
except ConnectionTimeout:
pass
except NoReverseMatch as e:
if 'account_inactive' in str(e):
login_user = login.account.user
tracked_login = events_api.track_login(login_user, '$failure',
request)
update_user_risk_score(login_user, tracked_login)
raise LoginError(None, 'Account is suspended')
except Exception as e:
error = LoginError(e, 'Login failed')
sentry.log_info(error, error=e)
if login:
deleted = self._delete_user_account(login.user, error=e)
if deleted and retry < 3:
return self.validate(attrs, retry=retry + 1)
sentry.log_error(error, base_error=e)
raise serializers.ValidationError(_("Incorrect value"))
if login and not login.is_existing:
# We have an account already signed up in a different flow
# with the same email address: raise an exception.
# This needs to be handled in the frontend. We can not just
# link up the accounts due to security constraints
if app_settings.UNIQUE_EMAIL:
# Do we have an account already with this email address?
account_exists = get_user_model().objects.filter(
email=login.user.email, ).exists()
if account_exists:
sentry.log_info('User already registered with this e-mail')
deleted = self._delete_user_account(login.user)
if deleted and retry < 3:
return self.validate(attrs, retry=retry + 1)
raise serializers.ValidationError(
_("User already registered with this e-mail address."))
login.lookup()
login.save(request, connect=True)
login_user = login.account.user
attrs['user'] = login_user
tracked_login = events_api.track_login(login_user, '$success', request)
update_user_risk_score(login_user, tracked_login)
try:
visits = WebsiteVisits.objects.get(uuid=attrs['uuid'])
visits.user = attrs['user']
visits.save()
except Exception as e:
print(e)
pass
try:
referral_code = attrs.get('referral_code')
if referral_code:
referral_user = User.objects.get(referral_code=referral_code)
user = attrs['user']
if referral_code and not user.invited_by and referral_user.id != user.id:
user.invited_by = referral_user
user.save()
except Exception as e:
print(e)
sentry.log_error(e)
pass
request = self._get_request()
x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
if x_forwarded_for:
ip = x_forwarded_for.split(',')[0]
else:
ip = request.META.get('REMOTE_ADDR')
user = attrs['user']
check_user_risk(user)
if user.is_authenticated and not user.probable_spammer:
try:
country = geo.country(ip)
user.country_code = country.get('country_code')
user.save()
except Exception as e:
print(e)
sentry.log_error(e)
return attrs