def _action_loop(self, model_name, timeout=5):
model_queue = self._model_queues[model_name]
try:
while not self.should_stop:
msgs = []
while len(msgs) < self.chunk_size:
try:
# If we have any messages queued up, push them through ASAP
msgs.append(model_queue.get(timeout=.1 if msgs else timeout))
except queue.Empty:
break
if not msgs:
logger.debug('%r: Recieved no messages to queue up', self)
continue
start = time.time()
logger.debug('%r: Preparing %d %ss to be indexed', self, len(msgs), model_name)
for msg, action in zip(msgs, ElasticsearchActionGenerator([self.index], msgs)):
# Keep blocking on put() until there's space in the queue or it's time to stop
while not self.should_stop:
try:
self._queue.put((msg, action), timeout=timeout)
break
except queue.Full:
continue
logger.info('%r: Prepared %d %ss to be indexed in %.02fs', self, len(msgs), model_name, time.time() - start)
except Exception as e:
client.captureException()
logger.exception('%r: _action_loop(%s) encountered an unexpected error', self, model_name)
self.stop()
def fetch_contact_field_results(org, contact_field, segment):
from ureport.polls.models import CACHE_ORG_FIELD_DATA_KEY, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
from ureport.polls.models import UREPORT_RUN_FETCHED_DATA_CACHE_TIME
start = time.time()
print "Fetching %s for %s with segment %s" % (contact_field, org.name, segment)
cache_time = UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
if segment and segment.get('location', "") == "District":
cache_time = UREPORT_RUN_FETCHED_DATA_CACHE_TIME
try:
segment = substitute_segment(org, segment)
this_time = datetime.now()
temba_client = org.get_temba_client()
client_results = temba_client.get_results(contact_field=contact_field, segment=segment)
results_data = temba_client_flow_results_serializer(client_results)
cleaned_results_data = results_data
print "Fetch took %ss" % (time.time() - start)
key = CACHE_ORG_FIELD_DATA_KEY % (org.pk, slugify(unicode(contact_field)), slugify(unicode(segment)))
cache.set(key, {'time': datetime_to_ms(this_time), 'results': cleaned_results_data}, cache_time)
except:
client.captureException()
import traceback
traceback.print_exc()
def wrapper(event, context):
tracing_id = get_tracing_id(event)
os.environ["TRACING_REQUEST_ID"] = tracing_id
if sentry:
# Provide additional metadata to sentry in case the exception
# gets trapped and reported within the function.
sentry.user_context({
"aws_log_group_name": getattr(context, "log_group_name"),
"aws_log_stream_name": getattr(context, "log_stream_name"),
"aws_function_name": getattr(context, "function_name"),
"tracing_id": tracing_id
})
try:
measurement = "%s_duration_ms" % (func.__name__)
with influx_timer(measurement, timestamp=now()):
return func(event, context)
except Exception as e:
logger.exception("Got an exception: %r", e)
if sentry:
logger.info("Inside sentry capture block.")
sentry.captureException()
else:
logger.info("Sentry is not available.")
raise
finally:
from django.db import connection
connection.close()
def report_exception(subject, info, exc_info=None):
"""
Formats an exception then posts it to sentry and if not in debug or
testing sends email to mail_admins.
"""
# Add hostname to subject mail
subject = "{0} - {1}".format(subject, settings.HOSTNAME)
if exc_info:
cls, err = exc_info[:2]
message = _(u"Exception in request:"
u" %(class)s: %(error)s")\
% {'class': cls.__name__, 'error': err}
message += u"".join(traceback.format_exception(*exc_info))
# send to sentry
try:
client.captureException(exc_info)
except Exception: # pylint: disable=broad-except
logging.exception(_(u'Sending to Sentry failed.'))
else:
message = u"%s" % info
if settings.DEBUG or settings.TESTING_MODE:
sys.stdout.write("Subject: %s\n" % subject)
sys.stdout.write("Message: %s\n" % message)
else:
mail_admins(subject=subject, message=message)
def fetch_results(self, segment=None):
from raven.contrib.django.raven_compat.models import client
cache_time = UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
if segment and segment.get('location', "") == "District":
cache_time = UREPORT_RUN_FETCHED_DATA_CACHE_TIME
try:
key = CACHE_POLL_RESULTS_KEY % (self.poll.org.pk, self.poll.pk, self.pk)
if segment:
segment = self.poll.org.substitute_segment(segment)
key += ":" + slugify(unicode(segment))
this_time = datetime.now()
temba_client = self.poll.org.get_temba_client()
client_results = temba_client.get_results(self.ruleset_uuid, segment=segment)
results = temba_client_flow_results_serializer(client_results)
cache.set(key, {'time': datetime_to_ms(this_time), 'results': results}, cache_time)
# delete the open ended cache
cache.delete('open_ended:%d' % self.id)
except: # pragma: no cover
client.captureException()
import traceback
traceback.print_exc()
def _index_loop(self):
try:
while not self.should_stop:
msgs = []
actions = self._actions(250, msgs)
stream = helpers.streaming_bulk(
self.es_client,
actions,
max_chunk_bytes=self.MAX_CHUNK_BYTES,
raise_on_error=False,
)
start = time.time()
for (ok, resp), msg in zip(stream, msgs):
if not ok and not (resp.get('delete') and resp['delete']['status'] == 404):
raise ValueError(ok, resp, msg)
assert len(resp.values()) == 1
_id = list(resp.values())[0]['_id']
assert msg.payload['ids'] == [util.IDObfuscator.decode_id(_id)], '{} {}'.format(msg.payload, util.IDObfuscator.decode_id(_id))
msg.ack()
if len(msgs):
logger.info('%r: Indexed %d documents in %.02fs', self, len(msgs), time.time() - start)
else:
logger.debug('%r: Recieved no messages for %.02fs', self, time.time() - start)
except Exception as e:
client.captureException()
logger.exception('%r: _index_loop encountered an unexpected error', self)
self.stop()
def process_response(self, request, response):
try:
if isinstance(response,HttpResponsePermanentRedirect):
return response
subdomain = request.META['HTTP_HOST'].split(settings.SESSION_COOKIE_DOMAIN)[0]
if subdomain not in ['portal','devel'] and request.META['HTTP_HOST'] not in settings.BASE_URL: # we are in a institutional portal
institutionportal = InstitutionPortal.objects.get(subdomain=subdomain)
request.session['institutionportal'] = institutionportal
if all(x not in request.path for x in ['login', 'done', 'scs_auth', 'media', 'static', 'api']) and request.user not in institutionportal.institution.user_set.all():
if request.user.is_anonymous() and request.path not in ['/','']:
request.session['errormessage'] = "Please login to access to the requested resource"
if request.path not in ['/','']:
return render_to_response("scs/login.html", {}, RequestContext(request))
else:
if request.path not in ['/','']:
request.session['errormessage'] = "You can't access if you are not memeber of the %s institution group" %institutionportal.institution.name
return render_to_response("scs/403.html", {}, RequestContext(request))
else:
if request.session.get('institutionportal', None):
del(request.session['institutionportal'])
return response
except Exception, e:
print e
print request.META['HTTP_HOST']
from raven.contrib.django.raven_compat.models import client
client.captureException()
if request.session.get('institutionportal', None):
del(request.session['institutionportal'])
return HttpResponsePermanentRedirect(settings.BASE_URL)
def authenticate( self, username = None, password= None, *args, **kwargs):
"""
Biomedtown backend authenticate method.
It delivery Authenticate request to biomedtown mod_auth_tkt service.
Arguments:
username (string) : Username.\n
password (string) : Password.\n
Return:
User (User instance) : Django User instance.\n
tkt64 (string) : return new ticket generation.\n
"""
if username is None or password is None:
return None
try:
service_response = self.service.rpc_login(username, password)
if service_response is not False:
user, tkt64 =self.userTicket(service_response)
if user is None:
return
return user, tkt64
return None
except Exception, e:
from raven.contrib.django.raven_compat.models import client
client.captureException()
return None
def authenticate( self, ticket=None,cip=None, *args, **kwargs):
"""
Authenticate User over given ticket.
Arguments:
ticket (string) : base 64 ticket.\n
Return:
User (User instance) : Django User instance.\n
tkt64 (string) : return new ticket generation.\n
"""
if ticket is None:
return
try:
user, tkt64 =self.userTicket(ticket,cip)
if user is None:
return
return user, tkt64
except Exception, e:
from raven.contrib.django.raven_compat.models import client
client.captureException()
return None
def delete(self, ticket, *args, **kwargs):
try:
WorkflowManager.deleteExecution(self.id, ticket)
except Exception, e:
from raven.contrib.django.raven_compat.models import client
client.captureException()
pass
def get_image(image, size_name=None, template_name="image.html", retina=False, **kwargs):
""" Templatetag to get the HTML for an image from a cropduster image object """
if image:
if CROPDUSTER_CROP_ONLOAD:
# If set, will check for thumbnail existence
# if not there, will create the thumb based on predefiend crop/size settings
thumb_path = image.thumbnail_path(size_name)
if not exists(thumb_path) and exists(image.image.path):
try:
size = image.size_set.size_set.get(slug=size_name)
except Size.DoesNotExist:
return ""
try:
image.create_thumbnail(size, force_crop=True)
except:
raven_client.captureException(exc_info=sys.exc_info())
return ""
if retina:
image_url = image.retina_thumbnail_url(size_name)
else:
image_url = image.thumbnail_url(size_name)
if not image_url:
return ""
try:
image_size = IMAGE_SIZE_MAP[(image.size_set_id, size_name)]
except KeyError:
return ""
# Set all the args that get passed to the template
kwargs["image_url"] = image_url
if hasattr(image_size, "auto_size") and image_size.auto_size != AUTO_SIZE:
kwargs["width"] = image_size.width if hasattr(image_size, "width") else ""
kwargs["height"] = image_size.height if hasattr(image_size, "height") else ""
if CROPDUSTER_PLACEHOLDER_MODE:
kwargs["image_url"] = "http://placehold.it/%sx%s" % (kwargs["width"], kwargs["height"])
kwargs["size_name"] = size_name
kwargs["attribution"] = image.attribution
if hasattr(image, "caption"): kwargs["alt"] = image.caption
if "title" not in kwargs: kwargs["title"] = kwargs["alt"]
tmpl = get_template("templatetags/" + template_name)
context = template.Context(kwargs)
return tmpl.render(context)
else:
return ""
def feedback_submit(request):
form = FeedbackForm(request.POST)
rate_limited = getattr(request, 'limited', False)
if form.is_valid() and not rate_limited:
from_address = settings.FEEDBACK_EMAIL_SENDER
to_addresses = [address.strip() for address in feedback_receiver_languages[get_language_from_request(request)]
.split(',')]
if from_address and to_addresses:
feedback_email = form.cleaned_data['feedback_email']
if not feedback_email:
feedback_email = "(no email supplied)"
message = EMAIL_MESSAGE.format(
form.cleaned_data['feedback_referer'],
request.META.get('HTTP_USER_AGENT','(unknown)'),
feedback_email,
form.cleaned_data['feedback_text'])
try:
send_mail('Feedback received for Court and Tribunal Finder',
message, from_address,
to_addresses, fail_silently=False)
except smtplib.SMTPException:
client.captureException()
return redirect('staticpages:feedback_sent')
def handle_notification(request):
""" Handle bounced emails via an SNS webhook. """
try:
notification = json.loads(request.body)
except ValueError, e:
client.captureException()
return HttpResponseBadRequest()
def __call__(instance, pk_str):
try:
flush_transaction()
pk = int(pk_str)
data = JobData.objects.get(pk=pk)
except (TypeError, ValueError):
error_msg = "Invalid value for pk"
error_extra = {
"pk tried": pk_str
}
raven_client.captureMessage(error_msg, extra=error_extra, stack=True)
except JobData.DoesNotExist:
error_msg = "Unable to find beanstalk job data."
error_extra = {
"pk tried": pk
}
raven_client.captureMessage(error_msg, extra=error_extra, stack=True)
else:
try:
val = instance.f(data)
if self.cleanup:
data.delete()
return val
except Exception:
raven_client.captureException()
def run(self, client):
try:
agg_func = self.AGGREGATOR_FUNCS[self.metric_aggregator]
try:
value = client.get_metric(self.target, aggregator=agg_func)
except (InvalidDataFormat, EmptyData) as exc:
return self.build_check_dict(Status.unknown, str(exc))
warning_func = self.OPERATOR_FUNCS[self.warning_operator]
critical_func = self.OPERATOR_FUNCS[self.critical_operator]
if critical_func(value, self.critical_value):
status = Status.critical
operator = self.get_critical_operator_display()
threshold_value = self.critical_value
output_fmt = self.PROBLEM_FMT
elif warning_func(value, self.warning_value):
status = Status.warning
operator = self.get_warning_operator_display()
threshold_value = self.warning_value
output_fmt = self.PROBLEM_FMT
else:
status = Status.passing
operator = '='
threshold_value = None
output_fmt = self.PASSING_FMT
output = output_fmt.format(status=status.name, metric=self.name,
operator=operator, threshold_value=threshold_value,
value=value)
return self.build_check_dict(status, output,
metrics={self.name: value})
except Exception as exc:
raven_client.captureException()
return self.build_check_dict(Status.unknown,
'unexpected error: %s' % exc)
def fetch_flows(org):
start = time.time()
#print "Fetching flows for %s" % org.name
try:
from ureport.polls.models import CACHE_ORG_FLOWS_KEY, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
this_time = datetime.now()
temba_client = org.get_temba_client()
flows = temba_client.get_flows()
all_flows = dict()
for flow in flows:
if flow.rulesets:
flow_json = dict()
flow_json['uuid'] = flow.uuid
flow_json['created_on'] = flow.created_on.strftime('%Y-%m-%d')
flow_json['name'] = flow.name
flow_json['participants'] = flow.participants
flow_json['runs'] = flow.runs
flow_json['completed_runs'] = flow.completed_runs
flow_json['rulesets'] = [
dict(uuid=elt.uuid, label=elt.label, response_type=elt.response_type) for elt in flow.rulesets]
all_flows[flow.uuid] = flow_json
all_flows_key = CACHE_ORG_FLOWS_KEY % org.pk
cache.set(all_flows_key,
{'time': datetime_to_ms(this_time), 'results': all_flows},
UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME)
except:
client.captureException()
import traceback
traceback.print_exc()
def retriveVtk(request):
if request.user.is_authenticated():
path = request.POST.get('path','')
if not path:
path = '/'
try:
webdav = easywebdav.connect(settings.LOBCDER_HOST, username='******',
password=request.ticket, protocol='https'
)
fileName = path.split('/')[-1]
fileToDownload = os.path.join(settings.LOBCDER_DOWNLOAD_DIR, fileName)
downloadChunks = webdav.downloadChunks(settings.LOBCDER_ROOT + path)
#remove file if exists
if os.path.exists(fileToDownload) and os.stat(fileToDownload)[6] != int(downloadChunks.raw.headers['content-length']):
os.remove(fileToDownload)
if not os.path.exists(fileToDownload):
webdav.download(settings.LOBCDER_ROOT+ path, fileToDownload)
content = json.dumps({'path': fileName}, sort_keys=False)
response = HttpResponse(content=content,
content_type='application/json')
return response
except Exception, e:
client.captureException()
response = HttpResponse(status=500)
response._is_string = True
return response
def get_instagram_photos(limit=6):
recent_media = []
cached_data = cache.get("instagram_photos")
if cached_data is not None:
return cached_data
try:
url = (
"https://api.instagram.com/v1/users/self/media/recent/?access_token=%s"
% settings.INSTAGRAM_ACCESS_TOKEN
)
response = requests.get(url)
if response.status_code == 200:
data = json.loads(response.content.decode())
for item in data["data"]:
recent_media.append(
{
"link": item["link"],
"image": item["images"]["thumbnail"]["url"],
"name": item["caption"]["text"].split("\n")[0],
}
)
cache.set("instagram_photos", recent_media[:limit], 60 * 60 * 24)
except Exception as e:
client.captureException()
logger.error(e)
return recent_media[:limit]
def get_context_data(self, **kwargs):
ctx = super(BlogPostsMixin, self).get_context_data(**kwargs)
blog = BlogPost.objects.all()
if self.blog_slugs:
if isinstance(self.blog_slugs, basestring):
blog_slugs = [self.blog_slugs]
else:
blog_slugs = self.blog_slugs
blog = blog.filter_by_blogs(*blog_slugs)
ctx['blog_slugs'] = blog_slugs
if self.blog_tags:
blog = blog.filter_by_tags(*self.blog_tags)
ctx['blog_tags'] = self.blog_tags
if self.blog_posts_limit:
blog = blog[:self.blog_posts_limit]
try:
# run the query here so that we can catch exceptions
# and render the page without blog posts
ctx[self.blog_posts_template_variable] = list(blog)
except Exception:
sentry_client.captureException()
ctx[self.blog_posts_template_variable] = []
return ctx
def has_valid_fxa_oauth(request, email):
if not email:
return False
# Grab the token out of the Authorization header
authorization = request.META.get('HTTP_AUTHORIZATION')
if not authorization:
return False
authorization = authorization.split(None, 1)
if authorization[0].lower() != 'bearer' or len(authorization) != 2:
return False
token = authorization[1].strip()
oauth, profile = get_fxa_clients()
# Validate the token with oauth-server and check for appropriate scope.
# This will raise an exception if things are not as they should be.
try:
oauth.verify_token(token, scope=['basket', 'profile:email'])
except fxa.errors.Error:
# security failure or server problem. can't validate. return invalid
sentry_client.captureException()
return False
try:
fxa_email = profile.get_email(token)
except fxa.errors.Error:
# security failure or server problem. can't validate. return invalid
sentry_client.captureException()
return False
return email == fxa_email
def fetch_reporter_group(org):
start = time.time()
print "Fetching reporter group for %s" % org.name
try:
from ureport.polls.models import CACHE_ORG_REPORTER_GROUP_KEY, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
this_time = datetime.now()
reporter_group = org.get_config('reporter_group')
if reporter_group:
temba_client = org.get_temba_client()
groups = temba_client.get_groups(name=reporter_group)
key = CACHE_ORG_REPORTER_GROUP_KEY % (org.pk, slugify(unicode(reporter_group)))
group_dict = dict()
if groups:
group = groups[0]
group_dict = dict(size=group.size, name=group.name, uuid=group.uuid)
cache.set(key,
{'time': datetime_to_ms(this_time), 'results': group_dict},
UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME)
except:
client.captureException()
import traceback
traceback.print_exc()
# delete the global count cache to force a recalculate at the end
cache.delete(GLOBAL_COUNT_CACHE_KEY)
print "Fetch %s reporter group took %ss" % (org.name, time.time() - start)
def fetch_flows(org, backend=None):
from ureport.polls.models import CACHE_ORG_FLOWS_KEY, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME
start = time.time()
logger.info("Fetching flows for %s" % org.name)
if backend:
backends = [backend]
else:
backends = org.backends.filter(is_active=True)
this_time = datetime.now()
org_flows = dict(time=datetime_to_ms(this_time), results=dict())
for backend_obj in backends:
backend = org.get_backend(backend_slug=backend_obj.slug)
try:
all_flows = backend.fetch_flows(org)
org_flows["results"] = all_flows
cache_key = CACHE_ORG_FLOWS_KEY % (org.pk, backend_obj.slug)
cache.set(cache_key, org_flows, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME)
except Exception:
client.captureException()
import traceback
traceback.print_exc()
logger.info("Fetch %s flows took %ss" % (org.name, time.time() - start))
if len(backends):
return org_flows.get("results", dict())
def filter_resources_by_facet(type, facet = None, value = None, page=1, numResults=10, orderBy = 'name', orderType='asc'):
"""
given a facet and a value return the list of the resources that match the query
"""
try:
if facet:
response = requests.get(FACETS_METADATA_API % (type, facet, value,numResults, page, orderBy, orderType), headers={'Accept' : 'application/json'})
else:
response = requests.get(TYPE_METADATA_API % (type, numResults, page, orderBy, orderType), headers={'Accept' : 'application/json'})
if response.status_code != 200:
raise AtosServiceException("Error while contacting Atos Service: status code = %s" % response.status_code)
try:
response = response.json()
resources = JSON_XPATH_PARSER.find(response)
pages = response["numTotalMetadata"] / numResults
if response["numTotalMetadata"] % numResults != 0:
pages += 1
response['resource_metadata'] = resources
response['pages'] = pages
return response
except Exception, e:
client.captureException()
return EMPTY_LIST.copy()
except BaseException, e:
raise AtosServiceException("Error while contacting Atos Service: %s" % e.message)
def feedback_submit(request):
form = FeedbackForm(request.POST)
if form.is_valid():
from_address = settings.FEEDBACK_EMAIL_SENDER
to_addresses = [address.strip() for address in settings.FEEDBACK_EMAIL_RECEIVER.split(',')]
if from_address and to_addresses:
feedback_email = form.cleaned_data['feedback_email']
if not feedback_email:
feedback_email = '(not provided)'
message = """
New feedback has arrived for courtfinder (https://courttribunalfinder.service.gov.uk/).
The user left feedback after seeing: %s
User's browser: %s
User's email: %s
Message: %s
""" % (form.cleaned_data['feedback_referer'],
request.META.get('HTTP_USER_AGENT','(unknown)'),
form.cleaned_data['feedback_email'],
form.cleaned_data['feedback_text'])
try:
nb_emails_sent = send_mail('Feedback received for Court and Tribunal Finder',
message, from_address,
to_addresses, fail_silently=False)
except smtplib.SMTPException:
client.captureException()
# do nothing else in case of error. User doesn't need to see.
return redirect(reverse('staticpages:feedback_sent'))
def create_tree(self, father, tree):
if isinstance(tree, dict):
for key, value in tree.items():
cat = Category.objects.create(
name=key,
slug="",
parent=father
)
self.create_tree(cat, value)
if isinstance(tree, str):
try:
course = Course.objects.get(slug=tree)
except Course.DoesNotExist:
if self.LOCAL_CACHE:
name = self.LOCAL_CACHE.get(tree, "Unknown course in cache")
else:
try:
ulb_course = ULBCourse.get_from_slug(tree, self.YEAR)
name = ulb_course.name
except Exception:
print("Slug %s failed" % tree)
client.captureException()
name = "Unknown course in cache"
course = Course.objects.create(name=name, slug=tree)
course.categories.add(father)
if isinstance(tree, list):
for subtree in tree:
self.create_tree(father, subtree)
def statusMessage(request):
"""
:type request: django.core.handlers.wsgi.WSGIRequest
:return:
"""
try:
if request.path == u'/done/':
return {}
if request.session.get('statusmessage', False):
message = request.session['statusmessage']
del request.session['statusmessage']
return {
'statusmessage': message,
}
if request.session.get('welcome', False):
message = request.session['welcome']
del request.session['welcome']
return {
'welcome': message,
}
if request.session.get('errormessage', False):
message = request.session['errormessage']
del request.session['errormessage']
return {
'errormessage': message,
}
except Exception, e:
client.captureException()
pass
def write(self, proteinbox=None, summary=None):
'''
Writes the wikitext representation of the protein box to MediaWiki.
Returns (result, None) if successful, or (None, Error) if not.
Arguments:
- `proteinbox`: an updated proteinbox to write
'''
page = self.get_page()
if not self.bots_allowed():
logger.warn('Bots Blocked', exc_info=True, extra={'page': page, 'bot': self})
try:
if proteinbox:
result = page.save(str(proteinbox), summary, minor=True)
self.text = page.edit()
else:
result = page.save(self.text, summary, minor=True)
self.force_update = False
self.save()
except MwClientError:
client.captureException()
def update_articles(update_list):
for pk in update_list:
try:
article = Article.objects.get(pk=pk)
article.update()
except Exception:
client.captureException()
def read(self, request, ticket="", group="", username=""):
"""
Promote a user as manager of a group
Arguments:
request (HTTP request istance): HTTP request send from client.
ticket (string) : base 64 ticket.
group (string) : the group name
username (string) : the username
Return:
Successes - Json/xml/yaml format response
Failure - 403 error
"""
try:
if request.GET.get("ticket"):
client_address = request.META["REMOTE_ADDR"]
user, tkt64 = authenticate(ticket=request.GET["ticket"], cip=client_address)
if user is not None:
group = VPHShareSmartGroup.objects.get(name=request.GET.get("group"))
user_to_promote = User.objects.get(username=request.GET.get("username"))
if not group.is_manager(user):
response = HttpResponse(status=403)
response._is_string = True
return response
# add user to the managers
group.managers.add(user_to_promote)
# add user to the group and all sub groups
while group is not None:
group.user_set.add(user_to_promote)
try:
group = VPHShareSmartGroup.objects.get(parent=group)
except ObjectDoesNotExist, e:
group = None
response = HttpResponse(status=200)
response._is_string = True
response.write("OK")
return response
else:
response = HttpResponse(status=403)
response._is_string = True
return response
except Exception, e:
from raven.contrib.django.raven_compat.models import client
client.captureException()
response = HttpResponse(status=500)
response._is_string = True
return response
def get_server_obj(self, uuid):
try:
return Server.objects.get(uuid__exact=uuid)
except Server.DoesNotExist:
return None
except Server.MultipleObjectsReturned:
client.captureException()
raise serializers.ValidationError("存在多条记录")
def error_handler(e):
log.exception(e)
if not settings.ENV_DEV:
sentry.captureException()
def send_partner_charity_reports(test=True):
# Create list of partners, combined the GiveDirectly entries
partners = {partner.name: [partner.id] for partner in PartnerCharity.objects.all()}
partners['GiveDirectly'] += partners['GiveDirectly Basic income research']
del partners['GiveDirectly Basic income research']
for partner, ids in partners.iteritems():
# Start time is when we last reported
last_report_datetime = PartnerCharityReport.objects.filter(partner__id=ids[0]
).aggregate(Max('date'))[
'date__max'] or datetime.datetime(2016, 1, 1, 0, 0, 0)
start = arrow.get(last_report_datetime).datetime
# End date is midnight yesterday (i.e. midnight between yesterday and today) UTC
# It could be the current time too (or rather a minute ago, to avoid races).
end = arrow.get(arrow.utcnow().date()).datetime
# Create spreadsheet
querysets = OrderedDict([
('New donations',
# For bank transactions, we use time_reconciled
Donation.objects.filter(Q(bank_transaction__time_reconciled__gte=start,
bank_transaction__time_reconciled__lt=end) |
Q(pin_transaction__isnull=False,
datetime__gte=start,
datetime__lt=end),
components__pledge_component__partner_charity__id__in=ids).order_by('datetime')),
('All donations',
Donation.objects.filter(Q(bank_transaction__time_reconciled__lt=end) |
Q(pin_transaction__isnull=False,
datetime__lt=end),
components__pledge_component__partner_charity__id__in=ids).order_by('datetime'))])
template = OrderedDict([
('Date', 'datetime'),
('Amount', 'components__amount'),
('Fees', 'components__fees'),
('Amount (net)', 'components__amount_net'),
('EAA Reference', 'reference'),
('First Name', 'pledge__first_name'),
('Last Name', 'pledge__last_name'),
('Email', 'pledge__email'),
('Payment method', 'payment_method'),
('Subscribe to marketing updates', 'pledge__subscribe_to_updates'),
('Designation', 'components__pledge_component__partner_charity__name')
])
filename = 'EAA donation report - {0} - {1}.xlsx'.format(partner,
# Avoid collisions of filename while testing
timezone.now())
location = os.path.join(settings.MEDIA_ROOT, 'partner_reports', filename)
write_spreadsheet(location, querysets, template)
# Create email
try:
partner_obj = PartnerCharity.objects.get(id=ids[0])
to = [partner_obj.email]
cc = [settings.EAA_INFO_EMAIL, settings.TESTING_EMAIL]
if partner_obj.email_cc:
cc.append(partner_obj.email_cc)
body = render_to_string('partner_report_message.txt', {'name': partner})
message = EmailMessage(
subject='Effective Altruism Australia donation report',
body=body,
to=to if not test else [settings.TESTING_EMAIL],
cc=cc if not test else [settings.TESTING_EMAIL],
# There is a filter in [email protected]
# from:(donations @ eaa.org.au) deliveredto:(info + receipts @ eaa.org.au)
# that automatically archives messages sent to info+receipt and adds the label 'receipts'
# bcc=["*****@*****.**", ],
bcc=[],
from_email=settings.POSTMARK_SENDER,
)
message.attach_file(location, mimetype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
message.send()
except Exception as e:
print e.message
client.captureException()
if not test:
partner = PartnerCharity.objects.get(id=ids[0])
PartnerCharityReport(partner=partner, date=end).save()
def send_summary_emails():
"""
We want to send them on Sunday at 8am in the local time of the group
So we run this each hour to check if any group needs their summary email sending.
We limit it to just the weekend (Saturday, Sunday) as this will cover any possible timezone offset (-11/+12)
In cron Sunday is day 0 and Saturday is day 6
"""
with timer() as t:
email_count = 0
recipient_count = 0
def is_8am_localtime(group):
with timezone.override(group.timezone):
localtime = timezone.localtime()
# Sunday is day 6 here, confusing!
return localtime.hour == 8 and localtime.weekday() == 6
groups = Group.objects.annotate(member_count=Count('members')).filter(
member_count__gt=0)
for group in groups:
if not is_8am_localtime(group):
continue
from_date, to_date = calculate_group_summary_dates(group)
if not group.sent_summary_up_to or group.sent_summary_up_to < to_date:
email_recipient_count = 0
context = prepare_group_summary_data(group, from_date, to_date)
if context['has_activity']:
for email in prepare_group_summary_emails(group, context):
try:
email.send()
email_count += 1
email_recipient_count += len(email.to)
except AnymailAPIError:
sentry_client.captureException()
# we save this even if some of the email sending fails, no retries right now basically...
# we also save if no emails were sent due to missing activity, to not try again over and over.
group.sent_summary_up_to = to_date
group.save()
group_summary_email(
group,
email_recipient_count=email_recipient_count,
feedback_count=context['feedbacks'].count(),
message_count=context['messages'].count(),
new_user_count=context['new_users'].count(),
pickups_done_count=context['pickups_done_count'],
pickups_missed_count=context['pickups_missed_count'],
has_activity=context['has_activity'],
)
recipient_count += email_recipient_count
stats_utils.periodic_task('group__send_summary_emails',
seconds=t.elapsed_seconds,
extra_fields={
'recipient_count': recipient_count,
'email_count': email_count,
})
def send_or_report_error(email):
try:
email.send()
except AnymailAPIError:
sentry_client.captureException()
def post(self, request):
"""
Receive conversation replies via e-mail
Request payload spec: https://developers.sparkpost.com/api/relay-webhooks/#header-relay-webhook-payload
"""
auth_key = request.META.get('HTTP_X_MESSAGESYSTEMS_WEBHOOK_TOKEN')
if auth_key is None or auth_key != settings.SPARKPOST_RELAY_SECRET:
return Response(
status=status.HTTP_403_FORBIDDEN,
data={
'message':
'Invalid HTTP_X_MESSAGESYSTEMS_WEBHOOK_TOKEN header'
})
for messages in [e['msys'].values() for e in request.data]:
for incoming_message in messages:
# 1. get email content and reply-to
reply_to = parseaddr(incoming_message['rcpt_to'])[1]
content = incoming_message['content']
# 2. check local part of reply-to and extract conversation and user (fail if they don't exist)
local_part = reply_to.split('@')[0]
try:
conversation_id, user_id, thread_id = parse_local_part(
local_part)
except (UnicodeDecodeError, binascii.Error):
sentry_client.captureException()
continue
user = get_user_model().objects.get(id=user_id)
thread = None
if thread_id is not None:
thread = ConversationMessage.objects.get(id=thread_id)
conversation = thread.conversation
else:
conversation = Conversation.objects.get(id=conversation_id)
# 3. extract the email reply text
# Try plain text first, most emails have that.
if 'text' in content:
# Try out both talon and discourse email_reply_trimmer
# Trimmers are conservative and sometimes keep more lines, leading to bloated replies.
# We choose the trimmed reply that has fewer lines.
text_content = content['text']
trimmed_talon, line_count_talon = trim_with_talon(
text_content)
trimmed_discourse, line_count_discourse = trim_with_discourse(
text_content)
reply_plain = trimmed_discourse if line_count_discourse <= line_count_talon else trimmed_talon
stats.incoming_email_trimmed({
'line_count_original':
len(text_content.splitlines()),
'line_count_talon':
line_count_talon,
'line_count_discourse':
line_count_discourse,
})
else:
# Fall back to html
try:
html_content = content['html']
except KeyError:
# Inform the user if we couldn't find any content
sentry_client.captureException()
notify_about_rejected_email(
user, 'Karrot could not find any reply text')
continue
reply_html = trim_html_with_talon(html_content)
reply_plain = generate_plaintext_from_html(reply_html)
stats.incoming_html_email_trimmed({
'length_original':
len(html_content.splitlines()),
'length_html_talon':
len(reply_html),
'length_plain_talon':
len(reply_plain),
})
# 4. add reply to conversation
if conversation.is_closed:
notify_about_rejected_email(user, reply_plain)
continue
if not conversation.participants.filter(id=user.id).exists():
notify_about_rejected_email(user, reply_plain)
continue
created_message = ConversationMessage.objects.create(
author=user,
conversation=conversation,
thread=thread,
content=reply_plain,
received_via='email',
)
IncomingEmail.objects.create(
user=user,
message=created_message,
payload=incoming_message,
)
return Response(status=status.HTTP_200_OK, data={})
def notify_about_rejected_email(user, content):
try:
prepare_incoming_email_rejected_email(user, content).send()
except AnymailAPIError:
sentry_client.captureException()
stats.incoming_email_rejected()
def profile_view(request, user_id=None):
"""Displays a view of a user's profile.
Args:
user_id
The ID of the user whose profile is being viewed. If not
specified, show the user's own profile.
"""
if request.user.is_eighthoffice and "full" not in request.GET and user_id is not None:
return redirect("eighth_profile", user_id=user_id)
if user_id is not None:
try:
profile_user = User.objects.get(id=user_id)
if profile_user is None:
raise Http404
except User.DoesNotExist:
raise Http404
else:
profile_user = request.user
num_blocks = 6
eighth_schedule = []
start_block = EighthBlock.objects.get_first_upcoming_block()
blocks = []
if start_block:
blocks = [start_block] + list(start_block.next_blocks(num_blocks - 1))
for block in blocks:
sch = {"block": block}
try:
sch["signup"] = EighthSignup.objects.get(scheduled_activity__block=block, user=profile_user)
except EighthSignup.DoesNotExist:
sch["signup"] = None
except MultipleObjectsReturned:
client.captureException()
sch["signup"] = None
eighth_schedule.append(sch)
if profile_user.is_eighth_sponsor:
sponsor = EighthSponsor.objects.get(user=profile_user)
start_date = get_start_date(request)
eighth_sponsor_schedule = (EighthScheduledActivity.objects.for_sponsor(sponsor).filter(block__date__gte=start_date).order_by(
"block__date", "block__block_letter"))
eighth_sponsor_schedule = eighth_sponsor_schedule[:10]
else:
eighth_sponsor_schedule = None
admin_or_teacher = (request.user.is_eighth_admin or request.user.is_teacher)
can_view_eighth = (profile_user.can_view_eighth or request.user == profile_user)
eighth_restricted_msg = (not can_view_eighth and admin_or_teacher)
if not can_view_eighth and not request.user.is_eighth_admin and not request.user.is_teacher:
eighth_schedule = []
has_been_nominated = profile_user.username in [
u.nominee.username for u in request.user.nomination_votes.filter(position__position_name=settings.NOMINATION_POSITION)
]
context = {
"profile_user": profile_user,
"eighth_schedule": eighth_schedule,
"can_view_eighth": can_view_eighth,
"eighth_restricted_msg": eighth_restricted_msg,
"eighth_sponsor_schedule": eighth_sponsor_schedule,
"nominations_active": settings.NOMINATIONS_ACTIVE,
"nomination_position": settings.NOMINATION_POSITION,
"has_been_nominated": has_been_nominated
}
return render(request, "users/profile.html", context)
def ready(self):
client.captureException()
def node_auth_view(request):
if request.method == "POST":
try:
if not request.POST.get("uid", None):
return JsonResponse({
"granted": False,
"error": "No user id sent to server!"
})
user = User.objects.get(id=int(request.POST.get("uid")))
siteid = request.POST.get("sid", None)
vmid = request.POST.get("vmid", None)
if user.access_token != request.POST.get("access_token"):
return JsonResponse({
"granted": False,
"error": "Invalid access token."
})
if siteid is not None and siteid != "":
site = Site.objects.get(id=int(siteid))
if not user.is_superuser and not site.group.users.filter(
id=user.id).exists():
return JsonResponse(
{
"granted":
False,
"error":
"User does not have permission to access this website."
},
status=403)
return JsonResponse({
"granted": True,
"site_homedir": site.path,
"site_user": site.user.username,
"user": user.username
})
if vmid is not None and vmid != "":
vm = VirtualMachine.objects.get(id=int(vmid))
if not user.is_superuser and not vm.owner == user and not vm.users.filter(
id=user.id).exists():
return JsonResponse(
{
"granted":
False,
"error":
"User does not have permission to access this virtual machine."
},
status=403)
if not vm.ip_address or not vm.password:
return JsonResponse({
"granted":
False,
"error":
"No IP address or root password set!"
})
return JsonResponse({
"granted": True,
"ip": vm.ip_address,
"password": vm.password
})
except Exception as e:
client.captureException()
return JsonResponse(
{
"granted": False,
"error": "Malformed request.",
"exception": str(e)
},
status=400)
else:
return JsonResponse({"error": "Method not allowed."}, status=405)
def thumb_picture(self):
try:
return get_thumbnailer(self.profile_picture)['pet_thumb']
except InvalidImageFormatError:
raven_client.captureException()
return self.profile_picture
def handle_server_error(request):
client.captureException(stack=True, tags={'url': request.path})
def exception() -> None:
if GeneralConfig.is_dev():
return
client.captureException()
def send_failure_alert(printer, is_warning=True, print_paused=False):
LOGGER.info(
f'Printer {printer.user.id} {"smells fishy" if is_warning else "is probably failing"}. Sending Alerts'
)
if not printer.current_print:
LOGGER.warn(
f'Trying to alert on printer without current print. printer_id: {printer.id}'
)
return
rotated_jpg_url = save_print_snapshot(
printer,
last_pic_of_print(printer.current_print, 'tagged'),
f'snapshots/{printer.id}/{printer.current_print.id}/{str(timezone.now().timestamp())}_rotated.jpg',
rotated=True,
to_long_term_storage=False)
# Calls wrapped in individual try/except because anyone of them could fail, and we still want the flow to continue
try:
mobile_notifications.send_failure_alert(printer, rotated_jpg_url,
is_warning, print_paused)
except:
sentryClient.captureException()
try:
if printer.user.alert_by_email:
send_failure_alert_email(printer, rotated_jpg_url, is_warning,
print_paused)
except:
sentryClient.captureException()
try:
send_failure_alert_pushbullet(printer, rotated_jpg_url, is_warning,
print_paused)
except:
sentryClient.captureException()
try:
send_failure_alert_pushover(printer, rotated_jpg_url, is_warning,
print_paused)
except:
sentryClient.captureException()
try:
send_failure_alert_telegram(printer, rotated_jpg_url, is_warning,
print_paused)
except:
sentryClient.captureException()
try:
if printer.user.is_pro and printer.user.alert_by_sms:
send_failure_alert_sms(printer, is_warning, print_paused)
except:
sentryClient.captureException()
try:
if printer.user.is_pro:
send_failure_alert_slack(printer, rotated_jpg_url, is_warning,
print_paused)
except:
sentryClient.captureException()
try:
send_failure_alert_discord(printer, rotated_jpg_url, is_warning,
print_paused)
except:
capture_exception()
def accounting_reconciliation(request):
if request.method == 'POST':
form = DateRangeSelector(request.POST)
if not form.is_valid():
return HttpResponseRedirect(reverse('accounting-reconciliation'))
else:
form = DateRangeSelector(last_month=True)
# There's gotta be a better way to do this
if hasattr(form, 'cleaned_data'):
start = form.cleaned_data['start']
end = form.cleaned_data['end']
else:
start = form.fields['start'].initial
end = form.fields['end'].initial
totals = {
partner.name: {
'bank':
total_donations_for_partner(start,
end,
partner,
payment_method='Bank transfer'),
'pin_payments':
total_donations_for_partner(start,
end,
partner,
payment_method='Pin'),
'pin_payments_after_fees':
total_donations_for_partner(start,
end,
partner,
payment_method='Pin',
after_fees=True),
'stripe':
total_donations_for_partner(start,
end,
partner,
payment_method='Stripe'),
'stripe_after_fees':
total_donations_for_partner(start,
end,
partner,
payment_method='Stripe',
after_fees=True),
'total':
total_donations_for_partner(start, end, partner)
}
for partner in PartnerCharity.objects.all().order_by('name')
}
grand_total = {
kind: sum(total[kind] for total in totals.values())
for kind in ('bank', 'pin_payments', 'pin_payments_after_fees',
'stripe', 'stripe_after_fees', 'total')
}
# This shouldn't/can't happen but it will mess up the reconciliation so let's check.
qs = BankTransaction.objects.filter(pledge__isnull=False,
do_not_reconcile=True)
if qs.exists():
message = "Error: transaction reconciled to pledge and also marked 'Do not reconcile', please check " \
"bank transactions with id: %s" % ', '.join(qs.values_list('id', flat=True))
client.captureException(
"Error: transaction reconciled to pledge and also marked 'Do not reconcile'"
)
return HttpResponse(message)
exceptions = BankTransaction.objects.filter(
date__gte=start,
date__lte=end).exclude(pledge__isnull=False).order_by('date')
return render(
request, 'reconciliation.html', {
'form': form,
'totals': sorted(totals.iteritems()),
'grand_total': grand_total,
'exceptions': exceptions
})
def emit_current_exception():
if os.environ.get("DEBUG") in ["1", 1, True, "true"]:
traceback.print_exc(file=sys.stdout)
else:
client.captureException()