def _action_loop(self, model_name, timeout=5): model_queue = self._model_queues[model_name] try: while not self.should_stop: msgs = [] while len(msgs) < self.chunk_size: try: # If we have any messages queued up, push them through ASAP msgs.append(model_queue.get(timeout=.1 if msgs else timeout)) except queue.Empty: break if not msgs: logger.debug('%r: Recieved no messages to queue up', self) continue start = time.time() logger.debug('%r: Preparing %d %ss to be indexed', self, len(msgs), model_name) for msg, action in zip(msgs, ElasticsearchActionGenerator([self.index], msgs)): # Keep blocking on put() until there's space in the queue or it's time to stop while not self.should_stop: try: self._queue.put((msg, action), timeout=timeout) break except queue.Full: continue logger.info('%r: Prepared %d %ss to be indexed in %.02fs', self, len(msgs), model_name, time.time() - start) except Exception as e: client.captureException() logger.exception('%r: _action_loop(%s) encountered an unexpected error', self, model_name) self.stop()
def fetch_contact_field_results(org, contact_field, segment): from ureport.polls.models import CACHE_ORG_FIELD_DATA_KEY, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME from ureport.polls.models import UREPORT_RUN_FETCHED_DATA_CACHE_TIME start = time.time() print "Fetching %s for %s with segment %s" % (contact_field, org.name, segment) cache_time = UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME if segment and segment.get('location', "") == "District": cache_time = UREPORT_RUN_FETCHED_DATA_CACHE_TIME try: segment = substitute_segment(org, segment) this_time = datetime.now() temba_client = org.get_temba_client() client_results = temba_client.get_results(contact_field=contact_field, segment=segment) results_data = temba_client_flow_results_serializer(client_results) cleaned_results_data = results_data print "Fetch took %ss" % (time.time() - start) key = CACHE_ORG_FIELD_DATA_KEY % (org.pk, slugify(unicode(contact_field)), slugify(unicode(segment))) cache.set(key, {'time': datetime_to_ms(this_time), 'results': cleaned_results_data}, cache_time) except: client.captureException() import traceback traceback.print_exc()
def wrapper(event, context): tracing_id = get_tracing_id(event) os.environ["TRACING_REQUEST_ID"] = tracing_id if sentry: # Provide additional metadata to sentry in case the exception # gets trapped and reported within the function. sentry.user_context({ "aws_log_group_name": getattr(context, "log_group_name"), "aws_log_stream_name": getattr(context, "log_stream_name"), "aws_function_name": getattr(context, "function_name"), "tracing_id": tracing_id }) try: measurement = "%s_duration_ms" % (func.__name__) with influx_timer(measurement, timestamp=now()): return func(event, context) except Exception as e: logger.exception("Got an exception: %r", e) if sentry: logger.info("Inside sentry capture block.") sentry.captureException() else: logger.info("Sentry is not available.") raise finally: from django.db import connection connection.close()
def report_exception(subject, info, exc_info=None): """ Formats an exception then posts it to sentry and if not in debug or testing sends email to mail_admins. """ # Add hostname to subject mail subject = "{0} - {1}".format(subject, settings.HOSTNAME) if exc_info: cls, err = exc_info[:2] message = _(u"Exception in request:" u" %(class)s: %(error)s")\ % {'class': cls.__name__, 'error': err} message += u"".join(traceback.format_exception(*exc_info)) # send to sentry try: client.captureException(exc_info) except Exception: # pylint: disable=broad-except logging.exception(_(u'Sending to Sentry failed.')) else: message = u"%s" % info if settings.DEBUG or settings.TESTING_MODE: sys.stdout.write("Subject: %s\n" % subject) sys.stdout.write("Message: %s\n" % message) else: mail_admins(subject=subject, message=message)
def fetch_results(self, segment=None): from raven.contrib.django.raven_compat.models import client cache_time = UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME if segment and segment.get('location', "") == "District": cache_time = UREPORT_RUN_FETCHED_DATA_CACHE_TIME try: key = CACHE_POLL_RESULTS_KEY % (self.poll.org.pk, self.poll.pk, self.pk) if segment: segment = self.poll.org.substitute_segment(segment) key += ":" + slugify(unicode(segment)) this_time = datetime.now() temba_client = self.poll.org.get_temba_client() client_results = temba_client.get_results(self.ruleset_uuid, segment=segment) results = temba_client_flow_results_serializer(client_results) cache.set(key, {'time': datetime_to_ms(this_time), 'results': results}, cache_time) # delete the open ended cache cache.delete('open_ended:%d' % self.id) except: # pragma: no cover client.captureException() import traceback traceback.print_exc()
def _index_loop(self): try: while not self.should_stop: msgs = [] actions = self._actions(250, msgs) stream = helpers.streaming_bulk( self.es_client, actions, max_chunk_bytes=self.MAX_CHUNK_BYTES, raise_on_error=False, ) start = time.time() for (ok, resp), msg in zip(stream, msgs): if not ok and not (resp.get('delete') and resp['delete']['status'] == 404): raise ValueError(ok, resp, msg) assert len(resp.values()) == 1 _id = list(resp.values())[0]['_id'] assert msg.payload['ids'] == [util.IDObfuscator.decode_id(_id)], '{} {}'.format(msg.payload, util.IDObfuscator.decode_id(_id)) msg.ack() if len(msgs): logger.info('%r: Indexed %d documents in %.02fs', self, len(msgs), time.time() - start) else: logger.debug('%r: Recieved no messages for %.02fs', self, time.time() - start) except Exception as e: client.captureException() logger.exception('%r: _index_loop encountered an unexpected error', self) self.stop()
def process_response(self, request, response): try: if isinstance(response,HttpResponsePermanentRedirect): return response subdomain = request.META['HTTP_HOST'].split(settings.SESSION_COOKIE_DOMAIN)[0] if subdomain not in ['portal','devel'] and request.META['HTTP_HOST'] not in settings.BASE_URL: # we are in a institutional portal institutionportal = InstitutionPortal.objects.get(subdomain=subdomain) request.session['institutionportal'] = institutionportal if all(x not in request.path for x in ['login', 'done', 'scs_auth', 'media', 'static', 'api']) and request.user not in institutionportal.institution.user_set.all(): if request.user.is_anonymous() and request.path not in ['/','']: request.session['errormessage'] = "Please login to access to the requested resource" if request.path not in ['/','']: return render_to_response("scs/login.html", {}, RequestContext(request)) else: if request.path not in ['/','']: request.session['errormessage'] = "You can't access if you are not memeber of the %s institution group" %institutionportal.institution.name return render_to_response("scs/403.html", {}, RequestContext(request)) else: if request.session.get('institutionportal', None): del(request.session['institutionportal']) return response except Exception, e: print e print request.META['HTTP_HOST'] from raven.contrib.django.raven_compat.models import client client.captureException() if request.session.get('institutionportal', None): del(request.session['institutionportal']) return HttpResponsePermanentRedirect(settings.BASE_URL)
def authenticate( self, username = None, password= None, *args, **kwargs): """ Biomedtown backend authenticate method. It delivery Authenticate request to biomedtown mod_auth_tkt service. Arguments: username (string) : Username.\n password (string) : Password.\n Return: User (User instance) : Django User instance.\n tkt64 (string) : return new ticket generation.\n """ if username is None or password is None: return None try: service_response = self.service.rpc_login(username, password) if service_response is not False: user, tkt64 =self.userTicket(service_response) if user is None: return return user, tkt64 return None except Exception, e: from raven.contrib.django.raven_compat.models import client client.captureException() return None
def authenticate( self, ticket=None,cip=None, *args, **kwargs): """ Authenticate User over given ticket. Arguments: ticket (string) : base 64 ticket.\n Return: User (User instance) : Django User instance.\n tkt64 (string) : return new ticket generation.\n """ if ticket is None: return try: user, tkt64 =self.userTicket(ticket,cip) if user is None: return return user, tkt64 except Exception, e: from raven.contrib.django.raven_compat.models import client client.captureException() return None
def delete(self, ticket, *args, **kwargs): try: WorkflowManager.deleteExecution(self.id, ticket) except Exception, e: from raven.contrib.django.raven_compat.models import client client.captureException() pass
def get_image(image, size_name=None, template_name="image.html", retina=False, **kwargs): """ Templatetag to get the HTML for an image from a cropduster image object """ if image: if CROPDUSTER_CROP_ONLOAD: # If set, will check for thumbnail existence # if not there, will create the thumb based on predefiend crop/size settings thumb_path = image.thumbnail_path(size_name) if not exists(thumb_path) and exists(image.image.path): try: size = image.size_set.size_set.get(slug=size_name) except Size.DoesNotExist: return "" try: image.create_thumbnail(size, force_crop=True) except: raven_client.captureException(exc_info=sys.exc_info()) return "" if retina: image_url = image.retina_thumbnail_url(size_name) else: image_url = image.thumbnail_url(size_name) if not image_url: return "" try: image_size = IMAGE_SIZE_MAP[(image.size_set_id, size_name)] except KeyError: return "" # Set all the args that get passed to the template kwargs["image_url"] = image_url if hasattr(image_size, "auto_size") and image_size.auto_size != AUTO_SIZE: kwargs["width"] = image_size.width if hasattr(image_size, "width") else "" kwargs["height"] = image_size.height if hasattr(image_size, "height") else "" if CROPDUSTER_PLACEHOLDER_MODE: kwargs["image_url"] = "http://placehold.it/%sx%s" % (kwargs["width"], kwargs["height"]) kwargs["size_name"] = size_name kwargs["attribution"] = image.attribution if hasattr(image, "caption"): kwargs["alt"] = image.caption if "title" not in kwargs: kwargs["title"] = kwargs["alt"] tmpl = get_template("templatetags/" + template_name) context = template.Context(kwargs) return tmpl.render(context) else: return ""
def feedback_submit(request): form = FeedbackForm(request.POST) rate_limited = getattr(request, 'limited', False) if form.is_valid() and not rate_limited: from_address = settings.FEEDBACK_EMAIL_SENDER to_addresses = [address.strip() for address in feedback_receiver_languages[get_language_from_request(request)] .split(',')] if from_address and to_addresses: feedback_email = form.cleaned_data['feedback_email'] if not feedback_email: feedback_email = "(no email supplied)" message = EMAIL_MESSAGE.format( form.cleaned_data['feedback_referer'], request.META.get('HTTP_USER_AGENT','(unknown)'), feedback_email, form.cleaned_data['feedback_text']) try: send_mail('Feedback received for Court and Tribunal Finder', message, from_address, to_addresses, fail_silently=False) except smtplib.SMTPException: client.captureException() return redirect('staticpages:feedback_sent')
def handle_notification(request): """ Handle bounced emails via an SNS webhook. """ try: notification = json.loads(request.body) except ValueError, e: client.captureException() return HttpResponseBadRequest()
def __call__(instance, pk_str): try: flush_transaction() pk = int(pk_str) data = JobData.objects.get(pk=pk) except (TypeError, ValueError): error_msg = "Invalid value for pk" error_extra = { "pk tried": pk_str } raven_client.captureMessage(error_msg, extra=error_extra, stack=True) except JobData.DoesNotExist: error_msg = "Unable to find beanstalk job data." error_extra = { "pk tried": pk } raven_client.captureMessage(error_msg, extra=error_extra, stack=True) else: try: val = instance.f(data) if self.cleanup: data.delete() return val except Exception: raven_client.captureException()
def run(self, client): try: agg_func = self.AGGREGATOR_FUNCS[self.metric_aggregator] try: value = client.get_metric(self.target, aggregator=agg_func) except (InvalidDataFormat, EmptyData) as exc: return self.build_check_dict(Status.unknown, str(exc)) warning_func = self.OPERATOR_FUNCS[self.warning_operator] critical_func = self.OPERATOR_FUNCS[self.critical_operator] if critical_func(value, self.critical_value): status = Status.critical operator = self.get_critical_operator_display() threshold_value = self.critical_value output_fmt = self.PROBLEM_FMT elif warning_func(value, self.warning_value): status = Status.warning operator = self.get_warning_operator_display() threshold_value = self.warning_value output_fmt = self.PROBLEM_FMT else: status = Status.passing operator = '=' threshold_value = None output_fmt = self.PASSING_FMT output = output_fmt.format(status=status.name, metric=self.name, operator=operator, threshold_value=threshold_value, value=value) return self.build_check_dict(status, output, metrics={self.name: value}) except Exception as exc: raven_client.captureException() return self.build_check_dict(Status.unknown, 'unexpected error: %s' % exc)
def fetch_flows(org): start = time.time() #print "Fetching flows for %s" % org.name try: from ureport.polls.models import CACHE_ORG_FLOWS_KEY, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME this_time = datetime.now() temba_client = org.get_temba_client() flows = temba_client.get_flows() all_flows = dict() for flow in flows: if flow.rulesets: flow_json = dict() flow_json['uuid'] = flow.uuid flow_json['created_on'] = flow.created_on.strftime('%Y-%m-%d') flow_json['name'] = flow.name flow_json['participants'] = flow.participants flow_json['runs'] = flow.runs flow_json['completed_runs'] = flow.completed_runs flow_json['rulesets'] = [ dict(uuid=elt.uuid, label=elt.label, response_type=elt.response_type) for elt in flow.rulesets] all_flows[flow.uuid] = flow_json all_flows_key = CACHE_ORG_FLOWS_KEY % org.pk cache.set(all_flows_key, {'time': datetime_to_ms(this_time), 'results': all_flows}, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME) except: client.captureException() import traceback traceback.print_exc()
def retriveVtk(request): if request.user.is_authenticated(): path = request.POST.get('path','') if not path: path = '/' try: webdav = easywebdav.connect(settings.LOBCDER_HOST, username='******', password=request.ticket, protocol='https' ) fileName = path.split('/')[-1] fileToDownload = os.path.join(settings.LOBCDER_DOWNLOAD_DIR, fileName) downloadChunks = webdav.downloadChunks(settings.LOBCDER_ROOT + path) #remove file if exists if os.path.exists(fileToDownload) and os.stat(fileToDownload)[6] != int(downloadChunks.raw.headers['content-length']): os.remove(fileToDownload) if not os.path.exists(fileToDownload): webdav.download(settings.LOBCDER_ROOT+ path, fileToDownload) content = json.dumps({'path': fileName}, sort_keys=False) response = HttpResponse(content=content, content_type='application/json') return response except Exception, e: client.captureException() response = HttpResponse(status=500) response._is_string = True return response
def get_instagram_photos(limit=6): recent_media = [] cached_data = cache.get("instagram_photos") if cached_data is not None: return cached_data try: url = ( "https://api.instagram.com/v1/users/self/media/recent/?access_token=%s" % settings.INSTAGRAM_ACCESS_TOKEN ) response = requests.get(url) if response.status_code == 200: data = json.loads(response.content.decode()) for item in data["data"]: recent_media.append( { "link": item["link"], "image": item["images"]["thumbnail"]["url"], "name": item["caption"]["text"].split("\n")[0], } ) cache.set("instagram_photos", recent_media[:limit], 60 * 60 * 24) except Exception as e: client.captureException() logger.error(e) return recent_media[:limit]
def get_context_data(self, **kwargs): ctx = super(BlogPostsMixin, self).get_context_data(**kwargs) blog = BlogPost.objects.all() if self.blog_slugs: if isinstance(self.blog_slugs, basestring): blog_slugs = [self.blog_slugs] else: blog_slugs = self.blog_slugs blog = blog.filter_by_blogs(*blog_slugs) ctx['blog_slugs'] = blog_slugs if self.blog_tags: blog = blog.filter_by_tags(*self.blog_tags) ctx['blog_tags'] = self.blog_tags if self.blog_posts_limit: blog = blog[:self.blog_posts_limit] try: # run the query here so that we can catch exceptions # and render the page without blog posts ctx[self.blog_posts_template_variable] = list(blog) except Exception: sentry_client.captureException() ctx[self.blog_posts_template_variable] = [] return ctx
def has_valid_fxa_oauth(request, email): if not email: return False # Grab the token out of the Authorization header authorization = request.META.get('HTTP_AUTHORIZATION') if not authorization: return False authorization = authorization.split(None, 1) if authorization[0].lower() != 'bearer' or len(authorization) != 2: return False token = authorization[1].strip() oauth, profile = get_fxa_clients() # Validate the token with oauth-server and check for appropriate scope. # This will raise an exception if things are not as they should be. try: oauth.verify_token(token, scope=['basket', 'profile:email']) except fxa.errors.Error: # security failure or server problem. can't validate. return invalid sentry_client.captureException() return False try: fxa_email = profile.get_email(token) except fxa.errors.Error: # security failure or server problem. can't validate. return invalid sentry_client.captureException() return False return email == fxa_email
def fetch_reporter_group(org): start = time.time() print "Fetching reporter group for %s" % org.name try: from ureport.polls.models import CACHE_ORG_REPORTER_GROUP_KEY, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME this_time = datetime.now() reporter_group = org.get_config('reporter_group') if reporter_group: temba_client = org.get_temba_client() groups = temba_client.get_groups(name=reporter_group) key = CACHE_ORG_REPORTER_GROUP_KEY % (org.pk, slugify(unicode(reporter_group))) group_dict = dict() if groups: group = groups[0] group_dict = dict(size=group.size, name=group.name, uuid=group.uuid) cache.set(key, {'time': datetime_to_ms(this_time), 'results': group_dict}, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME) except: client.captureException() import traceback traceback.print_exc() # delete the global count cache to force a recalculate at the end cache.delete(GLOBAL_COUNT_CACHE_KEY) print "Fetch %s reporter group took %ss" % (org.name, time.time() - start)
def fetch_flows(org, backend=None): from ureport.polls.models import CACHE_ORG_FLOWS_KEY, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME start = time.time() logger.info("Fetching flows for %s" % org.name) if backend: backends = [backend] else: backends = org.backends.filter(is_active=True) this_time = datetime.now() org_flows = dict(time=datetime_to_ms(this_time), results=dict()) for backend_obj in backends: backend = org.get_backend(backend_slug=backend_obj.slug) try: all_flows = backend.fetch_flows(org) org_flows["results"] = all_flows cache_key = CACHE_ORG_FLOWS_KEY % (org.pk, backend_obj.slug) cache.set(cache_key, org_flows, UREPORT_ASYNC_FETCHED_DATA_CACHE_TIME) except Exception: client.captureException() import traceback traceback.print_exc() logger.info("Fetch %s flows took %ss" % (org.name, time.time() - start)) if len(backends): return org_flows.get("results", dict())
def filter_resources_by_facet(type, facet = None, value = None, page=1, numResults=10, orderBy = 'name', orderType='asc'): """ given a facet and a value return the list of the resources that match the query """ try: if facet: response = requests.get(FACETS_METADATA_API % (type, facet, value,numResults, page, orderBy, orderType), headers={'Accept' : 'application/json'}) else: response = requests.get(TYPE_METADATA_API % (type, numResults, page, orderBy, orderType), headers={'Accept' : 'application/json'}) if response.status_code != 200: raise AtosServiceException("Error while contacting Atos Service: status code = %s" % response.status_code) try: response = response.json() resources = JSON_XPATH_PARSER.find(response) pages = response["numTotalMetadata"] / numResults if response["numTotalMetadata"] % numResults != 0: pages += 1 response['resource_metadata'] = resources response['pages'] = pages return response except Exception, e: client.captureException() return EMPTY_LIST.copy() except BaseException, e: raise AtosServiceException("Error while contacting Atos Service: %s" % e.message)
def feedback_submit(request): form = FeedbackForm(request.POST) if form.is_valid(): from_address = settings.FEEDBACK_EMAIL_SENDER to_addresses = [address.strip() for address in settings.FEEDBACK_EMAIL_RECEIVER.split(',')] if from_address and to_addresses: feedback_email = form.cleaned_data['feedback_email'] if not feedback_email: feedback_email = '(not provided)' message = """ New feedback has arrived for courtfinder (https://courttribunalfinder.service.gov.uk/). The user left feedback after seeing: %s User's browser: %s User's email: %s Message: %s """ % (form.cleaned_data['feedback_referer'], request.META.get('HTTP_USER_AGENT','(unknown)'), form.cleaned_data['feedback_email'], form.cleaned_data['feedback_text']) try: nb_emails_sent = send_mail('Feedback received for Court and Tribunal Finder', message, from_address, to_addresses, fail_silently=False) except smtplib.SMTPException: client.captureException() # do nothing else in case of error. User doesn't need to see. return redirect(reverse('staticpages:feedback_sent'))
def create_tree(self, father, tree): if isinstance(tree, dict): for key, value in tree.items(): cat = Category.objects.create( name=key, slug="", parent=father ) self.create_tree(cat, value) if isinstance(tree, str): try: course = Course.objects.get(slug=tree) except Course.DoesNotExist: if self.LOCAL_CACHE: name = self.LOCAL_CACHE.get(tree, "Unknown course in cache") else: try: ulb_course = ULBCourse.get_from_slug(tree, self.YEAR) name = ulb_course.name except Exception: print("Slug %s failed" % tree) client.captureException() name = "Unknown course in cache" course = Course.objects.create(name=name, slug=tree) course.categories.add(father) if isinstance(tree, list): for subtree in tree: self.create_tree(father, subtree)
def statusMessage(request): """ :type request: django.core.handlers.wsgi.WSGIRequest :return: """ try: if request.path == u'/done/': return {} if request.session.get('statusmessage', False): message = request.session['statusmessage'] del request.session['statusmessage'] return { 'statusmessage': message, } if request.session.get('welcome', False): message = request.session['welcome'] del request.session['welcome'] return { 'welcome': message, } if request.session.get('errormessage', False): message = request.session['errormessage'] del request.session['errormessage'] return { 'errormessage': message, } except Exception, e: client.captureException() pass
def write(self, proteinbox=None, summary=None): ''' Writes the wikitext representation of the protein box to MediaWiki. Returns (result, None) if successful, or (None, Error) if not. Arguments: - `proteinbox`: an updated proteinbox to write ''' page = self.get_page() if not self.bots_allowed(): logger.warn('Bots Blocked', exc_info=True, extra={'page': page, 'bot': self}) try: if proteinbox: result = page.save(str(proteinbox), summary, minor=True) self.text = page.edit() else: result = page.save(self.text, summary, minor=True) self.force_update = False self.save() except MwClientError: client.captureException()
def update_articles(update_list): for pk in update_list: try: article = Article.objects.get(pk=pk) article.update() except Exception: client.captureException()
def read(self, request, ticket="", group="", username=""): """ Promote a user as manager of a group Arguments: request (HTTP request istance): HTTP request send from client. ticket (string) : base 64 ticket. group (string) : the group name username (string) : the username Return: Successes - Json/xml/yaml format response Failure - 403 error """ try: if request.GET.get("ticket"): client_address = request.META["REMOTE_ADDR"] user, tkt64 = authenticate(ticket=request.GET["ticket"], cip=client_address) if user is not None: group = VPHShareSmartGroup.objects.get(name=request.GET.get("group")) user_to_promote = User.objects.get(username=request.GET.get("username")) if not group.is_manager(user): response = HttpResponse(status=403) response._is_string = True return response # add user to the managers group.managers.add(user_to_promote) # add user to the group and all sub groups while group is not None: group.user_set.add(user_to_promote) try: group = VPHShareSmartGroup.objects.get(parent=group) except ObjectDoesNotExist, e: group = None response = HttpResponse(status=200) response._is_string = True response.write("OK") return response else: response = HttpResponse(status=403) response._is_string = True return response except Exception, e: from raven.contrib.django.raven_compat.models import client client.captureException() response = HttpResponse(status=500) response._is_string = True return response
def get_server_obj(self, uuid): try: return Server.objects.get(uuid__exact=uuid) except Server.DoesNotExist: return None except Server.MultipleObjectsReturned: client.captureException() raise serializers.ValidationError("存在多条记录")
def error_handler(e): log.exception(e) if not settings.ENV_DEV: sentry.captureException()
def send_partner_charity_reports(test=True): # Create list of partners, combined the GiveDirectly entries partners = {partner.name: [partner.id] for partner in PartnerCharity.objects.all()} partners['GiveDirectly'] += partners['GiveDirectly Basic income research'] del partners['GiveDirectly Basic income research'] for partner, ids in partners.iteritems(): # Start time is when we last reported last_report_datetime = PartnerCharityReport.objects.filter(partner__id=ids[0] ).aggregate(Max('date'))[ 'date__max'] or datetime.datetime(2016, 1, 1, 0, 0, 0) start = arrow.get(last_report_datetime).datetime # End date is midnight yesterday (i.e. midnight between yesterday and today) UTC # It could be the current time too (or rather a minute ago, to avoid races). end = arrow.get(arrow.utcnow().date()).datetime # Create spreadsheet querysets = OrderedDict([ ('New donations', # For bank transactions, we use time_reconciled Donation.objects.filter(Q(bank_transaction__time_reconciled__gte=start, bank_transaction__time_reconciled__lt=end) | Q(pin_transaction__isnull=False, datetime__gte=start, datetime__lt=end), components__pledge_component__partner_charity__id__in=ids).order_by('datetime')), ('All donations', Donation.objects.filter(Q(bank_transaction__time_reconciled__lt=end) | Q(pin_transaction__isnull=False, datetime__lt=end), components__pledge_component__partner_charity__id__in=ids).order_by('datetime'))]) template = OrderedDict([ ('Date', 'datetime'), ('Amount', 'components__amount'), ('Fees', 'components__fees'), ('Amount (net)', 'components__amount_net'), ('EAA Reference', 'reference'), ('First Name', 'pledge__first_name'), ('Last Name', 'pledge__last_name'), ('Email', 'pledge__email'), ('Payment method', 'payment_method'), ('Subscribe to marketing updates', 'pledge__subscribe_to_updates'), ('Designation', 'components__pledge_component__partner_charity__name') ]) filename = 'EAA donation report - {0} - {1}.xlsx'.format(partner, # Avoid collisions of filename while testing timezone.now()) location = os.path.join(settings.MEDIA_ROOT, 'partner_reports', filename) write_spreadsheet(location, querysets, template) # Create email try: partner_obj = PartnerCharity.objects.get(id=ids[0]) to = [partner_obj.email] cc = [settings.EAA_INFO_EMAIL, settings.TESTING_EMAIL] if partner_obj.email_cc: cc.append(partner_obj.email_cc) body = render_to_string('partner_report_message.txt', {'name': partner}) message = EmailMessage( subject='Effective Altruism Australia donation report', body=body, to=to if not test else [settings.TESTING_EMAIL], cc=cc if not test else [settings.TESTING_EMAIL], # There is a filter in [email protected] # from:(donations @ eaa.org.au) deliveredto:(info + receipts @ eaa.org.au) # that automatically archives messages sent to info+receipt and adds the label 'receipts' # bcc=["*****@*****.**", ], bcc=[], from_email=settings.POSTMARK_SENDER, ) message.attach_file(location, mimetype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') message.send() except Exception as e: print e.message client.captureException() if not test: partner = PartnerCharity.objects.get(id=ids[0]) PartnerCharityReport(partner=partner, date=end).save()
def send_summary_emails(): """ We want to send them on Sunday at 8am in the local time of the group So we run this each hour to check if any group needs their summary email sending. We limit it to just the weekend (Saturday, Sunday) as this will cover any possible timezone offset (-11/+12) In cron Sunday is day 0 and Saturday is day 6 """ with timer() as t: email_count = 0 recipient_count = 0 def is_8am_localtime(group): with timezone.override(group.timezone): localtime = timezone.localtime() # Sunday is day 6 here, confusing! return localtime.hour == 8 and localtime.weekday() == 6 groups = Group.objects.annotate(member_count=Count('members')).filter( member_count__gt=0) for group in groups: if not is_8am_localtime(group): continue from_date, to_date = calculate_group_summary_dates(group) if not group.sent_summary_up_to or group.sent_summary_up_to < to_date: email_recipient_count = 0 context = prepare_group_summary_data(group, from_date, to_date) if context['has_activity']: for email in prepare_group_summary_emails(group, context): try: email.send() email_count += 1 email_recipient_count += len(email.to) except AnymailAPIError: sentry_client.captureException() # we save this even if some of the email sending fails, no retries right now basically... # we also save if no emails were sent due to missing activity, to not try again over and over. group.sent_summary_up_to = to_date group.save() group_summary_email( group, email_recipient_count=email_recipient_count, feedback_count=context['feedbacks'].count(), message_count=context['messages'].count(), new_user_count=context['new_users'].count(), pickups_done_count=context['pickups_done_count'], pickups_missed_count=context['pickups_missed_count'], has_activity=context['has_activity'], ) recipient_count += email_recipient_count stats_utils.periodic_task('group__send_summary_emails', seconds=t.elapsed_seconds, extra_fields={ 'recipient_count': recipient_count, 'email_count': email_count, })
def send_or_report_error(email): try: email.send() except AnymailAPIError: sentry_client.captureException()
def post(self, request): """ Receive conversation replies via e-mail Request payload spec: https://developers.sparkpost.com/api/relay-webhooks/#header-relay-webhook-payload """ auth_key = request.META.get('HTTP_X_MESSAGESYSTEMS_WEBHOOK_TOKEN') if auth_key is None or auth_key != settings.SPARKPOST_RELAY_SECRET: return Response( status=status.HTTP_403_FORBIDDEN, data={ 'message': 'Invalid HTTP_X_MESSAGESYSTEMS_WEBHOOK_TOKEN header' }) for messages in [e['msys'].values() for e in request.data]: for incoming_message in messages: # 1. get email content and reply-to reply_to = parseaddr(incoming_message['rcpt_to'])[1] content = incoming_message['content'] # 2. check local part of reply-to and extract conversation and user (fail if they don't exist) local_part = reply_to.split('@')[0] try: conversation_id, user_id, thread_id = parse_local_part( local_part) except (UnicodeDecodeError, binascii.Error): sentry_client.captureException() continue user = get_user_model().objects.get(id=user_id) thread = None if thread_id is not None: thread = ConversationMessage.objects.get(id=thread_id) conversation = thread.conversation else: conversation = Conversation.objects.get(id=conversation_id) # 3. extract the email reply text # Try plain text first, most emails have that. if 'text' in content: # Try out both talon and discourse email_reply_trimmer # Trimmers are conservative and sometimes keep more lines, leading to bloated replies. # We choose the trimmed reply that has fewer lines. text_content = content['text'] trimmed_talon, line_count_talon = trim_with_talon( text_content) trimmed_discourse, line_count_discourse = trim_with_discourse( text_content) reply_plain = trimmed_discourse if line_count_discourse <= line_count_talon else trimmed_talon stats.incoming_email_trimmed({ 'line_count_original': len(text_content.splitlines()), 'line_count_talon': line_count_talon, 'line_count_discourse': line_count_discourse, }) else: # Fall back to html try: html_content = content['html'] except KeyError: # Inform the user if we couldn't find any content sentry_client.captureException() notify_about_rejected_email( user, 'Karrot could not find any reply text') continue reply_html = trim_html_with_talon(html_content) reply_plain = generate_plaintext_from_html(reply_html) stats.incoming_html_email_trimmed({ 'length_original': len(html_content.splitlines()), 'length_html_talon': len(reply_html), 'length_plain_talon': len(reply_plain), }) # 4. add reply to conversation if conversation.is_closed: notify_about_rejected_email(user, reply_plain) continue if not conversation.participants.filter(id=user.id).exists(): notify_about_rejected_email(user, reply_plain) continue created_message = ConversationMessage.objects.create( author=user, conversation=conversation, thread=thread, content=reply_plain, received_via='email', ) IncomingEmail.objects.create( user=user, message=created_message, payload=incoming_message, ) return Response(status=status.HTTP_200_OK, data={})
def notify_about_rejected_email(user, content): try: prepare_incoming_email_rejected_email(user, content).send() except AnymailAPIError: sentry_client.captureException() stats.incoming_email_rejected()
def profile_view(request, user_id=None): """Displays a view of a user's profile. Args: user_id The ID of the user whose profile is being viewed. If not specified, show the user's own profile. """ if request.user.is_eighthoffice and "full" not in request.GET and user_id is not None: return redirect("eighth_profile", user_id=user_id) if user_id is not None: try: profile_user = User.objects.get(id=user_id) if profile_user is None: raise Http404 except User.DoesNotExist: raise Http404 else: profile_user = request.user num_blocks = 6 eighth_schedule = [] start_block = EighthBlock.objects.get_first_upcoming_block() blocks = [] if start_block: blocks = [start_block] + list(start_block.next_blocks(num_blocks - 1)) for block in blocks: sch = {"block": block} try: sch["signup"] = EighthSignup.objects.get(scheduled_activity__block=block, user=profile_user) except EighthSignup.DoesNotExist: sch["signup"] = None except MultipleObjectsReturned: client.captureException() sch["signup"] = None eighth_schedule.append(sch) if profile_user.is_eighth_sponsor: sponsor = EighthSponsor.objects.get(user=profile_user) start_date = get_start_date(request) eighth_sponsor_schedule = (EighthScheduledActivity.objects.for_sponsor(sponsor).filter(block__date__gte=start_date).order_by( "block__date", "block__block_letter")) eighth_sponsor_schedule = eighth_sponsor_schedule[:10] else: eighth_sponsor_schedule = None admin_or_teacher = (request.user.is_eighth_admin or request.user.is_teacher) can_view_eighth = (profile_user.can_view_eighth or request.user == profile_user) eighth_restricted_msg = (not can_view_eighth and admin_or_teacher) if not can_view_eighth and not request.user.is_eighth_admin and not request.user.is_teacher: eighth_schedule = [] has_been_nominated = profile_user.username in [ u.nominee.username for u in request.user.nomination_votes.filter(position__position_name=settings.NOMINATION_POSITION) ] context = { "profile_user": profile_user, "eighth_schedule": eighth_schedule, "can_view_eighth": can_view_eighth, "eighth_restricted_msg": eighth_restricted_msg, "eighth_sponsor_schedule": eighth_sponsor_schedule, "nominations_active": settings.NOMINATIONS_ACTIVE, "nomination_position": settings.NOMINATION_POSITION, "has_been_nominated": has_been_nominated } return render(request, "users/profile.html", context)
def ready(self): client.captureException()
def node_auth_view(request): if request.method == "POST": try: if not request.POST.get("uid", None): return JsonResponse({ "granted": False, "error": "No user id sent to server!" }) user = User.objects.get(id=int(request.POST.get("uid"))) siteid = request.POST.get("sid", None) vmid = request.POST.get("vmid", None) if user.access_token != request.POST.get("access_token"): return JsonResponse({ "granted": False, "error": "Invalid access token." }) if siteid is not None and siteid != "": site = Site.objects.get(id=int(siteid)) if not user.is_superuser and not site.group.users.filter( id=user.id).exists(): return JsonResponse( { "granted": False, "error": "User does not have permission to access this website." }, status=403) return JsonResponse({ "granted": True, "site_homedir": site.path, "site_user": site.user.username, "user": user.username }) if vmid is not None and vmid != "": vm = VirtualMachine.objects.get(id=int(vmid)) if not user.is_superuser and not vm.owner == user and not vm.users.filter( id=user.id).exists(): return JsonResponse( { "granted": False, "error": "User does not have permission to access this virtual machine." }, status=403) if not vm.ip_address or not vm.password: return JsonResponse({ "granted": False, "error": "No IP address or root password set!" }) return JsonResponse({ "granted": True, "ip": vm.ip_address, "password": vm.password }) except Exception as e: client.captureException() return JsonResponse( { "granted": False, "error": "Malformed request.", "exception": str(e) }, status=400) else: return JsonResponse({"error": "Method not allowed."}, status=405)
def thumb_picture(self): try: return get_thumbnailer(self.profile_picture)['pet_thumb'] except InvalidImageFormatError: raven_client.captureException() return self.profile_picture
def handle_server_error(request): client.captureException(stack=True, tags={'url': request.path})
def exception() -> None: if GeneralConfig.is_dev(): return client.captureException()
def send_failure_alert(printer, is_warning=True, print_paused=False): LOGGER.info( f'Printer {printer.user.id} {"smells fishy" if is_warning else "is probably failing"}. Sending Alerts' ) if not printer.current_print: LOGGER.warn( f'Trying to alert on printer without current print. printer_id: {printer.id}' ) return rotated_jpg_url = save_print_snapshot( printer, last_pic_of_print(printer.current_print, 'tagged'), f'snapshots/{printer.id}/{printer.current_print.id}/{str(timezone.now().timestamp())}_rotated.jpg', rotated=True, to_long_term_storage=False) # Calls wrapped in individual try/except because anyone of them could fail, and we still want the flow to continue try: mobile_notifications.send_failure_alert(printer, rotated_jpg_url, is_warning, print_paused) except: sentryClient.captureException() try: if printer.user.alert_by_email: send_failure_alert_email(printer, rotated_jpg_url, is_warning, print_paused) except: sentryClient.captureException() try: send_failure_alert_pushbullet(printer, rotated_jpg_url, is_warning, print_paused) except: sentryClient.captureException() try: send_failure_alert_pushover(printer, rotated_jpg_url, is_warning, print_paused) except: sentryClient.captureException() try: send_failure_alert_telegram(printer, rotated_jpg_url, is_warning, print_paused) except: sentryClient.captureException() try: if printer.user.is_pro and printer.user.alert_by_sms: send_failure_alert_sms(printer, is_warning, print_paused) except: sentryClient.captureException() try: if printer.user.is_pro: send_failure_alert_slack(printer, rotated_jpg_url, is_warning, print_paused) except: sentryClient.captureException() try: send_failure_alert_discord(printer, rotated_jpg_url, is_warning, print_paused) except: capture_exception()
def accounting_reconciliation(request): if request.method == 'POST': form = DateRangeSelector(request.POST) if not form.is_valid(): return HttpResponseRedirect(reverse('accounting-reconciliation')) else: form = DateRangeSelector(last_month=True) # There's gotta be a better way to do this if hasattr(form, 'cleaned_data'): start = form.cleaned_data['start'] end = form.cleaned_data['end'] else: start = form.fields['start'].initial end = form.fields['end'].initial totals = { partner.name: { 'bank': total_donations_for_partner(start, end, partner, payment_method='Bank transfer'), 'pin_payments': total_donations_for_partner(start, end, partner, payment_method='Pin'), 'pin_payments_after_fees': total_donations_for_partner(start, end, partner, payment_method='Pin', after_fees=True), 'stripe': total_donations_for_partner(start, end, partner, payment_method='Stripe'), 'stripe_after_fees': total_donations_for_partner(start, end, partner, payment_method='Stripe', after_fees=True), 'total': total_donations_for_partner(start, end, partner) } for partner in PartnerCharity.objects.all().order_by('name') } grand_total = { kind: sum(total[kind] for total in totals.values()) for kind in ('bank', 'pin_payments', 'pin_payments_after_fees', 'stripe', 'stripe_after_fees', 'total') } # This shouldn't/can't happen but it will mess up the reconciliation so let's check. qs = BankTransaction.objects.filter(pledge__isnull=False, do_not_reconcile=True) if qs.exists(): message = "Error: transaction reconciled to pledge and also marked 'Do not reconcile', please check " \ "bank transactions with id: %s" % ', '.join(qs.values_list('id', flat=True)) client.captureException( "Error: transaction reconciled to pledge and also marked 'Do not reconcile'" ) return HttpResponse(message) exceptions = BankTransaction.objects.filter( date__gte=start, date__lte=end).exclude(pledge__isnull=False).order_by('date') return render( request, 'reconciliation.html', { 'form': form, 'totals': sorted(totals.iteritems()), 'grand_total': grand_total, 'exceptions': exceptions })
def emit_current_exception(): if os.environ.get("DEBUG") in ["1", 1, True, "true"]: traceback.print_exc(file=sys.stdout) else: client.captureException()