def test_progress_export(self): """ Checks to make sure that identical requests (same payload, organization, user) are routed to the same ExportedData object, with a 200 status code """ with self.feature("organizations:data-export"): response1 = self.get_response(self.organization.slug, **self.payload) data_export = ExportedData.objects.get(id=response1.data["id"]) with self.feature("organizations:data-export"): response2 = self.get_valid_response(self.organization.slug, **self.payload) assert response2.data == { "id": data_export.id, "user": { "id": six.binary_type(self.user.id), "email": self.user.email, "username": self.user.username, }, "dateCreated": data_export.date_added, "dateFinished": data_export.date_finished, "dateExpired": data_export.date_expired, "query": { "type": ExportQueryType.as_str(data_export.query_type), "info": data_export.query_info, }, "status": data_export.status, }
class ExportedData(Model): """ Stores references to asynchronous data export jobs being stored in the Google Cloud Platform temporary storage solution. """ __core__ = False organization = FlexibleForeignKey("sentry.Organization") user = FlexibleForeignKey(settings.AUTH_USER_MODEL) file = FlexibleForeignKey("sentry.File", null=True, db_constraint=False) date_added = models.DateTimeField(default=timezone.now) date_finished = models.DateTimeField(null=True) date_expired = models.DateTimeField(null=True) query_type = BoundedPositiveIntegerField( choices=ExportQueryType.as_choices()) query_info = JSONField() @property def status(self): if self.date_finished is None: return ExportStatus.Early elif self.date_expired < timezone.now(): return ExportStatus.Expired else: return ExportStatus.Valid class Meta: app_label = "sentry" db_table = "sentry_exporteddata" __repr__ = sane_repr("data_id")
class ExportedData(Model): """ Stores references to asynchronous data export jobs being stored in the Google Cloud Platform temporary storage solution. """ __core__ = False organization = FlexibleForeignKey("sentry.Organization") user = FlexibleForeignKey(settings.AUTH_USER_MODEL, null=True, on_delete=models.SET_NULL) file = FlexibleForeignKey("sentry.File", null=True, db_constraint=False, on_delete=models.SET_NULL) date_added = models.DateTimeField(default=timezone.now) date_finished = models.DateTimeField(null=True) date_expired = models.DateTimeField(null=True, db_index=True) query_type = BoundedPositiveIntegerField( choices=ExportQueryType.as_choices()) query_info = JSONField() @property def status(self): if self.date_finished is None: return ExportStatus.Early elif self.date_expired < timezone.now(): return ExportStatus.Expired else: return ExportStatus.Valid def delete_file(self): if self.file: self.file.delete() def delete(self, *args, **kwargs): self.delete_file() super(ExportedData, self).delete(*args, **kwargs) def finalize_upload(self, file, expiration=DEFAULT_EXPIRATION): self.delete_file() current_time = timezone.now() expire_time = current_time + expiration self.update(file=file, date_finished=current_time, date_expired=expire_time) # TODO(Leander): Implement email notification class Meta: app_label = "sentry" db_table = "sentry_exporteddata" __repr__ = sane_repr("query_type", "query_info")
def serialize(self, obj, attrs, user, **kwargs): return { "id": obj.id, "user": attrs["user"], "dateCreated": obj.date_added, "dateFinished": obj.date_finished, "dateExpired": obj.date_expired, "query": { "type": ExportQueryType.as_str(obj.query_type), "info": obj.query_info }, "status": obj.status, }
def test_content(self): with self.feature("organizations:data-export"): response = self.get_valid_response(self.organization.slug, self.data_export.id) assert response.data["id"] == self.data_export.id assert response.data["user"] == { "id": six.binary_type(self.user.id), "email": self.user.email, "username": self.user.username, } assert response.data["dateCreated"] == self.data_export.date_added assert response.data["query"] == { "type": ExportQueryType.as_str(self.data_export.query_type), "info": self.data_export.query_info, }
def post(self, request, organization): """ Create a new asynchronous file export task, and email user upon completion, """ if not features.has("organizations:data-export", organization): return Response(status=404) serializer = ExportedDataSerializer(data=request.data, context={ "organization": organization, "user": request.user }) if not serializer.is_valid(): return Response(serializer.errors, status=400) data = serializer.validated_data try: # If this user has sent a sent a request with the same payload and organization, # we return them the latest one that is NOT complete (i.e. don't start another) query_type = ExportQueryType.from_str(data["query_type"]) data_export, created = ExportedData.objects.get_or_create( organization=organization, user=request.user, query_type=query_type, query_info=data["query_info"], date_finished=None, ) status = 200 if created: metrics.incr("dataexport.start", tags={"query_type": data["query_type"]}) assemble_download.delay(data_export_id=data_export.id) status = 201 except ValidationError as e: # This will handle invalid JSON requests metrics.incr("dataexport.invalid", tags={"query_type": data.get("query_type")}) return Response({"detail": six.text_type(e)}, status=400) return Response(serialize(data_export, request.user), status=status)
class ExportedDataSerializer(serializers.Serializer): max_value = len(ExportQueryType.as_choices()) - 1 query_type = serializers.IntegerField(required=True, min_value=0, max_value=max_value) query_info = serializers.JSONField(required=True)
def payload(self): payload = self.query_info.copy() payload["export_type"] = ExportQueryType.as_str(self.query_type) return payload
class ExportedData(Model): """ Stores references to asynchronous data export jobs being stored in the Google Cloud Platform temporary storage solution. """ __core__ = False organization = FlexibleForeignKey("sentry.Organization") user = FlexibleForeignKey(settings.AUTH_USER_MODEL, null=True, on_delete=models.SET_NULL) file = FlexibleForeignKey("sentry.File", null=True, db_constraint=False, on_delete=models.SET_NULL) date_added = models.DateTimeField(default=timezone.now) date_finished = models.DateTimeField(null=True) date_expired = models.DateTimeField(null=True, db_index=True) query_type = BoundedPositiveIntegerField( choices=ExportQueryType.as_choices()) query_info = JSONField() @property def status(self): if self.date_finished is None: return ExportStatus.Early elif self.date_expired < timezone.now(): return ExportStatus.Expired else: return ExportStatus.Valid @property def date_expired_string(self): if self.date_expired is None: return None return self.date_expired.strftime("%-I:%M %p on %B %d, %Y (%Z)") @property def payload(self): payload = self.query_info.copy() payload["export_type"] = ExportQueryType.as_str(self.query_type) return payload def delete_file(self): if self.file: self.file.delete() def delete(self, *args, **kwargs): self.delete_file() super(ExportedData, self).delete(*args, **kwargs) def finalize_upload(self, file, expiration=DEFAULT_EXPIRATION): self.delete_file() current_time = timezone.now() expire_time = current_time + expiration self.update(file=file, date_finished=current_time, date_expired=expire_time) self.email_success() def email_success(self): from sentry.utils.email import MessageBuilder # The following condition should never be true, but it's a safeguard in case someone manually calls this method if self.date_finished is None or self.date_expired is None or self.file is None: # TODO(Leander): Implement logging here return msg = MessageBuilder( subject="Your Download is Ready!", context={ "url": absolute_uri( reverse("sentry-data-export-details", args=[self.organization.slug, self.id])), "expiration": self.date_expired_string, }, template="sentry/emails/data-export-success.txt", html_template="sentry/emails/data-export-success.html", ) msg.send_async([self.user.email]) def email_failure(self, message): from sentry.utils.email import MessageBuilder msg = MessageBuilder( subject="Unable to Export Data", context={ "error_message": message, "payload": json.dumps(self.payload, indent=2, sort_keys=True), }, type="organization.export-data", template="sentry/emails/data-export-failure.txt", html_template="sentry/emails/data-export-failure.html", ) msg.send_async([self.user.email]) self.delete() class Meta: app_label = "sentry" db_table = "sentry_exporteddata" __repr__ = sane_repr("query_type", "query_info")
class ExportedDataSerializer(serializers.Serializer): query_type = serializers.ChoiceField( choices=ExportQueryType.as_str_choices(), required=True) # TODO(Leander): Implement query_info validation with jsonschema query_info = serializers.JSONField(required=True)