def test_progress_export(self):
"""
Checks to make sure that identical requests (same payload, organization, user)
are routed to the same ExportedData object, with a 200 status code
"""
with self.feature("organizations:data-export"):
response1 = self.get_response(self.organization.slug,
**self.payload)
data_export = ExportedData.objects.get(id=response1.data["id"])
with self.feature("organizations:data-export"):
response2 = self.get_valid_response(self.organization.slug,
**self.payload)
assert response2.data == {
"id": data_export.id,
"user": {
"id": six.binary_type(self.user.id),
"email": self.user.email,
"username": self.user.username,
},
"dateCreated": data_export.date_added,
"dateFinished": data_export.date_finished,
"dateExpired": data_export.date_expired,
"query": {
"type": ExportQueryType.as_str(data_export.query_type),
"info": data_export.query_info,
},
"status": data_export.status,
}
class ExportedData(Model):
"""
Stores references to asynchronous data export jobs being stored
in the Google Cloud Platform temporary storage solution.
"""
__core__ = False
organization = FlexibleForeignKey("sentry.Organization")
user = FlexibleForeignKey(settings.AUTH_USER_MODEL)
file = FlexibleForeignKey("sentry.File", null=True, db_constraint=False)
date_added = models.DateTimeField(default=timezone.now)
date_finished = models.DateTimeField(null=True)
date_expired = models.DateTimeField(null=True)
query_type = BoundedPositiveIntegerField(
choices=ExportQueryType.as_choices())
query_info = JSONField()
@property
def status(self):
if self.date_finished is None:
return ExportStatus.Early
elif self.date_expired < timezone.now():
return ExportStatus.Expired
else:
return ExportStatus.Valid
class Meta:
app_label = "sentry"
db_table = "sentry_exporteddata"
__repr__ = sane_repr("data_id")
class ExportedData(Model):
"""
Stores references to asynchronous data export jobs being stored
in the Google Cloud Platform temporary storage solution.
"""
__core__ = False
organization = FlexibleForeignKey("sentry.Organization")
user = FlexibleForeignKey(settings.AUTH_USER_MODEL,
null=True,
on_delete=models.SET_NULL)
file = FlexibleForeignKey("sentry.File",
null=True,
db_constraint=False,
on_delete=models.SET_NULL)
date_added = models.DateTimeField(default=timezone.now)
date_finished = models.DateTimeField(null=True)
date_expired = models.DateTimeField(null=True, db_index=True)
query_type = BoundedPositiveIntegerField(
choices=ExportQueryType.as_choices())
query_info = JSONField()
@property
def status(self):
if self.date_finished is None:
return ExportStatus.Early
elif self.date_expired < timezone.now():
return ExportStatus.Expired
else:
return ExportStatus.Valid
def delete_file(self):
if self.file:
self.file.delete()
def delete(self, *args, **kwargs):
self.delete_file()
super(ExportedData, self).delete(*args, **kwargs)
def finalize_upload(self, file, expiration=DEFAULT_EXPIRATION):
self.delete_file()
current_time = timezone.now()
expire_time = current_time + expiration
self.update(file=file,
date_finished=current_time,
date_expired=expire_time)
# TODO(Leander): Implement email notification
class Meta:
app_label = "sentry"
db_table = "sentry_exporteddata"
__repr__ = sane_repr("query_type", "query_info")
def serialize(self, obj, attrs, user, **kwargs):
return {
"id": obj.id,
"user": attrs["user"],
"dateCreated": obj.date_added,
"dateFinished": obj.date_finished,
"dateExpired": obj.date_expired,
"query": {
"type": ExportQueryType.as_str(obj.query_type),
"info": obj.query_info
},
"status": obj.status,
}
def test_content(self):
with self.feature("organizations:data-export"):
response = self.get_valid_response(self.organization.slug,
self.data_export.id)
assert response.data["id"] == self.data_export.id
assert response.data["user"] == {
"id": six.binary_type(self.user.id),
"email": self.user.email,
"username": self.user.username,
}
assert response.data["dateCreated"] == self.data_export.date_added
assert response.data["query"] == {
"type": ExportQueryType.as_str(self.data_export.query_type),
"info": self.data_export.query_info,
}
def post(self, request, organization):
"""
Create a new asynchronous file export task, and
email user upon completion,
"""
if not features.has("organizations:data-export", organization):
return Response(status=404)
serializer = ExportedDataSerializer(data=request.data,
context={
"organization": organization,
"user": request.user
})
if not serializer.is_valid():
return Response(serializer.errors, status=400)
data = serializer.validated_data
try:
# If this user has sent a sent a request with the same payload and organization,
# we return them the latest one that is NOT complete (i.e. don't start another)
query_type = ExportQueryType.from_str(data["query_type"])
data_export, created = ExportedData.objects.get_or_create(
organization=organization,
user=request.user,
query_type=query_type,
query_info=data["query_info"],
date_finished=None,
)
status = 200
if created:
metrics.incr("dataexport.start",
tags={"query_type": data["query_type"]})
assemble_download.delay(data_export_id=data_export.id)
status = 201
except ValidationError as e:
# This will handle invalid JSON requests
metrics.incr("dataexport.invalid",
tags={"query_type": data.get("query_type")})
return Response({"detail": six.text_type(e)}, status=400)
return Response(serialize(data_export, request.user), status=status)
class ExportedDataSerializer(serializers.Serializer):
max_value = len(ExportQueryType.as_choices()) - 1
query_type = serializers.IntegerField(required=True,
min_value=0,
max_value=max_value)
query_info = serializers.JSONField(required=True)
def payload(self):
payload = self.query_info.copy()
payload["export_type"] = ExportQueryType.as_str(self.query_type)
return payload
class ExportedData(Model):
"""
Stores references to asynchronous data export jobs being stored
in the Google Cloud Platform temporary storage solution.
"""
__core__ = False
organization = FlexibleForeignKey("sentry.Organization")
user = FlexibleForeignKey(settings.AUTH_USER_MODEL,
null=True,
on_delete=models.SET_NULL)
file = FlexibleForeignKey("sentry.File",
null=True,
db_constraint=False,
on_delete=models.SET_NULL)
date_added = models.DateTimeField(default=timezone.now)
date_finished = models.DateTimeField(null=True)
date_expired = models.DateTimeField(null=True, db_index=True)
query_type = BoundedPositiveIntegerField(
choices=ExportQueryType.as_choices())
query_info = JSONField()
@property
def status(self):
if self.date_finished is None:
return ExportStatus.Early
elif self.date_expired < timezone.now():
return ExportStatus.Expired
else:
return ExportStatus.Valid
@property
def date_expired_string(self):
if self.date_expired is None:
return None
return self.date_expired.strftime("%-I:%M %p on %B %d, %Y (%Z)")
@property
def payload(self):
payload = self.query_info.copy()
payload["export_type"] = ExportQueryType.as_str(self.query_type)
return payload
def delete_file(self):
if self.file:
self.file.delete()
def delete(self, *args, **kwargs):
self.delete_file()
super(ExportedData, self).delete(*args, **kwargs)
def finalize_upload(self, file, expiration=DEFAULT_EXPIRATION):
self.delete_file()
current_time = timezone.now()
expire_time = current_time + expiration
self.update(file=file,
date_finished=current_time,
date_expired=expire_time)
self.email_success()
def email_success(self):
from sentry.utils.email import MessageBuilder
# The following condition should never be true, but it's a safeguard in case someone manually calls this method
if self.date_finished is None or self.date_expired is None or self.file is None:
# TODO(Leander): Implement logging here
return
msg = MessageBuilder(
subject="Your Download is Ready!",
context={
"url":
absolute_uri(
reverse("sentry-data-export-details",
args=[self.organization.slug, self.id])),
"expiration":
self.date_expired_string,
},
template="sentry/emails/data-export-success.txt",
html_template="sentry/emails/data-export-success.html",
)
msg.send_async([self.user.email])
def email_failure(self, message):
from sentry.utils.email import MessageBuilder
msg = MessageBuilder(
subject="Unable to Export Data",
context={
"error_message": message,
"payload": json.dumps(self.payload, indent=2, sort_keys=True),
},
type="organization.export-data",
template="sentry/emails/data-export-failure.txt",
html_template="sentry/emails/data-export-failure.html",
)
msg.send_async([self.user.email])
self.delete()
class Meta:
app_label = "sentry"
db_table = "sentry_exporteddata"
__repr__ = sane_repr("query_type", "query_info")
class ExportedDataSerializer(serializers.Serializer):
query_type = serializers.ChoiceField(
choices=ExportQueryType.as_str_choices(), required=True)
# TODO(Leander): Implement query_info validation with jsonschema
query_info = serializers.JSONField(required=True)