def get_csv_from_s3(filename, is_owner): # pragma: no cover filename = get_downloads_path(is_owner) + filename + '.csv.zip' if S3.exists(filename): return S3.url_for(filename) return None
def write_csv_to_s3(data, is_owner, **kwargs): # pragma: no cover cwd = cd_temp() csv_file = csv_file_for(data, **kwargs) csv_file.close() zip_file_name = csv_file.name + '.zip' with zipfile.ZipFile(zip_file_name, 'w', zipfile.ZIP_DEFLATED) as zip_file: zip_file.write(csv_file.name) file_path = get_downloads_path(is_owner) + zip_file_name S3.upload_file(file_path) os.chdir(cwd) return S3.url_for(file_path)
def delete(self, request, *args, **kwargs): # pylint: disable=unused-argument user = request.user version = self.get_object() permitted = user.is_staff or user.is_superuser or user.is_admin_for(version.head) if not permitted: return HttpResponseForbidden() if version.has_export(): S3.remove(version.export_path) return Response(status=status.HTTP_200_OK) return Response(status=status.HTTP_204_NO_CONTENT)
def delete(self, using=None, keep_parents=False): if self.is_content_privately_referred(): raise ValidationError(dict(detail=CONTENT_REFERRED_PRIVATELY.format(self.mnemonic))) if not self.is_head and self.is_latest_version: prev_version = self.prev_version if not prev_version: raise ValidationError(dict(detail=CANNOT_DELETE_ONLY_VERSION)) prev_version.is_latest_version = True prev_version.save() generic_export_path = self.generic_export_path() super().delete(using=using, keep_parents=keep_parents) S3.delete_objects(generic_export_path)
def get(self, request, *args, **kwargs): # pylint: disable=unused-argument version = self.get_object() logger.debug( 'Export requested for %s version %s - Requesting AWS-S3 key', self.entity.lower(), version.version) if version.is_head: return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) key = version.export_path url = S3.url_for(key) if url: logger.debug(' URL and Key retrieved for %s version %s', self.entity.lower(), version.version) else: logger.debug(' Key does not exist for %s version %s', self.entity.lower(), version.version) return Response(status=status.HTTP_204_NO_CONTENT) response = Response(status=status.HTTP_303_SEE_OTHER) response['Location'] = url # Set headers to ensure sure response is not cached by a client response['Cache-Control'] = 'no-cache, no-store, must-revalidate' response['Pragma'] = 'no-cache' response['Expires'] = '0' response['Last-Updated'] = version.last_child_update.isoformat() response['Last-Updated-Timezone'] = settings.TIME_ZONE_PLACE return response
def write_csv_to_s3(data, is_owner, **kwargs): # pragma: no cover cwd = cd_temp() csv_file = csv_file_for(data, **kwargs) csv_file.close() zip_file_name = csv_file.name + '.zip' with zipfile.ZipFile(zip_file_name, 'w', zipfile.ZIP_DEFLATED) as zip_file: zip_file.write(csv_file.name) key = get_downloads_path(is_owner) + zip_file.filename S3.upload_file(key=key, file_path=os.path.abspath(zip_file.filename), binary=True, metadata=dict(ContentType='application/zip'), headers={'content-type': 'application/zip'}) os.chdir(cwd) return S3.url_for(key)
def delete(self, using=None, keep_parents=False): if self.is_content_privately_referred(): raise ValidationError(dict(detail=CONTENT_REFERRED_PRIVATELY.format(self.mnemonic))) generic_export_path = self.generic_export_path() if self.is_head: self.versions.exclude(id=self.id).delete() else: if self.is_latest_version: prev_version = self.prev_version if not prev_version: raise ValidationError(dict(detail=CANNOT_DELETE_ONLY_VERSION)) prev_version.is_latest_version = True prev_version.save() from core.pins.models import Pin Pin.objects.filter(resource_type__model=self.resource_type.lower(), resource_id=self.id).delete() super().delete(using=using, keep_parents=keep_parents) S3.delete_objects(generic_export_path)
def post(self, request, *args, **kwargs): # pylint: disable=unused-argument version = self.get_object() if version.is_head: return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) logger.debug('%s Export requested for version %s (post)', self.entity, version.version) status_code = status.HTTP_303_SEE_OTHER if not S3.exists(version.export_path): status_code = self.handle_export_version() return Response(status=status_code) response = Response(status=status_code) response['URL'] = version.uri + 'export/' return response
def write_export_file(version, resource_type, resource_serializer_type, logger): # pylint: disable=too-many-statements,too-many-locals cwd = cd_temp() logger.info('Writing export file to tmp directory: %s' % cwd) logger.info('Found %s version %s. Looking up resource...' % (resource_type, version.version)) resource = version.head logger.info('Found %s %s. Serializing attributes...' % (resource_type, resource.mnemonic)) resource_serializer = get_class(resource_serializer_type)(version) data = resource_serializer.data resource_string = json.dumps(data, cls=encoders.JSONEncoder) logger.info('Done serializing attributes.') batch_size = 1000 concepts_qs = version.concepts mappings_qs = version.mappings if resource_type != 'collection': concepts_qs = concepts_qs.filter(is_active=True) mappings_qs = mappings_qs.filter(is_active=True) total_concepts = concepts_qs.count() total_mappings = mappings_qs.count() with open('export.json', 'w') as out: out.write('%s, "concepts": [' % resource_string[:-1]) if total_concepts: logger.info('%s has %d concepts. Getting them in batches of %d...' % (resource_type.title(), total_concepts, batch_size)) concept_serializer_class = get_class( 'core.concepts.serializers.ConceptVersionDetailSerializer') for start in range(0, total_concepts, batch_size): end = min(start + batch_size, total_concepts) logger.info('Serializing concepts %d - %d...' % (start + 1, end)) concept_versions = concepts_qs.prefetch_related( 'names', 'descriptions').select_related('parent__organization', 'parent__user')[start:end] concept_serializer = concept_serializer_class(concept_versions, many=True) concept_data = concept_serializer.data concept_string = json.dumps(concept_data, cls=encoders.JSONEncoder) concept_string = concept_string[1:-1] with open('export.json', 'a') as out: out.write(concept_string) if end != total_concepts: out.write(', ') logger.info('Done serializing concepts.') else: logger.info('%s has no concepts to serialize.' % (resource_type.title())) with open('export.json', 'a') as out: out.write('], "mappings": [') if total_mappings: logger.info('%s has %d mappings. Getting them in batches of %d...' % (resource_type.title(), total_mappings, batch_size)) mapping_serializer_class = get_class( 'core.mappings.serializers.MappingDetailSerializer') for start in range(0, total_mappings, batch_size): end = min(start + batch_size, total_mappings) logger.info('Serializing mappings %d - %d...' % (start + 1, end)) mappings = mappings_qs.select_related( 'parent__organization', 'parent__user', 'from_concept', 'to_concept', 'from_source__organization', 'from_source__user', 'to_source__organization', 'to_source__user', )[start:end] mapping_serializer = mapping_serializer_class(mappings, many=True) mapping_data = mapping_serializer.data mapping_string = json.dumps(mapping_data, cls=encoders.JSONEncoder) mapping_string = mapping_string[1:-1] with open('export.json', 'a') as out: out.write(mapping_string) if end != total_mappings: out.write(', ') logger.info('Done serializing mappings.') else: logger.info('%s has no mappings to serialize.' % (resource_type.title())) with open('export.json', 'a') as out: out.write(']}') with zipfile.ZipFile('export.zip', 'w', zipfile.ZIP_DEFLATED) as _zip: _zip.write('export.json') file_path = os.path.abspath('export.zip') logger.info(file_path) logger.info('Done compressing. Uploading...') s3_key = version.export_path S3.upload_file(key=s3_key, file_path=file_path, binary=True) uploaded_path = S3.url_for(s3_key) logger.info('Uploaded to %s.' % uploaded_path) os.chdir(cwd)
def has_export(self): return S3.exists(self.export_path)
def get_export_url(self): return S3.url_for(self.export_path)
def upload_base64_logo(self, data, name): name = self.uri[1:] + name self.logo_path = S3.upload_base64(data, name, False, True) self.save()
def logo_url(self): url = None if self.logo_path: url = S3.public_url_for(self.logo_path) return url
def get_csv_from_s3(filename, is_owner): filename = get_downloads_path(is_owner) + filename + '.csv.zip' return S3.url_for(filename)