def retrieve(self, request, pk, *args, **kwargs): throttle = throttling.ImportDumpModeRateThrottle() if not throttle.allow_request(request, self): self.throttled(request, throttle.wait()) project = get_object_or_404(self.get_queryset(), pk=pk) self.check_permissions(request, 'export_project', project) dump_format = request.QUERY_PARAMS.get("dump_format", "plain") if settings.CELERY_ENABLED: task = tasks.dump_project.delay(request.user, project, dump_format) tasks.delete_project_dump.apply_async((project.pk, project.slug, task.id, dump_format), countdown=settings.EXPORTS_TTL) return response.Accepted({"export_id": task.id}) if dump_format == "gzip": path = "exports/{}/{}-{}.json.gz".format(project.pk, project.slug, uuid.uuid4().hex) storage_path = default_storage.path(path) with default_storage.open(storage_path, mode="wb") as outfile: services.render_project(project, gzip.GzipFile(fileobj=outfile)) else: path = "exports/{}/{}-{}.json".format(project.pk, project.slug, uuid.uuid4().hex) storage_path = default_storage.path(path) with default_storage.open(storage_path, mode="wb") as outfile: services.render_project(project, outfile) response_data = { "url": default_storage.url(path) } return response.Ok(response_data)
def image_ratio_ajax(request, upload_to=None, form_class=FileForm): """ Processes ajax post from imagescaler. """ form = form_class(request.POST, request.FILES) if form.is_valid(): uploaded_file = request.FILES['file'] if uploaded_file.content_type in IMAGE_TYPES: file_name, extension = os.path.splitext(uploaded_file.name) safe_name = '{0}{1}'.format(FILENAME_NORMALIZER(file_name), extension) name = os.path.join(upload_to or UPLOAD_PATH, safe_name) path = default_storage.save(name, uploaded_file) full_path = default_storage.path(path) try: os.chmod(full_path, 0660) except Exception: print sys.exc_info() if settings.TINY_PNG_ENABLED is True and uploaded_file.content_type in \ IMAGE_TYPES_FOR_TINY_PNG: compress_image(default_storage.path(path)) size = get_image_dimensions(default_storage.path(path), True) if size: width, height = size return HttpResponse(json.dumps({ 'url': default_storage.url(path), 'filename': path, 'data': {'width': width, 'height': height} })) return HttpResponse(status=403, content='Bad image format') return HttpResponse(status=403)
def write_uploaded_file_to_dataset(dataset, uploaded_file_post, subdir=None): """ Writes file POST data to the dataset directory in the file store :param dataset: dataset whose directory to be written to :type dataset: models.Model :param uploaded_file_post: uploaded file (either UploadedFile or File) :type uploaded_file_post: types.FileType :rtype: the path of the file written to """ filename = uploaded_file_post.name if subdir is not None: filename = path.join(subdir, filename) from django.core.files.storage import default_storage # Path on disk can contain subdirectories - but if the request # gets tricky with "../" or "/var" or something we strip them # out.. try: copyto = path.join(get_dataset_path(dataset), filename) default_storage.path(copyto) except (SuspiciousOperation, ValueError): copyto = path.join(get_dataset_path(dataset), path.basename(filename)) logger.debug("Writing uploaded file %s" % copyto) realcopyto = default_storage.save(copyto, uploaded_file_post) if copyto != realcopyto: logger.debug("Actually wrote uploaded file to %s" % copyto) return realcopyto
def check_availability(self): """ Perform check against Default Storage. """ try: name = default_storage.get_valid_name('Informer Storage') # Save data. content = ContentFile('File used by StorageInformer checking.') path = default_storage.save(name, content) # Check properties. default_storage.size(path) default_storage.url(path) default_storage.path(path) default_storage.get_accessed_time(path) default_storage.get_available_name(path) default_storage.get_created_time(path) default_storage.get_modified_time(path) default_storage.get_valid_name(path) # And remove file. default_storage.delete(path) storage = default_storage.__class__.__name__ except Exception as error: raise InformerException( f'An error occurred when trying to use your Storage: {error}') else: return True, f'Your {storage} is operational.'
def rpc_push(request): response = HttpResponse() response.write("<h1>upload data</h1>") response.write("curl -F upload_filename=@LICENSE http://localhost:8000/PUSH/") if request.method == "GET": return rpc_get(request, int(request.GET.get("id"))) if request.method == "POST": token = request.POST.get("token") server = check_auth_host(token) if server: current = datetime.now() for filename, mem_file in request.FILES.items(): filepath = default_storage.path('upload/tmp/%s' % filename) with open(filepath, 'wb+') as destination: for chunk in mem_file.chunks(): destination.write(chunk) new_file = "%s_%s.%s" % (current.date(), current.time(), filename.split(".")[-1]) logger.info("file %s was uploaded %s" % (new_file, datetime.now() - current)) new_file = default_storage.path('upload/%d/%s' % (server.id, new_file)) os.rename(filepath, new_file) else: response.write("<h2>Error: authorized token doesn't exist<h2>") response['Content-length'] = str(len(response.content)) return response
def decrypt(request): myEntry = Entry.objects.filter(slug=request.GET['slug'])[0] fileName = default_storage.path(myEntry.myFile) key = hashlib.sha256(request.GET['pass']).digest() try: CryptoLib.decrypt_file(key, fileName) dec_filePath = default_storage.path(fileName[:-4]) print dec_filePath wrapper = FileWrapper(file(dec_filePath)) response = HttpResponse(wrapper, content_type='application/zip') response['Content-Length'] = os.path.getsize(dec_filePath) default_storage.delete(fileName[:-4]) except: #nuke the media directory shutil.rmtree(default_storage.path('')) os.mkdir(default_storage.path('')) raise return response
def save_model(self, request, obj, form, change): if change == False and request.POST.__contains__('_saveasnew'): path_info = request.META['HTTP_REFERER'] id = path_info.split('/')[-2:-1] #old image id - table of size 1 s_file = StaticFile.objects.get(pk = int(id[0])) path = generate_file_path(None, request.POST['filename']) old_path = s_file.static_file img_path = 'uploads/'+str(old_path) result = img_path #urllib.urlretrieve(img_path) #uploads/folder/filename.ext if request.POST['crop_coords'] != "": crop_coords = map(int, request.POST['crop_coords'].split(',')) file = default_storage.open(old_path) img = Image.open(file) cropped_img = img.crop((crop_coords[0], crop_coords[1], crop_coords[0]+ crop_coords[2], crop_coords[1] + crop_coords[3])) cropped_img.save(default_storage.path(path)) obj.width, obj.height = cropped_img.size obj.crop_coords = '' else: file = default_storage.open(old_path) img = Image.open(file) img.save(default_storage.path(path)) obj.width, obj.height = img.size obj.static_file.save(path, File(open(default_storage.path(path))), save=True) obj.user = request.user obj.save() else: return super(FileAdmin, self).save_model(request, obj, form, change)
def create_thumbnail(image_name): image_path = default_storage.path(TWITTER_IMAGE_PATH + image_name) image = Image.open(image_path) image.thumbnail(THUMBNAIL_SIZE, Image.ANTIALIAS) thumbnail_path = default_storage.path(THUMBNAIL_PATH + image_name) #if (image_path.lower()).find('jpg') <= -1: # thumbnail_path=thumbnail_path+'.jpg' image.save(thumbnail_path,'JPEG') return thumbnail_path
def test_generate_attachments_zip_export(self): filenames = [ 'OSMWay234134797.osm', 'OSMWay34298972.osm', ] osm_fixtures_dir = os.path.realpath( os.path.join( os.path.dirname(api_tests.__file__), 'fixtures', 'osm')) paths = [ os.path.join(osm_fixtures_dir, filename) for filename in filenames ] xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') self._publish_xls_file_and_set_xform(xlsform_path) submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') count = Attachment.objects.filter(extension='osm').count() self._make_submission_w_attachment(submission_path, paths) self.assertTrue( Attachment.objects.filter(extension='osm').count() > count) options = {"extension": Export.ZIP_EXPORT} export = generate_attachments_zip_export( Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options) self.assertTrue(export.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r") zip_file.extractall(temp_dir) zip_file.close() for a in Attachment.objects.all(): self.assertTrue( os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir) # deleted submission submission = self.xform.instances.filter().first() submission.deleted_at = timezone.now() submission.save() export = generate_attachments_zip_export( Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options) self.assertTrue(export.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r") zip_file.extractall(temp_dir) zip_file.close() for a in Attachment.objects.all(): self.assertFalse( os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir)
def __init__(self, f): if not bool(f) or not default_storage.exists(f): self.file = '' self.last_modification = None self.size = size(0) self.url = "" else: self.file = basename(default_storage.path(f)) self.last_modification = getmtime(default_storage.path(f)) self.size = size(getsize(default_storage.path(f))) self.url = ""
def setUp(self): self.b1 = Book.objects.create(name="Seguranca em Redes Informaticas", author="Andre Zuquete", production_date="2013-05-01", original_file=default_storage.path(BASE_DIR + '/media/books/pg6598.txt')) self.b2 = Book.objects.create(name="Cyclopedia of Telephony & Telegraphy Vol. 2", author="Kempster Miller et. al.", production_date="2010-08-15", original_file=default_storage.path(BASE_DIR + '/media/books/pg33437.txt')) self.a1 = Account.objects.create(email='*****@*****.**', username='******', first_name='unit', last_name='test', user_data=self.ucd1)
def create(self, request, *args, **kwargs): """ B{Create} a private competition round B{URL:} ../api/v1/competitions/private/round/ :type competition_name: str :param competition_name: The competition name :type grid: str :param grid: The grid path from resources :type param_list: str :param param_list: The param_list from resources :type lab: str :param lab: The lab from resources """ serializer = self.serializer_class(data=request.data) if serializer.is_valid(): private_competition = get_object_or_404(Competition.objects.all(), name=serializer.validated_data['competition_name']) # this competition must be a private competition MustBePrivateCompetition(competition=private_competition) # verify if the team is enrolled in the competition UserCanAccessToThePrivateCompetition(competition=private_competition, user=request.user) # create a round for this competition try: has = Round.objects.filter(parent_competition=private_competition, grid_path=default_storage.path(serializer.validated_data['grid']), param_list_path=default_storage.path( serializer.validated_data['param_list']), lab_path=default_storage.path(serializer.validated_data['lab'])).count() if has > 0: return Response({'status': 'Bad request', 'message': 'You already have one solo trial with those files!'}, status=status.HTTP_400_BAD_REQUEST) with transaction.atomic(): r = Round.objects.create(name=uuid.uuid4(), parent_competition=private_competition) PrivateCompetitionRound.set_param(r, serializer.validated_data['grid'], 'grid') PrivateCompetitionRound.set_param(r, serializer.validated_data['param_list'], 'param_list') PrivateCompetitionRound.set_param(r, serializer.validated_data['lab'], 'lab') except IntegrityError, e: return Response({'status': 'Bad request', 'message': e.message}, status=status.HTTP_400_BAD_REQUEST) except DataError, e: return Response({'status': 'Bad request', 'message': 'Please specify correctly the files!'}, status=status.HTTP_400_BAD_REQUEST)
def to_representation(self, instance): grid = basename(default_storage.path(instance.grid_path)) lab = basename(default_storage.path(instance.lab_path)) param_list = basename(default_storage.path(instance.param_list_path)) return { 'name': instance.name, 'grid': grid, 'grid_path': "resources/"+default_storage.path(instance.grid_path).split('/media/resources/')[1], 'param_list': param_list, 'lab': lab, 'lab_path': "resources/" + default_storage.path(instance.lab_path).split('/media/resources/')[1], 'created_at': instance.created_at, 'updated_at': instance.updated_at }
def test_save(self): self.assertTrue(self.photo.md5sum) # prepare new upload image thumb = Image.new('RGB', (1024, 768), 'red') # the same size and color # as self.photo.image thumb_io = BytesIO() thumb.save(thumb_io, format='JPEG') # prevent the purposefully-thrown exception from breaking the entire # unittest's transaction with transaction.atomic(): self.assertRaisesMessage(IntegrityError, "UNIQUE constraint failed: " "loader_photo.md5sum", PhotoFactory, image = ContentFile(thumb_io.getvalue(), "test.jpg"), name = "Uploaded Photo 1", thumbnail = None # we won't generate # thumbnail image ) path = default_storage.path(name="photos/test.jpg") default_storage.delete(path) # remove photo created in 'media' folder # no problems with the new different image self.up_photo = PhotoFactory(name = "Uploaded Photo 1") # new blue image self.assertNotEqual(self.up_photo.md5sum, self.photo.md5sum)
def dump_project(self, user, project): mbuilder = MagicMailBuilder(template_mail_cls=InlineCSSTemplateMail) path = "exports/{}/{}-{}.json".format(project.pk, project.slug, self.request.id) storage_path = default_storage.path(path) try: url = default_storage.url(path) with default_storage.open(storage_path, mode="w") as outfile: render_project(project, outfile) except Exception: ctx = { "user": user, "error_subject": _("Error generating project dump"), "error_message": _("Error generating project dump"), "project": project } email = mbuilder.export_error(user, ctx) email.send() logger.error('Error generating dump %s (by %s)', project.slug, user, exc_info=sys.exc_info()) return deletion_date = timezone.now() + datetime.timedelta(seconds=settings.EXPORTS_TTL) ctx = { "url": url, "project": project, "user": user, "deletion_date": deletion_date } email = mbuilder.dump_project(user, ctx) email.send()
def serve_content(storage, name): """ Generate a response to server protected content. """ if not storage.exists(name): raise Http404 # Non-filesystem storages should re-direct to a temporary URL if not storage.__class__.__name__ == 'FileSystemStorage': return HttpResponseRedirect(storage.url(name)) # If behind a real server, use send-file if settings.GEDGO_SENDFILE_HEADER: response = HttpResponse() response[settings.GEDGO_SENDFILE_HEADER] = default_storage.path(name) # Otherwise, serve it ourselves, which should only happen in DEBUG mode else: wrapper = FileWrapper(storage.open(name)) response = HttpResponse(wrapper) # Set various file headers and return base = path.basename(name) response['Content-Type'] = mimetypes.guess_type(base)[0] response['Content-Length'] = storage.size(name) if response['Content-Type'] is None: response['Content-Disposition'] = "attachment; filename=%s;" % (base) return response
def __init__(self): super(Giedo, self).__init__(settings.GIEDO_SOCKET) self.l = logging.getLogger('giedo') self.last_sync_ts = 0 self.daan, self.cilia = None, None try: self.daan = WhimClient(settings.DAAN_SOCKET) except: self.l.exception("Couldn't connect to daan") try: self.cilia = WhimClient(settings.CILIA_SOCKET) except: self.l.exception("Couldn't connect to cilia") self.mirte = mirte.get_a_manager() self.threadPool = self.mirte.get_a('threadPool') self.operation_lock = threading.Lock() self.push_changes_event = threading.Event() self.openvpn_lock = threading.Lock() self.threadPool.execute(self.run_change_pusher) if default_storage.exists("villanet.pem"): self.villanet_key = RSA.load_pub_key(default_storage.path( "villanet.pem")) self.ss_actions = ( ('postfix', self.daan, self._gen_postfix), ('postfix-slm', self.daan, self._gen_postfix_slm), ('mailman', self.daan, self._gen_mailman), ('forum', self.daan, self._gen_forum), ('unix', self.cilia, self._gen_unix), ('wiki', self.daan, self._gen_wiki), ('ldap', self.daan, self._gen_ldap), ('wolk', self.cilia, self._gen_wolk), ('quassel', self.daan, self._gen_quassel)) self.push_changes_event.set()
def get(request, team_name, agent_name): """ B{Retrieve} the agent files as tar B{URL:} ../api/v1/agents/agent_file/<team_name>/<agent_name>/ Must be part of the team owner of the agent Server to Server only :type agent_name: str :param agent_name: The agent name :type team_name: str :param team_name: The team name """ team = get_object_or_404(Team.objects.all(), name=team_name) agent = get_object_or_404(Agent.objects.all(), team=team, agent_name=agent_name) if AgentFile.objects.filter(agent=agent).count() == 0: return Response({'status': 'Bad request', 'message': 'The agent doesn\'t have files.'}, status=status.HTTP_400_BAD_REQUEST) temp = tempfile.NamedTemporaryFile() with tarfile.open(temp.name, "w:gz") as tar: for file_obj in AgentFile.objects.filter(agent=agent): tar.add(default_storage.path(file_obj.file), arcname=file_obj.original_name) tar.close() wrapper = FileWrapper(temp) response = HttpResponse(wrapper, content_type="application/x-compressed") response['Content-Disposition'] = 'attachment; filename=' + agent_name + '.tar.gz' response['Content-Length'] = getsize(temp.name) temp.seek(0) return response
def handle(self, *args, **options): try: app_label, model_name, field_name, variant_name = args except ValueError: raise CommandError(self.USAGE) variant = get_variant(app_label, model_name, field_name, variant_name) in_database = set( getattr(getattr(x, field_name), variant_name).filename for x in variant.image.field.model._default_manager.all() ) base = os.path.join( variant.image.field.upload_to, variant.name, ) on_disk = set( os.path.join( variant.image.field.upload_to, variant.name, x, ) for x in os.listdir(default_storage.path(base)) ) for x in on_disk.difference(in_database): print("I: Can be deleted: %s" % x)
def get_sapelli_dir_path(user=None): """ Creates the Sapelli working directory. Parameters ---------- user : geokey.users.models.User User who uploaded the project (optional). Returns ------- str: Absolute path to the Sapelli working directory. Raises ------ SapelliException: When the working directory could not be created. """ sapelli_dir_path = os.path.join(default_storage.path('sapelli'), '') # joining with '' adds the trailing / or \ if user: sapelli_dir_path = os.path.join(sapelli_dir_path, slugify(str(user.id) + '_' + user.display_name), '') if not os.path.exists(sapelli_dir_path): # Create the directory if it doesn't exist: try: os.makedirs(sapelli_dir_path) except BaseException, e: raise SapelliException('Failed to create Sapelli working directory (%s): %s' % (sapelli_dir_path, str(e)))
def _get_size(self, image_file, ratiofield): ratio = ratiofield.ratio_width / float(ratiofield.ratio_height) try: width, height = get_image_dimensions(default_storage.path(image_file), True) size = (height * ratio, height) option = self.option if option: if option[0] == 'scale': width = size[0] * option[1] height = size[1] * option[1] elif option[0] == 'width': width = option[1] height = size[1] * width / size[0] elif option[0] == 'height': height = option[1] width = height * size[0] / size[1] elif option[0] == 'max_size': max_width, max_height = option[1] width, height = size # recalculate height if needed if max_width < width: height = height * max_width / width width = max_width # recalculate width if needed if max_height < height: width = max_height * width / height height = max_height size = (width, height) return size except IOError: print "image %s does not exist" % image_file return None
def change_file(new_file_name, custom_gain): def timeout(args): print "[Timer expired] Deleting " + args os.remove(default_storage.path(args)) try: tags = ReplayGain(default_storage.path(new_file_name), custom_gain) print tags new_file = FileWrapper(open(default_storage.path(new_file_name))) t = Timer(10, timeout, args=[new_file_name]) t.start() return new_file #~ except: message = 'Error: is the file a proper MP3?'
def attach_upload(request): if not request.user.is_authenticated(): return HttpResponse(get_script_response(code=403, message='Unauthorized')) if 'userfile' not in request.FILES: return HttpResponse(get_script_response(code=400, message='File expected')) f = request.FILES['userfile'] # file is present only if request method is POST and form type was specified if not f.content_type.startswith('image/'): return HttpResponse(get_script_response(code=400, message='File format not supported!')) hash_base = "%s-%s-%s-%s" % (f.name, datetime.datetime.now().isoformat(' '), request.META['HTTP_USER_AGENT'], request.META['REMOTE_ADDR']) fname_base = hashlib.md5(hash_base.encode('utf-8')).hexdigest() ext = f.name.split('.')[-1] if len(ext) > 5: ext = ext[0:2] full_file = "uploads/%s.%s" % (fname_base, ext) thumb_file = 'uploads/thumb_%s.jpg' % fname_base log.debug("Generated name for upload: %s" % fname_base) log.debug("Generated filename for thumbnail: %s" % thumb_file) fs.save(full_file, f) try: im = Image.open(fs.path(full_file)) im.thumbnail(settings.IMG_THUMBS_SIZE, Image.ANTIALIAS) im.save(fs.path(thumb_file), "JPEG") except IOError as e: log.error("Failed to create thumbnail. %s" % e.message) fs.save(thumb_file, f) #save names in session uid = generate_uid() while uid in request.session: uid = generate_uid() request.session[uid] = (fname_base, full_file, thumb_file, ext) #return url to it response = HttpResponse( get_script_response(code=0, message=uid, thumb_url=fs.url(thumb_file) ) ) return response
def test_delete_webp_folder(self): webp_image = WebPImage.objects.create( static_path='django-test-image.png', quality=60) webp_image.save_image() webp_path = default_storage.path(webp_settings.WEBP_CONVERTER_PREFIX) self.assertTrue(os.path.exists(webp_path)) utils.delete_webp_folder() self.assertFalse(os.path.exists(webp_path))
def generate_thumbnail(image, max_size=None, add_watermark=False): """ Generates a thumbnail from a `models.Image`. If `max_size` is None, the default value is used. Returns the bytes of the thumbnail. """ if max_size is None: max_size = 100 assert image is not None assert isinstance(max_size, int) if image.thumbnail_image: img = PilImage.open(default_storage.path(image.thumbnail_image.path)) else: img = PilImage.open(default_storage.path(image.image.path)) img.thumbnail((max_size, max_size,), PilImage.ANTIALIAS) output_file = BytesIO() if add_watermark: angle = 20 opacity = 0.8 image_width, image_height = img.size text = image.studio.watermark_text or image.studio.name watermark_font, watermark_text_width, watermark_text_height = _get_font_for_image(img, text, max_size) watermark = PilImage.new('RGBA', img.size, (0, 0, 0, 0)) draw = ImageDraw.Draw(watermark, 'RGBA') pos_x = int((image_width - watermark_text_width) / 2) pos_y = int(((image_height - watermark_text_height) / 2)) draw.text([pos_x, pos_y - watermark_text_height], text, font=watermark_font) draw.text([pos_x, pos_y + watermark_text_height], text, font=watermark_font) watermark = watermark.rotate(angle, PilImage.BICUBIC) alpha = watermark.split()[3] alpha = ImageEnhance.Brightness(alpha).enhance(opacity) watermark.putalpha(alpha) PilImage.composite(watermark, img, watermark).save(output_file, 'JPEG') else: img.save(output_file, "JPEG") del img # no need to close it return output_file.getvalue()
def handle_noargs(self, **options): info = options.get('info') strip = options.get('strip') del_count = 0 strip_count = 0 for (path, dirs, files) in os.walk(storage.path('pictures')): rel_path = os.path.relpath(path, settings.MEDIA_ROOT) for f in files: f_path = os.path.join(rel_path, f) extension = f.split('.')[-1].lower() format = imghdr.what(os.path.join(path, f)) if not format: self.stdout.write("Unknown format: %s\n" % f_path) if not Picture.objects.filter(image=f_path).exists(): if info: self.stdout.write("Detected orphaned media file '%s'\n" % f_path) else: self.stdout.write("Deleting orphaned media file '%s'\n" % f_path) storage.delete(f_path) del_count = del_count + 1 else: # Detect PNG images saved as JPG if format == 'png' and extension == 'jpg': p = Picture.objects.get(image=f_path) self.stdout.write("Renaming %s image\n" % p) p.image.save(f.replace('.jpg', '.png'), p.image) elif strip and extension in ['jpg', 'jpeg']: try: # Process EXIF data (strip all except copyright if present) exif_writer = MinimalExifWriter(storage.path(f_path)) exif_writer.removeExif() exif_writer.process() strip_count = strip_count + 1 except Exception as e: self.stdout.write("Unable to strip EXIF data from '%s': %s\n" % (f_path, str(e))) if info: self.stdout.write("Detected %d orphaned media files\n" % del_count) else: self.stdout.write("Deleted %d orphaned media files\n" % del_count) if strip: self.stdout.write("Stripped EXIF data from %d pictures\n" % strip_count)
def output(request, slug, extension): assert settings.DEBUG, "Overridden by nginx configuration" return serve( request, 'output.%s' % extension, document_root=default_storage.path(slug), )
def getShow(s): url = s.MediaURL fileName = s.getAudioFileName() save_path = default_storage.path(fileName) if not os.path.isfile(save_path): downloadTo(url, save_path) s.AudioFile.name = fileName s.save()
def test_thumbnail_created_at_right_path(self): image_name = 'test.jpg' created_path = create_thumbnail(image_name) thumbnail_path = 'collected_twitter_images/thumbnails/' + image_name expected_path = default_storage.path(thumbnail_path) self.assertEqual(created_path, expected_path)
def _render(self, path, val, caller): if val == 'email_embedded_media': fullpath = default_storage.path(path) elif val == 'email_embedded_static': fullpath = staticfiles_storage.path(path) else: fullpath = path return 'cid:' + self.environment.email_object_instance.attach_related_file(fullpath)
def prepare_offline_file(session, local_path): replay_path = default_storage.path(local_path) current_dir = os.getcwd() dir_path = os.path.dirname(replay_path) replay_filename = os.path.basename(replay_path) meta_filename = '{}.json'.format(session.id) offline_filename = '{}.tar'.format(session.id) os.chdir(dir_path) with open(meta_filename, 'wt') as f: f.write(model_to_json(session)) with tarfile.open(offline_filename, 'w') as f: f.add(replay_filename) f.add(meta_filename) file = open(offline_filename, 'rb') os.chdir(current_dir) return file
def convertir_audio_de_archivo_de_audio_globales(self, archivo_de_audio): """Realiza la conversión y actualiza la instancia de ArchivoDeAudio. Esta funcion debe usarse en el Alta y Modificacioin de ArchivoDeAudio. :param archivo_de_audio: ArchivoDeAudio para la cual hay que convertir el audio :type archivo_de_audio: ominicontacto_app.models.ArchivoDeAudio :raises: OmlAudioConversionError """ assert isinstance(archivo_de_audio, ArchivoDeAudio) # chequea archivo original (a convertir) wav_full_path = default_storage.path( archivo_de_audio.audio_original.name) assert os.path.exists(wav_full_path) # genera nombre del archivo de salida _template = ConversorDeAudioService.\ TEMPLATE_NOMBRE_AUDIO_ASTERISK_PREDEFINIDO filename = _template.format( archivo_de_audio.descripcion, settings.TMPL_OML_AUDIO_CONVERSOR_EXTENSION) # Creamos directorios si no existen abs_output_dir = os.path.join( settings.MEDIA_ROOT, ConversorDeAudioService.DIR_AUDIO_PREDEFINIDO) self._crear_directorios(abs_output_dir) # Creamos archivo si no existe abs_output_filename = os.path.join(abs_output_dir, filename) self._crear_archivo(abs_output_filename) assert os.path.exists(abs_output_filename) # convierte archivo self._convertir_audio(wav_full_path, abs_output_filename) # guarda ref. a archivo convertido archivo_de_audio.audio_asterisk = os.path.join( ConversorDeAudioService.DIR_AUDIO_PREDEFINIDO, filename) archivo_de_audio.save()
def create_perma_wb_router(config={}): """ Configure server. This should do basically the same stuff as pywb.webapp.pywb_init.create_wb_router() """ # paths script_path = os.path.dirname(__file__) # Get root storage location for warcs. # archive_path should be the location pywb can find warcs, like 'file://generated/' or 'http://perma.s3.amazonaws.com/generated/' # We can get it by requesting the location of a blank file from default_storage. # default_storage may use disk or network storage depending on config, so we look for either a path() or url() try: archive_path = 'file://' + default_storage.path('') + '/' except NotImplementedError: archive_path = default_storage.url('/') archive_path = archive_path.split('?', 1)[0] # remove query params query_handler = QueryHandler.init_from_config(PermaCDXSource()) # pywb template vars (used in templates called by pywb, such as head_insert.html, but not our ErrorTemplateView) add_env_globals({'static_path': settings.STATIC_URL}) # use util func to create the handler wb_handler = create_wb_handler( query_handler, dict(archive_paths=[archive_path], wb_handler_class=PermaGUIDHandler, buffer_response=True, head_insert_html=os.path.join(script_path, 'head_insert.html'), enable_memento=True, redir_to_exact=False)) wb_handler.replay.content_loader.record_loader.loader = CachedLoader() route = PermaRoute(GUID_REGEX, wb_handler) router = create_wb_router(config) router.error_view = PermaTemplateView('archive-error.html') router.routes.insert(0, route) return router
def get_store_URL(request, *args, **kwargs): # these urls are relative for s3 rel_url = request.GET.get('rel_url', None) # just need to convert it to s3 url. That's all. # Note: storage starts from INSIDE 'media' folder, because that's # the media url provided in the settings. # print("...storage.exists? {}".format(storage.listdir(""))) # I have NO idea where I am # print("...converting url: {}".format(rel_url)) rel_obj_path = get_rel_path(rel_url, "frame_images") print("------ rel_obj_path: {}".format(rel_obj_path)) abs_obj_path = '' store_path = '' if rel_obj_path and storage.exists(str(rel_obj_path)): print("EXISTS! Object at {}".format(rel_obj_path)) # print("Path check: {}".format(storage.path(str(rel_obj_path)) )) # fails here on heroku if hasattr(settings, 'USE_S3') and settings.USE_S3 == True: # Generate path for S3 if hasattr(settings, 'MEDIA_URL'): store_path = "{}{}".format(settings.MEDIA_URL, rel_obj_path) else: return JsonResponse({ 'status':'false', 'message': "Cannot get S3 media url from settings." }, status=500 ) else: store_path = PurePosixPath(storage.path(str(rel_obj_path))) # abs_obj_path = store_path.joinpath(rel_obj_path) # no need? abs_obj_path = store_path print("------- joined path = {}".format(store_path)) else: print("ERROR: Could not find object at {}".format(rel_obj_path)) # This does not reject request. It returns "" as URL. # Response return JsonResponse({'url': str(abs_obj_path)})
def image_registration(request, pk): if request.method == 'POST': image = request.POST['autentication_image'] user = get_object_or_404(Student, pk=pk) user_image = convert64toImage(image, user.user.first_name) temppath = default_storage.save('temp.png', content=user_image) temp_filepath = default_storage.path(temppath) face_accepted = encoded_face(temp_filepath) if face_accepted: user.image = user_image user.save() default_storage.delete(temppath) return redirect('course:user_dashboard') else: default_storage.delete(temppath) messages.warning(request, 'Face Not decteted.') default_storage.delete(temppath) return render(request, 'auth_image.html')
def _get_video_info(self): """ Returns basic information about the video as dictionary. """ if not hasattr(self, '_info_cache'): encoding_backend = get_backend() try: _path = getattr(self, 'path', self.name) except NotImplementedError: _path = self.name try: path = default_storage.path(_path) except NotImplementedError: path = default_storage.url(_path) try: self._info_cache = encoding_backend.get_media_info(path) except FFmpegError: self._info_cache = {} return self._info_cache
def _test_logfile_upload(self, user, uuid): # Upload crashreport device_local_id = self.upload_crashreport(user, uuid) # Upload a logfile for the crashreport self.upload_logfile(user, uuid, device_local_id) logfile_instance = (Device.objects.get(uuid=uuid).crashreports.get( device_local_id=device_local_id).logfiles.last()) uploaded_logfile_path = crashreport_file_name( logfile_instance, os.path.basename(Dummy.DEFAULT_LOG_FILE_PATHS[0])) self.assertTrue(default_storage.exists(uploaded_logfile_path)) # The files are not 100% equal, because the server adds some extra # bytes. However, we mainly care that the contents are equal: self._assert_zip_file_contents_equal( default_storage.path(uploaded_logfile_path), Dummy.DEFAULT_LOG_FILE_PATHS[0], )
def upload_drive_task(filename, tmp_storage_file, submission_obj, parent): print(filename, tmp_storage_file, submission_obj, parent) meta = {"name": filename, "parents": [parent]} file_path = default_storage.path(tmp_storage_file) print(file_path) try: creds = Credentials(token=DRIVE_TOKENS.get('access')) service = build('drive', 'v3', credentials=creds) media_body = MediaFileUpload(filename=file_path, chunksize=1024 * 1024 * 50, resumable=True) file = service.files().create(body=meta, media_body=media_body).execute() except RefreshError: refresh_token() creds = Credentials(token=DRIVE_TOKENS.get('access')) service = build('drive', 'v3', credentials=creds) media_body = MediaFileUpload(filename=file_path, chunksize=1024 * 1024 * 50, resumable=True) file = service.files().create(body=meta, media_body=media_body).execute() url = 'https://drive.google.com/file/d/{}/view'.format(file.get('id')) # submission_obj = Submission.objects.get(id=submission_id) submission_obj.submission_url = url submission_obj.task_submitted = True submission_obj.save() if not media_body._fd.closed: media_body._fd.close() default_storage.delete(tmp_storage_file)
def upload_session_replay_to_external_storage(session_id): logger.info(f'Start upload session to external storage: {session_id}') session = Session.objects.filter(id=session_id).first() if not session: logger.error(f'Session db item not found: {session_id}') return local_path, foobar = find_session_replay_local(session) if not local_path: logger.error(f'Session replay not found, may be upload error: {local_path}') return abs_path = default_storage.path(local_path) remote_path = session.get_relative_path_by_local_path(abs_path) ok, err = server_replay_storage.upload(abs_path, remote_path) if not ok: logger.error(f'Session replay upload to external error: {err}') return try: default_storage.delete(local_path) except: pass return
def initial_input(self, *args): directory = args[0] initial = Collective.objects.create(community=self, schema={}) directories, files = default_storage.listdir(directory) for file in files: full_name = directory + '/' + file size = default_storage.size(full_name) if size < 3 * 1024: continue properties = { "size": size, "name": file, "file": default_storage.path(full_name), "url": default_storage.url(full_name) } Individual.objects.create(community=self, collective=initial, properties=properties, schema={}) return initial
def prepare(request): keys = list(request.POST.keys()) if len(keys) != 1: return 'msg', 'Invalid request' testName = keys[0] if testName in tests: dialog_id = MH.create_dialog() ls = tests[testName] ls[0].to_pickle(default_storage.path(str(dialog_id))) for r in ls[1:-1]: if r[0] == 'Q': MH.append_query(dialog_id, r[1]) elif r[0] == 'G': MH.append_graph(dialog_id, r[1]) elif r[0] == 'P': MH.set_predicate(dialog_id, r[1]) MH.set_target_graph_json(dialog_id, ls[ls[-1]][1]) return 'url', str(dialog_id) return 'msg', 'Invalid request'
def process_file(path): """Process single file""" # failsafe copy of file copy = default_storage.open(path, 'rb') default_storage.save(path + timestamp, copy) copy.close() try: path = default_storage.path(path) squeeze(path) except NotImplementedError: if path[-1:] != os.sep: pf = default_storage.open(path, 'rwb') image = pf.read() tmpfilehandle, tmpfilepath = tempfile.mkstemp() tmpfilehandle = os.fdopen(tmpfilehandle, 'wb') tmpfilehandle.write(image) tmpfilehandle.close() squeeze(tmpfilepath) tmpfilehandle = open(tmpfilepath) pf.close() default_storage.save(path, tmpfilehandle) os.remove(tmpfilepath)
class TestSapelliLoader(TestCase): def setUp(self): self.user = UserFactory.create() def tearDown(self): # delete project(s): for sapelli_project in SapelliProject.objects.filter(sapelli_id__in=[ horniman_sapelli_project_info['sapelli_id'], 1337 ]): try: sapelli_project.geokey_project.delete( ) # will also delete sapelli_project except BaseException, e: pass # delete sapelli/user folder try: shutil.rmtree( join(default_storage.path('sapelli'), slugify(str(self.user.id) + '_' + self.user.display_name), '')) except BaseException, e: pass
def __init__(self): super(Giedo, self).__init__(settings.GIEDO_SOCKET) self.last_sync_ts = 0 self.daan = WhimClient(settings.DAAN_SOCKET) self.cilia = WhimClient(settings.CILIA_SOCKET) self.mirte = mirte.get_a_manager() self.threadPool = self.mirte.get_a('threadPool') self.operation_lock = threading.Lock() self.push_changes_event = threading.Event() self.threadPool.execute(self.run_change_pusher) if default_storage.exists("villanet.pem"): self.villanet_key = RSA.load_pub_key( default_storage.path("villanet.pem")) self.ss_actions = (('postfix', self.daan, self._gen_postfix), ('postfix-slm', self.daan, self._gen_postfix_slm), ('mailman', self.daan, self._gen_mailman), ('forum', self.daan, self._gen_forum), ('unix', self.cilia, self._gen_unix), ('wiki', self.daan, self._gen_wiki), ('ldap', self.daan, self._gen_ldap)) self.push_changes_event.set()
def handle(self, *args, **options): webp_path = default_storage.path(WEBP_CONVERTER_PREFIX) if options['interactive']: confirm_message = ( 'This will delete all files in {webp_path}\n\n' 'Are you sure you want to do this?\n\n' "Type 'yes' to continue, or 'no' to cancel: ").format( webp_path=webp_path) if input(confirm_message) != 'yes': raise CommandError("Clearing webp cache cancelled.") self.stdout.write("Deleting cache entries...") for webp_image in WebPImage.objects.all().iterator(): key = make_image_key(webp_image.static_path, webp_image.quality) cache.delete(key) self.stdout.write("Deleting WebPImage models...") WebPImage.objects.all().delete() self.stdout.write("Deleting images...") if os.path.exists(webp_path): shutil.rmtree(webp_path) self.stdout.write("Successfully cleared cache")
def handle(self, *args, **options): for partner in Partner.objects.all(): self.stdout.write("Process files in the outbox directory for " 'partner "%s".' % partner.as2_name) for org in Organization.objects.all(): if settings.DATA_DIR: outbox_folder = os.path.join( settings.DATA_DIR, "messages", partner.as2_name, "outbox", org.as2_name, ) else: outbox_folder = os.path.join("messages", partner.as2_name, "outbox", org.as2_name) # Check of the directory exists and if not create it try: _, pending_files = default_storage.listdir(outbox_folder) except FileNotFoundError: pending_files = [] os.makedirs(default_storage.path(outbox_folder)) # For each file found call send message to send it to the server pending_files = filter(lambda x: x != ".", pending_files) for pending_file in pending_files: pending_file = os.path.join(outbox_folder, pending_file) self.stdout.write( 'Sending file "%s" from organization "%s" to partner ' '"%s".' % (pending_file, org.as2_name, partner.as2_name)) call_command( "sendas2message", org.as2_name, partner.as2_name, pending_file, delete=True, )
def create_perma_pywb_app(config): """ Configure server. """ query_handler = QueryHandler.init_from_config(settings.CDX_SERVER_URL) # Get root storage location for warcs. # archive_path should be the location pywb can find warcs, like 'file://generated/' or 'http://perma.s3.amazonaws.com/generated/' # We can get it by requesting the location of a blank file from default_storage. # default_storage may use disk or network storage depending on config, so we look for either a path() or url() try: archive_path = 'file://' + default_storage.path('') + '/' except NotImplementedError: archive_path = default_storage.url('/') archive_path = archive_path.split('?', 1)[0] # remove query params # use util func to create the handler wb_handler = create_wb_handler(query_handler, dict(archive_paths=[archive_path], wb_handler_class=Handler, buffer_response=True, head_insert_html='ui/head_insert.html', template_globals={'static_path': 'static/js'}, redir_to_exact=False)) # Finally, create wb router return archivalrouter.ArchivalRouter( { Route(r'([a-zA-Z0-9\-]+)', wb_handler) }, # Specify hostnames that pywb will be running on # This will help catch occasionally missed rewrites that fall-through to the host # (See archivalrouter.ReferRedirect) hostpaths=['http://localhost:8000/'], port=8000 )
def load_from_sap(sap_file, user): """ Loads & saves a SapelliProject from the given SAP file. Parameters ---------- sap_file : django.core.files.File Uploaded (suspected) SAP file. user : geokey.users.models.User User who uploaded the project. Returns ------- SapelliProject: SapelliProject instance for the parsed project. Raises ------ SapelliException: In case of a configuration problem. SapelliSAPException: When project loading fails. SapelliDuplicateException: When the project has already been uploaded. """ # Check if we got a file at all: if sap_file is None: raise SapelliSAPException('No file provided.') # Store copy of file on disk (as it probably is an "in memory" file uploaded in an HTTP request): try: filename, extension = os.path.splitext(os.path.basename(sap_file.name)) relative_sap_file_path = default_storage.save( os.path.join(get_sapelli_dir_path(user), 'SAPs', '') + filename + extension, ContentFile(sap_file.read())) sap_file_path = default_storage.path(relative_sap_file_path) except BaseException, e: raise SapelliSAPException('Failed to store uploaded file: ' + str(e))
def serve_image_or_thumb(request, image): assert isinstance(image, models.Image) file_obj = image.get_image_or_thumbnail_file() full_filename = default_storage.path(file_obj.path) filesize = os.path.getsize(full_filename) content_type = image.content_type # TODO: send in chunks to avoid loading the file in memory # from django.core.servers.basehttp import FileWrapper # with open(full_filename) as f: # fw = FileWrapper(f) # response = HttpResponse(fw, content_type=content_type) # response['Content-Length'] = filesize # response['Content-Disposition'] = 'attachment; filename="{0}"'.format( # filename_to_user) # return response with open(full_filename, mode='r+b') as f: file_contents = f.read() response = HttpResponse(file_contents, content_type=content_type) response['Content-Length'] = filesize return response
def retrieve(self, request, pk, *args, **kwargs): throttle = throttling.ImportDumpModeRateThrottle() if not throttle.allow_request(request, self): self.throttled(request, throttle.wait()) project = get_object_or_404(self.get_queryset(), pk=pk) self.check_permissions(request, 'export_project', project) if settings.CELERY_ENABLED: task = tasks.dump_project.delay(request.user, project) tasks.delete_project_dump.apply_async( (project.pk, project.slug), countdown=settings.EXPORTS_TTL) return response.Accepted({"export_id": task.id}) path = "exports/{}/{}-{}.json".format(project.pk, project.slug, uuid.uuid4().hex) storage_path = default_storage.path(path) with default_storage.open(storage_path, mode="w") as outfile: service.render_project(project, outfile) response_data = {"url": default_storage.url(path)} return response.Ok(response_data)
def dump_project(self, user, project): path = "exports/{}/{}-{}.json".format(project.pk, project.slug, self.request.id) storage_path = default_storage.path(path) try: url = default_storage.url(path) with default_storage.open(storage_path, mode="w") as outfile: render_project(project, outfile) except Exception: # Error ctx = { "user": user, "error_subject": _("Error generating project dump"), "error_message": _("Error generating project dump"), "project": project } email = mail_builder.export_error(user, ctx) email.send() logger.error('Error generating dump %s (by %s)', project.slug, user, exc_info=sys.exc_info()) else: # Success deletion_date = timezone.now() + datetime.timedelta( seconds=settings.EXPORTS_TTL) ctx = { "url": url, "project": project, "user": user, "deletion_date": deletion_date } email = mail_builder.dump_project(user, ctx) email.send()
def handle_command(dialog_id, command): j, cmds = __get(dialog_id) MH.append_command(dialog_id, command) if __is_draw_command(command): cmds.append(command) success, err_msgs = j.handle_next(command) for msg in err_msgs: MH.append_query(dialog_id, msg) if success: MH.append_query(dialog_id, 'Here is the graph I generated...') MH.append_graph(dialog_id, j.get().to_json()) if __is_graph_equal_to_target(dialog_id, j.get().to_json()): MH.append_query(dialog_id, 'Congratulations! You have generated the target graph.') MH.append_query(dialog_id, 'What else would you like to do?') else: command = command.lower() if command == 'undo': j = __init_jarvis(dialog_id) cmds = cmds[:-1] print(cmds) for c in cmds: j.handle_next(c) __cache[dialog_id] = j, cmds MH.append_query(dialog_id, 'Undo done!') MH.append_graph(dialog_id, j.get().to_json()) elif command == 'reset': j = __init_jarvis(dialog_id) __cache[dialog_id] = j, [] MH.append_query(dialog_id, 'Reset done!') MH.append_graph(dialog_id, j.get().to_json()) elif command == 'sample': MH.append_query(dialog_id, 'Here are samples of the data...') MH.append_query(dialog_id, str(pd.read_pickle(default_storage.path(str(dialog_id))).sample(n=5, random_state=1))) if __is_graph_equal_to_target(dialog_id, j.get().to_json()): MH.append_query(dialog_id, 'Congratulations! You have generated the target graph.') MH.append_query(dialog_id, 'What else would you like to do?')
def setUp(self): self.site = Site.objects.get_current() # # setup the protected dir; since we're using the default storage class, # this will point to # # /path/to/static/protected/ # # where "/path/to/static/" is your settings.MEDIA_ROOT and "protected" # is your PRODUCT.PROTECTED_DIR setting. # self.protected_dir = default_storage.path( config_value('PRODUCT', 'PROTECTED_DIR') ) if not os.path.exists(self.protected_dir): os.makedirs(self.protected_dir) # setup a temporary file in the protected dir: this is the file that # django will use during this test, but we won't use it; close and # remove it. _file, _abs_path = mkstemp(dir=self.protected_dir) os.close(_file) os.remove(_abs_path) self.file_name = os.path.basename(_abs_path) # setup a temporary source dir and source file, using the same file name # generated eariler. self.dir = mkdtemp() self.file = open(os.path.join(self.dir, self.file_name), "w") # a fake SHA self.key = "".join(["12abf" for i in range(8)]) # setup a contact c, _created = Contact.objects.get_or_create( first_name="Jim", last_name="Tester", email="*****@*****.**", ) ad, _created = AddressBook.objects.get_or_create( contact=c, description="home", street1 = "test", state="OR", city="Portland", country = Country.objects.get(iso2_code__iexact = 'US'), is_default_shipping=True, is_default_billing=True, ) # setup a order o, _created = Order.objects.get_or_create( contact=c, shipping_cost=Decimal('6.00'), site=self.site ) # setup download self.product, _created = DownloadableProduct.objects.get_or_create( product=Product.objects.get(slug='dj-rocks'), file=File(self.file), num_allowed_downloads=3, expire_minutes=1, ) self.product_link, _created = DownloadLink.objects.get_or_create( downloadable_product=self.product, order=o, key=self.key, num_attempts=0, time_stamp=timezone.now() ) # setup client self.domain = 'satchmoserver' self.client = Client(SERVER_NAME=self.domain) # go through the verification step self.pd_url = urlresolvers.reverse( 'satchmo_download_send', kwargs= {'download_key': self.key} ) pd_process_url = urlresolvers.reverse( 'satchmo_download_process', kwargs= {'download_key': self.key} ) # first, hit the url. response = self.client.get(self.pd_url) self.assertEqual(response['Location'], 'http://%s%s' % (self.domain, pd_process_url) ) # follow the redirect to "process" the key. response = self.client.get(response['Location']) self.assertEqual(self.client.session.get('download_key', None), self.key)
def create_document_thumbnail(self, object_id): """ Create thumbnail for a document. """ logger.debug("Generating thumbnail for document #{}.".format(object_id)) try: document = Document.objects.get(id=object_id) except Document.DoesNotExist: logger.error("Document #{} does not exist.".format(object_id)) return image_path = None image_file = None if document.is_image: if not os.path.exists(storage.path(document.doc_file.name)): from shutil import copyfile copyfile(document.doc_file.path, storage.path(document.doc_file.name)) image_file = storage.open(document.doc_file.name, 'rb') elif document.is_video or document.is_audio: image_file = open(document.find_placeholder(), 'rb') elif document.is_file: try: document_location = storage.path(document.doc_file.name) except NotImplementedError as e: logger.debug(e) document_location = storage.url(document.doc_file.name) try: image_path = render_document(document_location) if image_path is not None: try: image_file = open(image_path, 'rb') except Exception as e: logger.debug(e) logger.debug( "Failed to render document #{}".format(object_id)) else: logger.debug("Failed to render document #{}".format(object_id)) except ConversionError as e: logger.debug("Could not convert document #{}: {}.".format( object_id, e)) except NotImplementedError as e: logger.debug("Failed to render document #{}: {}".format( object_id, e)) thumbnail_content = None try: try: thumbnail_content = generate_thumbnail_content(image_file) except Exception as e: logger.error( "Could not generate thumbnail, falling back to 'placeholder': {}" .format(e)) thumbnail_content = generate_thumbnail_content( document.find_placeholder()) except Exception as e: logger.error("Could not generate thumbnail: {}".format(e)) return finally: if image_file is not None: image_file.close() if image_path is not None: os.remove(image_path) if not thumbnail_content: logger.warning("Thumbnail for document #{} empty.".format(object_id)) filename = 'document-{}-thumb.png'.format(document.uuid) document.save_thumbnail(filename, thumbnail_content) logger.debug("Thumbnail for document #{} created.".format(object_id))
def _storage_filename(wfm_id): fname = f'{wfm_id}-{uuid.uuid1()}-fetch.dat' return default_storage.path(fname)
def download(request, resourceid, sender=Layer): instance = resolve_object( request, sender, {'pk': resourceid}, permission='base.download_resourcebase', permission_msg=_( "You are not permitted to save or edit this resource.")) if isinstance(instance, Layer): try: upload_session = instance.get_upload_session() layer_files = [ item for idx, item in enumerate( LayerFile.objects.filter(upload_session=upload_session)) ] # Create Target Folder dirpath = tempfile.mkdtemp() dir_time_suffix = get_dir_time_suffix() target_folder = os.path.join(dirpath, dir_time_suffix) if not os.path.exists(target_folder): os.makedirs(target_folder) # Copy all Layer related files into a temporary folder for l in layer_files: if storage.exists(l.file): geonode_layer_path = storage.path(l.file) base_filename, original_ext = os.path.splitext( geonode_layer_path) shutil.copy2(geonode_layer_path, target_folder) # Let's check for associated SLD files (if any) try: for s in instance.styles.all(): sld_file_path = os.path.join(target_folder, "".join([s.name, ".sld"])) sld_file = open(sld_file_path, "w") sld_file.write(s.sld_body.strip()) sld_file.close() try: sld_file = open(sld_file_path, "r") response = requests.get(s.sld_url, timeout=TIMEOUT) sld_remote_content = response.text sld_file_path = os.path.join( target_folder, "".join([s.name, "_remote.sld"])) sld_file = open(sld_file_path, "w") sld_file.write(sld_remote_content.strip()) sld_file.close() except BaseException: traceback.print_exc() tb = traceback.format_exc() logger.debug(tb) except BaseException: traceback.print_exc() tb = traceback.format_exc() logger.debug(tb) # Let's dump metadata target_md_folder = os.path.join(target_folder, ".metadata") if not os.path.exists(target_md_folder): os.makedirs(target_md_folder) try: links = Link.objects.filter(resource=instance.resourcebase_ptr) for link in links: link_name = custom_slugify(link.name) link_file = os.path.join( target_md_folder, "".join([link_name, ".%s" % link.extension])) if link.link_type in ('data'): # Skipping 'data' download links continue elif link.link_type in ('metadata', 'image'): # Dumping metadata files and images link_file = open(link_file, "wb") try: response = requests.get(link.url, stream=True, timeout=TIMEOUT) response.raw.decode_content = True shutil.copyfileobj(response.raw, link_file) except BaseException: traceback.print_exc() tb = traceback.format_exc() logger.debug(tb) finally: link_file.close() elif link.link_type.startswith('OGC'): # Dumping OGC/OWS links link_file = open(link_file, "w") link_file.write(link.url.strip()) link_file.close() except BaseException: traceback.print_exc() tb = traceback.format_exc() logger.debug(tb) # ZIP everything and return target_file_name = "".join([instance.name, ".zip"]) target_file = os.path.join(dirpath, target_file_name) zip_dir(target_folder, target_file) response = HttpResponse(content=open(target_file), status=200, content_type="application/zip") response[ 'Content-Disposition'] = 'attachment; filename="%s"' % target_file_name return response except NotImplementedError: traceback.print_exc() tb = traceback.format_exc() logger.debug(tb) return HttpResponse(json.dumps({'error': 'file_not_found'}), status=404, content_type="application/json") return HttpResponse(json.dumps({'error': 'unauthorized_request'}), status=403, content_type="application/json")
def handle_uploaded_file(f): save_path = os.path.join('uploads', f.name) path = default_storage.save(save_path, f) return default_storage.path(path)
def test_generate_filtered_attachments_zip_export(self): """Test media zip file export filters attachments""" filenames = [ 'OSMWay234134797.osm', 'OSMWay34298972.osm', ] osm_fixtures_dir = os.path.realpath( os.path.join(os.path.dirname(api_tests.__file__), 'fixtures', 'osm')) paths = [ os.path.join(osm_fixtures_dir, filename) for filename in filenames ] xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') self._publish_xls_file_and_set_xform(xlsform_path) submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') count = Attachment.objects.filter(extension='osm').count() self._make_submission_w_attachment(submission_path, paths) self._make_submission_w_attachment(submission_path, paths) self.assertTrue( Attachment.objects.filter(extension='osm').count() > count) options = { "extension": Export.ZIP_EXPORT, "query": u'{"_submission_time": {"$lte": "2019-01-13T00:00:00"}}' } filter_query = options.get("query") instance_ids = query_data(self.xform, fields='["_id"]', query=filter_query) export = generate_attachments_zip_export(Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options) self.assertTrue(export.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r") zip_file.extractall(temp_dir) zip_file.close() filtered_attachments = Attachment.objects.filter( instance__xform_id=self.xform.pk).filter( instance_id__in=[i_id['_id'] for i_id in instance_ids]) self.assertNotEqual(Attachment.objects.count(), filtered_attachments.count()) for a in filtered_attachments: self.assertTrue( os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir) # export with no query options.pop('query') export1 = generate_attachments_zip_export(Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options) self.assertTrue(export1.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export1.filepath), "r") zip_file.extractall(temp_dir) zip_file.close() for a in Attachment.objects.all(): self.assertTrue( os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir)
def get_context_data(self, *args, **kwargs): ctx = super(PDFReportView, self).get_context_data(*args, **kwargs) r = self.request randomizer = self.request.GET.get('r') or '' ctx['app'] = app = self.get_app() ctx['kwargs'] = k = self.kwargs report_uri = app.url_for('index') client_kwargs = k.copy() client_kwargs.pop('app', None) context = self.get_context_url(_full=True, **k) ctx['context'] = { 'url': context, 'parts': self.get_further_resources_inputs(**k) } fr_map = self.get_further_resources(inputs=ctx['context']['parts'], **k) further_resources = [] for fr_key, fr_list in fr_map.items(): for fr_item in fr_list: # we could do it with set(), but we want to preserve order if fr_item in further_resources: continue further_resources.append(fr_item) ctx['context']['further_resources'] = further_resources ctx['risk_analysis'] = risk_analysis = RiskAnalysis.objects.get( id=k['an']) def p(val): # for test we need full fs path if settings.TEST: return default_storage.path(val) # otherwise, we need nice absolute url _path = default_storage.url(val) return r.build_absolute_uri(_path) ctx['paths'] = { 'map': p( os.path.join( context, 'map_{}.png'.format(randomizer) if randomizer else 'map.png')), 'charts': [], 'legend': p( os.path.join( context, 'legend_{}.png'.format(randomizer) if randomizer else 'legend.png')) } for cidx in range(0, 4): chart_path = os.path.join( context, 'chart_{}_{}.png'.format(cidx, randomizer) if randomizer else 'chart_{}.png'.format(cidx)) if not os.path.exists(default_storage.path(chart_path)): continue chart_f = p(chart_path) ctx['paths']['charts'].append(chart_f) ctx['resources'] = {} ctx['resources']['permalink'] = '{}?init={}'.format( r.build_absolute_uri(report_uri), self.get_client_url(app, **client_kwargs)) for resname in ( 'permalink', 'dims', 'dimsVal', ): _fname = os.path.join( context, '{}_{}.txt'.format(resname, randomizer) if randomizer else '{}.txt'.format(resname)) fname = default_storage.path(_fname) if os.path.exists(fname): with open(fname, 'rt') as f: data = json.loads(f.read()) ctx['resources'][resname] = data ctx['dimensions'] = self.get_dimensions(risk_analysis, ctx['resources']) return ctx
def get_arp_root(): if not default_storage.exists(settings.ARP_ROOT): os.mkdir(default_storage.path(settings.ARP_ROOT)) return settings.ARP_ROOT