def render(self, context): # text typed in the tag token = self.args["file"] try: filename = resolve_path(token, self.parser, context) except PathResolutionException as e: return self.make_error_msg(f"Path Resolution failed: {e}") challenge_short_name = context["site"].short_name filepath = os.path.join(settings.MEDIA_ROOT, challenge_short_name, filename) filepath = os.path.abspath(filepath) filepath = self.make_canonical_path(filepath) # when all rendering is done, check if the final path is still not getting # into places it should not go. if not self.is_inside_project_data_folder(filepath, context["site"]): error_msg = "'{}' cannot be opened because it is outside the current project.".format( filepath) return self.make_error_msg(error_msg) storage = DefaultStorage() try: with storage.open(filepath, "r") as f: contents = f.read() except Exception as e: return self.make_error_msg("error opening file:" + str(e)) # TODO check content safety return contents
def handle(self, *args, **options): storage = DefaultStorage() for model in _get_models(['shapes', 'photos', 'shapes']): has_images = False # transfer image fields for f in model._meta.fields: if isinstance(f, models.ImageField): has_images = True if hasattr(storage, 'transfer'): filenames = model.objects.all() \ .values_list(f.name, flat=True) print '%s: %s' % (model, f) for filename in progress.bar(filenames): if filename and storage.local.exists(filename): storage.transfer(filename) # transfer thumbs if has_images: print '%s: thumbnails' % model ids = model.objects.all().values_list('id', flat=True) ct_id = ContentType.objects.get_for_model(model).id for id in progress.bar(ids): ensure_thumbs_exist_task.delay(ct_id, id)
def upload_file(request): callback = request.POST.get("fd-callback", None) if len(request.FILES): file = request.FILES.get("fd-file") file_name = file.name else: file_name = request.META.get("HTTP_X_FILE_NAME") file_temp = NamedTemporaryFile() file = File(file_temp) chunk = request.read(1024) while chunk: file_temp.write(chunk) chunk = request.read(1024) file_temp.flush() default_storage = DefaultStorage() file_path = get_file_path(file_name) file_url = "%s%s" % (settings.MEDIA_URL, file_path) default_storage.save(file_path, file) output = simplejson.dumps({'success': True, 'file': file_url}) if callback: html = '<!DOCTYPE html><html><head></head><body><script type="text/javascript">try{window.top.'+callback+'('+output+')}catch(e){}</script></body></html>' return HttpResponse(html) else: return HttpResponse(output, content_type='application/json; charset="utf-8"')
def post(self, request, **kwargs): file_obj = request.FILES.get('file') filename = request.POST['key'] extension = filename.split(".")[-1].lower() if extension not in ('png', 'jpeg', 'jpg', 'svg', 'gif'): return HttpResponseBadRequest("Files of type " + extension + " are not supported") # TODO: Add validation here e.g. file size/type check # TODO: Potentially resize image # organize a path for the file in bucket file_path = '{uuid}/{userid}.{extension}'.\ format(userid=request.user.id, uuid=uuid4(), extension=extension) media_storage = DefaultStorage() media_storage.save(file_path, file_obj) file_url = media_storage.url(file_path) # The file url contains a signature, which expires in a few hours # In our case, we have made the S3 file public for anyone who has the url # Which means, the file is accessible without the signature # So we simply strip out the signature from the url - i.e. everything after the ? file_url = file_url.split('?')[0] return JsonResponse({ 'message': 'OK', 'fileUrl': file_url, })
def random_image(field): color1 = random_rgb() color2 = random_rgb() color3 = random_rgb() color4 = random_rgb() size = (random.randint(300, 900), random.randint(300, 900)) im = Image.new("RGB", size) # create the image draw = ImageDraw.Draw(im) # create a drawing object that is draw.rectangle([(0, 0), ((size[0] / 2), (size[1] / 2))], fill=color1) draw.rectangle([((size[0] / 2), 0), ((size[1] / 2), size[0])], fill=color2) draw.rectangle([(0, (size[1] / 2)), ((size[0] / 2), size[1])], fill=color3) draw.rectangle([((size[0] / 2), (size[1] / 2)), (size[0], size[1])], fill=color4) filename = "%s.png" % uuid.uuid4().hex[:10] filename = field.generate_filename(None, filename) storage = DefaultStorage() full_path = storage.path(filename) directory = os.path.dirname(full_path) try: os.makedirs(directory) except OSError, e: if e.errno != errno.EEXIST: raise
def go(): problem = get_or_none(Problem, id=id) if not problem: return {"success": False, "error": u"존재하지 않는 문제입니다."} checker = ObjectPermissionChecker(request.user) if not checker.has_perm('edit_problem', problem) and problem.user != request.user: return {"success": False, "error": u"권한이 없습니다."} if request.method != "POST": return {"success": False, "error": u"POST 접근하셔야 합니다."} file = request.FILES["file"] md5 = md5file(file) target_path = os.path.join("judge-attachments", md5, file.name) storage = DefaultStorage() storage.save(target_path, file) new_attachment = Attachment(problem=problem, file=target_path) new_attachment.save() # 해당 오브젝트에 대해 아무 퍼미션이나 있으면 처리됨. 문제의 경우 PUBLISHED 일 때는 이 권한을 사용하지 않아서 안전하다 visible_users = get_users_with_perms(problem, with_group_users=False) visible_groups = get_groups_with_perms(problem) publish("problem-attachment-%s" % datetime.now().strftime('%s.%f'), "problem", "problem-attachment", actor=request.user, target=problem, timestamp=datetime.now(), visible_users=visible_users, visible_groups=visible_groups, verb=u"문제 {target}에 첨부파일 %s 을 추가했습니다." % file.name) return {"success": True}
def applyImport(self, request, queryset): storage = DefaultStorage() for __datafile in queryset: # restore campus data f1 = storage.open(__datafile.data_file_campus.name, mode='rb') for deserialized_object in serializers.deserialize( "json", f1.read(), ignorenonexistent=True): print(str(deserialized_object)) deserialized_object.save() del f1 # restore colleges data f2 = storage.open(__datafile.data_file_college.name, mode='rb') for deserialized_object in serializers.deserialize( "json", f2.read(), ignorenonexistent=True): print(str(deserialized_object)) deserialized_object.save() del f2 # restore department data f3 = storage.open(__datafile.data_file_department.name, mode='rb') for deserialized_object in serializers.deserialize( "json", f3.read(), ignorenonexistent=True): print(str(deserialized_object)) deserialized_object.save() del f3
def get_etag_data(self, user): """Return the ETag data for the user's avatar. Args: user (django.contrib.auth.models.User): The user. Returns: list of unicode: The uniquely identifying information for the user's avatar. """ settings_manager = self._settings_manager_class(user) configuration = \ settings_manager.configuration_for(self.avatar_service_id) file_hash = configuration.get('file_hash') if not file_hash: storage = DefaultStorage() file_hash = md5() with storage.open(configuration['file_path'], 'rb') as f: file_hash = file_hash.update(f.read()) configuration['file_hash'] = file_hash.hexdigest() settings_manager.save() return [self.avatar_service_id, file_hash]
def render(self, context): # text typed in the tag token = self.args["file"] try: filename = resolve_path(token, self.parser, context) except PathResolutionException as e: return self.make_error_msg(f"Path Resolution failed: {e}") challenge = context["site"] try: filepath = safe_join(challenge.get_project_data_folder(), filename) except SuspiciousFileOperation: return self.make_error_msg( f"'{filename}' cannot be opened because it is outside the current challenge." ) storage = DefaultStorage() try: with storage.open(filepath, "r") as f: contents = f.read() except Exception as e: return self.make_error_msg("error opening file:" + str(e)) # TODO check content safety return contents
def get_avatar_urls_uncached(self, user, size): """Return the avatar URLs for the requested user. Args: user (django.contrib.auth.models.User): The user whose avatar URLs are to be fetched. size (int): The size (in pixels) the avatar is to be rendered at. Returns dict: A dictionary containing the URLs of the user's avatars at normal- and high-DPI. """ storage = DefaultStorage() settings_manager = self._settings_manager_class(user) configuration = \ settings_manager.configuration_for(self.avatar_service_id) if not configuration: return {} return { '1x': storage.url(configuration['file_path']) }
def go(): problem = get_or_none(Problem, id=id) if not problem: return {"success": False, "error": u"존재하지 않는 문제입니다."} if not request.user.is_superuser and problem.user != request.user: return {"success": False, "error": u"권한이 없습니다."} if request.method != "POST": return {"success": False, "error": u"POST 접근하셔야 합니다."} file = request.FILES["file"] md5 = md5file(file) target_path = os.path.join("judge-attachments", md5, file.name) storage = DefaultStorage() storage.save(target_path, file) new_attachment = Attachment(problem=problem, file=target_path) new_attachment.save() publish( "problem-attachment-%s" % datetime.now().strftime("%s.%f"), "problem", "problem-attachment", actor=request.user, target=problem, timestamp=datetime.now(), admin_only=True, verb=u"문제 {target}에 첨부파일 %s 을 추가했습니다." % file.name, ) return {"success": True}
def _try_create_comicsite(self, user, short_name, description="test project"): """ split this off from create_comicsite because sometimes you just want to assert that creation fails """ url = reverse("challenges:create") factory = RequestFactory() storage = DefaultStorage() banner = storage._open(settings.COMIC_PUBLIC_FOLDER_NAME + "/fakefile2.jpg") data = { "short_name": short_name, "description": description, "skin": "fake_test_dir/fakecss.css", "logo": "fakelogo.jpg", "banner": banner, "prefix": "form", "page_set-TOTAL_FORMS": "0", "page_set-INITIAL_FORMS": "0", "page_set-MAX_NUM_FORMS": "", } success = self._login(user) response = self.client.post(url, data) return response
def export_buildings(export_id, export_name, export_type, building_ids, export_model='seed.BuildingSnapshot', selected_fields=None): model = get_model(*export_model.split(".")) selected_buildings = model.objects.filter(pk__in=building_ids) def _row_cb(i): cache.set("export_buildings__%s" % export_id, i) my_exporter = getattr(exporter, "export_%s" % export_type, None) if not my_exporter: _row_cb(-1) # this means there was an error return exported_filename = my_exporter(selected_buildings, selected_fields, _row_cb) exported_file = open(exported_filename) s3_keyname = exporter._make_export_filename(export_id, export_name, export_type) s3_key = DefaultStorage().bucket.new_key(s3_keyname) s3_key.set_contents_from_file(exported_file) exported_file.close() os.remove(exported_filename) _row_cb(selected_buildings.count()) # means we're done!
def _fetch_image_and_get_file(url, allowed_mime_types, upload_to=''): status_code, storage_name = fetch_remote_file_to_storage( url, upload_to=upload_to, allowed_mime_types=allowed_mime_types) if status_code == 200: image = DefaultStorage().open(storage_name) image.name = storage_name return image
def render(self, context): challenge: Challenge = context["currentpage"].challenge try: projectpath = safe_join(challenge.get_project_data_folder(), self.path) except SuspiciousFileOperation: return self.make_dataset_error_msg( "path is outside the challenge folder.") storage = DefaultStorage() try: filenames = storage.listdir(projectpath)[1] except OSError as e: return self.make_dataset_error_msg(str(e)) filenames.sort() # if extensionsFilter is given, show only filenames with those extensions if "extensionFilter" in self.args.keys(): extensions = self.args["extensionFilter"].split(",") filenames = filter_by_extension(filenames, extensions) links = [] for filename in filenames: downloadlink = reverse( "root-serving:challenge-file", kwargs={ "challenge_name": challenge.short_name, "path": f"{self.path}/{filename}", }, ) links.append('<li><a href="' + downloadlink + '">' + filename + " </a></li>") htmlOut = '<ul class="dataset">' + "".join(links) + "</ul>" return htmlOut
def render(self, context): # text typed in the tag token = self.args["file"] try: filename = resolve_path(token, self.parser, context) except PathResolutionException as e: return self.make_error_msg(f"Path Resolution failed: {e}") challenge = context["challenge"] try: filepath = safe_join(challenge.get_project_data_folder(), filename) except SuspiciousFileOperation: return self.make_error_msg( f"'{filename}' cannot be opened because it is outside the current challenge." ) storage = DefaultStorage() try: with storage.open(filepath, "r") as f: contents = f.read() except Exception as e: return self.make_error_msg("error opening file:" + str(e)) # TODO check content safety return contents
def build_custom_pdf(book_id, customizations, file_name, waiter_id=None): """Builds a custom PDF file.""" try: from django.core.files import File from django.core.files.storage import DefaultStorage from catalogue.models import Book task_logger.info(DefaultStorage().path(file_name)) if not DefaultStorage().exists(file_name): kwargs = { 'cover': True, } if 'nocover' in customizations: kwargs['cover'] = False customizations.remove('nocover') wldoc = Book.objects.get(pk=book_id).wldocument() pdf = wldoc.as_pdf(customizations=customizations, morefloats=settings.LIBRARIAN_PDF_MOREFLOATS, ilustr_path=gallery_path( wldoc.book_info.url.slug), **kwargs) DefaultStorage().save(file_name, File(open(pdf.get_filename()))) finally: if waiter_id is not None: WaitedFile.objects.filter(pk=waiter_id).delete()
def get_doc(self, geschaft_id): geschaft = Geschaft.objects.get(id=geschaft_id) if self.type == 'XML': template = self.xml_file.read().decode("utf-8") context = geschaft.variables #Replace special chars replacements = {'&': '&', '<': '<', '>': '>'} for key, value in replacements.items(): context = context.replace(key, value) data = DocxCreator().create_docx(template, [json.loads(context)]) return data elif self.type == 'PDF': storage = DefaultStorage() template = self.xml_file.read().decode("utf-8") self.pdf_file.open() pdf = PdfWriter(file=storage.open(self.pdf_file.name)) pdf.render(template, json.loads(geschaft.variables)) data = pdf.save() self.pdf_file.close() return data
def render(self, context): challenge_short_name = context.page.challenge.short_name projectpath = challenge_short_name + "/" + self.path storage = DefaultStorage() try: filenames = storage.listdir(projectpath)[1] except OSError as e: return self.make_dataset_error_msg(str(e)) filenames.sort() # if extensionsFilter is given, show only filenames with those extensions if "extensionFilter" in self.args.keys(): extensions = self.args["extensionFilter"].split(",") filenames = filter_by_extension(filenames, extensions) links = [] for filename in filenames: downloadlink = reverse( "serving:challenge-file", kwargs={ "challenge_short_name": challenge_short_name, "path": f"{self.path}/{filename}", }, ) links.append('<li><a href="' + downloadlink + '">' + filename + " </a></li>") htmlOut = '<ul class="dataset">' + "".join(links) + "</ul>" return htmlOut
def get_etag_data(self, user): """Return the ETag data for the user's avatar. Args: user (django.contrib.auth.models.User): The user. Returns: list of unicode: The uniquely identifying information for the user's avatar. """ settings_manager = self._settings_manager_class(user) configuration = \ settings_manager.configuration_for(self.avatar_service_id) file_hash = configuration.get('file_hash') if not file_hash: storage = DefaultStorage() file_hash = md5() with storage.open(configuration['file_path'], 'rb') as f: file_hash.update(f.read()) configuration['file_hash'] = file_hash.hexdigest() settings_manager.save() return [self.avatar_service_id, file_hash]
def render_to_response(self, form, **kwargs): '''Render Datatables expected JSON format''' storage = DefaultStorage() page = self.get_page(form) # My hack # Add image url to all fields indicated in the view class # using image_fields property try: for item in self.image_fields: for object_item in page.object_list: object_item[item] = storage.url(object_item[item]) except AttributeError: pass # Add hyperlink url to all fields indicated in the view class # using url_fields property try: for item in self.url_fields: for object_item in page.object_list: object_item[item['field']] = [ object_item[item['field']], reverse(item['target'], args=[object_item[item['arg']]]), ] except AttributeError: pass data = { 'iTotalRecords': page.paginator.count, 'iTotalDisplayRecords': page.paginator.count, 'sEcho': form.cleaned_data['sEcho'], 'aaData': self.get_rows(page.object_list), } return self.json_response(data)
def serve(request, project_name, path, document_root=None,override_permission=""): """ Serve static file for a given project. This is meant as a replacement for the inefficient debug only 'django.views.static.serve' way of serving files under /media urls. """ if document_root == None: document_root = settings.MEDIA_ROOT path = posixpath.normpath(unquote(path)) path = path.lstrip('/') newpath = '' for part in path.split('/'): if not part: # Strip empty path components. continue drive, part = os.path.splitdrive(part) head, part = os.path.split(part) if part in (os.curdir, os.pardir): # Strip '.' and '..' in path. continue newpath = os.path.join(newpath, part).replace('\\', '/') if newpath and path != newpath: return HttpResponseRedirect(newpath) fullpath = os.path.join(document_root,project_name, newpath) storage = DefaultStorage() if not storage.exists(fullpath): # On case sensitive filesystems you can have problems if the project # nameurl in the url is not exactly the same case as the filepath. # find the correct case for projectname then. # Also case sensitive file systems are weird. # who would ever want to have a folder 'data' and 'Data' contain # different files? projectlist = ComicSite.objects.filter(short_name=project_name) if projectlist == []: raise Http404(_("project '%s' does not exist" % project_name )) project_name = projectlist[0].short_name fullpath = os.path.join(document_root,project_name, newpath) if not storage.exists(fullpath): raise Http404(_('"%(path)s" does not exist') % {'path': fullpath}) if can_access(request.user,path,project_name,override_permission): f = storage.open(fullpath, 'rb') file = File(f) # create django file object # Do not offer to save images, but show them directly return serve_file(request, file, save_as=True) else: return HttpResponseForbidden("This file is not available without " "credentials")
def render(self, context): filename_raw = self.args["file"] filename_clean = substitute(filename_raw, context["request"].GET.items()) # If any url parameters are still in filename they were not replaced. This filename # is missing information.. if re.search(r"{{\w+}}", filename_clean): missed_parameters = re.findall(r"{{\w+}}", filename_clean) found_parameters = context["request"].GET.items() if not found_parameters: found_parameters = "None" error_msg = ( "I am missing required url parameter(s) %s, url parameter(s) found: %s " "" % (missed_parameters, found_parameters)) return self.make_error_msg(error_msg) challenge: Challenge = context["currentpage"].challenge try: filename = safe_join(challenge.get_project_data_folder(), filename_clean) except SuspiciousFileOperation: return self.make_error_msg("file is outside the challenge folder.") storage = DefaultStorage() try: contents = storage.open(filename, "r").read() except Exception as e: return self.make_error_msg(str(e)) # TODO check content safety try: render_function = getrenderer(self.args["type"]) # (table,headers) = read_function(filename) except Exception as e: return self.make_error_msg("getrenderer: %s" % e) RENDER_FRIENDLY_ERRORS = True # FRIENDLY = on template tag error, replace template tag with red error # text # NOT SO FRIENDLY = on template tag error, stop rendering, show full # debug page try: svg_data = render_function(filename) except Exception as e: if RENDER_FRIENDLY_ERRORS: return self.make_error_msg( str("Error in render funtion '%s()' : %s" % (render_function.__name__, traceback.format_exc(0)))) else: raise # self.get_graph_svg(table,headers) # html_out = "A graph rendered! source: '%s' <br/><br/> %s" %(filename_clean,svg_data) html_out = svg_data # rewrite relative links return html_out
def cached_large_cover_url(self): if self.large_cover_url: digest = md5(self.large_cover_url).hexdigest() store = DefaultStorage() storage_name = "imdb/image/{}".format(digest) if store.exists(storage_name): return "{}{}".format(settings.MEDIA_URL, storage_name) return "{}?size=full".format(reverse('movie_cover', kwargs={'movieID': self.imdb_id}))
def create_from_file(self, image_filename, ideal_relative_name, defaults): # Import the file to media root and create the ORM # objects. storage = DefaultStorage() desired_storage_path = join("images", ideal_relative_name) with open(image_filename, "rb") as f: storage_filename = storage.save(desired_storage_path, f) return self.create(image=storage_filename, **defaults)
def upload_to_s3(fp, key_name): """ Upload the contents of file handle 'fp' to the S3 bucket specified by AWS_STORAGE_BUCKET_NAME, under the given filename. Return the public URL. """ storage = DefaultStorage() saved_name = storage.save(key_name, fp) return storage.url(saved_name)
def cached_cover_url(self): if self.cover_url: digest = md5(self.cover_url).hexdigest() store = DefaultStorage() storage_name = "imdb/image/{}".format(digest) if store.exists(storage_name): return "{}{}".format(settings.MEDIA_URL, storage_name) return reverse('movie_cover', kwargs={'movieID': self.imdb_id})
def get_dirnames(path): """ Get all directory names in path as list of strings Raises: OSError if directory can not be found """ storage = DefaultStorage() dirnames = storage.listdir(path)[0] dirnames.sort() return dirnames
def save_upload_file(filename, content): if -1 != filename.find("/media/"): filename = filename.split("/media/")[1] if -1 != filename.find("/static/"): filename = filename.split("/static/")[1] f = DefaultStorage() file = f.save(filename, content) file_url = f.url(file) return file_url
def serve(request, challenge_short_name, path, document_root=None): """ Serve static file for a given project. This is meant as a replacement for the inefficient debug only 'django.views.static.serve' way of serving files under /media urls. """ if document_root is None: document_root = settings.MEDIA_ROOT path = posixpath.normpath(unquote(path)) path = path.lstrip('/') newpath = '' for part in path.split('/'): if not part: # Strip empty path components. continue drive, part = os.path.splitdrive(part) head, part = os.path.split(part) if part in (os.curdir, os.pardir): # Strip '.' and '..' in path. continue newpath = os.path.join(newpath, part).replace('\\', '/') if newpath and path != newpath: return HttpResponseRedirect(newpath) fullpath = os.path.join(document_root, challenge_short_name, newpath) storage = DefaultStorage() if not storage.exists(fullpath): # On case sensitive filesystems you can have problems if the project # nameurl in the url is not exactly the same case as the filepath. # find the correct case for projectname then. projectlist = Challenge.objects.filter( short_name__iexact=challenge_short_name) if not projectlist: raise Http404("project '%s' does not exist" % challenge_short_name) challenge_short_name = projectlist[0].short_name fullpath = os.path.join(document_root, challenge_short_name, newpath) if not storage.exists(fullpath): raise Http404('"%(path)s" does not exist' % {'path': fullpath}) if can_access(request.user, path, challenge_short_name): try: f = storage.open(fullpath, 'rb') file = File(f) # create django file object except IOError: return HttpResponseForbidden("This is not a file") # Do not offer to save images, but show them directly return serve_file(file, save_as=True) else: return HttpResponseForbidden("This file is not available without " "credentials")
def upload(req): if req.method == 'POST': uploaded_file = req.FILES['files'] file_extension = uploaded_file.name.split(".")[-1] random_filename = str(uuid.uuid4()) final_filename = random_filename + "." + file_extension store = DefaultStorage() store.save(final_filename, uploaded_file) return JsonResponse({'filename': final_filename})
def download_image(self, image_url): store = DefaultStorage() storage_name = 'imdb/image/{}'.format(md5(image_url).hexdigest()) if not store.exists(storage_name): r = requests.get(image_url, stream=True) if r.status_code == 200: r.raw.decode_content = True store.save(storage_name, r.raw) return storage_name
def save(self, **kwargs): entry = super(FormEntryForm, self).save() # entry.record_entry(self) for field_key in self.fields: if self.form_field_prefix in field_key: field = self.fields[field_key] raw_value = self._raw_value(field_key) value = self.cleaned_value(raw_value) model_field = field.widget.model_field # TODO-- add secure file handling... if value and field.widget.model_field.is_multipart: type = raw_value.__class__.__name__ # print 'TYPE? %s'%(type) if isinstance(raw_value, InMemoryUploadedFile) or isinstance(raw_value, TemporaryUploadedFile): file_upload_path = join( "form_uploads", str(self.form_schema.slug), str(entry.pk), str(model_field.slug), raw_value.name, ) # if settings.DEBUG: # print 'FILE UPLOAD PATH: %s'%(file_upload_path) try: if model_field.type == FormField.SECURE_FILE: secure_file_storage = import_by_path(settings.SECURE_MEDIA_STORAGE)() value = secure_file_storage.save(file_upload_path, raw_value) key_name = "%s/%s" % (settings.AWS_MEDIA_FOLDER, value) BaseSecureAtom.make_private(settings.AWS_STORAGE_BUCKET_NAME_MEDIA_SECURE, key_name) else: file_storage = DefaultStorage() value = file_storage.save(file_upload_path, raw_value) except: print "Error uploading file to %s/%s" % (file_upload_path, raw_value) value = file_upload_path field_entry, created = self.field_model.objects.get_or_create( form_entry=entry, form_field=field.widget.model_field ) # if created: # print 'created new field entry: %s, %s, %s'%(field_entry, entry.pk, field.widget.model_field.title) # else: # print 'Update field entry %s'%(value) field_entry.value = field_entry.get_compressed_value(value) field_entry.save() return entry
def setUp(self): super().setUp() person_extra = PersonExtraFactory.create(base__id="2009", base__name="Tessa Jowell") dulwich_not_stand = PersonExtraFactory.create(base__id="4322", base__name="Helen Hayes") edinburgh_candidate = PersonExtraFactory.create( base__id="818", base__name="Sheila Gilmore") edinburgh_winner = PersonExtraFactory.create( base__id="5795", base__name="Tommy Sheppard") edinburgh_may_stand = PersonExtraFactory.create( base__id="5163", base__name="Peter McColl") MembershipFactory.create( person=person_extra.base, post=self.dulwich_post_extra.base, on_behalf_of=self.labour_party_extra.base, post_election=self.dulwich_post_extra_pee, ) MembershipFactory.create( person=person_extra.base, organization=self.labour_party_extra.base, post_election=self.edinburgh_east_post_extra_pee, ) MembershipFactory.create( person=dulwich_not_stand.base, post=self.dulwich_post_extra.base, on_behalf_of=self.labour_party_extra.base, post_election=self.dulwich_post_extra_pee_earlier, ) dulwich_not_stand.not_standing.add(self.election) MembershipFactory.create( person=edinburgh_winner.base, post=self.edinburgh_east_post_extra.base, on_behalf_of=self.labour_party_extra.base, elected=True, post_election=self.edinburgh_east_post_extra_pee, ) MembershipFactory.create( person=edinburgh_candidate.base, post=self.edinburgh_east_post_extra.base, on_behalf_of=self.labour_party_extra.base, post_election=self.edinburgh_east_post_extra_pee, ) MembershipFactory.create( person=edinburgh_may_stand.base, post=self.edinburgh_east_post_extra.base, on_behalf_of=self.labour_party_extra.base, post_election=self.edinburgh_east_post_extra_pee_earlier, ) self.storage = DefaultStorage()
def save_file_to_storage(uploaded_file): """ Saves an uploaded file to default storage and returns its filename. """ storage = DefaultStorage() filename = storage.get_valid_name(uploaded_file.name) with storage.open(filename, 'wb') as f: for chunk in uploaded_file.chunks(): f.write(chunk) return filename
def value_from_datadict(self, data, files, name): s3_path = data.get(name, None) if s3_path is None: return None mode = 'r' storage = DefaultStorage() storage.location = '' name = urllib.unquote_plus(urlparse(s3_path).path.lstrip('/')) input_file = S3BotoStorageFile(name, mode, storage) return input_file
def done(self, form_list, **kwargs): form_data = [form.cleaned_data for form in form_list] name = form_data[0]['name'] description = form_data[0]['description'] mime_type = form_data[0]['mime_type'] is_resource = form_data[0]['is_resource'] original_file = form_data[0]['original_file'] file = form_data[0]['file'] type = self.storage.get_step_data('select_file').get( 'select_file-type') entity_types = self.storage.get_step_data('select_file').getlist( 'select_file-entity_types') path = self.file_storage.path(file.name) map_attrs_data = self.storage.get_step_data('map_attributes') project = self.get_project() org = project.organization config_dict = { 'project': project, 'file': path, 'type': type, 'entity_types': entity_types.copy(), 'party_name_field': form_data[2]['party_name_field'], 'party_type_field': form_data[2]['party_type_field'], 'location_type_field': form_data[2]['location_type_field'], 'geometry_field': form_data[2]['geometry_field'], 'attributes': map_attrs_data.getlist('attributes', None), } importer = self._get_importer(type, path) importer.import_data(config_dict) if is_resource: default_storage = DefaultStorage() file.seek(0) ext = file.name[file.name.rfind('.'):] resource = Resource(name=name, description=description, original_file=original_file, mime_type=mime_type, contributor=self.request.user, project=self.get_project()) upload_to = getattr(resource.file.field, 'upload_to') url = '' while not url: temp_url = upload_to + '/' + random_id() + ext if not Resource.objects.filter( file__contains=temp_url).exists(): url = default_storage.save(temp_url, file.read()) resource.file.url = url resource.save() ContentObject.objects.create(resource=resource, content_object=resource.project) return redirect('organization:project-dashboard', organization=org.slug, project=project.slug)
async def __call__(self, in_q, out_q): """ The coroutine for this stage. Args: in_q (:class:`asyncio.Queue`): The queue to receive :class:`~pulpcore.plugin.stages.DeclarativeContent` objects from. out_q (:class:`asyncio.Queue`): The queue to put :class:`~pulpcore.plugin.stages.DeclarativeContent` into. Returns: The coroutine for this stage. """ storage_backend = DefaultStorage() shutdown = False batch = [] while not shutdown: try: content = in_q.get_nowait() except asyncio.QueueEmpty: if not batch: content = await in_q.get() batch.append(content) continue else: batch.append(content) continue artifacts_to_save = [] for declarative_content in batch: if declarative_content is None: shutdown = True break for declarative_artifact in declarative_content.d_artifacts: if declarative_artifact.artifact.pk is None: src_path = str(declarative_artifact.artifact.file) dst_path = declarative_artifact.artifact.storage_path( None) with open(src_path, mode='rb') as input_file: django_file_obj = File(input_file) storage_backend.save(dst_path, django_file_obj) declarative_artifact.artifact.file = dst_path artifacts_to_save.append(declarative_artifact.artifact) if artifacts_to_save: Artifact.objects.bulk_create(artifacts_to_save) for declarative_content in batch: if declarative_content is None: continue await out_q.put(declarative_content) batch = [] await out_q.put(None)
def parse_php_arrays(filename): """ Parse a php page containing only php arrays like $x=(1,2,3). Created to parse anode09 eval results. Returns: dict{"varname1",array1,....}, array1 is a float array """ verbose = False output = {} storage = DefaultStorage() with storage.open(filename, "r") as f: content = f.read() content = content.replace("\n", "") php = re.compile(r"\<\?php(.*?)\?\>", re.DOTALL) s = php.search(content) assert s is not None, ( "trying to parse a php array, but could not find anything like <? php /?> in '%s'" % filename ) phpcontent = s.group(1) phpvars = phpcontent.split("$") phpvars = [x for x in phpvars if x != ""] # remove empty if verbose: print("found %d php variables in %s. " % (len(phpvars), filename)) print("parsing %s into int arrays.. " % filename) # check whether this looks like a php var phpvar = re.compile( r"([a-zA-Z]+[a-zA-Z0-9]*?)=array\((.*?)\);", re.DOTALL ) for var in phpvars: result = phpvar.search(var) # TODO Log these messages as info if result is None: msg = ( "Could not match regex pattern '%s' to '%s'\ " % (phpvar.pattern, var) ) continue if len(result.groups()) != 2: msg = ( "Expected to find varname and content,\ but regex '%s' found %d items:%s " % ( phpvar.pattern, len(result.groups()), "[" + ",".join(result.groups()) + "]", ) ) continue (varname, varcontent) = result.groups() output[varname] = [float(x) for x in varcontent.split(",")] return output
def cleanup_screenshot_files(): """Remove stale screenshots""" storage = DefaultStorage() try: files = storage.listdir('screenshots')[1] except OSError: return for name in files: fullname = os.path.join('screenshots', name) if not Screenshot.objects.filter(image=fullname).exists(): storage.delete(fullname)
async def __call__(self, in_q, out_q): """ The coroutine for this stage. Args: in_q (:class:`asyncio.Queue`): The queue to receive :class:`~pulpcore.plugin.stages.DeclarativeContent` objects from. out_q (:class:`asyncio.Queue`): The queue to put :class:`~pulpcore.plugin.stages.DeclarativeContent` into. Returns: The coroutine for this stage. """ storage_backend = DefaultStorage() shutdown = False batch = [] while not shutdown: try: content = in_q.get_nowait() except asyncio.QueueEmpty: if not batch: content = await in_q.get() batch.append(content) continue else: batch.append(content) continue artifacts_to_save = [] for declarative_content in batch: if declarative_content is None: shutdown = True break for declarative_artifact in declarative_content.d_artifacts: if declarative_artifact.artifact.pk is None: src_path = str(declarative_artifact.artifact.file) dst_path = declarative_artifact.artifact.storage_path(None) with open(src_path, mode='rb') as input_file: django_file_obj = File(input_file) storage_backend.save(dst_path, django_file_obj) declarative_artifact.artifact.file = dst_path artifacts_to_save.append(declarative_artifact.artifact) if artifacts_to_save: Artifact.objects.bulk_create(artifacts_to_save) for declarative_content in batch: if declarative_content is None: continue await out_q.put(declarative_content) batch = [] await out_q.put(None)
def done(self, form_list, **kwargs): form_data = [form.cleaned_data for form in form_list] name = form_data[0]['name'] description = form_data[0]['description'] mime_type = form_data[0]['mime_type'] is_resource = form_data[0]['is_resource'] original_file = form_data[0]['original_file'] file = form_data[0]['file'] type = self.storage.get_step_data( 'select_file').get('select_file-type') entity_types = self.storage.get_step_data('select_file').getlist( 'select_file-entity_types' ) path = self.file_storage.path(file.name) map_attrs_data = self.storage.get_step_data('map_attributes') project = self.get_project() org = project.organization config_dict = { 'project': project, 'file': path, 'type': type, 'entity_types': entity_types.copy(), 'party_name_field': form_data[2]['party_name_field'], 'party_type_field': form_data[2]['party_type_field'], 'location_type_field': form_data[2]['location_type_field'], 'geometry_field': form_data[2]['geometry_field'], 'attributes': map_attrs_data.getlist('attributes', None), } importer = self._get_importer(type, path) importer.import_data(config_dict) if is_resource: default_storage = DefaultStorage() file.seek(0) ext = file.name[file.name.rfind('.'):] resource = Resource( name=name, description=description, original_file=original_file, mime_type=mime_type, contributor=self.request.user, project=self.get_project()) upload_to = getattr(resource.file.field, 'upload_to') url = '' while not url: temp_url = upload_to + '/' + random_id() + ext if not Resource.objects.filter( file__contains=temp_url).exists(): url = default_storage.save(temp_url, file.read()) resource.file.url = url resource.save() ContentObject.objects.create(resource=resource, content_object=resource.project) return redirect('organization:project-dashboard', organization=org.slug, project=project.slug)
def cleanup_files(self): """Remove stale screenshots""" storage = DefaultStorage() try: files = storage.listdir('screenshots')[1] except OSError: return for name in files: fullname = os.path.join('screenshots', name) if not Screenshot.objects.filter(image=fullname).exists(): storage.delete(fullname)
def cleanup(self, user): """Clean up the uploaded file. This will delete the uploaded file from the storage. Args: user (django.contrib.auth.models.User): The user. """ settings_manager = self._settings_manager_class(user) configuration = settings_manager.configuration_for(self.avatar_service_id) storage = DefaultStorage() storage.delete(configuration["file_path"])
def serve_fullpath(*, fullpath): storage = DefaultStorage() if not (os.path.abspath(fullpath) == fullpath) or not storage.exists( fullpath ): raise Http404("File not found.") try: f = storage.open(fullpath, "rb") file = File(f) return serve_file(file, save_as=True) except IOError: raise Http404("File not found.")
def value_from_datadict(self, data, files, name): url = data.get(name) upload = files.get(name, False) if url: if url == 'initial': return None storage = DefaultStorage() filename = urllib2.unquote(urlparse(url).path) try: file = storage.open(filename) return file except IOError: return False return upload
def save(self): """Save the file and return the configuration. Returns: dict: The avatar service configuration. """ storage = DefaultStorage() file_path = self.cleaned_data["avatar_upload"].name file_path = storage.get_valid_name(file_path) with storage.open(file_path, "wb") as f: f.write(self.cleaned_data["avatar_upload"].read()) return {"absolute_url": storage.url(file_path), "file_path": file_path}
def fetch_mdb_dump(filename, table): storage = DefaultStorage() with storage.open(filename, 'rb') as mdb_file: response = requests.post( settings.MDB_DUMP_URL, auth=(settings.MDB_DUMP_USERNAME, settings.MDB_DUMP_PASSWORD), data={'table_name': table}, files={'mdb_file': mdb_file}, ) if not 200 <= response.status_code < 300: mail_admins( f'Unable to export table "{table}" from database', f'MDB Dump returned error: "{response.text}"' ) raise ValueError('Unable to export table from database') return response.text
def random_image(field): # Delay import to avoid hard dependency from PIL import Image, ImageDraw color1 = random_rgb() color2 = random_rgb() color3 = random_rgb() color4 = random_rgb() size = (random.randint(300, 900), random.randint(300, 900)) im = Image.new("RGB", size) # create the image draw = ImageDraw.Draw(im) # create a drawing object that is draw.rectangle( [(0, 0), ((size[0] / 2), (size[1] / 2))], fill=color1 ) draw.rectangle( [((size[0] / 2), 0), ((size[1] / 2), size[0])], fill=color2 ) draw.rectangle( [(0, (size[1] / 2)), ((size[0] / 2), size[1])], fill=color3 ) draw.rectangle( [((size[0] / 2), (size[1] / 2)), (size[0], size[1])], fill=color4 ) filename = "%s.png" % uuid.uuid4().hex[:10] filename = field.generate_filename(None, filename) storage = DefaultStorage() full_path = storage.path(filename) directory = os.path.dirname(full_path) try: os.makedirs(directory) except OSError as e: if e.errno != errno.EEXIST: raise filehandle = storage.open(filename, mode="w") im.save(filehandle, "PNG") filehandle.close() return filename # and we"re done!
def fetch_remote_file_to_storage(remote_url, upload_to=''): """ Fetches a remote url, and stores it in DefaultStorage :return: (status_code, new_storage_name) """ store = DefaultStorage() r = requests.get(remote_url, stream=True) if r.status_code == 200: name, ext = os.path.splitext(urlparse.urlparse(r.url).path) storage_name = '{upload_to}/cached/{filename}{ext}'.format( upload_to=upload_to, filename=hashlib.md5(remote_url).hexdigest(), ext=ext) if not store.exists(storage_name): buf = StringIO.StringIO(r.content) store.save(storage_name, buf) return r.status_code, storage_name return r.status_code, None
def test_load_with_file(self): archive_stream = io.BytesIO() archive = tarfile.TarFile(fileobj=archive_stream, mode='w') in_stream = [ {'test1': 'item', 'readme': {'__type__':'File', 'storage_path':'assets/readme.txt', 'path':'assets/readme.txt',} }, {'test2': 'item2', 'readme': {'__type__':'File', 'storage_path':'assets/readme2.txt', 'path':'assets/readme2.txt',} }, ] tarextfile = WritableTarExtFile(archive, 'manifest.json', json.dumps(in_stream)) tarextfile.save() tarextfile = WritableTarExtFile(archive, 'assets/readme.txt', 'readme1') tarextfile.save() tarextfile = WritableTarExtFile(archive, 'assets/readme2.txt', 'readme2') tarextfile.save() archive.close() archive_stream.seek(0) tap = TarFileDataTap(StreamDataTap(archive_stream)) items = list(tap) self.assertEqual(len(items), 2) self.assertEqual(items[0]['test1'], 'item') self.assertTrue(isinstance(items[0]['readme'], File)) self.assertEqual(items[1]['test2'], 'item2') self.assertTrue(isinstance(items[1]['readme'], File)) #test that the file object passed back can be properly saved file_to_save = items[0]['readme'] storage = DefaultStorage() result = storage.save(file_to_save.name, file_to_save) assert result tap.close()
def go(): problem = get_or_none(Problem, id=id) if not problem: return {"success": False, "error": u"존재하지 않는 문제입니다."} if not request.user.is_superuser and problem.user != request.user: return {"success": False, "error": u"권한이 없습니다."} if request.method != "POST": return {"success": False, "error": u"POST 접근하셔야 합니다."} file = request.FILES["file"] md5 = md5file(file) target_path = os.path.join("judge-attachments", md5, file.name) storage = DefaultStorage() storage.save(target_path, file) new_attachment = Attachment(problem=problem, file=target_path) new_attachment.save() return {"success": True}
def export(self, buildings, fields, row_cb): """ The main method of export. Uses the export type defined by the initializer :param buildings: Array of building ids to export :param fields: Array of fields to export :param row_cb: ID for row cache :return: """ export_method = getattr(self, "export_%s" % self.export_type, None) if export_method: export_method(buildings, fields, row_cb) else: return None # save the tempfile to the file storage location (s3 or local) if not self.tempfile == None: if 'FileSystemStorage' in settings.DEFAULT_FILE_STORAGE: # This is non-ideal. We should just save the file in the right location to start with # or return the file from the "export". This was done to avoid changing the exporter code 'too much'. file_storage = DefaultStorage() f = open(self.tempfile, 'r') file_storage.save(self.filename(), f) f.close() else: s3_key = DefaultStorage().bucket.new_key(self.filename()) f = open(self.tempfile) s3_key.set_contents_from_file(f) f.close() os.remove(self.tempfile) return self.filename
def __init__(self, relative_source, requested_size, opts=None, quality=None, basedir=None, subdir=None, prefix=None, relative_dest=None, processors=None, extension=None, storage_server=None): if not storage_server: storage_server = DefaultStorage() relative_source = force_unicode(relative_source) # Set the absolute filename for the source file with self._get_data_as_tempfile(relative_source, storage_server) as source: quality = get_thumbnail_setting('QUALITY', quality) convert_path = get_thumbnail_setting('CONVERT') wvps_path = get_thumbnail_setting('WVPS') if processors is None: processors = dynamic_import(get_thumbnail_setting('PROCESSORS')) # Call super().__init__ now to set the opts attribute. generate() won't # get called because we are not setting the dest attribute yet. super(DjangoThumbnail, self).__init__(source, requested_size, opts=opts, quality=quality, convert_path=convert_path, wvps_path=wvps_path, processors=processors) # Get the relative filename for the thumbnail image, then set the # destination filename if relative_dest is None: relative_dest = \ self._get_relative_thumbnail(relative_source.name, basedir=basedir, subdir=subdir, prefix=prefix, extension=extension) with NamedTemporaryFile() as dest: self.dest = dest.name self.generate() dest.seek(0) data = dest.read() f = StringIO(data) f.name = relative_dest f.size = len(data) self.relative_url = storage_server.save(relative_dest, File(f)) self.absolute_url = os.path.join(settings.MEDIA_URL, self.relative_url)
def random_image(field): color1 = random_rgb() color2 = random_rgb() color3 = random_rgb() color4 = random_rgb() size = (random.randint(300, 900), random.randint(300, 900)) im = Image.new("RGB", size) # create the image draw = ImageDraw.Draw(im) # create a drawing object that is draw.rectangle( [(0, 0), ((size[0] / 2), (size[1] / 2))], fill=color1 ) draw.rectangle( [((size[0] / 2), 0), ((size[1] / 2), size[0])], fill=color2 ) draw.rectangle( [(0, (size[1] / 2)), ((size[0] / 2), size[1])], fill=color3 ) draw.rectangle( [((size[0] / 2), (size[1] / 2)), (size[0], size[1])], fill=color4 ) filename = "%s.png" % uuid.uuid4().hex[:10] filename = field.generate_filename(None, filename) storage = DefaultStorage() full_path = storage.path(filename) directory = os.path.dirname(full_path) try: os.makedirs(directory) except OSError, e: if e.errno != errno.EEXIST: raise
def render(self, context): challenge: Challenge = context["currentpage"].challenge try: projectpath = safe_join( challenge.get_project_data_folder(), self.path ) except SuspiciousFileOperation: return self.make_dataset_error_msg( "path is outside the challenge folder." ) storage = DefaultStorage() try: filenames = storage.listdir(projectpath)[1] except OSError as e: return self.make_dataset_error_msg(str(e)) filenames.sort() # if extensionsFilter is given, show only filenames with those extensions if "extensionFilter" in self.args.keys(): extensions = self.args["extensionFilter"].split(",") filenames = filter_by_extension(filenames, extensions) links = [] for filename in filenames: downloadlink = reverse( "root-serving:challenge-file", kwargs={ "challenge_name": challenge.short_name, "path": f"{self.path}/{filename}", }, ) links.append( '<li><a href="' + downloadlink + '">' + filename + " </a></li>" ) htmlOut = '<ul class="dataset">' + "".join(links) + "</ul>" return htmlOut