def run_test(self, filename, content='Lorem ipsum dolar sit amet'): content = UnicodeContentFile(content) filename = default_storage.save(filename, content) self.assert_(default_storage.exists(filename)) self.assertEqual(default_storage.size(filename), content.size) now = datetime.utcnow() delta = timedelta(minutes=5) mtime = default_storage.getmtime(filename) self.assert_(mtime > mktime((now - delta).timetuple())) self.assert_(mtime < mktime((now + delta).timetuple())) file = default_storage.open(filename) self.assertEqual(file.size, content.size) fileurl = force_unicode(file).replace('\\', '/') fileurl = urlquote_plus(fileurl, '/') if fileurl.startswith('/'): fileurl = fileurl[1:] self.assertEqual( MEDIA_URL+fileurl, default_storage.url(filename) ) file.close() default_storage.delete(filename) self.assert_(not default_storage.exists(filename))
def sign(version_id, reviewer=False): version = Version.objects.get(pk=version_id) app = version.addon log.info('Signing version: %s of app: %s' % (version_id, app)) if not app.type == amo.ADDON_WEBAPP: log.error('Attempt to sign something other than an app.') raise SigningError('Not an app') if not app.is_packaged: log.error('Attempt to sign a non-packaged app.') raise SigningError('Not packaged') try: file_obj = version.all_files[0] except IndexError: log.error('Attempt to sign an app with no files in version.') raise SigningError('No file') path = (file_obj.signed_reviewer_file_path if reviewer else file_obj.signed_file_path) if storage.exists(path): log.info('Already signed app exists.') return path with statsd.timer('services.sign.app'): try: sign_app(file_obj.file_path, path, reviewer) except SigningError: if storage.exists(path): storage.delete(path) raise log.info('Signing complete.') return path
def task_join_catalogs(job_id): """Union of unique rows in two catalogs""" job = JoinCatalogs.objects.get(pk=job_id) job.job_status = 'started' job.save() if default_storage.exists(job.left_table.handle.name): left = csv.DictReader(default_storage.open(job.left_table.handle.name)) if default_storage.exists(job.right_table.handle.name): right = csv.DictReader(default_storage.open(job.right_table.handle.name)) keys = set(left.fieldnames + right.fieldnames) left = [r for r in left] right = [r for r in right] joinedCatalogs, columns = join(left, right, job.fk_field) path = os.path.join(BASE_DIR, MEDIA_ROOT, 'catalogs', '{}.csv'.format(job.results_label)) handler = open(path,'w') handler = csv.DictWriter(handler, fieldnames=keys) handler.writeheader() handler.writerows(joinedCatalogs) job.completed = 1 job.results_handle = 'catalogs/{}.csv'.format(job.results_label) result = Catalog( name=job.results_label, handle='catalogs/{}.csv'.format(job.results_label) ) result.save() job.complete = True job.completed_date = timezone.now() job.job_status = 'complete' job.save() return True
def test_delete_image(self): """Test that two images with same size and same name can be stored on disk""" def create_image(): filepath = os.path.join(settings.BASE_DIR, 'big.jpeg') with open(filepath) as f: # prepare form data image = InMemoryUploadedFile( f, 'image', 'big.jpeg', 'image/jpeg', 42, # not significant for the test 'utf-8' ) files = MultiValueDict() files['image'] = image post = MultiValueDict() post['ptype'] = 1 post['name'] = 'test with big.jpeg' # create form form = ImageForm(TestImage, post, files) # validate resize operation form.is_valid() # execute resize operation image = form.save() return image # create two times the same image: one = create_image() self.assertTrue(default_storage.exists(one.og['filepath'])) one.delete() self.assertFalse(default_storage.exists(one.og['filepath']))
def test_upload_sign_error_existing(self, sign_app_mock): sign_app_mock.side_effect = SigningError langpack = self.create_langpack() eq_(LangPack.objects.count(), 1) original_uuid = langpack.uuid original_file_path = langpack.file_path original_file_version = langpack.file_version original_version = langpack.version # create_langpack() doesn't create a fake file, let's add one. storage.open(langpack.file_path, 'w').close() upload = self.upload('langpack') with self.assertRaises(SigningError): LangPack.from_upload(upload, instance=langpack) # Test that we didn't delete the upload file ok_(storage.exists(upload.path)) # Test that we didn't delete the existing filename or alter the # existing langpack in the database. eq_(LangPack.objects.count(), 1) langpack.reload() eq_(original_uuid, langpack.uuid) eq_(langpack.file_path, original_file_path) eq_(original_file_version, langpack.file_version) eq_(original_version, langpack.version) ok_(storage.exists(langpack.file_path)) # Cleanup storage.delete(langpack.file_path)
def test_doesnt_delete_file_referenced_by_orphan_and_nonorphan_nodes(self): """ Make sure we don't delete a file, as long as it's referenced by a non-orphan node. """ # Our orphan, to be taken soon from this world orphan_node = _create_expired_contentnode() # our legit node, standing proud and high with its non-orphaned status legit_node = ContentNode.objects.create( kind_id=content_kinds.VIDEO, ) f = File.objects.create( contentnode=legit_node, checksum="aaa", ) forphan = File.objects.create( contentnode=orphan_node, checksum="aaa", ) # The file they both share. This has the same checksum and contents. # Alas, a file cannot have an orphan and non-orphan reference. This must # not be deleted. f.file_on_disk.save("aaa.jpg", ContentFile("aaa")) forphan.file_on_disk.save("aaa.jpg", ContentFile("aaa")) # check that our file exists in object storage assert default_storage.exists("storage/a/a/aaa.jpg") clean_up_contentnodes() assert default_storage.exists("storage/a/a/aaa.jpg")
def delete_or_disable_related_content_exclude_addons_with_other_devs(self): addon = Addon.objects.latest('pk') user = UserProfile.objects.get(pk=55021) user.update(picture_type='image/png') AddonUser.objects.create(addon=addon, user=user_factory()) # Create a photo so that we can test deletion. with storage.open(user.picture_path, 'wb') as fobj: fobj.write('test data\n') with storage.open(user.picture_path_original, 'wb') as fobj: fobj.write('original test data\n') assert user.addons.count() == 1 rating = Rating.objects.create( user=user, addon=addon, version=addon.current_version) Rating.objects.create( user=user, addon=addon, version=addon.current_version, reply_to=rating) Collection.objects.create(author=user) # Now that everything is set up, disable/delete related content. user.delete_or_disable_related_content() # The add-on should not have been touched, it has another dev. assert user.addons.exists() addon.reload() assert addon.status == amo.STATUS_PUBLIC assert not user._ratings_all.exists() # Even replies. assert not user.collections.exists() assert not storage.exists(user.picture_path) assert not storage.exists(user.picture_path_original)
def delete_or_disable_related_content_actually_delete(self): addon = Addon.objects.latest('pk') user = UserProfile.objects.get(pk=55021) user.update(picture_type='image/png') # Create a photo so that we can test deletion. with storage.open(user.picture_path, 'wb') as fobj: fobj.write('test data\n') with storage.open(user.picture_path_original, 'wb') as fobj: fobj.write('original test data\n') assert user.addons.count() == 1 rating = Rating.objects.create( user=user, addon=addon, version=addon.current_version) Rating.objects.create( user=user, addon=addon, version=addon.current_version, reply_to=rating) Collection.objects.create(author=user) # Now that everything is set up, delete related content. user.delete_or_disable_related_content(delete=True) assert not user.addons.exists() assert not user._ratings_all.exists() # Even replies. assert not user.collections.exists() assert not storage.exists(user.picture_path) assert not storage.exists(user.picture_path_original)
def _user_detail(request, user): ctx = _entity_detail(request, user) ctx['photosUrl'] = reverse('fotos', kwargs={'path':''}) + \ '?q=tag:'+str(user.name) photos_path = path.join(settings.SMOELEN_PHOTOS_PATH, str(user.name)) if default_storage.exists(photos_path + '.jpg'): img = Image.open(default_storage.open(photos_path + '.jpg')) width, height = img.size if default_storage.exists(photos_path + '.orig'): # smoel was created using newer strategy. Shrink until it fits the # requirements. width, height = resize_proportional(img.size[0], img.size[1], settings.SMOELEN_WIDTH, settings.SMOELEN_HEIGHT) elif width > settings.SMOELEN_WIDTH: # smoel was created as high-resolution image, probably 600px wide width /= 2 height /= 2 else: # smoel was created as normal image, probably 300px wide pass ctx.update({ 'hasPhoto': True, 'photoWidth': width, 'photoHeight': height}) return render_to_response('leden/user_detail.html', ctx, context_instance=RequestContext(request))
def attachment_remove(r): if not r.user.is_authenticated(): return HttpResponse(get_json_response(code=403, message='Unauthorized')) try: if 'base_name' in r.POST and len(r.POST['base_name']) > 0: # deleting recently uploaded image if not re.match('[a-z0-9]+\.[a-z]+', r.POST['base_name']): raise ValueError(u'Передано некорректное имя файла') fname_base = r.POST['base_name'] uploaded_file = 'uploads/%s' % fname_base thumb_file = "uploads/%s" % NewsPost.get_thumbname_from_base(fname_base) if fs.exists(uploaded_file): try: fs.delete(uploaded_file) except: pass if fs.exists(thumb_file): try: fs.delete(thumb_file) except: pass except ValueError as e: HttpResponse(get_json_response(code=1, message=e.message)) return HttpResponse(get_json_response(code=0))
def attach_delete(request): if not request.user.is_authenticated(): return HttpResponse(get_json_response(code=403, message='Unauthorized')) try: if 'uid' in request.POST and len(request.POST['uid']) > 0: if not re.match('[a-zA-Z0-9]+', request.POST['uid']): raise ValueError(u'Передано некорректное имя файла') uid = request.POST['uid'] if not uid in request.session: raise ValueError(u'Сессия завершена!') uploaded_file = request.session[uid][1] thumb_file = request.session[uid][2] del request.session[uid] if fs.exists(uploaded_file): try: fs.delete(uploaded_file) except IOError as e: log.error("Failed to delete file '%s'. %s" % (uploaded_file, e.message)) if fs.exists(thumb_file): try: fs.delete(thumb_file) except IOError as e: log.error("Failed to delete file '%s'. %s" % (thumb_file, e.message)) except ValueError as e: HttpResponse(get_json_response(code=400, message=e.message)) return HttpResponse(get_json_response(code=200))
def delete(self, hard=False): # Recursive import from olympia.users.tasks import delete_photo # Cache the values in case we do a hard delete and loose # reference to the user-id. picture_path = self.picture_path original_picture_path = self.picture_path_original if hard: super(UserProfile, self).delete() else: log.info( u'User (%s: <%s>) is being anonymized.' % (self, self.email)) self.email = None self.fxa_id = None self.display_name = None self.homepage = '' self.location = '' self.deleted = True self.picture_type = None self.auth_id = generate_auth_id() self.last_login_attempt = None self.last_login_attempt_ip = '' self.last_login_ip = '' self.anonymize_username() self.save() if storage.exists(picture_path): delete_photo.delay(picture_path) if storage.exists(original_picture_path): delete_photo.delay(original_picture_path)
def sign(version_id, reviewer=False, resign=False, **kw): version = Version.objects.get(pk=version_id) app = version.addon log.info('Signing version: %s of app: %s' % (version_id, app)) if not app.type == amo.ADDON_WEBAPP: log.error('[Webapp:%s] Attempt to sign something other than an app.' % app.id) raise SigningError('Not an app') if not app.is_packaged: log.error('[Webapp:%s] Attempt to sign a non-packaged app.' % app.id) raise SigningError('Not packaged') try: file_obj = version.all_files[0] except IndexError: log.error( '[Webapp:%s] Attempt to sign an app with no files in version.' % app.id) raise SigningError('No file') path = (file_obj.signed_reviewer_file_path if reviewer else file_obj.signed_file_path) if storage.exists(path) and not resign: log.info('[Webapp:%s] Already signed app exists.' % app.id) return path if resign: z = zipfile.ZipFile(file_obj.file_path, 'r') if 'META-INF/ids.json' in z.namelist(): # This zip is broken due to previously used bad signing # code. rebuild it. (This code can be deleted once all # broken packages are re-signed.) tempf = tempfile.NamedTemporaryFile() zout = zipfile.ZipFile(tempf, 'w', zipfile.ZIP_DEFLATED) for f in sorted(z.infolist()): if f.filename != 'META-INF/ids.json': zout.writestr(f, z.read(f.filename)) zout.close() shutil.copyfile(tempf.name, file_obj.file_path) tempf.close() ids = json.dumps({ 'id': app.guid, 'version': version_id }) with statsd.timer('services.sign.app'): try: sign_app(file_obj.file_path, path, ids, reviewer) except SigningError: log.info('[Webapp:%s] Signing failed' % app.id) if storage.exists(path): storage.delete(path) raise log.info('[Webapp:%s] Signing complete.' % app.id) return path
def test_extra_methodes(self): default_storage.mkdir('test_directory') self.assertTrue(default_storage.exists('test_directory')) default_storage.mv('test_directory', 'moved_directory') self.assertTrue(default_storage.exists('moved_directory')) default_storage.delete('moved_directory') self.assertFalse(default_storage.exists('moved_directory'))
def test_populate_e10s_feature_compatibility(): # Create addons... # One must have no latest file object. addon_unreviewed = addon_factory( name='no current version', status=amo.STATUS_UNREVIEWED) addon_unreviewed.update(_current_version=None) assert addon_unreviewed.get_latest_file() is None # One must have a latest file object with no file on the filesystem. addon_no_file = addon_factory(name='no file') assert not storage.exists(addon_no_file.get_latest_file().file_path) # One must have a file, and be e10s incompatible addon = addon_factory(guid='guid@xpi', name='not e10s compatible') AMOPaths().xpi_copy_over(addon.get_latest_file(), 'extension.xpi') assert storage.exists(addon.get_latest_file().file_path) # One must have a file, and be e10s compatible addon_compatible = addon_factory( guid='guid-e10s@xpi', name='e10s compatible') AMOPaths().xpi_copy_over( addon_compatible.get_latest_file(), 'extension_e10s.xpi') assert storage.exists(addon_compatible.get_latest_file().file_path) # One must have a file, and be a web extension addon_webextension = addon_factory( guid='@webextension-guid', name='web extension') AMOPaths().xpi_copy_over( addon_webextension.get_latest_file(), 'webextension.xpi') assert storage.exists(addon_webextension.get_latest_file().file_path) # One must be unlisted, and compatible. addon_compatible_unlisted = addon_factory( guid='unlisted-guid-e10s@xpi', name='unlisted e10s compatible webext', is_listed=False) AMOPaths().xpi_copy_over( addon_compatible_unlisted.get_latest_file(), 'webextension_no_id.xpi') assert storage.exists( addon_compatible_unlisted.get_latest_file().file_path) # Call the command ! call_command('process_addons', task='populate_e10s_feature_compatibility') assert AddonFeatureCompatibility.objects.count() == 3 addon.reload() assert addon.feature_compatibility.pk assert addon.feature_compatibility.e10s == amo.E10S_UNKNOWN addon_compatible.reload() assert addon_compatible.feature_compatibility.pk assert addon_compatible.feature_compatibility.e10s == amo.E10S_COMPATIBLE addon_webextension.reload() assert addon_webextension.feature_compatibility.pk assert (addon_webextension.feature_compatibility.e10s == amo.E10S_COMPATIBLE_WEBEXTENSION)
def test_extract_header_img_missing(): file_obj = os.path.join( settings.ROOT, 'src/olympia/devhub/tests/addons/static_theme.zip') data = {'images': {'headerURL': 'missing_file.png'}} dest_path = tempfile.mkdtemp() header_file = dest_path + '/missing_file.png' assert not default_storage.exists(header_file) utils.extract_header_img(file_obj, data, dest_path) assert not default_storage.exists(header_file)
def setup_files(self): # Clean out any left over stuff. if storage.exists(self.file.signed_file_path): storage.delete(self.file.signed_file_path) # Make sure the source file is there. if not storage.exists(self.file.file_path): os.makedirs(os.path.dirname(self.file.file_path)) shutil.copyfile(self.packaged_app_path('mozball.zip'), self.file.file_path)
def test_doesnt_remove_non_empty_directories(self): # Add an extra disabled file. The approved one should move, but not the # other, so the directory should be left intact. self.disabled_file = file_factory( version=self.version, status=amo.STATUS_DISABLED) self.addon.update(status=amo.STATUS_APPROVED) self.file_.update(status=amo.STATUS_APPROVED) with storage.open(self.file_.guarded_file_path, 'wb') as fp: fp.write(b'content') assert not storage.exists(self.file_.file_path) assert storage.exists(self.file_.guarded_file_path) with storage.open(self.disabled_file.guarded_file_path, 'wb') as fp: fp.write(b'disabled content') assert not storage.exists(self.disabled_file.file_path) assert storage.exists(self.disabled_file.guarded_file_path) cron.unhide_disabled_files() assert storage.exists(self.file_.file_path) assert not storage.exists(self.file_.guarded_file_path) # The disabled file shouldn't have moved. assert not storage.exists(self.disabled_file.file_path) assert storage.exists(self.disabled_file.guarded_file_path) # The directory in guarded file path should still exist. assert storage.exists(os.path.dirname(self.file_.guarded_file_path))
def test_media_storage(self): tmp_storage = MediaStorage() tmp_storage.save(self.test_string) name = tmp_storage.name tmp_storage = MediaStorage(name=name) self.assertEqual(self.test_string, tmp_storage.read()) self.assertTrue(default_storage.exists(tmp_storage.get_full_path())) tmp_storage.remove() self.assertFalse(default_storage.exists(tmp_storage.get_full_path()))
def test_download__success(self): """ Scenario: Download image, should download the image Expected: - image saved in collected images and - its thumbnail is generated """ url = 'http://s1.proxy03.twitpic.com/photos/large/399879761.jpg' download(url) self.assertTrue(default_storage.exists(self.image_path)) self.assertTrue(default_storage.exists(self.thumb_image_path))
def check_img_url(value, thumb = False): if not thumb: if default_storage.exists(value.file_path): return value.file_path.url else: return "http://placehold.it/%sx%s" % (value.file_height, value.file_width) else: if default_storage.exists(value.file_path): return value.thumb_path.url else: return "http://placehold.it/%sx%s" % (value.thumb_height, value.thumb_width)
def post(self, request, competition_name, round_name, param): """ B{Set} the round file B{URL:} ../api/v1/set_round_file/<competition_name>/<round_name>/<param>/ Upload resource :type competition_name: str :param competition_name: The competition name :type round_name: str :param round_name: The round name :type param: str :param param: grid, lab or param_list :type path: str :param path: The path to the file """ path = request.data.get('path', None) if path is None: return Response({'status': 'Bad request', 'message': 'The file that you requested doesn\'t exists!'}, status=status.HTTP_400_BAD_REQUEST) if not default_storage.exists(path): return Response({'status': 'Bad request', 'message': 'The file that you requested doesn\'t exists!'}, status=status.HTTP_400_BAD_REQUEST) # verify if is a valid path if not path.startswith('resources/'): return Response({'status': 'Bad request', 'message': 'Invalid file!'}, status=status.HTTP_400_BAD_REQUEST) if param not in ['grid', 'lab', 'param_list']: return Response({'status': 'Bad request', 'message': 'Please provide one of those params: grid, lab or param_list'}, status=status.HTTP_400_BAD_REQUEST) competition = get_object_or_404(Competition.objects.all(), name=competition_name) r = get_object_or_404(Round.objects.all(), name=round_name, parent_competition=competition) if bool(getattr(r, param + '_path', '')) and default_storage.exists(getattr(r, param + '_path', '')) \ and getattr(r, param + "_can_delete"): default_storage.delete(getattr(r, param + "_path", None)) setattr(r, param + "_path", default_storage.path(path)) setattr(r, param + "_can_delete", False) r.save() return Response({'status': 'Uploaded', 'message': 'The file has been associated!'}, status=status.HTTP_201_CREATED)
def deletion(self, isMenu): if isMenu: if default_storage.exists(os.path.join(settings.MEDIA_ROOT,''.join([self.menuName]))): shutil.rmtree(''.join([settings.MEDIA_ROOT,self.menuName]), ignore_errors=True) return if default_storage.exists(os.path.join(settings.MEDIA_ROOT,self.logo.name)): if 'default.png' not in self.logo.name: os.remove(os.path.join(settings.MEDIA_ROOT,self.logo.name)) if default_storage.exists(os.path.join(settings.MEDIA_ROOT,self.thumbnail.name)): if 'default.png' not in self.thumbnail.name: os.remove(os.path.join(settings.MEDIA_ROOT,self.thumbnail.name)) return
def check_delete(self, file_, filename): """Test that when the File object is deleted, it is removed from the filesystem.""" try: with storage.open(filename, 'w') as f: f.write('sample data\n') assert storage.exists(filename) file_.delete() assert not storage.exists(filename) finally: if storage.exists(filename): storage.delete(filename)
def test_file_handling(self): cf = CachedFile() val = SimpleUploadedFile("testfile.txt", b"file_content") cf.file.save("testfile.txt", val) cf.type = "text/plain" cf.filename = "testfile.txt" cf.save() assert default_storage.exists(cf.file.name) with default_storage.open(cf.file.name, 'r') as f: assert f.read().strip() == "file_content" cf.delete() assert not default_storage.exists(cf.file.name)
def test_delete_by_version(self): """Test that version (soft)delete doesn't delete the file.""" f = File.objects.get(pk=67442) try: with storage.open(f.file_path, 'w') as fi: fi.write('sample data\n') assert storage.exists(f.file_path) f.version.delete() assert storage.exists(f.file_path) finally: if storage.exists(f.file_path): storage.delete(f.file_path)
def handle_noargs(self, **options): # First, clear all adjusted images that reference nonexistant # storage paths. storage_paths = AdjustedImage.objects.values_list('storage_path', flat=True).distinct() nonexistant = [path for path in storage_paths if not default_storage.exists(path)] self._delete_queryset(AdjustedImage.objects.filter(storage_path__in=nonexistant)) # Second, clear all areas that reference nonexistant storage paths. storage_paths = Area.objects.values_list('storage_path', flat=True).distinct() nonexistant = [path for path in storage_paths if not default_storage.exists(path)] self._delete_queryset(Area.objects.filter(storage_path__in=nonexistant)) # Now clear all duplicate adjusted images. fields = ( 'storage_path', 'requested_width', 'requested_height', 'requested_crop', 'requested_adjustment', 'requested_max_width', 'requested_max_height' ) kwargs_list = AdjustedImage.objects.values(*fields ).annotate(count=models.Count('id') ).filter(count__gt=1 ).values(*fields) duplicate_pks = [] for kwargs in kwargs_list: pks = AdjustedImage.objects.filter(**kwargs).values_list('pk', flat=True) duplicate_pks.extend(list(pks)[1:]) self._delete_queryset(AdjustedImage.objects.filter(pk__in=duplicate_pks), reason='is a duplicate', reason_plural='are duplicates') # Now clean up files that aren't referenced by any adjusted images. known_paths = set(AdjustedImage.objects.values_list('adjusted', flat=True).distinct()) orphaned_count = 0 for dirpath, dirnames, filenames in self._walk('daguerre', topdown=False): for filename in filenames: filepath = os.path.join(dirpath, filename) if filepath not in known_paths: orphaned_count += 1 try: default_storage.delete(filepath) except IOError: pass if not orphaned_count: self.stdout.write("No orphaned files found.\n") else: self.stdout.write("Deleted {0} orphaned file{1}.\n".format(orphaned_count, pluralize(orphaned_count))) self.stdout.write("\n")
def test_remove_local_files(self): file_path = join(config.TEMP_DIR, 'test_remove_local.txt') with open(file_path, 'w') as wfile: wfile.write('dummy') new_path = utils.move_to_storage(storage, file_path, 'tests') local = models.LocalContent( url='http://whatever', local_path=new_path, ) self.assertEquals(storage.exists(new_path), True) local.remove_files() self.assertEqual(local.local_path, '') self.assertEquals(storage.exists(new_path), False)
def image_status(request, addon_id, addon, icon_size=64): # Default icon needs no checking. if not addon.icon_type or addon.icon_type.split('/')[0] == 'icon': icons = True else: icons = storage.exists( os.path.join(addon.get_icon_dir(), '%s-%s.png' % ( addon.id, icon_size))) previews = all(storage.exists(p.thumbnail_path) for p in addon.get_previews()) return {'overall': icons and previews, 'icons': icons, 'previews': previews}
def test_ban_and_disable_related_content_bulk(self, hide_disabled_mock): user_sole = user_factory(email='*****@*****.**', fxa_id='13579') addon_sole = addon_factory(users=[user_sole]) self.setup_user_to_be_have_content_disabled(user_sole) user_multi = user_factory(email='*****@*****.**', fxa_id='24680') innocent_user = user_factory() addon_multi = addon_factory( users=UserProfile.objects.filter( id__in=[user_multi.id, innocent_user.id])) self.setup_user_to_be_have_content_disabled(user_multi) # Now that everything is set up, disable/delete related content. UserProfile.ban_and_disable_related_content_bulk( [user_sole, user_multi]) addon_sole.reload() addon_multi.reload() # if sole dev should have been disabled, but the author retained assert addon_sole.status == amo.STATUS_DISABLED assert list(addon_sole.authors.all()) == [user_sole] # shouldn't have been disabled as it has another author assert addon_multi.status != amo.STATUS_DISABLED assert list(addon_multi.authors.all()) == [innocent_user] # the File objects have been disabled assert not File.objects.filter(version__addon=addon_sole).exclude( status=amo.STATUS_DISABLED).exists() # But not for the Add-on that wasn't disabled assert File.objects.filter(version__addon=addon_multi).exclude( status=amo.STATUS_DISABLED).exists() assert not user_sole._ratings_all.exists() # Even replies. assert not user_sole.collections.exists() assert not user_multi._ratings_all.exists() # Even replies. assert not user_multi.collections.exists() assert not storage.exists(user_sole.picture_path) assert not storage.exists(user_sole.picture_path_original) assert not storage.exists(user_multi.picture_path) assert not storage.exists(user_multi.picture_path_original) assert user_sole.deleted assert user_sole.email == '*****@*****.**' assert user_sole.auth_id assert user_sole.fxa_id == '13579' assert user_multi.deleted assert user_multi.email == '*****@*****.**' assert user_multi.auth_id assert user_multi.fxa_id == '24680' hide_disabled_mock.assert_not_called()
def test_copy_to_mirror(self): f = File.objects.get(id=67442) self.clean_files(f) f.copy_to_mirror() assert storage.exists(f.mirror_file_path)
def save_to_files(request): if request.method == "POST": print("POST: ", request.POST) print("FILES: ", request.FILES) try: form = UploadFileForm(data=request.POST, files=request.FILES) if form.is_valid(): # get md5 value. Note: consider (file + parameters) as a whole md5 form_file = form.cleaned_data['file'] str_parameters = form.cleaned_data['parameters'] print(str_parameters) print("OK here0") parameters = json.loads(str_parameters) print(parameters) print("OK here1") b_file = form_file.read() print("OK here 2") file_parameters = str_parameters.encode('utf-8') + b_file md5 = hashlib.md5(file_parameters).hexdigest() sub_base = "md5_data/" path = sub_base + md5 # check if the file exists md5ob = get_object_or_None(UploadParametersMD5, md5=md5) print("Md5 existed in DB?: ", md5ob) if md5ob: status_old = md5ob.status print("existed in databse, code: ", status_old) time_ = md5ob.time if status_old == 200: return_json = [{ 'md5': md5, 'time': time_, 'save_status': 'Finished' }] return JsonResponse(return_json, safe=False) ##elif status_old == 201: ## return_json = [{'md5': md5, 'time': time_, 'save_status': True}] ## return JsonResponse(return_json, safe=False) elif status_old == 202: return_json = [{ 'md5': md5, 'time': time_, 'save_status': "Running" }] return JsonResponse(return_json, safe=False) # check if the file existed in filesystem if md5ob: print("Previous saving failed, Re-saving now...") if default_storage.exists(path): default_storage.delete(path) time_ = time.time() return_json = [{'md5': md5, 'time': time_}] # store md5 value and parameters into database, store file print("saving upload file...") path = default_storage.save( sub_base + md5, form_file ) # note this path doesnot include the media root, e.g. it is actually stored in "media/data/xxxxxx" file_path = settings.MEDIA_ROOT + '/' + path # distribute parameters details file_type = parameters['filetype'] species = parameters['species'] # insert to data base the info of file, paramerter and time # initial code 202 print("create model instance") a = UploadParametersMD5(md5=md5, status=202, file_type=file_type, species=species, time=time_, path=file_path) a.save() return_json = [{ 'md5': md5, 'time': time_, 'save_status': True }] return JsonResponse(return_json, safe=False) else: print("Form not valid") print("Error: ", form.errors) except Exception as e: print("Save to file Failed: ", e) return_json = [{'md5': "", 'time': 0.0, 'save_status': False}] return JsonResponse(return_json, safe=False)
def download(request, resourceid, sender=Layer): instance = resolve_object( request, sender, {'pk': resourceid}, permission='base.download_resourcebase', permission_msg=_( "You are not permitted to save or edit this resource.")) if isinstance(instance, Layer): try: upload_session = instance.get_upload_session() layer_files = [ item for idx, item in enumerate( LayerFile.objects.filter(upload_session=upload_session)) ] # Create Target Folder dirpath = tempfile.mkdtemp() dir_time_suffix = get_dir_time_suffix() target_folder = os.path.join(dirpath, dir_time_suffix) if not os.path.exists(target_folder): os.makedirs(target_folder) # Copy all Layer related files into a temporary folder for l in layer_files: if storage.exists(l.file): geonode_layer_path = storage.path(l.file) base_filename, original_ext = os.path.splitext( geonode_layer_path) shutil.copy2(geonode_layer_path, target_folder) # Let's check for associated SLD files (if any) try: for s in instance.styles.all(): sld_file = os.path.join(target_folder, "".join([s.name, ".sld"])) sld_file = open(sld_file, "w") sld_file.write(s.sld_body.strip()) sld_file.close() sld_file = open(sld_file, "w") try: response = requests.get(s.sld_url, timeout=TIMEOUT) sld_remote_content = response.text sld_file = os.path.join( target_folder, "".join([s.name, "_remote.sld"])) sld_file.write(sld_remote_content.strip()) except: traceback.print_exc() tb = traceback.format_exc() logger.debug(tb) finally: sld_file.close() except: traceback.print_exc() tb = traceback.format_exc() logger.debug(tb) # Let's dump metadata target_md_folder = os.path.join(target_folder, ".metadata") if not os.path.exists(target_md_folder): os.makedirs(target_md_folder) try: links = Link.objects.filter(resource=instance.resourcebase_ptr) for link in links: link_name = custom_slugify(link.name) link_file = os.path.join( target_md_folder, "".join([link_name, ".%s" % link.extension])) if link.link_type in ('metadata', 'data', 'image'): link_file = open(link_file, "wb") try: response = requests.get(link.url, stream=True, timeout=TIMEOUT) response.raw.decode_content = True shutil.copyfileobj(response.raw, link_file) except: traceback.print_exc() tb = traceback.format_exc() logger.debug(tb) finally: link_file.close() elif link.link_type.startswith('OGC'): link_file = open(link_file, "w") link_file.write(link.url.strip()) link_file.close() except: traceback.print_exc() tb = traceback.format_exc() logger.debug(tb) # ZIP everything and return target_file_name = "".join([instance.name, ".zip"]) target_file = os.path.join(dirpath, target_file_name) zip_dir(target_folder, target_file) response = HttpResponse(content=open(target_file), status=200, content_type="application/zip") response[ 'Content-Disposition'] = 'attachment; filename="%s"' % target_file_name return response except NotImplementedError: traceback.print_exc() tb = traceback.format_exc() logger.debug(tb) return HttpResponse(json.dumps({'error': 'file_not_found'}), status=404, content_type="application/json") return HttpResponse(json.dumps({'error': 'unauthorized_request'}), status=403, content_type="application/json")
def testExistsRelative(self): self.assertFalse( default_storage.exists("admin/css/../img/sorting-icons.svg")) with self.save_file("admin/img/sorting-icons.svg"): self.assertTrue( default_storage.exists("admin/css/../img/sorting-icons.svg"))
def testExists(self): self.assertFalse(default_storage.exists("foo.txt")) with self.save_file(): self.assertTrue(default_storage.exists("foo.txt")) self.assertFalse(default_storage.exists("fo"))
def _unprep_package(self, name): package = packager_path(name) if storage.exists(package): storage.delete(package)
def test_move_chunking(self): src = self.newfile('src.txt', '<contents>') dest = self.path('somedir/dest.txt') move_stored_file(src, dest, chunk_size=1) assert self.contents(dest) == b'<contents>' assert not storage.exists(src)
def create_thumbnail(instance, thumbnail_remote_url, thumbnail_create_url=None, check_bbox=False, ogc_client=None, overwrite=False, width=240, height=200): thumbnail_dir = os.path.join(settings.MEDIA_ROOT, 'thumbs') if not os.path.exists(thumbnail_dir): os.makedirs(thumbnail_dir) thumbnail_name = None if isinstance(instance, Layer): thumbnail_name = 'layer-%s-thumb.png' % instance.uuid elif isinstance(instance, Map): thumbnail_name = 'map-%s-thumb.png' % instance.uuid thumbnail_path = os.path.join(thumbnail_dir, thumbnail_name) if overwrite or not storage.exists(thumbnail_path): BBOX_DIFFERENCE_THRESHOLD = 1e-5 if not thumbnail_create_url: thumbnail_create_url = thumbnail_remote_url if check_bbox: # Check if the bbox is invalid valid_x = (float(instance.bbox_x0) - float(instance.bbox_x1))**2 > BBOX_DIFFERENCE_THRESHOLD valid_y = (float(instance.bbox_y1) - float(instance.bbox_y0))**2 > BBOX_DIFFERENCE_THRESHOLD else: valid_x = True valid_y = True image = None if valid_x and valid_y: Link.objects.get_or_create(resource=instance.get_self_resource(), url=thumbnail_remote_url, defaults=dict( extension='png', name="Remote Thumbnail", mime='image/png', link_type='image', )) ResourceBase.objects.filter(id=instance.id) \ .update(thumbnail_url=thumbnail_remote_url) # Download thumbnail and save it locally. if not ogc_client: ogc_client = http_client if ogc_client: try: params = {'width': width, 'height': height} # Add the bbox param only if the bbox is different to [None, None, # None, None] if None not in instance.bbox: params['bbox'] = instance.bbox_string params['crs'] = instance.srid for _p in params.keys(): if _p.lower() not in thumbnail_create_url.lower(): thumbnail_create_url = thumbnail_create_url + '&%s=%s' % ( _p, params[_p]) resp, image = ogc_client.request(thumbnail_create_url) if 'ServiceException' in image or \ resp.status_code < 200 or resp.status_code > 299: msg = 'Unable to obtain thumbnail: %s' % image logger.error(msg) # Replace error message with None. image = None except BaseException as e: logger.exception(e) # Replace error message with None. image = None if check_ogc_backend(geoserver.BACKEND_PACKAGE): if image is None and instance.bbox: instance_bbox = instance.bbox[0:4] request_body = { 'bbox': [str(coord) for coord in instance_bbox], 'srid': instance.srid, 'width': width, 'height': height } if thumbnail_create_url: request_body[ 'thumbnail_create_url'] = thumbnail_create_url elif instance.alternate: request_body['layers'] = instance.alternate image = _prepare_thumbnail_body_from_opts(request_body) if image is not None: instance.save_thumbnail(thumbnail_name, image=image) else: msg = 'Unable to obtain thumbnail for: %s' % instance logger.error(msg)
def thumbnail(image_url, width, height, quality=95): """ Given the URL to an image, resizes the image using the given width and height on the first time it is requested, and returns the URL to the new resized image. if width or height are zero then original ratio is maintained. """ if not image_url: return "" image_url = unquote(unicode(image_url)) if image_url.startswith(settings.MEDIA_URL): image_url = image_url.replace(settings.MEDIA_URL, "", 1) image_dir, image_name = os.path.split(image_url) image_prefix, image_ext = os.path.splitext(image_name) filetype = {".png": "PNG", ".gif": "GIF"}.get(image_ext, "JPEG") thumb_name = "%s-%sx%s%s" % (image_prefix, width, height, image_ext) thumb_dir = os.path.join(settings.MEDIA_ROOT, image_dir, settings.THUMBNAILS_DIR_NAME) if not os.path.exists(thumb_dir): os.makedirs(thumb_dir) thumb_path = os.path.join(thumb_dir, thumb_name) thumb_url = "%s/%s" % (settings.THUMBNAILS_DIR_NAME, quote(thumb_name.encode("utf-8"))) image_url_path = os.path.dirname(image_url) if image_url_path: thumb_url = "%s/%s" % (image_url_path, thumb_url) try: thumb_exists = os.path.exists(thumb_path) except UnicodeEncodeError: # The image that was saved to a filesystem with utf-8 support, # but somehow the locale has changed and the filesystem does not # support utf-8. from mezzanine.core.exceptions import FileSystemEncodingChanged raise FileSystemEncodingChanged() if thumb_exists: # Thumbnail exists, don't generate it. return thumb_url elif not default_storage.exists(image_url): # Requested image does not exist, just return its URL. return image_url image = Image.open(default_storage.open(image_url)) image_info = image.info width = int(width) height = int(height) # If already right size, don't do anything. if width == image.size[0] and height == image.size[1]: return image_url # Set dimensions. if width == 0: width = image.size[0] * height / image.size[1] elif height == 0: height = image.size[1] * width / image.size[0] if image.mode not in ("L", "RGBA"): image = image.convert("RGBA") # Required for progressive jpgs. ImageFile.MAXBLOCK = image.size[0] * image.size[1] try: image = ImageOps.fit(image, (width, height), Image.ANTIALIAS) image = image.save(thumb_path, filetype, quality=quality, **image_info) # Push a remote copy of the thumbnail if MEDIA_URL is # absolute. if "://" in settings.MEDIA_URL: with open(thumb_path, "r") as f: default_storage.save(thumb_url, File(f)) except Exception: # If an error occurred, a corrupted image may have been saved, # so remove it, otherwise the check for it existing will just # return the corrupted image next time it's requested. try: os.remove(thumb_path) except Exception: pass return image_url return thumb_url
def import_dir_locked(): arp_root = get_arp_root() return default_storage.exists(settings.ARP_IMPORT_LOCK)
def get_arp_root(): if not default_storage.exists(settings.ARP_ROOT): os.mkdir(default_storage.path(settings.ARP_ROOT)) return settings.ARP_ROOT
def create_paths(self): if not storage.exists(self.file.file_path): with storage.open(self.file.file_path, 'w') as f: f.write('test data\n')
def upload_to_internet_archive(self, link_guid): try: link = Link.objects.get(guid=link_guid) if link.internet_archive_upload_status == 'failed_permanently': return except: print "Link %s does not exist" % link_guid return if not settings.UPLOAD_TO_INTERNET_ARCHIVE: return if not link.can_upload_to_internet_archive(): print "Not eligible for upload." return metadata = { "collection": settings.INTERNET_ARCHIVE_COLLECTION, "title": '%s: %s' % (link_guid, truncatechars(link.submitted_title, 50)), "mediatype": 'web', "description": 'Perma.cc archive of %s created on %s.' % ( link.submitted_url, link.creation_timestamp, ), "contributor": 'Perma.cc', "submitted_url": link.submitted_url, "perma_url": "http://%s/%s" % (settings.HOST, link_guid), "external-identifier": 'urn:X-perma:%s' % link_guid, } identifier = settings.INTERNET_ARCHIVE_IDENTIFIER_PREFIX + link_guid try: if default_storage.exists(link.warc_storage_file()): item = internetarchive.get_item(identifier) # if item already exists (but has been removed), # ia won't update its metadata in upload function if item.exists and item.metadata['title'] == 'Removed': item.modify_metadata( metadata, access_key=settings.INTERNET_ARCHIVE_ACCESS_KEY, secret_key=settings.INTERNET_ARCHIVE_SECRET_KEY, ) warc_name = os.path.basename(link.warc_storage_file()) # copy warc to local disk storage for upload temp_warc_file = tempfile.TemporaryFile() copy_file_data(default_storage.open(link.warc_storage_file()), temp_warc_file) temp_warc_file.seek(0) success = internetarchive.upload( identifier, {warc_name: temp_warc_file}, metadata=metadata, access_key=settings.INTERNET_ARCHIVE_ACCESS_KEY, secret_key=settings.INTERNET_ARCHIVE_SECRET_KEY, retries=10, retries_sleep=60, verbose=True, ) if success: link.internet_archive_upload_status = 'completed' link.save() else: link.internet_archive_upload_status = 'failed' self.retry( exc=Exception("Internet Archive reported upload failure.")) return else: link.internet_archive_upload_status = 'failed_permanently' link.save() except requests.ConnectionError as e: logger.exception( "Upload to Internet Archive task failed because of a connection error. \nLink GUID: %s\nError: %s" % (link.pk, e)) return
def file_exists(self): return default_storage.exists(str(self.image))
def handle_upload_validation_result(results, upload_pk, channel): """Annotate a set of validation results and save them to the given FileUpload instance.""" upload = FileUpload.objects.get(pk=upload_pk) if (not upload.addon_id or not upload.addon.find_latest_version(channel=channel, exclude=())): # Legacy submission restrictions apply if: # - It's the very first upload (there is no addon id yet) # - It's the first upload in that channel results = annotate_new_legacy_addon_restrictions(results=results) if upload.addon_id and upload.version: results = annotate_webext_incompatibilities( results=results, file_=None, addon=upload.addon, version_string=upload.version, channel=channel) results = skip_signing_warning_if_signing_server_not_configured(results) upload.validation = json.dumps(results) upload.save() # We want to hit the custom save(). # Track the time it took from first upload through validation # until the results were processed and saved. upload_start = utc_millesecs_from_epoch(upload.created) now = datetime.datetime.now() now_ts = utc_millesecs_from_epoch(now) delta = now_ts - upload_start statsd.timing('devhub.validation_results_processed', delta) if not storage.exists(upload.path): # TODO: actually fix this so we can get stats. It seems that # the file maybe gets moved but it needs more investigation. log.warning('Scaled upload stats were not tracked. File is ' 'missing: {}'.format(upload.path)) return size = Decimal(storage.size(upload.path)) megabyte = Decimal(1024 * 1024) # Stash separate metrics for small / large files. quantifier = 'over' if size > megabyte else 'under' statsd.timing( 'devhub.validation_results_processed_{}_1mb'.format(quantifier), delta) # Scale the upload / processing time by package size (in MB) # so we can normalize large XPIs which naturally take longer to validate. scaled_delta = None size_in_mb = size / megabyte if size > 0: # If the package is smaller than 1MB, don't scale it. This should # help account for validator setup time. unit = size_in_mb if size > megabyte else Decimal(1) scaled_delta = Decimal(delta) / unit statsd.timing('devhub.validation_results_processed_per_mb', scaled_delta) log.info('Time to process and save upload validation; ' 'upload.pk={upload}; processing_time={delta}; ' 'scaled_per_mb={scaled}; upload_size_in_mb={size_in_mb}; ' 'created={created}; now={now}'.format(delta=delta, upload=upload.pk, created=upload.created, now=now, scaled=scaled_delta, size_in_mb=size_in_mb))
def testExistsDir(self): self.assertFalse(default_storage.exists("foo/")) with self.save_file(name="foo/bar.txt"): self.assertTrue(default_storage.exists("foo/"))
def file_exists(self): return default_storage.exists(self.file.name)
def thumbnail( image_url, width, height, upscale=True, quality=95, left=0.5, top=0.5, padding=False, padding_color="#fff", ): """ Given the URL to an image, resizes the image using the given width and height on the first time it is requested, and returns the URL to the new resized image. If width or height are zero then original ratio is maintained. When ``upscale`` is False, images smaller than the given size will not be grown to fill that size. The given width and height thus act as maximum dimensions. """ if not image_url: return "" try: from PIL import Image, ImageFile, ImageOps except ImportError: return "" image_url = unquote(str(image_url)).split("?")[0] if image_url.startswith(settings.MEDIA_URL): image_url = image_url.replace(settings.MEDIA_URL, "", 1) image_dir, image_name = os.path.split(image_url) image_prefix, image_ext = os.path.splitext(image_name) filetype = {".png": "PNG", ".gif": "GIF"}.get(image_ext.lower(), "JPEG") thumb_name = "%s-%sx%s" % (image_prefix, width, height) if not upscale: thumb_name += "-no-upscale" if left != 0.5 or top != 0.5: left = min(1, max(0, left)) top = min(1, max(0, top)) thumb_name = "%s-%sx%s" % (thumb_name, left, top) thumb_name += "-padded-%s" % padding_color if padding else "" thumb_name = "%s%s" % (thumb_name, image_ext) # `image_name` is used here for the directory path, as each image # requires its own sub-directory using its own name - this is so # we can consistently delete all thumbnails for an individual # image, which is something we do in filebrowser when a new image # is written, allowing us to purge any previously generated # thumbnails that may match a new image name. thumb_dir = os.path.join( settings.MEDIA_ROOT, image_dir, settings.THUMBNAILS_DIR_NAME, image_name ) if not os.path.exists(thumb_dir): try: os.makedirs(thumb_dir) except OSError: pass thumb_path = os.path.join(thumb_dir, thumb_name) thumb_url = "%s/%s/%s" % ( settings.THUMBNAILS_DIR_NAME, quote(image_name.encode("utf-8")), quote(thumb_name.encode("utf-8")), ) image_url_path = os.path.dirname(image_url) if image_url_path: thumb_url = "%s/%s" % (image_url_path, thumb_url) try: thumb_exists = os.path.exists(thumb_path) except UnicodeEncodeError: # The image that was saved to a filesystem with utf-8 support, # but somehow the locale has changed and the filesystem does not # support utf-8. from mezzanine.core.exceptions import FileSystemEncodingChanged raise FileSystemEncodingChanged() if thumb_exists: # Thumbnail exists, don't generate it. return thumb_url elif not default_storage.exists(image_url): # Requested image does not exist, just return its URL. return image_url f = default_storage.open(image_url) try: image = Image.open(f) except: # noqa # Invalid image format. return image_url image_info = image.info # Transpose to align the image to its orientation if necessary. # If the image is transposed, delete the exif information as # not all browsers support the CSS image-orientation: # - http://caniuse.com/#feat=css-image-orientation try: orientation = image._getexif().get(0x0112) except: # noqa orientation = None if orientation: methods = { 2: (Image.FLIP_LEFT_RIGHT,), 3: (Image.ROTATE_180,), 4: (Image.FLIP_TOP_BOTTOM,), 5: (Image.FLIP_LEFT_RIGHT, Image.ROTATE_90), 6: (Image.ROTATE_270,), 7: (Image.FLIP_LEFT_RIGHT, Image.ROTATE_270), 8: (Image.ROTATE_90,), }.get(orientation, ()) if methods: image_info.pop("exif", None) for method in methods: image = image.transpose(method) to_width = int(width) to_height = int(height) from_width = image.size[0] from_height = image.size[1] if not upscale: to_width = min(to_width, from_width) to_height = min(to_height, from_height) # Set dimensions. if to_width == 0: to_width = from_width * to_height // from_height elif to_height == 0: to_height = from_height * to_width // from_width if image.mode not in ("P", "L", "RGBA") and filetype not in ("JPG", "JPEG"): try: image = image.convert("RGBA") except: # noqa return image_url # Required for progressive jpgs. ImageFile.MAXBLOCK = 2 * (max(image.size) ** 2) # Padding. if padding and to_width and to_height: from_ratio = float(from_width) / from_height to_ratio = float(to_width) / to_height pad_size = None if to_ratio < from_ratio: pad_height = int(to_height * (float(from_width) / to_width)) pad_size = (from_width, pad_height) pad_top = (pad_height - from_height) // 2 pad_left = 0 elif to_ratio > from_ratio: pad_width = int(to_width * (float(from_height) / to_height)) pad_size = (pad_width, from_height) pad_top = 0 pad_left = (pad_width - from_width) // 2 if pad_size is not None: pad_container = Image.new("RGBA", pad_size, padding_color) pad_container.paste(image, (pad_left, pad_top)) image = pad_container # Create the thumbnail. to_size = (to_width, to_height) to_pos = (left, top) try: image = ImageOps.fit(image, to_size, Image.ANTIALIAS, 0, to_pos) image = image.save(thumb_path, filetype, quality=quality, **image_info) # Push a remote copy of the thumbnail if MEDIA_URL is # absolute. if "://" in settings.MEDIA_URL: with open(thumb_path, "rb") as f: default_storage.save(unquote(thumb_url), File(f)) except Exception: # If an error occurred, a corrupted image may have been saved, # so remove it, otherwise the check for it existing will just # return the corrupted image next time it's requested. try: os.remove(thumb_path) except Exception: pass return image_url return thumb_url
def test_moderated_upload(self): """ Test if moderation flag works """ with self.settings(ADMIN_MODERATE_UPLOADS=False): self.client.login(username=self.user, password=self.passwd) input_path = self._get_input_path() with open(input_path, 'rb') as f: data = { 'title': 'document title', 'doc_file': f, 'resource': '', 'extension': 'txt', 'permissions': '{}', } resp = self.client.post(self.document_upload_url, data=data) self.assertEqual(resp.status_code, 200) dname = 'document title' _d = Document.objects.get(title=dname) self.assertTrue(_d.is_published) uuid = _d.uuid _d.delete() from geonode.documents.utils import delete_orphaned_document_files _, document_files_before = storage.listdir( os.path.join("documents", "document")) deleted = delete_orphaned_document_files() _, document_files_after = storage.listdir( os.path.join("documents", "document")) self.assertTrue(len(deleted) > 0) self.assertEqual( set(deleted), set(document_files_before) - set(document_files_after)) from geonode.base.utils import delete_orphaned_thumbs thumb_files_before = get_thumbs() deleted = delete_orphaned_thumbs() thumb_files_after = get_thumbs() if len(thumb_files_before): self.assertTrue( len(deleted) > 0, f"before: {thumb_files_before} - deleted: {deleted} - after: {thumb_files_after}" ) self.assertEqual( set(deleted), set(thumb_files_before) - set(thumb_files_after), f"deleted: {deleted} vs {set(thumb_files_before) - set(thumb_files_after)}" ) fn = os.path.join(os.path.join("documents", "document"), os.path.basename(input_path)) self.assertFalse(storage.exists(fn)) files = [thumb for thumb in get_thumbs() if uuid in thumb] self.assertEqual(len(files), 0) with self.settings(ADMIN_MODERATE_UPLOADS=True): self.client.login(username=self.user, password=self.passwd) norman = get_user_model().objects.get(username="******") group = GroupProfile.objects.get(slug="bar") input_path = self._get_input_path() with open(input_path, 'rb') as f: data = { 'title': 'document title', 'doc_file': f, 'resource': '', 'extension': 'txt', 'permissions': '{}', } resp = self.client.post(self.document_upload_url, data=data) self.assertEqual(resp.status_code, 200) dname = 'document title' _d = Document.objects.get(title=dname) self.assertFalse(_d.is_approved) self.assertTrue(_d.is_published) group.join(norman) self.assertFalse(group.user_is_role(norman, "manager")) GroupMember.objects.get(group=group, user=norman).promote() self.assertTrue(group.user_is_role(norman, "manager")) self.client.login(username="******", password="******") resp = self.client.get(reverse('document_detail', args=(_d.id, ))) # Forbidden self.assertEqual(resp.status_code, 403) _d.group = group.group _d.save() resp = self.client.get(reverse('document_detail', args=(_d.id, ))) # Allowed - edit permissions self.assertEqual(resp.status_code, 200) perms_list = get_perms(norman, _d.get_self_resource()) + get_perms( norman, _d) self.assertTrue('change_resourcebase_metadata' in perms_list) GroupMember.objects.get(group=group, user=norman).demote() self.assertFalse(group.user_is_role(norman, "manager")) resp = self.client.get(reverse('document_detail', args=(_d.id, ))) # Allowed - no edit self.assertEqual(resp.status_code, 200) perms_list = get_perms(norman, _d.get_self_resource()) + get_perms( norman, _d) self.assertFalse('change_resourcebase_metadata' in perms_list) group.leave(norman)
def delete_photo(self, photo): if default_storage.exists(photo.name): default_storage.delete(photo.name)
def run_validator(path, for_appversions=None, test_all_tiers=False, overrides=None, compat=False, listed=True): """A pre-configured wrapper around the addon validator. *file_path* Path to addon / extension file to validate. *for_appversions=None* An optional dict of application versions to validate this addon for. The key is an application GUID and its value is a list of versions. *test_all_tiers=False* When False (default) the validator will not continue if it encounters fatal errors. When True, all tests in all tiers are run. See bug 615426 for discussion on this default. *overrides=None* Normally the validator gets info from the manifest but there are a few things we need to override. See validator for supported overrides. Example: {'targetapp_maxVersion': {'<app guid>': '<version>'}} *compat=False* Set this to `True` when performing a bulk validation. This allows the validator to ignore certain tests that should not be run during bulk validation (see bug 735841). *listed=True* If the addon is unlisted, treat it as if it was a self hosted one (don't fail on the presence of an updateURL). To validate the addon for compatibility with Firefox 5 and 6, you'd pass in:: for_appversions={amo.FIREFOX.guid: ['5.0.*', '6.0.*']} Not all application versions will have a set of registered compatibility tests. """ from validator.validate import validate apps = dump_apps.Command.JSON_PATH if not os.path.exists(apps): call_command('dump_apps') with NamedTemporaryFile(suffix='_' + os.path.basename(path)) as temp: if path and not os.path.exists(path) and storage.exists(path): # This file doesn't exist locally. Write it to our # currently-open temp file and switch to that path. copyfileobj(storage.open(path), temp.file) path = temp.name with statsd.timer('devhub.validator'): json_result = validate( path, for_appversions=for_appversions, format='json', # When False, this flag says to stop testing after one # tier fails. determined=test_all_tiers, approved_applications=apps, overrides=overrides, compat_test=compat, listed=listed ) track_validation_stats(json_result) return json_result
def has_been_copied(self): """Checks if file has been copied to mirror""" if not self.mirror_file_path: return False return storage.exists(self.mirror_file_path)
def test_file_on_mirror(self): # Make sure the mirror dir is clear. if storage.exists(os.path.dirname(self.file.mirror_file_path)): rmtree(os.path.dirname(self.file.mirror_file_path)) new_file = tasks.repackage_jetpack(self.builder_data()) assert storage.exists(new_file.mirror_file_path)
def test_cachedir_tag(self): self.assertTrue(default_storage.exists(PHOTOLOGUE_CACHEDIRTAG)) content = default_storage.open(PHOTOLOGUE_CACHEDIRTAG).read() self.assertEquals(content, b"Signature: 8a477f597d28d172789f06886806bc55")
def create(self, request, *args, **kwargs): """ 简化版大文件上传接口, 同时可处理大文件上传, 但不支持断点续传 """ chunk_file = request.data.get('chunk_file', None) # 分块文件 chunk_md5 = request.data.get('chunk_md5', None) # 分块文件MD5 chunk_index = request.data.get('chunk_index', None) # 分块文件顺序 chunks_num = request.data.get('chunks_num', None) # 完整文件分块数目,用于确定当前分块是否已经全部上传完毕 file_name = request.data.get('file_name', None) # 完整文件的文件名称,用于所有分块上传完毕后合并的名称 file_md5 = request.data.get('file_md5', None) # 完整文件的文件md5,用于确定文件合并后是否正确 group_id = request.data.get('group', None) file_type = request.data.get('file_type', None) if not chunk_file or not chunk_index or not chunks_num or not file_name: result = {'success': False, 'messages': '上传失败, 缺少指定参数值: chunk_file/chunk_index/chunks_num/file_name'} return Response(result, status=status.HTTP_400_BAD_REQUEST) file_type_name = FileType.get(int(file_type), 'default') try: group = FileGroup.objects.get(pk=int(group_id)) group_name = group.name except: group = None group_name = 'None' file_path = f'{file_type_name}/{request.user.username}/{group_name}' base_path = create_or_get_directory(f'{settings.MEDIA_ROOT}/{file_path}') base_chunk_path = create_or_get_directory(f'{base_path}/{file_name}-tmp') chunk_path = os.path.join(base_chunk_path, f'{file_name}.part{chunk_index}') # default_storage不会覆盖文件, 若文件存在, 删除后重新上传 if default_storage.exists(chunk_path): default_storage.delete(chunk_path) # 保存 default_storage.save(chunk_path, ContentFile(chunk_file.read())) # 验证分块MD5是否正确 if chunk_md5: chunk_file_md5 = check_md5_sum(file_name=chunk_path) # 保存的分块文件内容不一致 if chunk_file_md5 != chunk_md5: result = {'success': False, 'messages': '文件上传发生错误, md5值不一致', 'results': {'upload_md5': chunk_md5, 'save_md5': chunk_file_md5}} return Response(result, status=status.HTTP_400_BAD_REQUEST) if int(chunk_index) == int(chunks_num): uploaded = True save_file_path = os.path.join(base_path, file_name) # 文件已经存在, 添加时间戳 if os.path.exists(save_file_path): save_file_path = os.path.join(base_path, f'{int(time.time())}-{file_name}') with open(save_file_path, 'wb') as uploaded_file: for index in range(int(chunks_num)): chunk_file = os.path.join(base_chunk_path, f'{file_name}.part{index + 1}') try: chunk_file = open(chunk_file, 'rb') # 按序打开每个分片 uploaded_file.write(chunk_file.read()) # 读取分片内容写入新文件 chunk_file.close() except Exception as error: print(f'合并文件:{file_name} form {base_chunk_path}失败:{error}') uploaded = False # 检查合并后的MD5 uploaded_file_md5 = check_md5_sum(save_file_path) if uploaded_file_md5 != file_md5: uploaded = False if uploaded: attachment = FileAttachment.objects.create(group=group, file_name=file_name, file_path=os.path.join(file_path, file_name), file_size=os.path.getsize(save_file_path), file_type=file_type, creator=request.user) shutil.rmtree(base_chunk_path) action_log(request=request, user=request.user, action_type=UploadAction, old_instance=None, instance=attachment, action_info=f'上传素材:{attachment.__str__()}') serializer = self.get_serializer(attachment) results = serializer.data results['uploaded'] = True result = {'success': True, 'messages': f'新增素材:{attachment.__str__()}', 'results': results} return Response(result, status=status.HTTP_200_OK) else: result = {'success': False, 'messages': '合并文件失败, 请重新上传'} return Response(result, status=status.HTTP_400_BAD_REQUEST) else: result = {'success': True, 'messages': f'成功上传文件:{file_name}, 分块:{chunk_index}', 'results': {'uploaded': False, 'chunk_index': chunk_index, 'file_name': file_name, } } return Response(result, status=status.HTTP_200_OK)
def remove_icons(destination): for size in ADDON_ICON_SIZES: filename = '%s-%s.png' % (destination, size) if storage.exists(filename): storage.delete(filename)
def photo_size(request, id, size, crop=False, quality=90, download=False, constrain=False): """ Renders image and returns response Does not use template Saves resized image within cache system Returns 404 if if image rendering fails """ if isinstance(quality, str) and quality.isdigit(): quality = int(quality) cache_key = generate_image_cache_key(file=id, size=size, pre_key=PHOTO_PRE_KEY, crop=crop, unique_key=id, quality=quality, constrain=constrain) cached_image = cache.get(cache_key) if cached_image: return redirect('{0}{1}'.format( get_setting('site', 'global', 'siteurl'), cached_image)) photo = get_object_or_404(Image, id=id) size = [int(s) for s in size.split('x')] size = aspect_ratio(photo.image_dimensions(), size, constrain) # check permissions if not has_perm(request.user, 'photos.view_image', photo): raise Http403 attachment = '' if download: attachment = 'attachment;' if not photo.image or not default_storage.exists(photo.image.name): raise Http404 # At this point, we didn't get the image from the cache. # Check if this particular thumbnail already exists on file system. # If it's there, no need to rebuild it from the original image! file_name = photo.image_filename() file_path = 'cached%s%s' % (request.path, file_name) if default_storage.exists(file_path): image = get_image_from_path( os.path.join(settings.MEDIA_ROOT, file_path)) else: # gets resized image from cache or rebuild image = get_image(photo.image, size, PHOTO_PRE_KEY, crop=crop, quality=quality, unique_key=str(photo.pk), constrain=constrain) # if image not rendered; quit if not image: raise Http404 response = HttpResponse(content_type='image/jpeg') response['Content-Disposition'] = '%s filename="%s"' % ( attachment, photo.image_filename()) image.convert('RGB').save(response, "JPEG", quality=quality) if photo.is_public_photo() and photo.is_public_photoset(): if not default_storage.exists(file_path): default_storage.save(file_path, ContentFile(response.content)) full_file_path = "%s%s" % (settings.MEDIA_URL, file_path) cache.set(cache_key, full_file_path) cache_group_key = "photos_cache_set.%s" % photo.pk cache_group_list = cache.get(cache_group_key) if cache_group_list is None: cache.set(cache_group_key, [cache_key]) else: cache_group_list += [cache_key] cache.set(cache_group_key, cache_group_list) return response
def create_thumbnail(instance, thumbnail_remote_url, thumbnail_create_url=None, check_bbox=False, ogc_client=None, overwrite=False): thumbnail_dir = os.path.join(settings.MEDIA_ROOT, 'thumbs') if not os.path.exists(thumbnail_dir): os.makedirs(thumbnail_dir) thumbnail_name = None if isinstance(instance, Layer): thumbnail_name = 'layer-%s-thumb.png' % instance.uuid elif isinstance(instance, Map): thumbnail_name = 'map-%s-thumb.png' % instance.uuid thumbnail_path = os.path.join(thumbnail_dir, thumbnail_name) if overwrite is True or storage.exists(thumbnail_path) is False: if not ogc_client: ogc_client = http_client BBOX_DIFFERENCE_THRESHOLD = 1e-5 if not thumbnail_create_url: thumbnail_create_url = thumbnail_remote_url if check_bbox: # Check if the bbox is invalid valid_x = (float(instance.bbox_x0) - float(instance.bbox_x1))**2 > BBOX_DIFFERENCE_THRESHOLD valid_y = (float(instance.bbox_y1) - float(instance.bbox_y0))**2 > BBOX_DIFFERENCE_THRESHOLD else: valid_x = True valid_y = True image = None if valid_x and valid_y: Link.objects.get_or_create(resource=instance.get_self_resource(), url=thumbnail_remote_url, defaults=dict( extension='png', name="Remote Thumbnail", mime='image/png', link_type='image', )) ResourceBase.objects.filter(id=instance.id) \ .update(thumbnail_url=thumbnail_remote_url) # Download thumbnail and save it locally. try: resp, image = ogc_client.request(thumbnail_create_url) if 'ServiceException' in image or \ resp.status < 200 or resp.status > 299: msg = 'Unable to obtain thumbnail: %s' % image raise Exception(msg) except BaseException: import traceback logger.debug(traceback.format_exc()) # Replace error message with None. image = None if image is not None: instance.save_thumbnail(thumbnail_name, image=image)
def testDelete(self): with self.save_file(): self.assertTrue(default_storage.exists("foo.txt")) default_storage.delete("foo.txt") self.assertFalse(default_storage.exists("foo.txt"))
def clean_files(self, f): if f.mirror_file_path and storage.exists(f.mirror_file_path): storage.delete(f.mirror_file_path) if not storage.exists(f.file_path): with storage.open(f.file_path, 'w') as fp: fp.write('sample data\n')