def process_file(sender, **kwargs): """ what we do here: - determine file type, set it. - generate file hash - check for file renames """ instance = kwargs.get("instance") if instance.file: name, extension = os.path.splitext(instance.file.name) if len(extension) > 1: instance.extension = extension[1:].lower() else: instance.extension = '' instance.type = OTHER_TYPE for type, definition in conf.FILE_TYPES.items(): if instance.extension in definition.get("extensions"): instance.type = type instance.generate_file_hash() if instance.id: old_instance = File.objects.get(pk=instance.id) if not old_instance.filename == instance.filename: # rename! new_file = DjangoFile(open(instance.file.path, mode='rb')) instance.file.delete(False) # remove including thumbs instance.file.save(instance.filename, new_file, save=False)
def test_extract_and_commit_source_from_version(settings): addon = addon_factory( file_kw={'filename': 'webextension_no_id.xpi'}, version_kw={'version': '0.1'}) # Generate source file source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH) with zipfile.ZipFile(source, 'w') as zip_file: zip_file.writestr('manifest.json', '{}') source.seek(0) addon.current_version.source = DjangoFile(source) addon.current_version.save() repo = AddonGitRepository.extract_and_commit_source_from_version( addon.current_version) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source') assert os.listdir(repo.git_repository_path) == ['.git'] # Verify via subprocess to make sure the repositories are properly # read by the regular git client output = _run_process('git branch', repo) assert 'listed' in output output = _run_process('git log listed', repo) expected = 'Create new version {} ({}) for {} from source file'.format( repr(addon.current_version), addon.current_version.id, repr(addon)) assert expected in output
def test_extract_and_commit_source_from_version_no_dotgit_clash(settings): addon = addon_factory( file_kw={'filename': 'webextension_no_id.xpi'}, version_kw={'version': '0.1'}) # Generate source file source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH) with zipfile.ZipFile(source, 'w') as zip_file: zip_file.writestr('manifest.json', '{}') zip_file.writestr('.git/config', '') source.seek(0) addon.current_version.source = DjangoFile(source) addon.current_version.save() with mock.patch('olympia.lib.git.uuid.uuid4') as uuid4_mock: uuid4_mock.return_value = mock.Mock( hex='b236f5994773477bbcd2d1b75ab1458f') repo = AddonGitRepository.extract_and_commit_source_from_version( addon.current_version) assert repo.git_repository_path == os.path.join( settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source') assert os.listdir(repo.git_repository_path) == ['.git'] # Verify via subprocess to make sure the repositories are properly # read by the regular git client output = _run_process('git ls-tree -r --name-only listed', repo) assert set(output.split()) == { 'extracted/manifest.json', 'extracted/.git.b236f599/config'}
def export(self, encode=False, export_file=False): task = self.getTask() rows = super(CSVMixin, self).export(encode=True) if export_file: with tempfile.NamedTemporaryFile() as csvfile: writer = csv.writer(csvfile) for row in rows: writer.writerow(row) dfile = DjangoFile(csvfile) f = File(creator=task.workflow.owner, file=dfile, filename='%s_result.csv' % task.title.lower(), linked=True) f.save() return f pseudo_buffer = Echo() writer = csv.writer(pseudo_buffer) response = StreamingHttpResponse( (writer.writerow(row) for row in rows), content_type="text/csv") response[ 'Content-Disposition'] = 'attachment; filename="%s.csv"' % filename( str(task)) return response
def test(self): f = open("base/apps.py", "rb") file = DjangoFile(f) assert file.name == "base/apps.py" file.open() file_hash = compute_file_hash(file) assert file_hash assert file.size
def setUp(self): self.addon = Addon.objects.get(pk=3615) self.version = self.addon._latest_version tdir = temp.gettempdir() self.source_file = temp.NamedTemporaryFile(suffix=".zip", dir=tdir) self.source_file.write('a' * (2**21)) self.source_file.seek(0) self.version.source = DjangoFile(self.source_file) self.version.save() self.user = UserProfile.objects.get(email="*****@*****.**") self.group = Group.objects.create(name='Editors BinarySource', rules='Editors:BinarySource') self.url = reverse('downloads.source', args=(self.version.pk, ))
def test_existing_source_link(self): tmp_file = temp.NamedTemporaryFile with tmp_file(suffix=".zip", dir=temp.gettempdir()) as source_file: source_file.write('a' * (2**21)) source_file.seek(0) self.version.source = DjangoFile(source_file) self.version.save() response = self.client.get(self.url) doc = pq(response.content) link = doc('.current-source-link') assert link assert link.text() == 'View current' assert link[0].attrib['href'] == reverse('downloads.source', args=(self.version.pk, ))
def export(self, encode=False, export_file=False): task = self.getTask() rows = super(XLSXMixin, self).export(encode=True) wb = Workbook() ws = wb.worksheets[0] for row in rows: tmp = [] for line in row: if line == None: tmp.append('') else: tmp.append(line) ws.append(tmp) if export_file: tfile = '/tmp/' + uuid.uuid4().hex wb.save(tfile) wkbook = open(tfile) tempFile = tempfile.NamedTemporaryFile() copyfileobj(wkbook, tempFile) wkbook.close() remove(tfile) dfile = DjangoFile(tempFile) f = File(creator=task.workflow.owner, file=dfile, filename='%s_result.xlsx' % task.title.lower(), linked=True) f.save() return f response = HttpResponse(save_virtual_workbook(wb), content_type='application/vnd.ms-excel') response[ 'Content-Disposition'] = 'attachment; filename="%s.xlsx"' % filename( str(task)) return response
def setUp(self): super(TestDownloadSource, self).setUp() self.addon = Addon.objects.get(pk=3615) # Make sure non-ascii is ok. self.addon.update(slug=u'crosswarpex-확장') self.version = self.addon.current_version tdir = temp.gettempdir() self.source_file = temp.NamedTemporaryFile(suffix=".zip", dir=tdir) self.source_file.write(b'a' * (2**21)) self.source_file.seek(0) self.version.source = DjangoFile(self.source_file) self.version.save() self.filename = os.path.basename(self.version.source.path) self.user = UserProfile.objects.get(email="*****@*****.**") self.group = Group.objects.create(name='Editors BinarySource', rules='Editors:BinarySource') self.url = reverse('downloads.source', args=(self.version.pk, ))
def test_existing_source_link(self): with temp.NamedTemporaryFile(suffix='.zip', dir=temp.gettempdir()) as source_file: with zipfile.ZipFile(source_file, 'w') as zip_file: zip_file.writestr('foo', 'a' * (2**21)) source_file.seek(0) self.version.source.save(os.path.basename(source_file.name), DjangoFile(source_file)) self.version.save() response = self.client.get(self.url) doc = pq(response.content) link = doc('.current-source-link') assert link assert link.text() == 'View current' assert link[0].attrib['href'] == reverse('downloads.source', args=(self.version.pk, ))
def store_file(self, profile): """ Add the encoded input file to the ``output_files`` field. :param profile: The :py:class:`EncodingProfile` instance that contains the encoding data. :type profile: :py:class:`EncodingProfile` :raises: :py:class:`~encode.UploadError`: Something went wrong while uploading the file or the file does not exist. """ # use random name for (encoded) output file file_name = get_random_filename(profile.container) path = self.output_path(profile) logger.info("Saving encoded file {0} in storage as {1}".format( short_path(path), file_name)) if os.path.exists(path): # put encoded file in external storage with open(path, 'rb') as encoded_file: media_file = MediaFile(title=file_name) try: media_file.file.save(file_name, DjangoFile(encoded_file)) self.output_files.add(media_file) logger.info("Stored {0} at {1}".format( file_name, media_file.file.url)) except socket.error as error: # pragma: no cover raise UploadError(error) else: raise UploadError("{} does not exist".format(path)) # when all output_files have been saved (cq. uploaded) if self.ready: self.encoded = True self.encoding = False self.uploaded = True self.save()
def test_extract_and_commit_source_from_version_rename_dotgit_files(settings): addon = addon_factory( file_kw={'filename': 'webextension_no_id.xpi'}, version_kw={'version': '0.1'}) # Generate source file source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH) with zipfile.ZipFile(source, 'w') as zip_file: zip_file.writestr('manifest.json', '{}') zip_file.writestr('.gitattributes', '') zip_file.writestr('.gitignore', '') zip_file.writestr('.gitmodules', '') zip_file.writestr('some/directory/.gitattributes', '') zip_file.writestr('some/directory/.gitignore', '') zip_file.writestr('some/directory/.gitmodules', '') source.seek(0) addon.current_version.source = DjangoFile(source) addon.current_version.save() with mock.patch('olympia.lib.git.uuid.uuid4') as uuid4_mock: uuid4_mock.return_value = mock.Mock( hex='b236f5994773477bbcd2d1b75ab1458f') repo = AddonGitRepository.extract_and_commit_source_from_version( addon.current_version) # Verify via subprocess to make sure the repositories are properly # read by the regular git client output = _run_process('git ls-tree -r --name-only listed', repo) assert set(output.split()) == { 'extracted/manifest.json', 'extracted/.gitattributes.b236f599', 'extracted/.gitignore.b236f599', 'extracted/.gitmodules.b236f599', 'extracted/some/directory/.gitattributes.b236f599', 'extracted/some/directory/.gitignore.b236f599', 'extracted/some/directory/.gitmodules.b236f599', }
def get_uploaded_file(file_path): f = open(file_path, "rb") file = DjangoFile(f) uploaded_file = InMemoryUploadedFile(file, None, file_path, "text/plain", file.size, None, None) return uploaded_file
def pre_process_files(self, data): from django.core.files.base import File as DjangoFile for key, value in data.items(): if isinstance(value, File): data[key] = DjangoFile(open(value.path))