def process_file(sender, **kwargs):
"""
what we do here:
- determine file type, set it.
- generate file hash
- check for file renames
"""
instance = kwargs.get("instance")
if instance.file:
name, extension = os.path.splitext(instance.file.name)
if len(extension) > 1:
instance.extension = extension[1:].lower()
else:
instance.extension = ''
instance.type = OTHER_TYPE
for type, definition in conf.FILE_TYPES.items():
if instance.extension in definition.get("extensions"):
instance.type = type
instance.generate_file_hash()
if instance.id:
old_instance = File.objects.get(pk=instance.id)
if not old_instance.filename == instance.filename:
# rename!
new_file = DjangoFile(open(instance.file.path, mode='rb'))
instance.file.delete(False) # remove including thumbs
instance.file.save(instance.filename, new_file, save=False)
def test_extract_and_commit_source_from_version(settings):
addon = addon_factory(
file_kw={'filename': 'webextension_no_id.xpi'},
version_kw={'version': '0.1'})
# Generate source file
source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH)
with zipfile.ZipFile(source, 'w') as zip_file:
zip_file.writestr('manifest.json', '{}')
source.seek(0)
addon.current_version.source = DjangoFile(source)
addon.current_version.save()
repo = AddonGitRepository.extract_and_commit_source_from_version(
addon.current_version)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source')
assert os.listdir(repo.git_repository_path) == ['.git']
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git branch', repo)
assert 'listed' in output
output = _run_process('git log listed', repo)
expected = 'Create new version {} ({}) for {} from source file'.format(
repr(addon.current_version), addon.current_version.id, repr(addon))
assert expected in output
def test_extract_and_commit_source_from_version_no_dotgit_clash(settings):
addon = addon_factory(
file_kw={'filename': 'webextension_no_id.xpi'},
version_kw={'version': '0.1'})
# Generate source file
source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH)
with zipfile.ZipFile(source, 'w') as zip_file:
zip_file.writestr('manifest.json', '{}')
zip_file.writestr('.git/config', '')
source.seek(0)
addon.current_version.source = DjangoFile(source)
addon.current_version.save()
with mock.patch('olympia.lib.git.uuid.uuid4') as uuid4_mock:
uuid4_mock.return_value = mock.Mock(
hex='b236f5994773477bbcd2d1b75ab1458f')
repo = AddonGitRepository.extract_and_commit_source_from_version(
addon.current_version)
assert repo.git_repository_path == os.path.join(
settings.GIT_FILE_STORAGE_PATH, id_to_path(addon.id), 'source')
assert os.listdir(repo.git_repository_path) == ['.git']
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git ls-tree -r --name-only listed', repo)
assert set(output.split()) == {
'extracted/manifest.json', 'extracted/.git.b236f599/config'}
def export(self, encode=False, export_file=False):
task = self.getTask()
rows = super(CSVMixin, self).export(encode=True)
if export_file:
with tempfile.NamedTemporaryFile() as csvfile:
writer = csv.writer(csvfile)
for row in rows:
writer.writerow(row)
dfile = DjangoFile(csvfile)
f = File(creator=task.workflow.owner,
file=dfile,
filename='%s_result.csv' % task.title.lower(),
linked=True)
f.save()
return f
pseudo_buffer = Echo()
writer = csv.writer(pseudo_buffer)
response = StreamingHttpResponse(
(writer.writerow(row) for row in rows), content_type="text/csv")
response[
'Content-Disposition'] = 'attachment; filename="%s.csv"' % filename(
str(task))
return response
def test(self):
f = open("base/apps.py", "rb")
file = DjangoFile(f)
assert file.name == "base/apps.py"
file.open()
file_hash = compute_file_hash(file)
assert file_hash
assert file.size
def setUp(self):
self.addon = Addon.objects.get(pk=3615)
self.version = self.addon._latest_version
tdir = temp.gettempdir()
self.source_file = temp.NamedTemporaryFile(suffix=".zip", dir=tdir)
self.source_file.write('a' * (2**21))
self.source_file.seek(0)
self.version.source = DjangoFile(self.source_file)
self.version.save()
self.user = UserProfile.objects.get(email="*****@*****.**")
self.group = Group.objects.create(name='Editors BinarySource',
rules='Editors:BinarySource')
self.url = reverse('downloads.source', args=(self.version.pk, ))
def test_existing_source_link(self):
tmp_file = temp.NamedTemporaryFile
with tmp_file(suffix=".zip", dir=temp.gettempdir()) as source_file:
source_file.write('a' * (2**21))
source_file.seek(0)
self.version.source = DjangoFile(source_file)
self.version.save()
response = self.client.get(self.url)
doc = pq(response.content)
link = doc('.current-source-link')
assert link
assert link.text() == 'View current'
assert link[0].attrib['href'] == reverse('downloads.source',
args=(self.version.pk, ))
def export(self, encode=False, export_file=False):
task = self.getTask()
rows = super(XLSXMixin, self).export(encode=True)
wb = Workbook()
ws = wb.worksheets[0]
for row in rows:
tmp = []
for line in row:
if line == None:
tmp.append('')
else:
tmp.append(line)
ws.append(tmp)
if export_file:
tfile = '/tmp/' + uuid.uuid4().hex
wb.save(tfile)
wkbook = open(tfile)
tempFile = tempfile.NamedTemporaryFile()
copyfileobj(wkbook, tempFile)
wkbook.close()
remove(tfile)
dfile = DjangoFile(tempFile)
f = File(creator=task.workflow.owner,
file=dfile,
filename='%s_result.xlsx' % task.title.lower(),
linked=True)
f.save()
return f
response = HttpResponse(save_virtual_workbook(wb),
content_type='application/vnd.ms-excel')
response[
'Content-Disposition'] = 'attachment; filename="%s.xlsx"' % filename(
str(task))
return response
def setUp(self):
super(TestDownloadSource, self).setUp()
self.addon = Addon.objects.get(pk=3615)
# Make sure non-ascii is ok.
self.addon.update(slug=u'crosswarpex-확장')
self.version = self.addon.current_version
tdir = temp.gettempdir()
self.source_file = temp.NamedTemporaryFile(suffix=".zip", dir=tdir)
self.source_file.write(b'a' * (2**21))
self.source_file.seek(0)
self.version.source = DjangoFile(self.source_file)
self.version.save()
self.filename = os.path.basename(self.version.source.path)
self.user = UserProfile.objects.get(email="*****@*****.**")
self.group = Group.objects.create(name='Editors BinarySource',
rules='Editors:BinarySource')
self.url = reverse('downloads.source', args=(self.version.pk, ))
def test_existing_source_link(self):
with temp.NamedTemporaryFile(suffix='.zip',
dir=temp.gettempdir()) as source_file:
with zipfile.ZipFile(source_file, 'w') as zip_file:
zip_file.writestr('foo', 'a' * (2**21))
source_file.seek(0)
self.version.source.save(os.path.basename(source_file.name),
DjangoFile(source_file))
self.version.save()
response = self.client.get(self.url)
doc = pq(response.content)
link = doc('.current-source-link')
assert link
assert link.text() == 'View current'
assert link[0].attrib['href'] == reverse('downloads.source',
args=(self.version.pk, ))
def store_file(self, profile):
"""
Add the encoded input file to the ``output_files`` field.
:param profile: The :py:class:`EncodingProfile` instance that contains
the encoding data.
:type profile: :py:class:`EncodingProfile`
:raises: :py:class:`~encode.UploadError`: Something went wrong while
uploading the file or the file does not exist.
"""
# use random name for (encoded) output file
file_name = get_random_filename(profile.container)
path = self.output_path(profile)
logger.info("Saving encoded file {0} in storage as {1}".format(
short_path(path), file_name))
if os.path.exists(path):
# put encoded file in external storage
with open(path, 'rb') as encoded_file:
media_file = MediaFile(title=file_name)
try:
media_file.file.save(file_name, DjangoFile(encoded_file))
self.output_files.add(media_file)
logger.info("Stored {0} at {1}".format(
file_name, media_file.file.url))
except socket.error as error: # pragma: no cover
raise UploadError(error)
else:
raise UploadError("{} does not exist".format(path))
# when all output_files have been saved (cq. uploaded)
if self.ready:
self.encoded = True
self.encoding = False
self.uploaded = True
self.save()
def test_extract_and_commit_source_from_version_rename_dotgit_files(settings):
addon = addon_factory(
file_kw={'filename': 'webextension_no_id.xpi'},
version_kw={'version': '0.1'})
# Generate source file
source = temp.NamedTemporaryFile(suffix='.zip', dir=settings.TMP_PATH)
with zipfile.ZipFile(source, 'w') as zip_file:
zip_file.writestr('manifest.json', '{}')
zip_file.writestr('.gitattributes', '')
zip_file.writestr('.gitignore', '')
zip_file.writestr('.gitmodules', '')
zip_file.writestr('some/directory/.gitattributes', '')
zip_file.writestr('some/directory/.gitignore', '')
zip_file.writestr('some/directory/.gitmodules', '')
source.seek(0)
addon.current_version.source = DjangoFile(source)
addon.current_version.save()
with mock.patch('olympia.lib.git.uuid.uuid4') as uuid4_mock:
uuid4_mock.return_value = mock.Mock(
hex='b236f5994773477bbcd2d1b75ab1458f')
repo = AddonGitRepository.extract_and_commit_source_from_version(
addon.current_version)
# Verify via subprocess to make sure the repositories are properly
# read by the regular git client
output = _run_process('git ls-tree -r --name-only listed', repo)
assert set(output.split()) == {
'extracted/manifest.json',
'extracted/.gitattributes.b236f599',
'extracted/.gitignore.b236f599',
'extracted/.gitmodules.b236f599',
'extracted/some/directory/.gitattributes.b236f599',
'extracted/some/directory/.gitignore.b236f599',
'extracted/some/directory/.gitmodules.b236f599',
}
def get_uploaded_file(file_path):
f = open(file_path, "rb")
file = DjangoFile(f)
uploaded_file = InMemoryUploadedFile(file, None, file_path, "text/plain",
file.size, None, None)
return uploaded_file
def pre_process_files(self, data):
from django.core.files.base import File as DjangoFile
for key, value in data.items():
if isinstance(value, File):
data[key] = DjangoFile(open(value.path))