def download(self):
self.path_cache = Path(mkdtemp())
# Use .css to keep the same ContentType, otherwise the webserver
# will use the generic octet and the browser will ignore the import
# https://www.w3.org/TR/2013/CR-css-cascade-3-20131003/#content-type
self.path_import_file = self.path_cache / 'import.css'
self.font_files = []
with self.path_import_file.open(mode='w') as file_object:
for agent_name, agent_string in self.user_agents.items():
import_file = force_text(
requests.get(self.url,
headers={
'User-Agent': agent_string
}).content)
for line in import_file.split('\n'):
if 'url' in line:
font_url = line.split(' ')[-2][4:-1]
url = furl(force_text(font_url))
font_filename = url.path.segments[-1]
path_font_filename = self.path_cache / font_filename
with path_font_filename.open(
mode='wb') as font_file_object:
with requests.get(font_url,
stream=True) as response:
shutil.copyfileobj(fsrc=response.raw,
fdst=font_file_object)
line = line.replace(font_url, font_filename)
file_object.write(line)
def _process(self, document_version):
if self.command_exiftool:
temporary_folder = mkdtemp()
path_temporary_file = Path(temporary_folder,
document_version.document.label)
try:
with path_temporary_file.open(
mode='xb') as temporary_fileobject:
document_version.save_to_file(
file_object=temporary_fileobject)
temporary_fileobject.seek(0)
try:
result = self.command_exiftool(
temporary_fileobject.name)
except sh.ErrorReturnCode_1 as exception:
result = json.loads(s=exception.stdout)[0]
if result.get('Error', '') == 'Unknown file type':
# Not a fatal error
return result
else:
return json.loads(s=result.stdout)[0]
except Exception as exception:
logger.error('Error processing document version: %s; %s',
document_version, exception)
raise
finally:
fs_cleanup(filename=str(path_temporary_file))
else:
logger.warning(
'EXIFTool binary not found, not processing document '
'version: %s', document_version)
def test_file_patching(self):
test_replace_text = 'replaced_text'
temporary_directory = mkdtemp()
path_temporary_directory = Path(temporary_directory)
path_test_file = path_temporary_directory / 'test_file.css'
with path_test_file.open(mode='w') as file_object:
file_object.write(
'@import url("https://fonts.googleapis.com/css?family=Lato:400,700,400italic");'
)
dependency = TestDependency(name='test_dependency', module=__name__)
replace_list = [
{
'filename_pattern': '*',
'content_patterns': [
{
'search': '"https://fonts.googleapis.com/css?family=Lato:400,700,400italic"',
'replace': test_replace_text,
}
]
}
]
dependency.patch_files(path=temporary_directory, replace_list=replace_list)
with path_test_file.open(mode='r') as file_object:
final_text = file_object.read()
shutil.rmtree(temporary_directory, ignore_errors=True)
self.assertEqual(final_text, '@import url({});'.format(test_replace_text))
def download(self):
self.path_cache = mkdtemp()
with requests.get(self.version_metadata['dist']['tarball'], stream=True) as response:
response.raise_for_status()
with self.get_tar_file_path().open(mode='wb') as file_object:
shutil.copyfileobj(fsrc=response.raw, fdst=file_object)
def extract(self, replace_list=None):
temporary_directory = mkdtemp()
path_compressed_file = self.get_tar_file_path()
with tarfile.open(name=force_text(s=path_compressed_file),
mode='r') as file_object:
file_object.extractall(path=temporary_directory)
self.patch_files(path=temporary_directory, replace_list=replace_list)
path_install = self.get_install_path()
# Clear the installation path of previous content
shutil.rmtree(path=force_text(s=path_install), ignore_errors=True)
# Scoped packages are nested under a parent directory
# create it to avoid rename errors.
path_install.mkdir(parents=True)
# Copy the content under the dependency's extracted content folder
# 'package' to the final location.
# We do a copy and delete instead of move because os.rename doesn't
# support renames across filesystems.
path_uncompressed_package = Path(temporary_directory, 'package')
shutil.rmtree(force_text(s=path_install))
shutil.copytree(force_text(s=path_uncompressed_package),
force_text(s=path_install))
shutil.rmtree(force_text(s=path_uncompressed_package))
# Clean up temporary directory used for download
shutil.rmtree(path=temporary_directory, ignore_errors=True)
shutil.rmtree(path=self.path_cache, ignore_errors=True)
def _create_test_staging_folder(self):
self.test_staging_folder = StagingFolderSource.objects.create(
label=TEST_SOURCE_LABEL,
folder_path=mkdtemp(),
preview_width=TEST_STAGING_PREVIEW_WIDTH,
uncompress=TEST_SOURCE_UNCOMPRESS_N,
)
self.test_staging_folders.append(self.test_staging_folder)
def setUp(self):
super(CacheTestMixin, self).setUp()
self.temporary_directory = mkdtemp()
DefinedStorage(
dotted_path='django.core.files.storage.FileSystemStorage',
label='File caching test storage',
name=STORAGE_NAME_FILE_CACHING_TEST_STORAGE,
kwargs={'location': self.temporary_directory})
def setUp(self):
super().setUp()
self.temporary_directory = mkdtemp()
shutil.copy(TEST_NON_ASCII_DOCUMENT_PATH, self.temporary_directory)
self.test_filename = os.path.basename(TEST_NON_ASCII_DOCUMENT_PATH)
self.test_staging_folder = MockStagingFolder()
self.test_staging_folder.folder_path = self.temporary_directory
self.test_staging_files = []
def setUp(self):
self.old_value = setting_backend_arguments.value
super().setUp()
self.document_search_model = SearchModel.get_for_model(
instance=Document
)
setting_backend_arguments.set(
value={'index_path': mkdtemp()}
)
self.search_backend = SearchBackend.get_instance()
def _request_test_staging_folder_create_api_view(self):
return self.post(viewname='rest_api:stagingfolder-list',
data={
'label': TEST_SOURCE_LABEL,
'folder_path': mkdtemp(),
'preview_width': TEST_STAGING_PREVIEW_WIDTH,
'uncompress': TEST_SOURCE_UNCOMPRESS_N,
})
self.test_staging_folder = StagingFolderSource.objects.first()
self.test_staging_folders.append(self.test_staging_folder)
def gpg_command(self, function, **kwargs):
temporary_directory = mkdtemp()
os.chmod(temporary_directory, 0x1C0)
gpg = gnupg.GPG(gnupghome=temporary_directory,
gpgbinary=self.kwargs['gpg_path'])
result = function(gpg=gpg, **kwargs)
shutil.rmtree(temporary_directory)
return result
def setUp(self):
super(PatchFilesTestCase, self).setUp()
self.temporary_directory = mkdtemp()
self.path_temporary_directory = Path(self.temporary_directory)
self.path_test_file = self.path_temporary_directory / 'test_file.txt'
with self.path_test_file.open(mode='w') as file_object:
file_object.writelines([
'line 1\n',
' line 2\n',
'line 3\n',
])
def setUp(self):
super(DependencyClassTestCase, self).setUp()
self.test_replace_text = 'replaced_text'
self.temporary_directory = mkdtemp()
self.path_temporary_directory = Path(self.temporary_directory)
self.path_test_file = self.path_temporary_directory / 'test_file.css'
with self.path_test_file.open(mode='w') as file_object:
file_object.write(
'@import url("https://fonts.googleapis.com/css?family=Lato:400,700,400italic");'
)
self.test_dependency = TestDependency(name='test_dependency',
module=__name__)
def test_unicode_staging_file(self):
temporary_directory = mkdtemp()
shutil.copy(TEST_NON_ASCII_DOCUMENT_PATH, temporary_directory)
filename = os.path.basename(TEST_NON_ASCII_DOCUMENT_PATH)
staging_file_1 = StagingFile(staging_folder=MockStagingFolder(),
filename=filename)
staging_file_2 = StagingFile(
staging_folder=MockStagingFolder(),
encoded_filename=staging_file_1.encoded_filename)
self.assertEqual(filename, staging_file_2.filename)
shutil.rmtree(temporary_directory)
def setUp(self):
super(StagingFolderViewTestCase, self).setUp()
self.temporary_directory = mkdtemp()
shutil.copy(src=TEST_SMALL_DOCUMENT_PATH, dst=self.temporary_directory)
self.filename = os.path.basename(TEST_SMALL_DOCUMENT_PATH)
def soffice(self):
"""
Executes LibreOffice as a sub process
"""
if not self.command_libreoffice:
raise OfficeConversionError(
_('LibreOffice not installed or not found.'))
with NamedTemporaryFile() as temporary_file_object:
# Copy the source file object of the converter instance to a
# named temporary file to be able to pass it to the LibreOffice
# execution.
self.file_object.seek(0)
shutil.copyfileobj(fsrc=self.file_object,
fdst=temporary_file_object)
self.file_object.seek(0)
temporary_file_object.seek(0)
libreoffice_home_directory = mkdtemp()
args = (
temporary_file_object.name,
'--outdir',
setting_temporary_directory.value,
'-env:UserInstallation=file://{}'.format(
os.path.join(libreoffice_home_directory,
'LibreOffice_Conversion')),
)
kwargs = {'_env': {'HOME': libreoffice_home_directory}}
if self.mime_type == 'text/plain':
kwargs.update({'infilter': 'Text (encoded):UTF8,LF,,,'})
try:
self.command_libreoffice(*args, **kwargs)
except sh.ErrorReturnCode as exception:
temporary_file_object.close()
raise OfficeConversionError(exception)
except Exception as exception:
temporary_file_object.close()
logger.error('Exception launching Libre Office; %s', exception)
raise
finally:
fs_cleanup(filename=libreoffice_home_directory)
# LibreOffice return a PDF file with the same name as the input
# provided but with the .pdf extension.
# Get the converted output file path out of the temporary file
# name plus the temporary directory
filename, extension = os.path.splitext(
os.path.basename(temporary_file_object.name))
logger.debug('filename: %s', filename)
logger.debug('extension: %s', extension)
converted_file_path = os.path.join(
setting_temporary_directory.value,
os.path.extsep.join((filename, 'pdf')))
logger.debug('converted_file_path: %s', converted_file_path)
# Don't use context manager with the NamedTemporaryFile on purpose
# so that it is deleted when the caller closes the file and not
# before.
temporary_converted_file_object = NamedTemporaryFile()
# Copy the LibreOffice output file to a new named temporary file
# and delete the converted file
with open(converted_file_path, mode='rb') as converted_file_object:
shutil.copyfileobj(fsrc=converted_file_object,
fdst=temporary_converted_file_object)
fs_cleanup(filename=converted_file_path)
temporary_converted_file_object.seek(0)
return temporary_converted_file_object
def setUp(self):
super().setUp()
self.temporary_directory = mkdtemp()
def setUp(self):
super(ZipCompressedPassthroughStorageTestCase, self).setUp()
self.temporary_directory = mkdtemp()
def setUp(self):
super(EncryptedPassthroughStorageTestCase, self).setUp()
self.temporary_directory = mkdtemp()
def setUp(self):
super(WatchFolderTestCase, self).setUp()
self.temporary_directory = mkdtemp()
def setUp(self):
super(IntervalSourceTestCase, self).setUp()
self.temporary_directory = mkdtemp()
def setUp(self):
self.old_value = setting_backend_arguments.value
super().setUp()
setting_backend_arguments.set(value={'index_path': mkdtemp()})
self.search_backend = SearchBackend.get_instance()
def setUp(self):
super(CacheTestMixin, self).setUp()
global test_storage
self.temporary_directory = mkdtemp()
test_storage = FileSystemStorage(location=self.temporary_directory)
