def test_backup(self):
with TemporaryUploadedFile("test_backup.txt", "text/plain", 0,
"UTF-8") as tmp_file:
tmp_file.write(b"content one")
tmp_file.flush()
tmp_file.seek(0)
instance1 = OverwriteFileSystemStorageModel.objects.create(
file=tmp_file)
# create a second instance with the same filename and different content:
with TemporaryUploadedFile("test_backup.txt", "text/plain", 0,
"UTF-8") as tmp_file:
tmp_file.write(b"content two")
tmp_file.flush()
tmp_file.seek(0)
instance2 = OverwriteFileSystemStorageModel.objects.create(
file=tmp_file)
assert_is_file(instance1.file.path)
assert_is_file(instance2.file.path)
assert instance1.file.path == instance2.file.path
assert instance1.file.name == "test_backup.txt"
assert instance2.file.name == "test_backup.txt"
assert_filenames_and_content(
path=temp_storage_location,
reference=[("test_backup.txt", b"content two"),
("test_backup.txt.bak01", b"content one")],
)
def temp_uploaded_file(file):
file_size = os.path.getsize(file.name)
file_content = file.read()
temp = TemporaryUploadedFile(file.name, 'text', file_size, None)
temp.write(file_content)
temp.seek(0)
return temp
def _make_form(self, project_path, instance=None, name=None):
payload = open(project_path, 'rb').read()
if name is None:
name = os.path.basename(project_path)
qgis_file = TemporaryUploadedFile(name, 'application/xml',
len(payload), 'utf-8')
qgis_file.file.write(payload)
qgis_file.file.seek(0)
form = QdjangoProjectForm(
data={
'feature_count_wms': 10,
'group_slug': self.project_group.slug,
'toc_tab_default': 'layers',
'toc_layers_init_status': 'not_collapsed',
'toc_themes_init_status': 'collapsed',
'legend_position': 'tab',
'multilayer_query': 'single',
'multilayer_querybybbox': 'single',
'multilayer_querybypolygon': 'single'
},
files={
'qgis_file': qgis_file
},
group=self.project_group,
request=type('', (object,), {'user': self.test_user1})(),
instance=instance,
initial={}
)
return form
def setUp(self):
self.text = "Spam Spam Spam Spam.\n"
self.digest = hashlib.sha1(self.text).hexdigest()
self.tempfile = TemporaryUploadedFile('spam4.txt', 'text/plain',
len(self.text), 'utf-8')
self.tempfile.file.write(self.text)
self.tempfile.file.seek(0)
def test_get_attachment(self):
user_specified_filename = 'a'
internal_filename = 'b'
file = TemporaryUploadedFile(internal_filename, 'text/plain', 10000,
'utf-8')
attachment = ReportAttachment.objects.create(
file=file,
user=self.user,
filename=user_specified_filename,
report=self.report)
attachment.save()
response = self.client.get(
reverse('crt_forms:get-report-attachment',
kwargs={
'id': self.pk,
'attachment_id': attachment.pk
}), )
# we should reply with a redirect to a presigned s3 url
self.assertEqual(response.status_code, 302)
# the presigned url should target the private S3 bucket
self.assertTrue('/crt-private/' in str(response.url))
# the presigned url should have a 30 second expiration
self.assertTrue('Expires=30' in str(response.url))
# the presigned url should target the internal (not user specified) filename
self.assertTrue(
f'/attachments/{internal_filename}' in str(response.url))
# the response-content-disposition should be set so that the file downloads with the user specified filename
self.assertTrue(
f'response-content-disposition=attachment%3Bfilename%3D{user_specified_filename}'
in str(response.url))
def test_default_server_url(self):
server = CommCareServer.objects.get()
project = CommCareProject(server=server, domain="foo")
account = CommCareAccount(server=server,
username="******",
api_key="P@ssWord")
database = ExportDatabase(
name="Test DB",
connection_string="postgresql://*****:*****@123.4.0.0/test")
config_file = TemporaryUploadedFile(name="config_file",
content_type="application/xml",
size=100,
charset="utf-8")
export_config = ExportConfig(
name="Test Config",
project=project,
account=account,
database=database,
config_file=config_file,
extra_args="",
)
command = _compile_export_command(export_config, project, force=False)
self.assertIn('https://www.commcarehq.org', command)
def process_zip(self):
with file_storage.get_document_as_local_fn(self.source_path) as (local_file_path, _):
with zipfile.ZipFile(local_file_path) as zip_file:
zip_file_filelist = zip_file.filelist
self.log_info('Start extracting {} documents from {}'.format(
len(zip_file_filelist), local_file_path))
for n, a_file in enumerate(zip_file_filelist):
if a_file.is_dir():
continue
file_size = a_file.file_size
file_name = os.path.basename(a_file.filename)
mime_type = self.get_mime_type(file_name)
self.log_info(
'Extract/start LoadDocument for {} of {} files: name={}, size={}, mime_type={}'.format(
n + 1, len(zip_file_filelist), file_name, file_size, mime_type))
with TemporaryUploadedFile(file_name, mime_type, file_size, 'utf-8') as tempfile:
tempfile.file = zip_file.open(a_file)
tempfile.file.seek = lambda *args: 0
self.upload_file(
file_name=file_name,
file_size=file_size,
contents=tempfile,
directory_path=self.directory_path)
def process_tar(self):
with file_storage.get_document_as_local_fn(self.source_path) as (local_file_path, _):
with tarfile.TarFile(local_file_path) as tar_file:
tar_file_members = tar_file.getmembers()
self.log_info('Start extracting {} documents from {}'.format(
len(tar_file_members), local_file_path))
for n, a_file in enumerate(tar_file_members):
if a_file.isdir():
continue
file_size = a_file.size
file_name = os.path.basename(a_file.name)
mime_type = self.get_mime_type(file_name)
self.log_info(
'Extract/start LoadDocument for {} of {} files: name={}, size={}, mime_type={}'.format(
n + 1, len(tar_file_members), file_name, file_size, mime_type))
with TemporaryUploadedFile(file_name, mime_type, file_size, 'utf-8') as tempfile:
tempfile.file = tar_file.extractfile(a_file)
self.upload_file(
file_name=file_name,
file_size=file_size,
contents=tempfile,
directory_path=self.directory_path)
def test_save_temporary_file(self):
name = "foo/bar/toto.txt"
content = TemporaryUploadedFile(
"temporary", "application/binary",
(settings.FILE_UPLOAD_MAX_MEMORY_SIZE + 1), None)
self.storage.save(name, content)
self.assertEqual(self.storage.open(name).read(), b'')
def validate_file_link(self, value):
if not value:
return
r = requests.head(value, allow_redirects=True)
headers = r.headers
mime_type = headers.get('content-type')
file_size = headers.get('content-length')
file_name = None
if "Content-Disposition" in r.headers.keys():
file_name_list = re.findall("filename=(.+)", r.headers["Content-Disposition"])
if len(file_name_list) > 0:
file_name = file_name_list[0]
if not file_name:
file_name = value.split("/")[-1]
tf = TemporaryUploadedFile(
file_name,
mime_type,
file_size,
'utf-8'
)
r = requests.get(value, stream=True)
for chunk in r.iter_content(chunk_size=4096):
tf.write(chunk)
tf.seek(0)
self._file = tf
def stage_replica(replica):
from django.core.files.uploadedfile import TemporaryUploadedFile
from tardis.tardis_portal.models import Replica, Location
if not replica.location.type == 'external':
raise ValueError('Only external replicas can be staged')
if getattr(settings, "DEEP_DATASET_STORAGE", False):
relurl = path.relpath(replica.url[7:], settings.SYNC_TEMP_PATH)
spliturl = relurl.split(os.sep)[1:]
subdir = path.dirname(path.join(*spliturl))
else:
subdir = None
with TemporaryUploadedFile(replica.datafile.filename, None, None,
None) as tf:
if replica.verify(tempfile=tf.file):
if not replica.stay_remote:
tf.file.flush()
target_replica = {
'datafile':
replica.datafile,
'url':
write_uploaded_file_to_dataset(replica.datafile.dataset,
tf,
subdir=subdir),
'location':
Location.get_default_location(),
'verified':
True,
'protocol':
''
}
Replica.objects.filter(id=replica.id).update(**target_replica)
return True
else:
return False
def test_file_size_uploadble(self):
small_file = TemporaryUploadedFile('file.txt', b'this is a small file', 10000, 'utf-8')
try:
validate_file_size(small_file)
except ValidationError:
self.fail('validate_file_size unexpectedly raised ValidationError!')
def new_file(self, *args, **kwargs):
"""
Create the file object to append to as data is coming in.
"""
super().new_file(*args, **kwargs)
self.file = TemporaryUploadedFile(self.file_name, self.content_type, 0,
self.charset,
self.content_type_extra)
def new_file(self, file_name, *args, **kwargs):
"""
Create the file object to append to as data is coming in.
Ignores and overwrites most of the arguments and relies on exsiting request
"""
super(FeedUploadHandler, self).new_file(file_name, *args, **kwargs)
self._validate_file()
self.file = TemporaryUploadedFile(self.file_name, self.content_type, 0, self.charset)
def test_file_extension_ok(self):
file = TemporaryUploadedFile('file.txt', 'text/plain', 5000, 'utf-8')
try:
validate_file_extension(file)
except ValidationError:
self.fail('validate_file_extension unexpectedly raised ValidationError!')
def stage_file(datafile):
from django.core.files.uploadedfile import TemporaryUploadedFile
with TemporaryUploadedFile(datafile.filename, None, None, None) as tf:
if datafile.verify(tempfile=tf.file):
tf.file.flush()
datafile.url = write_uploaded_file_to_dataset(datafile.dataset, tf)
datafile.protocol = ''
datafile.save()
def new_file(self, *args, **kwargs):
super().new_file(*args, **kwargs)
self.file = TemporaryUploadedFile(self.file_name, self.content_type, 0,
self.charset,
self.content_type_extra)
job = JobModel.objects.get(job_id=self.request.GET["job_id"])
job.job_status = JobModel.STARTED
job.save()
def new_file(self, file_name, *args, **kwargs):
"""
Create the file object to append to as data is coming in.
"""
super(TemporaryFileUploadHandler,
self).new_file(file_name, *args, **kwargs)
self.file = TemporaryUploadedFile(self.file_name, self.content_type, 0,
self.charset)
def test_media_root_pathlib(self):
with tempfile.TemporaryDirectory() as tmp_dir:
with override_settings(MEDIA_ROOT=Path(tmp_dir)):
with TemporaryUploadedFile('foo.txt', 'text/plain', 1,
'utf-8') as tmp_file:
Document.objects.create(myfile=tmp_file)
self.assertTrue(
os.path.exists(
os.path.join(tmp_dir, 'unused', 'foo.txt')))
def test_move_temporary_file(self):
"""
The temporary uploaded file is moved rather than copied to the
destination.
"""
with TemporaryUploadedFile('something.txt', 'text/plain', 0, 'UTF-8') as tmp_file:
tmp_file_path = tmp_file.temporary_file_path()
Document.objects.create(myfile=tmp_file)
self.assertFalse(os.path.exists(tmp_file_path), 'Temporary file still exists')
def _make_tempfile(self, filename, content):
fileobj = TemporaryUploadedFile(
name=filename + ".tempfile",
content_type='text/plain',
size=0,
charset='utf8',
)
fileobj.write(content)
fileobj.flush()
return fileobj
def test_uploaded_file_name(self):
tmp_file = TemporaryUploadedFile('whatever',
'application/octet-stream', 0, 'utf-8')
with utils.uploaded_file_name(tmp_file) as name:
self.assertEquals(name, tmp_file.file.name)
mem_file = SimpleUploadedFile('whatever', 'hello42')
with utils.uploaded_file_name(mem_file) as name:
self.assertTrue(name.endswith('whatever'))
self.assertEqual(open(name, 'rb').read(), 'hello42')
self.assertFalse(os.path.exists(name))
def parse_distutils_request(request):
"""
Due to a bug in the Python distutils library, the request post is sent using \n
as a separator instead of the \r\n that the HTTP spec demands. This breaks the Django
form parser and therefore we have to write a custom parser.
This bug was fixed in the Python 2.7.4 and 3.4:
http://bugs.python.org/issue10510
"""
body = request.body.decode('latin-1')
if not body.endswith('\r\n'):
sep = body.splitlines()[1]
request.POST = QueryDict('', mutable=True)
try:
request._files = MultiValueDict()
except Exception:
pass
for part in filter(lambda e: e.strip(), body.split(sep)):
try:
header, content = part.lstrip().split('\n', 1)
except Exception:
continue
if content.startswith('\n'):
content = content[1:]
if content.endswith('\n'):
content = content[:-1]
headers = parse_header(header)
if "name" not in headers:
continue
if "filename" in headers and headers['name'] == 'content':
dist = TemporaryUploadedFile(name=headers["filename"],
size=len(content),
content_type="application/gzip",
charset='utf-8')
dist.write(content.encode('utf-8'))
dist.seek(0)
request.FILES.appendlist('distribution', dist)
else:
request.POST.appendlist(headers["name"], content)
else:
request.FILES['distribution'] = request.FILES['content']
# Distutils sends UNKNOWN for empty fields (e.g platform)
for key, value in request.POST.items():
if value == 'UNKNOWN':
request.POST[key] = None
def to_python(self, value):
# handle previously existing file
try:
current_file = None
if ':' in value['current_file']:
current_file = self.signer.unsign(value['current_file'])
except signing.BadSignature:
raise ValidationError("Got bogus upstream data")
except (KeyError, TypeError):
pass
# handle new uploaded image
try:
obj = ''
if ':' in value['temp_name']:
temp_name = self.signer.unsign(value['temp_name'])
temp_file = self.storage.open(temp_name, 'rb')
file_size = self.storage.size(temp_name)
if file_size < settings.FILE_UPLOAD_MAX_MEMORY_SIZE:
obj = InMemoryUploadedFile(
file=temp_file,
field_name=None,
name=value['file_name'],
charset=value['charset'],
content_type=value['content_type'],
content_type_extra=value['content_type_extra'],
size=file_size,
)
else:
obj = TemporaryUploadedFile(
value['file_name'],
value['content_type'],
0,
value['charset'],
content_type_extra=value['content_type_extra'],
)
while True:
chunk = temp_file.read(0x10000)
if not chunk:
break
obj.file.write(chunk)
obj.file.seek(0)
obj.file.size = file_size
self.storage.delete(temp_name)
self.remove_current(current_file)
elif value['temp_name'] == 'delete':
self.remove_current(current_file)
except signing.BadSignature:
raise ValidationError("Got bogus upstream data")
except (IOError, KeyError, TypeError):
obj = current_file
except Exception as excp:
raise ValidationError("File upload failed. {}: {}".format(
excp.__class__.__name__, excp))
return obj
class SoftwareFileUploadHandler(FileUploadHandler):
"""
Upload handler that streams data into a temporary file.
"""
def __init__(self, *args, **kwargs):
super(SoftwareFileUploadHandler, self).__init__(*args, **kwargs)
#if 'software_formhash' in self.request.GET:
#self.software_formhash = self.request.POST['software_formhash']
#cache.set(self.software_formhash, { 'progress_percentage': 0 })
#self.activated = True
#else:
# self.activated = False
def new_file(self, file_name, *args, **kwargs):
"""
Create the file object to append to as data is coming in.
"""
super(SoftwareFileUploadHandler,
self).new_file(file_name, *args, **kwargs)
if 'software_uuid' not in self.request.GET:
self.activated = False
else:
try:
software = Software.objects.filter(
uuid=self.request.GET['software_uuid'])
if software:
self.activated = False
else:
try:
content_length = int(
self.request.META.get('CONTENT_LENGTH', 0))
except Exception, e:
content_length = 0
LOG.error(
'Request META CONTENT_LENGTH format error. %s' % e)
created_at = datetime.datetime.now(tz=utc)
software = Software(uuid=self.request.GET['software_uuid'],
status=SOFTWARE_STATE_UPLOADING,
created_at=created_at,
content_name=self.file_name,
content_type=self.content_type,
content_total=content_length,
classify='unknown')
software.save()
self.activated = True
except Exception, e:
LOG.error('software upload error. %s', e)
self.activated = False
if self.activated:
self.file = TemporaryUploadedFile(self.file_name,
self.content_type, 0,
self.charset)
def test_file_name_valid(self):
valid_filename = TemporaryUploadedFile(
'file-name_acceptable 03152021.txt', b'This filename is supported',
5000, 'utf-8')
try:
validate_filename(valid_filename)
except ValidationError:
self.fail(
'validate_file_size unexpectedly raised ValidationError!')
def test_corrupted_image(self):
f = ImageField()
img_file = SimpleUploadedFile('not_an_image.jpg', b'not an image')
msg = ('Upload a valid image. The file you uploaded was either not an '
'image or a corrupted image.')
with self.assertRaisesMessage(ValidationError, msg):
f.clean(img_file)
with TemporaryUploadedFile('not_an_image_tmp.png', 'text/plain', 1,
'utf-8') as tmp_file:
with self.assertRaisesMessage(ValidationError, msg):
f.clean(tmp_file)
def test_upload_file(self):
host_name = "{0}:{1}".format(os.environ['HOSTNAME'], "8989")
resp = requests.post('http://' + host_name + '/api/v1/type/imagefile/base/mes' \
+ '/table/' + self.__class__.rand_name + '/label/1/')
temp_file = TemporaryUploadedFile("img_test_data", "byte", 66666,
"xxxxxxxxxxxxxxxxxxxxxxxxxx")
self.assertEqual(
ImageManager().put_data("mes", self.__class__.rand_name.join, "1",
[temp_file]), 1)
tfmsa_logger("==========PASS==========")
def test_basic_save(self):
with TemporaryUploadedFile("test_basic_save.txt", "text/plain", 0,
"UTF-8") as tmp_file:
instance = OverwriteFileSystemStorageModel.objects.create(
file=tmp_file)
print(instance.file.path)
assert_is_file(instance.file.path)
assert instance.file.name == "test_basic_save.txt"
assert_filenames_and_content(path=temp_storage_location,
reference=[("test_basic_save.txt", b"")])
def handle(f, g):
nama, ext = os.path.splitext(g)
j = TemporaryUploadedFile(g, f.content_type, f.size, f.charset)
for chunk in f.chunks():
j.write(chunk)
if ext.lower() == ".json":
savejson(j)
elif ext.lower() == ".csv":
savecsv(j)
elif ext.lower() == ".xls" or ".xlsx":
savexls(j)
