def _create_form(app_id, xmlns, with_attachment):
xml = FORM_XML.format(doc_id=random_hex(), xmlns=xmlns)
attachments = {}
if with_attachment:
attachment = open(
'./corehq/ex-submodules/casexml/apps/case/tests/data/attachments/fruity.jpg',
'rb')
attachments = {'pic.jpg': UploadedFile(attachment, 'pic.jpg')}
submit_form_locally(xml,
domain=DOMAIN,
app_id=app_id,
attachments=attachments)
def test_uploaded_logo(self):
logo_src = os.path.join(askbot.get_install_directory(), 'tests',
'images', 'logo.gif')
logo_file = open(logo_src, 'r')
new_logo = UploadedFile(file=logo_file)
askbot_settings.update('SITE_LOGO_URL', new_logo)
logo_url = askbot_settings.SITE_LOGO_URL
self.assertTrue(
logo_url.startswith('/' +
django_settings.ASKBOT_UPLOADED_FILES_URL))
response = self.client.get(logo_url)
self.assertTrue(response.status_code == 200)
def attach_images_to_case(case_id, files):
"""
Handle case submission for the vscan endpoint
"""
xform = render_vscan_xform(case_id, files)
file_dict = {}
for f in files:
identifier = os.path.split(f)[-1]
file_dict[identifier] = UploadedFile(files[f], identifier)
submit_form_locally(xform, attachments=file_dict, domain=UTH_DOMAIN)
def download_incoming_media(self, media_url):
file_id = media_url.rsplit('/', 1)[-1]
headers = {
'Authorization': f'App {self.config.auth_token}',
'Accept': 'application/json'
}
response = requests.get(media_url, headers=headers)
uploaded_file = UploadedFile(
BytesIO(response.content),
file_id,
content_type=response.headers.get('content-type'))
return file_id, uploaded_file
def test_storage_content_type_long_name(storage):
data = b"some initial byte data"
test_file = io.BytesIO(data)
file_name = 't' * (settings.MINIO_STORAGE_MAX_FILE_NAME_LEN + 100) + '.jpg'
upload_file = UploadedFile(file=test_file, name=file_name, size=len(data))
file_i = file_info(upload_file)
storage.file_put(file_i)
file_resp = storage.file_get(file_i.path)
assert file_resp.headers.get('Content-Type') == 'image/jpeg'
def test_file_is_not_a_zip(self):
"""
Test that the uploaded file has the .zip extension.
"""
upload = {
"workspace": self.workspace,
"local": UploadedFile(name="SWAT_Model.tar.gz"),
"aws": {}
}
with self.assertRaises(NotAZipError):
SWATModelZip(upload)
def test_metadata_on_content_file_delete(self):
# test that some of the metadata is not deleted on content file deletion
# adding a valid netcdf file should generate some core metadata and all extended metadata
files = [
UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)
]
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=True)
# there should be 1 content files
self.assertEqual(self.resNetcdf.files.all().count(), 2)
# there should be 1 format elements
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 2)
# delete content file that we added above
hydroshare.delete_resource_file(self.resNetcdf.short_id,
self.netcdf_file_name, self.user)
# there should no content file
self.assertEqual(self.resNetcdf.files.all().count(), 0)
# there should be a title element
self.assertNotEquals(self.resNetcdf.metadata.title, None)
# there should be abstract element
self.assertNotEquals(self.resNetcdf.metadata.description, None)
# there should be 2 creator element
self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 2)
# there should be 1 contributor element
self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 1)
# there should be no coverage element
self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0)
# there should be no format element
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0)
# there should be subject element
self.assertNotEquals(self.resNetcdf.metadata.subjects.all().count(), 0)
# testing extended metadata elements
self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0)
self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0)
def setUpTestData(cls):
star_wars = m.Account.objects.create(name="Star Wars")
sw_source = m.DataSource.objects.create(name="Evil Empire")
cls.sw_source = sw_source
sw_version = m.SourceVersion.objects.create(data_source=sw_source,
number=1)
star_wars.default_version = sw_version
star_wars.save()
cls.yoda = cls.create_user_with_profile(username="******",
account=star_wars,
permissions=["iaso_forms"])
cls.jedi_council = m.OrgUnitType.objects.create(name="Jedi Council",
short_name="Cnc")
cls.jedi_council_corruscant = m.OrgUnit.objects.create(
name="Corruscant Jedi Council")
cls.project = m.Project.objects.create(
name="Hydroponic gardens",
app_id="stars.empire.agriculture.hydroponics",
account=star_wars)
cls.form_1 = m.Form.objects.create(name="Hydroponics study",
form_id="hydro_1",
period_type=m.MONTH,
single_per_period=True)
cls.form_version_1 = m.FormVersion.objects.create(
form=cls.form_1,
version_id="1",
file=UploadedFile(
open("iaso/tests/fixtures/hydroponics_test_upload.xml")))
cls.create_form_instance(form=cls.form_1,
period="202001",
org_unit=cls.jedi_council_corruscant,
project=cls.project,
uuid="uuid-1")
cls.project.unit_types.add(cls.jedi_council)
cls.project.forms.add(cls.form_1)
cls.project.save()
cls.maxDiff = None
with open("iaso/tests/fixtures/hydroponics_test_upload.xml"
) as modified_xml:
instance = cls.form_1.instances.first()
instance.file = SimpleUploadedFile("file.txt",
modified_xml.read().encode())
instance.get_and_save_json_of_xml()
def test_currupt_image_file(self):
"""
Test that when corrupt file is provided to validate_video_image, it gives proper error message.
"""
with open(settings.MEDIA_ROOT + '/test-corrupt-image.png', 'w+') as file:
image_file = UploadedFile(
file,
content_type='image/png',
size=settings.VIDEO_IMAGE_SETTINGS['VIDEO_IMAGE_MIN_BYTES']
)
error = validate_video_image(image_file)
self.assertEquals(error, 'There is a problem with this image file. Try to upload a different file.')
def setUpClass(cls):
super().setUpClass()
cls.catalog = Catalog.objects.create(name="Catalogue de vêtements", )
cls.image = UploadedFile()
cls.product = Product.objects.create(
name="Débardeur homme",
image=cls.image,
is_available=True,
is_free=False,
price=25,
catalog=cls.catalog,
)
def test_schema_after_version_change(client):
data = SuppliedData.objects.create()
with open(
os.path.join(
"tests",
"fixtures",
"tenders_releases_1_release_with_invalid_extensions.json",
)) as fp:
data.original_file.save("test.json", UploadedFile(fp))
resp = client.post(data.get_absolute_url(), {"version": "1.1"})
assert resp.status_code == 200
with open(os.path.join(data.upload_dir(),
"extended_schema.json")) as extended_release_fp:
assert ("mainProcurementCategory" in json.load(extended_release_fp)
["definitions"]["Tender"]["properties"])
with open(
os.path.join(data.upload_dir(),
"validation_errors-3.json")) as validation_errors_fp:
assert "'version' is missing but required" in validation_errors_fp.read(
)
# test link is still there.
resp = client.get(data.get_absolute_url())
assert resp.status_code == 200
assert "extended_schema.json" in resp.content.decode()
with open(os.path.join(data.upload_dir(),
"extended_schema.json")) as extended_release_fp:
assert ("mainProcurementCategory" in json.load(extended_release_fp)
["definitions"]["Tender"]["properties"])
with open(
os.path.join(data.upload_dir(),
"validation_errors-3.json")) as validation_errors_fp:
assert "'version' is missing but required" in validation_errors_fp.read(
)
resp = client.post(data.get_absolute_url(), {"version": "1.0"})
assert resp.status_code == 200
with open(os.path.join(data.upload_dir(),
"extended_schema.json")) as extended_release_fp:
assert ("mainProcurementCategory" not in json.load(extended_release_fp)
["definitions"]["Tender"]["properties"])
with open(
os.path.join(data.upload_dir(),
"validation_errors-3.json")) as validation_errors_fp:
assert "'version' is missing but required" not in validation_errors_fp.read(
)
def multiuploader(request):
if request.method == 'POST':
log.info('received POST to main multiuploader view')
if request.FILES == None:
return HttpResponseBadRequest('Must have files attached!')
#getting file data for farther manipulations
file = request.FILES[u'files[]']
wrapped_file = UploadedFile(file)
filename = wrapped_file.name
file_size = wrapped_file.file.size
log.info('Got file: "' + str(filename) + '"')
#writing file manually into model
#because we don't need form of any type.
image = Image()
image.title = str(filename)
image.image = file
image.save()
log.info('File saving done')
#getting url for photo deletion
file_delete_url = '/delete/'
#getting file url here
file_url = '/'
#getting thumbnail url using sorl-thumbnail
im = get_thumbnail(image, "80x80", quality=50)
thumb_url = im.url
#generating json response array
result = []
result.append({
"name": filename,
"size": file_size,
"url": file_url,
"thumbnail_url": thumb_url,
"delete_url": file_delete_url + str(image.pk) + '/',
"delete_type": "POST",
})
response_data = simplejson.dumps(result)
return HttpResponse(response_data, content_type='application/json')
else: #GET
return render_to_response(
'multiuploader_main.html',
{
'static_url': settings.MEDIA_URL,
'open_tv': u'{{',
'close_tv': u'}}'
},
)
def test_metadata_extraction_on_content_file_add(self):
# test the core metadata at this point
self.assertEqual(
self.resNetcdf.metadata.title.value,
'Snow water equivalent estimation at TWDEF site from Oct 2009 to June 2010'
)
# there shouldn't any abstract element
self.assertEqual(self.resNetcdf.metadata.description, None)
# there shouldn't any coverage element
self.assertEqual(self.resNetcdf.metadata.coverages.all().count(), 0)
# there shouldn't any format element
self.assertEqual(self.resNetcdf.metadata.formats.all().count(), 0)
# there shouldn't any subject element
self.assertEqual(self.resNetcdf.metadata.subjects.all().count(), 0)
# there shouldn't any contributor element
self.assertEqual(self.resNetcdf.metadata.contributors.all().count(), 0)
# there shouldn't any source element
self.assertEqual(self.resNetcdf.metadata.sources.all().count(), 0)
# there shouldn't any relation element
self.assertEqual(
self.resNetcdf.metadata.relations.all().filter(
type='cites').count(), 0)
# there should be 1 creator
self.assertEqual(self.resNetcdf.metadata.creators.all().count(), 1)
# there shouldn't any extended metadata
self.assertEqual(self.resNetcdf.metadata.ori_coverage.all().count(), 0)
self.assertEqual(self.resNetcdf.metadata.variables.all().count(), 0)
# adding a valid netcdf file should generate some core metadata and all extended metadata
files = [
UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name)
]
utils.resource_file_add_pre_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
utils.resource_file_add_process(resource=self.resNetcdf,
files=files,
user=self.user,
extract_metadata=False)
super(TestNetcdfMetaData,
self).netcdf_metadata_extraction(expected_creators_count=2)
def save_attachments(post, path, filename):
"""Create zip file of all attachments of post."""
if not os.path.exists(path):
os.makedirs(path)
my_file = SimpleUploadedFile(path / filename, '')
uploaded = UploadedFile(file=my_file)
with ZipFile(uploaded.file, 'w') as z:
for file in post.attachments.all():
filename = pathlib.PurePosixPath(file.file.path).name
z.write(file.file.path, filename)
return uploaded
def test_max_upload_size_fail(self):
document = base.create_document()
file_field = self.FileField(document.file.field)
file_size = file_field.get_max_upload_size()-1
document.file.file = \
UploadedFile(file=document.file.file, name=document.title, content_type='txt', size=file_size)
# fudge the max_upload_size to fall below file size.
file_field.set_max_upload_size(document.file.size - 1)
with self.assertRaises(forms.ValidationError):
file_field.field.clean(value=document.file, model_instance=document)
# Cleanup
file_field.restore_max_upload_size()
os.remove(document.file.path)
def new_file(self,
field_name,
file_name,
content_type,
content_length,
charset=None):
self.md5 = hashlib.md5()
self.data = ''
self.file = UploadedFile(name=file_name,
content_type=content_type,
charset=charset)
self.file.size = 0
self.file.hashmap = []
def get_step_files(self, step):
wizard_files = self.data[self.step_files_key].get(step, {})
if wizard_files and not self.file_storage:
raise NoFileStorageConfigured
files = {}
for field, field_dict in six.iteritems(wizard_files):
field_dict = field_dict.copy()
tmp_name = field_dict.pop('tmp_name')
files[field] = UploadedFile(file=self.file_storage.open(tmp_name),
**field_dict)
return files or None
def test_file_does_not_exist(self):
"""
Test that the uploaded file exists.
"""
# "workspace" key points to file that does not exist
upload = {
"workspace": os.path.join(self.workspace),
"local": UploadedFile(name="SWAT_Model_Does_Not_Exist.zip"),
"aws": {}
}
with self.assertRaises(FileNotFoundError):
SWATModelZip(upload)
def _create_composite_resource(self, file_obj):
uploaded_file = UploadedFile(file=file_obj,
name=os.path.basename(file_obj.name))
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.user,
title='Test Raster File Type Metadata',
files=(uploaded_file,)
)
# set the generic logical file type
resource_post_create_actions(resource=self.composite_resource, user=self.user,
metadata=self.composite_resource.metadata)
def create_css_from_manifest(tmp_dir, theme, manifest):
for item in manifest:
if "url" in item:
theme.css.create(**item, order=theme.css.count())
else:
with open(item["path"], "rb") as fp:
file_obj = UploadedFile(
fp,
name=item["name"],
content_type="text/css",
size=os.path.getsize(item["path"]),
)
create_css(theme, file_obj)
def handle(self, *args, **options):
all_products = Product.objects.all()
for product in all_products:
if product.image:
image = UploadedFile(product.image.file,
product.image.file.name)
product.cloudinary_image = image
product.save()
print(f"Product image for {product.name} migrated")
else:
print(f"No Product image for {product.name} to migrate")
def test_pd_compare(self):
df = []
for fn in self.files:
with open(fn,'rb') as fh:
fu = UploadedFile(fh)
fm = FileModel(imported_file=fu)
df.append(pd_read(fm.imported_file))
res_a = pd_compare(df[0], df[0])
self.assertEquals(res_a, 'ok')
res_b = pd_compare(df[0], df[1])
self.assertEquals(res_b, expected_diff)
def new_file(self,
field_name,
file_name,
content_type,
content_length,
charset=None):
self.checksum_compute = NoChecksum() if not UPDATE_MD5 else Checksum()
self.data = ''
self.file = UploadedFile(name=file_name,
content_type=content_type,
charset=charset)
self.file.size = 0
self.file.hashmap = []
def test_thumbnail(self):
file_name = 'test_image.jpg'
with open(os.path.join(os.path.dirname(__file__), file_name),
'rb') as f:
test_image = UploadedFile(File(f), content_type='image/jpeg')
category = Category.objects.create(name='Test Category',
slug='test-category',
thumbnail=test_image)
self.assertEqual(category.pk, 1)
self.assertEqual(category.thumbnail_width, 640)
self.assertEqual(category.thumbnail_height, 480)
def test_post_second_step_local_invalid(self):
with open('./texts/test_input/test_upload.txt', 'rb') as file:
uf = UploadedFile(file=file, name='namestring')
valid_data = {
'1-uri': 'uristring',
'0-method': 'local',
'text_wizard-current_step': 4
}
response = self.client.post(add_path, valid_data)
observed_error = response.context['form']._errors
expected_error = {}
self.assertEqual(expected_error, observed_error)
def _create_composite_resource(self, title='Test Ref Time Series File Type Metadata'):
uploaded_file = UploadedFile(file=self.refts_file_obj,
name=os.path.basename(self.refts_file_obj.name))
self.composite_resource = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.user,
title=title,
files=(uploaded_file,)
)
# set the generic logical file as part of resource post create signal
resource_post_create_actions(resource=self.composite_resource, user=self.user,
metadata=self.composite_resource.metadata)
def test_metadata_file_hash(self):
self._publish_transportation_form()
src = os.path.join(self.this_directory, "fixtures", "transportation",
"screenshot.png")
uf = UploadedFile(file=open(src), content_type='image/png')
count = MetaData.objects.count()
MetaData.media_upload(self.xform, uf)
# assert successful insert of new metadata record
self.assertEqual(MetaData.objects.count(), count + 1)
md = MetaData.objects.get(xform=self.xform,
data_value='screenshot.png')
# assert checksum string has been generated, hash length > 1
self.assertTrue(len(md.hash) > 16)
def test_add_xml_file_one(self):
# test that if a .xml file gets added then the resource abstract and keywords get
# updated. Abstract gets updated only if the there is no abstract already
# this zip file has only the required 3 files (.shp, .shx and .dbf)
files = []
target = 'hs_geographic_feature_resource/tests/states_required_files.zip'
files.append(UploadedFile(file=open(target, 'r'), name='states_required_files.zip'))
self.resGeoFeature = hydroshare.create_resource(
resource_type='GeographicFeatureResource',
owner=self.user,
title='Test Geographic Feature (shapefiles)',
keywords=['kw1', 'kw2'],
files=files
)
# uploaded file validation and metadata extraction happens in post resource
# creation handler
hydroshare.utils.resource_post_create_actions(resource=self.resGeoFeature, user=self.user,
metadata=[])
# check that the resource has 3 files
self.assertEqual(self.resGeoFeature.files.count(), 3)
# there should not be any abstract
self.assertEqual(self.resGeoFeature.metadata.description, None)
# there should be 2 keywords
self.assertEqual(self.resGeoFeature.metadata.subjects.count(), 2)
# now add the .shp.xml file
files = []
target = 'hs_geographic_feature_resource/tests/states.shp.xml'
files.append(UploadedFile(file=open(target, 'r'),
name='states.shp.xml'))
hydroshare.utils.resource_file_add_process(self.resGeoFeature, files, self.user)
# check that the resource has 4 files
self.assertEqual(self.resGeoFeature.files.count(), 4)
# there should be abstract now (abstract came from the xml file)
self.assertNotEqual(self.resGeoFeature.metadata.description, None)
# there should be 4 (2 keywords came from the xml file) keywords
self.assertEqual(self.resGeoFeature.metadata.subjects.count(), 4)
self.resGeoFeature.delete()
def setUp(self):
super().setUp()
# create a dummy image file in memory for upload
image_file = BytesIO()
image = Image.new('RGBA', size=(50, 50), color=(256, 0, 0))
image.save(image_file, 'png')
image_file.seek(0)
self.profile.image = UploadedFile(image_file, "filename.png",
"image/png",
len(image_file.getvalue()))
self.profile.save(update_image=True)
def _process_uploaded_file(resource, validate_files_dict):
log = logging.getLogger()
res_file = resource.files.all().first()
if res_file:
# get the file from irods to temp dir
temp_file = utils.get_file_from_irods(res_file)
# validate the file
error_info, files_to_add_to_resource = raster.raster_file_validation(
raster_file=temp_file)
if not error_info:
log.info("Geo raster file validation successful.")
# extract metadata
temp_dir = os.path.dirname(temp_file)
temp_vrt_file_path = [
os.path.join(temp_dir, f) for f in os.listdir(temp_dir)
if '.vrt' == os.path.splitext(f)[1]
].pop()
metadata = raster.extract_metadata(temp_vrt_file_path)
# delete the original resource file
file_name = delete_resource_file_only(resource, res_file)
delete_format_metadata_after_delete_file(resource, file_name)
# add all extracted files (tif and vrt)
for f in files_to_add_to_resource:
uploaded_file = UploadedFile(file=open(f, 'rb'),
name=os.path.basename(f))
utils.add_file_to_resource(resource, uploaded_file)
# use the extracted metadata to populate resource metadata
for element in metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
resource.metadata.create_element(k, **v)
log_msg = "Geo raster resource (ID:{}) - extracted metadata was saved to DB"
log_msg = log_msg.format(resource.short_id)
log.info(log_msg)
else:
# delete the invalid file just uploaded
delete_resource_file_only(resource, res_file)
validate_files_dict['are_files_valid'] = False
err_msg = "Uploaded file was not added to the resource. "
err_msg += ", ".join(msg for msg in error_info)
validate_files_dict['message'] = err_msg
log_msg = "File validation failed for raster resource (ID:{})."
log_msg = log_msg.format(resource.short_id)
log.error(log_msg)
# cleanup the temp file directory
if os.path.exists(temp_file):
shutil.rmtree(os.path.dirname(temp_file))