def load(self, file_storage): field_dict = self.data field_dict = field_dict.copy() tmp_name = field_dict.pop('tmp_name') file_obj = UploadedFile(file=file_storage.open(tmp_name), **field_dict) file_obj.url = file_storage.url(tmp_name) return file_obj
def testFileUpload(self): from django.http import QueryDict, HttpRequest from tardis.tardis_portal.views import upload from django.core.files import File from django.core.files.uploadedfile import UploadedFile from django.utils.datastructures import MultiValueDict from os import path # create request.FILES object django_file = File(self.f1) uploaded_file = UploadedFile(file=django_file) uploaded_file.name = self.filename uploaded_file.size = self.f1_size post_data = [("enctype", "multipart/form-data")] post = QueryDict("&".join(["%s=%s" % (k, v) for (k, v) in post_data])) files = MultiValueDict({"Filedata": [uploaded_file]}) request = HttpRequest() request.FILES = files request.POST = post request.method = "POST" response = upload(request, self.dataset.id) test_files_db = models.Dataset_File.objects.filter(dataset__id=self.dataset.id) self.assertTrue(path.exists(path.join(self.dataset_path, self.filename))) self.assertTrue(self.dataset.id == 1) self.assertTrue(test_files_db[0].url == "tardis://testfile.txt")
def test_unicode_file_names(self): """ Regression test for #8156: files with unicode names I can't quite figure out the encoding situation between doctest and this file, but the actual repr doesn't matter; it just shouldn't return a unicode object. """ uf = UploadedFile(name=u'¿Cómo?',content_type='text') self.assertEqual(type(uf.__repr__()), str)
def __init__(self, path, file_name, content_type, charset): UploadedFile.__init__(self) self.path = path self.name = file_name self.content_type = content_type self.charset = charset self.file_handle = None self.mode = None
def stitch_chunks(self): f = open(os.path.join(settings.MEDIA_ROOT, cloud_path(self, self.filename)), "wb") for chunk in self.chunks.all().order_by("pk"): f.write(chunk.chunk.read()) f.close() f = UploadedFile(open(f.name, "rb")) self.upload.save(self.filename, f) self.state = Upload.STATE_COMPLETE self.save() f.close()
def document_validator(filepath, ex_first_row, ex_first_col): try: with open(os.path.join(settings.MEDIA_ROOT, filepath), 'r') as f: file = UploadedFile(f) dialect = csv.Sniffer().sniff(file.readline(), delimiters=[';', ',', '\t']) mimetype = magic.from_buffer(file.readline(), mime=True) file.seek(0) reader = csv.reader(file, dialect) temp_list = [] for line in iter(reader): if reader.line_num == 1: # save first row temp_list.append(line) # save last row temp_list.append(line) print ex_first_row if not ex_first_row: print "Ciao" # check char in first row and first col if not ex_first_row and not float(temp_list[0][-1]): print 'Hola' raise ValueError if not ex_first_col and not float(temp_list[-1][0]): print 'Hello' raise ValueError ncol = (len(temp_list[0]) - 1) if ex_first_col else len(temp_list[0]) nrow = (reader.line_num - 1) if ex_first_col else reader.line_num if nrow <= 2: print 'Hey' raise ValueError is_cubic = True if (ncol == nrow) else False return_value = {'is_valid': True, 'nrow': nrow, 'ncol': ncol, 'separator': dialect.delimiter, 'mimetype': mimetype, 'is_cubic': is_cubic} except csv.Error: print "Csv" return_value = {'is_valid': False} file = None except Exception: return_value = {'is_valid': False} print "Exc" file = None except ValueError: return_value = {'is_valid': False} print "Value" file = file return return_value, file
def save(self, *args, **kwargs): s = BytesIO() data = zipfile.ZipFile(s,'a') projectfiles = fileobject.objects.filter(project=self.project) for filedata in projectfiles: filed = filedata.filename.read() pathAndName = str(self.project.title)+filedata.subfolder+os.path.split(str(filedata.filename))[1] #### this is where subfolders will be added to inside the zip file. data.writestr(pathAndName, filed) data.close() s.seek(0) filedata = UploadedFile(s) filedata.name = self.project.title+".zip" self.filename = filedata super(zippedObjectProxy, self,).save(generate=False, *args, **kwargs)
def value_from_datadict(self, data, files, name): # we cache the return value of this function, since it is called a # bunch of time, and is expensive if self._cached_value is self._sentinel: upload = super().value_from_datadict(data, files, name) if upload != FILE_INPUT_CONTRADICTION: self.signed_path = data.get(self.signed_path_field_name(name), None) data_uri = data.get(self.data_uri_field_name(name)) has_file = (upload or data_uri) # the path to the cached uploaded file path = None # if we have a cache key, and no file, fetch the file from the cache if self.signed_path and not has_file: try: path = self.signer.unsign(self.signed_path) except BadSignature: # False means the field value should be cleared, which # is the best thing we can do in this case. It # shouldn't happen anyways. self.signed_path = "" self._cached_value = None return self._cached_value elif has_file: # we have a file, so write it to disk, just in case form validation fails with NamedTemporaryFile(prefix="".join(CHOICES[x % 64] for x in os.urandom(16)), suffix=".jpg", dir=self.tmp_dir, delete=False) as f: # write the uploaded file to disk, or the data from the dataURI try: if upload: f.write(upload.read()) else: f.write(b64decode(data_uri[data_uri.find(",")+1:])) except Error: pass else: path = f.name self.signed_path = self.signer.sign(f.name) if path: upload = UploadedFile(open(path, "rb"), name=path, size=os.path.getsize(path)) # tack on a URL attribute so the parent Widget thinks it # has an initial value upload.url = settings.MEDIA_URL + os.path.relpath(upload.file.name, settings.MEDIA_ROOT) self._cached_value = upload return self._cached_value
def upload_save_process(request): """ save file into local storage """ f = request.FILES["file"] wrapper_f = UploadedFile(f) name, filetype = split_name(wrapper_f.name) obj = ProcessedFile() obj.title = name + str(uuid.uuid4()) + "." + filetype wrapper_f.name = obj.title obj.file_obj = f obj.file_type = filetype if filetype != " " else "unknown" obj.save() return wrapper_f
def smart_load_from_upload(classname: str, f: UploadedFile) -> BasePriceList: ''' Attempt to intelligently load the given Django UploadedFile, interpreting it as a price list for the given schedule class name. If interpreting it under the preferred schedule results in either a ValidationError or no valid rows, attempts will be made to re-interpret the file under different schedules. The first found that yields better interpretations of the data will be returned. If no better matches are found, the original result or exception (from interpreting the data under the preferred price list) will be returned. ''' original_error = None pricelist: Optional[BasePriceList] = None try: pricelist = load_from_upload(classname, f) except ValidationError as e: original_error = e if original_error or (pricelist and not pricelist.valid_rows): # See if any of our other registered schedules can make better sense # of it. next_best_pricelist = None for fallback, _ in CHOICES: if fallback == classname: continue try: f.seek(0) next_best_pricelist = load_from_upload(fallback, f) if next_best_pricelist.valid_rows: pricelist = next_best_pricelist break except ValidationError as e: pass if pricelist is None: default_error = ValidationError('Unrecognized price list!') raise original_error or default_error return pricelist
def parse_file(self, file: UploadedFile, _) -> Iterable[Article]: data = file.read() try: for para in self.split_file(data): yield self.parse_document(para) except ApaError: log.error("APA parse attempt failed.") if settings.DEBUG: log.error("The generated HTML can be found in /tmp/apa_unrtf.html") raise
def fileupload(request): if request.method == 'POST': cc = request.POST.get('cc'); myexcel = request.FILES['files[]'] excel_obj = UploadedFile(myexcel) workbook = xlrd.open_workbook(file_contents = excel_obj.read()) all_worksheets = workbook.sheet_names() worksheet_name = all_worksheets[0] worksheet = workbook.sheet_by_name(worksheet_name) for rownum in xrange(worksheet.nrows): tmp = [] for entry in worksheet.row_values(rownum): tmp.append(entry) print tmp return JsonResponse({'status': 'fileupload_ok'})
def file_complete(self, file_size): mp_file_size = sum([part.size for part in self.mp]) if file_size > self.max_file_size or mp_file_size > self.max_file_size: raise FileExceedsSizeLimit if file_size != mp_file_size: raise Exception("Uploaded file size doesn't match computed file size.") self.mp.complete_upload() # save local metadata self.row.size = file_size self.row.save() self.row.releaseLock() uploaded_file = UploadedFile() uploaded_file.download_key = self.row.getKey() uploaded_file.name = self.row.filename return uploaded_file
def _decode_files(self, files): """ Helper method that when given *files* -- a ``dict`` with the structure:: { "<field_name>": { "file_storage_key": "<unicode>", "name": "<unicode>", "content_type": "<unicode>", "size": "<int>", "charset": "<unicode>", }, ... } a new ``dict`` it returned with the structure:: { "<field_name>": <UploadedFile object>, ... } """ if files is None: return None decoded = {} for name, data in files.iteritems(): key = data.pop('file_storage_key') uploaded_file = UploadedFile(file=self.file_storage.open(key), **data) # In order to ensure that files aren't repeatedly saved to the file # storage, the filename of each file in the file storage is added # to ``UploadedFile`` objects as a ``_wizard_file_storage_key`` # attribute when they're decoded. This acts as a marker to indicate # that the file already exists in the file storage. uploaded_file._wizard_file_storage_key = key decoded[name] = uploaded_file return decoded
def save_document(request_file, content_subdir, related_obj, ashash = True): uploadedfile = UploadedFile(request_file) file_content = uploadedfile.read() doc_obj = Document() doc_obj.filehash = md5(file_content).hexdigest() doc_obj.urlencfilename = quote(uploadedfile.name) doc_obj.filename = uploadedfile.name doc_obj.content_type = uploadedfile.file.content_type if ashash: doc_obj.filepath = settings.BASE_DIR + content_subdir + doc_obj.filehash else: doc_obj.filepath = settings.BASE_DIR + content_subdir + doc_obj.filename if related_obj.__class__.__name__.lower() == "queryset": if len(related_obj) == 1: setattr(doc_obj, related_obj[0].__class__.__name__.lower(), related_obj[0]) else: print "ERROR: The queryset object had %s elements to it" % str(len(related_obj)) else: setattr(doc_obj, related_obj.__class__.__name__.lower(), related_obj) doc_obj.save() wfile = open(doc_obj.filepath, "w") wfile.write(file_content) wfile.close()
def upload(req): if req.method == 'POST': if 'file' in req.FILES: file = req.FILES['file'] filename = file._name image = UploadedFile.read(file) print image # apps.ImageTrans.getGrayscale(file) image2 = cv2.imread(image,0) print image2 fp = open('%s/%s' % ('image/data', filename) , 'wb') for chunk in file.chunks(): fp.write(chunk) fp.close() return HttpResponse('File Uploaded') return HttpResponse('Failed to Upload File')
def seed_instances(self, source_version, form, periods, mapping_version, fixed_instance_count=None): for org_unit in source_version.orgunit_set.all(): instances = [] for period in periods: if fixed_instance_count and "Clinic" in org_unit.name: instance_by_ou_periods = randint(1, fixed_instance_count) else: instance_by_ou_periods = 2 if randint(1, 100) == 50 else 1 with_location = randint(1, 3) == 2 # print("generating", form.name, org_unit.name, instance_by_ou_periods) for instance_count in range(0, instance_by_ou_periods): instance = Instance(project=self.project) instance.created_at = parse_datetime("2018-02-16T11:00:00+00") instance.org_unit = org_unit instance.period = period instance.file_name = "fake_it_until_you_make_it.xml" instance.uuid = str(uuid4()) if with_location: instance.location = Point(-11.7868289 + (2 * random()), 8.4494988 + (2 * random()), 0) test_data = {"_version": 1} if mapping_version and "question_mappings" in mapping_version.json: # quality or quantity for key in mapping_version.json["question_mappings"]: test_data[key] = randint(1, 10) else: # CVS test_data["cs_304"] = randint(1, 100) instance.json = test_data instance.form = form if mapping_version.mapping.is_event_tracker(): instance.json.clear() instance.json = { "DE_2005736": "2.5", "DE_2006098": "5", "DE_2006101": "1", "DE_2006103": "Exclusive", "DE_2006104": "3", "DE_2008294": "NVP only", "DE_391382": "dsd", "DE_424405": "", "MMD_PER_NAM": "kid " + str(randint(1, 10000)), "gender": "Male" if randint(1, 10) < 5 else "Female", "is_existing": "0", "last_name": "Skywalker", "_version": 1, "households_note": "", "hh_repeat": [ { "hh_name": "household 1", "hh_gender": "Male" if randint(1, 10) < 5 else "Female", "hh_age": randint(18, 65), "hh_street": "streeet 1", "hh_number": "44b", "hh_city": "bxl", }, { "hh_name": "household 2", "hh_gender": "Male" if randint(1, 10) < 5 else "Female", "hh_age": randint(18, 65), "hh_street": "street b", "hh_number": "45", "hh_city": "Namur", }, ], "instanceID": "uuid:" + instance.uuid, } self.generate_xml_file(instance, form.latest_version) else: instance.json["instanceID"] = "uuid:" + str(uuid4()) xml_string = ( open("./testdata/seed-data-command-instance.xml") .read() .replace("REPLACEUUID", instance.json["instanceID"]) ) buffer = BytesIO(xml_string.encode("utf-8")) buffer.seek(0, 2) file = InMemoryUploadedFile( file=buffer, field_name="file", name=instance.file_name, content_type="application/xml", size=buffer.tell(), charset="utf-8", ) instance.file = file UploadedFile() instances.append(instance) Instance.objects.bulk_create(instances)
def _save(self, name, content): name = self._normalise_name(name) name = self._prepend_prefix(name) content = UploadedFile(content, name) response = self._upload(name, content) return response['public_id']
def test_clean_valid_mime(self): form = ResumableForm() f = UploadedFile(file=None, name="sound.ogg", content_type="audio/ogg") self.assertEqual(f, form.fields.get('file').clean(None, f))
def test_case(self): response = self.client.post(reverse('polygon:revision_case_create', kwargs=self.kwargs), data={ "option": "text", "input_text": "this is input", "output_text": "this is output", "in_samples": False, "output_lock": False, "case_number": 0, "activated": True }) self.assertEqual(self.revision.cases.count(), 1) my_case = self.revision.cases.first() self.assertNotEqual(my_case.fingerprint, "invalid") self.assertEqual(my_case.input_preview, "this is input") self.assertEqual(my_case.output_preview, "this is output") self.assertEqual(my_case.case_number, 1) response = self.client.post( reverse('polygon:revision_case_create', kwargs=self.kwargs), data={ "option": "single", "input_file": SimpleUploadedFile("input", b"this is input 3"), "output_file": SimpleUploadedFile("output", b"this is output 3"), "case_number": 3, "activated": True }) self.assertEqual(self.revision.cases.count(), 2) my_case = self.revision.cases.get(pk=2) self.assertNotEqual(my_case.fingerprint, "invalid") self.assertEqual(my_case.input_preview, "this is input 3") self.assertEqual(my_case.output_preview, "this is output 3") self.assertEqual(my_case.case_number, 3) zip_location = path.join(BASE_LOCATION, "asset", "d.zip") with open(zip_location, "rb") as f: response = self.client.post(reverse('polygon:revision_case_create', kwargs=self.kwargs), data={ "option": "batch", "batch_file": UploadedFile(f), "case_number": 2, "activated": True }) self.assertEqual(self.revision.cases.count(), 4) mmap = {1: 1, 2: 4, 3: 2, 4: 3, 5: 4} input_map = {1: "this", 2: "this", 3: "1 2", 4: "2 2", 5: "this"} for case in self.revision.cases.all(): self.assertNotEqual(case.pk, 2) self.assertEqual(mmap[case.pk], case.case_number) self.assertIn(input_map[case.pk], case.input_preview) with open(zip_location, "rb") as f: response = self.client.post(reverse('polygon:revision_case_create', kwargs=self.kwargs), data={ "option": "batch", "batch_file": UploadedFile(f), "case_number": 0, "activated": True }) self.assertEqual(self.revision.cases.count(), 6) self.assertEqual( len( set(self.revision.cases.all().values_list("fingerprint", flat=True))), 4) # update case p_kwargs = {"cpk": 1} p_kwargs.update(self.kwargs) response = self.client.post(reverse( 'polygon:revision_case_update_file', kwargs=p_kwargs), data={ "option": "text", "input_text": "new input", "output_text": "new output" }) # print(response, response.__dict__) self.assertEqual(self.revision.cases.count(), 6) for case in self.revision.cases.all(): # print(case.pk, case.parent_id) self.assertNotEqual(case.pk, 1)
def calculate_checksum(uploaded_file: UploadedFile) -> str: md5 = hashlib.md5() for chunk in uploaded_file.chunks(): md5.update(chunk) return md5.hexdigest()
def accept_uploaded_photo(request, album_id): """ Main Multiuploader module. Parses data from jQuery plugin and makes database changes. """ if request.method == 'POST': logid = random.randint(0,1000) log.info('[%s] received POST to main multiuploader view' % logid) if request.FILES == None: return HttpResponseBadRequest('Must have files attached!') #getting file data for farther manipulations file = request.FILES[u'files[]'] wrapped_file = UploadedFile(file) filename = wrapped_file.name file_size = wrapped_file.file.size log.info ('[%s] Got file: "%s"' % (logid, str(filename))) # Write out file to disk as a temp file randnumber = logid # use the random number here too temp_filename = '%stmp%s_%s' % (settings.TEMP_DIRECTORY,randnumber, filename) log.info('[%s] Writing out to: %s' % (logid, temp_filename)) destination = open(temp_filename, 'wb+') if wrapped_file.multiple_chunks(): for chunk in wrapped_file.chunks(): destination.write(chunk) else: destination.write(wrapped_file.read()) destination.close() # Dump out EXIF Tags # im = Image.open(temp_filename) # if hasattr( im, '_getexif' ): # exifinfo = im._getexif() # if exifinfo: # for tag, value in exifinfo.items(): # decoded = ExifTags.TAGS.get(tag, tag) # log.info('Found tag: %s, value: %s' % (decoded,value)) orientation = None date_taken = None # Make full size and thumbsize try: im = Image.open(temp_filename) except IOError: log.info('[%s] Error opening file %s: %s %s' % (logid, temp_filename, e.errno, e)) return HttpResponseBadRequest('Could not read file') if hasattr( im, '_getexif' ): exifinfo = im._getexif() if exifinfo: for tag, value in exifinfo.items(): decoded = ExifTags.TAGS.get(tag, tag) # if decoded != 'MakerNote': # if decoded != 'UserComment': # log.info('Found tag: %s, value: %s' % (decoded,value)) if decoded == 'Orientation': orientation = value log.info('[%s] Found tag: %s, value: %s' % (logid,decoded,value)) elif decoded == 'DateTime': date_taken = datetime.strptime(value, "%Y:%m:%d %H:%M:%S") log.info('[%s] Found tag: %s, value: %s, date_taken=%s' % (logid,decoded,value,date_taken)) # We rotate regarding to the EXIF orientation information if orientation: if orientation == 1: # Nothing log.info('[%s] Orientation: No rotation necessary' % logid) pass elif orientation == 2: # Vertical Mirror log.info('[%s] Orientation: Vertical flip' % logid) im = im.transpose(Image.FLIP_LEFT_RIGHT) elif orientation == 3: # Rotation 180 log.info('[%s] Orientation: Rotation 180' % logid) im = im.transpose(Image.ROTATE_180) elif orientation == 4: # Horizontal Mirror log.info('[%s] Orientation: Horizontal Mirror' % logid) im = im.transpose(Image.FLIP_TOP_BOTTOM) elif orientation == 5: # Horizontal Mirror + Rotation 270 log.info('[%s] Orientation: Flip top bottom, rot 270' % logid) im = im.transpose(Image.FLIP_TOP_BOTTOM).transpose(Image.ROTATE_270) elif orientation == 6: # Rotation 270 log.info('[%s] Orientation: Rotate 270' % logid) im = im.transpose(Image.ROTATE_270) elif orientation == 7: # Vertical Mirror + Rotation 270 log.info('[%s] Orientation: Flip left right, rotate 270' % logid) im = im.transpose(Image.FLIP_LEFT_RIGHT).transpose(Image.ROTATE_270) elif orientation == 8: # Rotation 90 log.info('[%s] Orientation: Rotate 90' % logid) im = im.transpose(Image.ROTATE_90) #------------------ # Save the transposed image to disk orig_path = '%stmp%s_mod%s' % (settings.TEMP_DIRECTORY,randnumber, filename) # keep 100% fidelity on the image try: log.info('[%s] Writing corrected photo to path %s' % (logid,orig_path)) im.save(orig_path, "JPEG", quality=100) except IOError: log.info('[%s] Error saving file %s: %s %s' % (logid, orig_path, e.errno, e)) return HttpResponseBadRequest('Could not save file') #------------------ # Save the photo object into the database album = Album.objects.get(id=album_id) photo = Photo() photo.album = album log.info('[%s] Determining photo order' % logid) #------------------ # Determine where in the photo order this picture needs to be photo.order = 0 if date_taken: photo.photodate = date_taken log.info('[%s] Date Taken is %s' % (logid,date_taken)) # Now try to insert the photo by date taken in the order list prev_photo = photo.prev_photo_by_photodate() if prev_photo: log.info('got prev photo. id=%s, photodate=%s, order=%s' % (prev_photo.id,prev_photo.photodate,prev_photo.order)) photo.order = prev_photo.order else: # First in album photo.order = 0 else: # Last in album photo.order = album.photo_set.count() + 1 log.info('[%s] Writing photo entry to database' % logid) #------------------ # Now finally write the entry to the db photo.save() log.info('[%s] Photo object saved. id = %s, order = %s' % (logid, photo.id,photo.order)) #album.reorder_photos() log.info('[%s] Attempting to save file %s to django model id %s' % (logid, orig_path, photo.id)) f = open(orig_path, 'r') photo.filename.save('%s.jpg' % photo.id, File(f)) f.close() log.info('[%s] Cleaning up files' % logid) #clean up temp file unlink(temp_filename) unlink(orig_path) #settings imports file_delete_url = 'multi_delete/' thumbnail_options = dict(size=(200, 200), crop=True) thumb_url = get_thumbnailer(photo.filename).get_thumbnail(thumbnail_options).url #generating json response array result = [] result.append({"name":filename, "size":file_size, "url": thumb_url, "thumbnail_url":thumb_url, "delete_url":'/', "delete_type":"POST",}) response_data = simplejson.dumps(result) #checking for json data type #big thanks to Guy Shapiro if "application/json" in request.META['HTTP_ACCEPT_ENCODING']: mimetype = 'application/json' else: mimetype = 'text/plain' return HttpResponse(response_data, mimetype=mimetype) else: #GET return HttpResponse('Only POST accepted')
def handle_files(files, time, board): """ Check and save files. :param files: files fot handling :param time: current time :param board: post's board :return: json list of files features """ _files = [] for file in files.items(): size = file[1].size if size > config['max_filesize']: return False name = file[1].name ext = name.split('.')[-1] if not ext.lower() in config['allowed_ext']: return False # file saving index = file[0].replace('file', '') # equal 0 for first file and so on path = choose_path(board, 'src', time, ext, index) with open(path, 'wb+') as destination: for chunk in file[1].chunks(): destination.write(chunk) destination.close() # TODO: Refactor all this hell if ext.lower() == 'webm': temp_file = NamedTemporaryFile() temp_path = temp_file.name + '.png' call(["ffmpeg", "-i", path, "-vframes", "1", temp_path]) temp_file.close() temp_th = open(temp_path, 'rb+') preview = UploadedFile(file=temp_th) thumb = make_thumb(preview) preview.close() image = Image.open(temp_path) else: image = Image.open(path) thumb = make_thumb(file[1]) path = choose_path(board, 'thumb', time, 'jpg', index) destination = open(path, 'wb+') destination.write(thumb.read()) destination.close() thumb = Image.open(path) filename = '{0}-{1}.{2}'.format(time, index, ext) file_data = { "name": name, "type": 0, # content_type, "tmp_name": ".", # ??? "error": 0, "size": size, "filename": name, "extension": ext, "file_id": time, "file": filename, "thumb": '{0}-{1}.jpg'.format(time, index), "is_an_image": 0, # content_type.split('/')[0] == 'image', "hash": "c5c76d11ff82103d18c3c9767bcb881e", # TODO hash "width": image.width, "height": image.height, "thumbwidth": thumb.width, "thumbheight": thumb.height, "file_path": '{0}/src/{1}'.format(board, filename), "thumb_path": '{0}/thumb/{1}-{2}.jpg'.format(board, time, index) } image.close() thumb.close() _files.append(file_data) return _files
def mock_upload(filename): with open(get_testing_img_path(filename), 'rb') as img: uploaded_img = UploadedFile(img, 'test_img.png') return models.Illustration.upload_illustration(uploaded_img)
def test_shp_set_file_type_to_geo_feature_required(self): # here we are using a shp file for setting it # to Geo Feature file type which includes metadata extraction self._create_composite_resource() # add the 3 required files to the resource files = [] shp_temp_file = os.path.join(self.temp_dir, self.states_shp_file_name) shutil.copy(self.states_shp_file, shp_temp_file) shx_temp_file = os.path.join(self.temp_dir, self.states_shx_file_name) shutil.copy(self.states_shx_file, shx_temp_file) dbf_temp_file = os.path.join(self.temp_dir, self.states_dbf_file_name) shutil.copy(self.states_dbf_file, dbf_temp_file) files.append( UploadedFile(file=open(shp_temp_file, 'r'), name=self.states_shp_file_name)) files.append( UploadedFile(file=open(shx_temp_file, 'r'), name=self.states_shx_file_name)) files.append( UploadedFile(file=open(dbf_temp_file, 'r'), name=self.states_dbf_file_name)) hydroshare.utils.resource_file_add_process(self.composite_resource, files, self.user) self.assertEqual(self.composite_resource.files.all().count(), 3) res_file = self.composite_resource.files.first() expected_folder_name = res_file.file_name[:-4] # check that the resource file is associated with GenericLogicalFile self.assertEqual(res_file.has_logical_file, True) self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile") # check that there is 3 GenericLogicalFile object self.assertEqual(GenericLogicalFile.objects.count(), 3) # set the shp file to GeoFeatureFile type shp_res_file = [ f for f in self.composite_resource.files.all() if f.extension == '.shp' ][0] GeoFeatureLogicalFile.set_file_type(self.composite_resource, shp_res_file.id, self.user) # test files in the file type self.assertEqual(self.composite_resource.files.count(), 3) # check that there is no GenericLogicalFile object self.assertEqual(GenericLogicalFile.objects.count(), 0) # check that there is one GeoFeatureLogicalFile object self.assertEqual(GeoFeatureLogicalFile.objects.count(), 1) logical_file = GeoFeatureLogicalFile.objects.first() self.assertEqual(logical_file.files.count(), 3) # check that the 3 resource files are now associated with GeoFeatureLogicalFile for res_file in self.composite_resource.files.all(): self.assertEqual(res_file.logical_file_type_name, "GeoFeatureLogicalFile") self.assertEqual(res_file.has_logical_file, True) self.assertTrue( isinstance(res_file.logical_file, GeoFeatureLogicalFile)) # check that we put the 3 files in a new folder for res_file in self.composite_resource.files.all(): file_path, base_file_name, _ = get_resource_file_name_and_extension( res_file) expected_file_path = "{}/data/contents/{}/{}" res_file.file_folder = expected_folder_name expected_file_path = expected_file_path.format( self.composite_resource.root_path, expected_folder_name, base_file_name) self.assertEqual(file_path, expected_file_path) # test extracted raster file type metadata # there should not be any resource level coverage self.assertEqual(self.composite_resource.metadata.coverages.count(), 0) self.assertNotEqual(logical_file.metadata.geometryinformation, None) self.assertEqual( logical_file.metadata.geometryinformation.featureCount, 51) self.assertEqual( logical_file.metadata.geometryinformation.geometryType, "MULTIPOLYGON") self.assertNotEqual(logical_file.metadata.originalcoverage, None) self.assertEqual(logical_file.metadata.originalcoverage.datum, 'unknown') self.assertEqual( logical_file.metadata.originalcoverage.projection_name, 'unknown') self.assertGreater( len(logical_file.metadata.originalcoverage.projection_string), 0) self.assertEqual(logical_file.metadata.originalcoverage.unit, 'unknown') self.assertEqual(logical_file.metadata.originalcoverage.eastlimit, -66.9692712587578) self.assertEqual(logical_file.metadata.originalcoverage.northlimit, 71.406235393967) self.assertEqual(logical_file.metadata.originalcoverage.southlimit, 18.921786345087) self.assertEqual(logical_file.metadata.originalcoverage.westlimit, -178.217598362366) # there should not be any file level keywords self.assertEqual(logical_file.metadata.keywords, []) self.composite_resource.delete() # there should be no GeoFeatureLogicalFile object at this point self.assertEqual(GeoFeatureLogicalFile.objects.count(), 0) # there should be no GenericFileMetaData object at this point self.assertEqual(GeoFeatureFileMetaData.objects.count(), 0)
def multiuploader(request): """ Main Multiuploader module. Parses data from jQuery plugin and makes database changes. """ if request.method == 'POST': log.info('received POST to main multiuploader view') if request.FILES == None: return HttpResponseBadRequest('Must have files attached!') #getting file data for farther manipulations file = request.FILES[u'files[]'] wrapped_file = UploadedFile(file) filename = wrapped_file.name file_size = wrapped_file.file.size log.info ('Got file: "%s"' % str(filename)) log.info('Content type: "$s" % file.content_type') #writing file manually into model #because we don't need form of any type. image = MultiuploaderImage() image.filename=str(filename) image.image=file image.key_data = image.key_generate image.save() log.info('File saving done') #getting thumbnail url using sorl-thumbnail if 'image' in file.content_type.lower(): im = get_thumbnail(image, "80x80", quality=50) thumb_url = im.url else: thumb_url = '' #settings imports try: file_delete_url = settings.MULTI_FILE_DELETE_URL+'/' file_url = settings.MULTI_IMAGE_URL+'/'+image.key_data+'/' except AttributeError: file_delete_url = 'multi_delete/' file_url = 'multi_image/'+image.key_data+'/' """ is actually: [{"name": "Screenshot from 2012-11-14 16:17:46.png", "url": "multi_image/95925526541943247735000327303075602114185579370918344597903504067450818566531/", "thumbnail_url": "/media/cache/f8/bd/f8bd83aadeba651ff9c040bb394ce117.jpg", "delete_type": "POST", "delete_url": "multi_delete/7/", "size": 38520}] should be: {"files":[{"url":"http://jquery-file-upload.appspot.com/AMIfv9734HSTDGd3tIybbnKVru--IjhjULKvNcIGUL2lvfqA93RNCAizDbvP-RQJNbh-N9m8UXsk-90jFFYSp8TlbZYhEcNN6Vb9HzQVQtdmF83H6sE_XkdnlI2V8lHX5V3Y4AamdX6VMbAt9sNWNx2BVGzhTfAYkRLYmRE1VzzWSe9C8c8Fu8g/Screenshot%20from%202012-11-14%2016%3A17%3A46.png","thumbnail_url":"http://lh5.ggpht.com/fcjVNT6qUGoMDtqqaNDNtU4mghy34qlzfj2GujikLgC7Nj5Bs4LUT_DWG_Q8OWujqvYHsKbeQ9pkvoAW4WiaubmqQxobIPyt=s80","name":"Screenshot from 2012-11-14 16:17:46.png","type":"image/png","size":38520,"delete_url":"http://jquery-file-upload.appspot.com/AMIfv9734HSTDGd3tIybbnKVru--IjhjULKvNcIGUL2lvfqA93RNCAizDbvP-RQJNbh-N9m8UXsk-90jFFYSp8TlbZYhEcNN6Vb9HzQVQtdmF83H6sE_XkdnlI2V8lHX5V3Y4AamdX6VMbAt9sNWNx2BVGzhTfAYkRLYmRE1VzzWSe9C8c8Fu8g/Screenshot%20from%202012-11-14%2016%3A17%3A46.png?delete=true","delete_type":"DELETE"}]} """ #generating json response array result = { 'files': [ {"name":filename, "size":file_size, "url":file_url, "thumbnail_url":thumb_url, "delete_url":file_delete_url+str(image.pk)+'/', "delete_type":"POST",} ] } response_data = simplejson.dumps(result) #checking for json data type #big thanks to Guy Shapiro if "application/json" in request.META['HTTP_ACCEPT_ENCODING']: mimetype = 'application/json' else: mimetype = 'text/plain' return HttpResponse(response_data, mimetype=mimetype) else: #GET return HttpResponse('Only POST accepted')
def multiuploader(request, noajax=False): """ Main Multiuploader module. Parses data from jQuery plugin and makes database changes. """ if request.method == 'POST': log.info('received POST to main multiuploader view') if request.FILES is None: response_data = [{"error": _('Must have files attached!')}] return HttpResponse(json.dumps(response_data)) if not u'form_type' in request.POST: response_data = [{ "error": _("Error when detecting form type, form_type is missing") }] return HttpResponse(json.dumps(response_data)) signer = Signer() try: form_type = signer.unsign(request.POST.get(u"form_type")) except BadSignature: response_data = [{"error": _("Tampering detected!")}] return HttpResponse(json.dumps(response_data)) form = MultiUploadForm(request.POST, request.FILES, form_type=form_type) if not form.is_valid(): error = _("Unknown error") if "file" in form._errors and len(form._errors["file"]) > 0: error = form._errors["file"][0] response_data = [{"error": error}] return HttpResponse(json.dumps(response_data)) file = request.FILES[u'file'] wrapped_file = UploadedFile(file) filename = wrapped_file.name file_size = wrapped_file.file.size log.info('Got file: "%s"' % filename) #writing file manually into model #because we don't need form of any type. fl = MultiuploaderFile() fl.filename = filename fl.file = file fl.save() log.info('File saving done') thumb_url = "" try: im = get_thumbnail(fl.file, "80x80", quality=50) thumb_url = im.url except Exception as e: log.error(e) #generating json response array result = [{ "id": fl.id, "name": filename, "size": file_size, "url": reverse('multiuploader_file_link', args=[fl.pk]), "thumbnail_url": thumb_url, "delete_url": reverse('multiuploader_delete', args=[fl.pk]), "delete_type": "POST", }] response_data = json.dumps(result) #checking for json data type #big thanks to Guy Shapiro if noajax: if request.META['HTTP_REFERER']: redirect(request.META['HTTP_REFERER']) if "application/json" in request.META['HTTP_ACCEPT_ENCODING']: mimetype = 'application/json' else: mimetype = 'text/plain' return HttpResponse(response_data, content_type="{0}; charset=utf-8".format(mimetype)) else: # GET return HttpResponse('Only POST accepted')
def partial_update(self, request, *args, **kwargs): """ Transfer file data """ # PATCHED: Validate upload session url kwarg upload_session_id = kwargs.get('upload_session_id') try: UploadSession.objects.get(pk=upload_session_id) except (UploadSession.DoesNotExist, ValidationError): return Response('Wrong upload session uid.', status=status.HTTP_400_BAD_REQUEST) # Validate tus header if not has_required_tus_header(request): return Response('Missing "{}" header.'.format('Tus-Resumable'), status=status.HTTP_400_BAD_REQUEST) # Validate content type if not self._is_valid_content_type(request): return Response( 'Invalid value for "Content-Type" header: {}. Expected "{}".'. format(request.META['CONTENT_TYPE'], TusUploadStreamParser.media_type), status=status.HTTP_400_BAD_REQUEST) # Retrieve object upload = self.get_object() # Get upload_offset upload_offset = getattr(request, constants.UPLOAD_OFFSET_NAME) # Validate upload_offset if upload_offset != upload.upload_offset: raise Conflict # Make sure there is a tempfile for the upload assert upload.get_or_create_temporary_file() # Change state if upload.state == states.INITIAL: upload.start_receiving() upload.save() # Get chunk from request chunk_bytes = self.get_chunk(request) # Check for data if not chunk_bytes: return Response('No data.', status=status.HTTP_400_BAD_REQUEST) # Check checksum (http://tus.io/protocols/resumable-upload.html#checksum) upload_checksum = getattr(request, constants.UPLOAD_CHECKSUM_FIELD_NAME, None) if upload_checksum is not None: if upload_checksum[0] not in tus_api_checksum_algorithms: return Response('Unsupported Checksum Algorithm: {}.'.format( upload_checksum[0]), status=status.HTTP_400_BAD_REQUEST) elif not checksum_matches(upload_checksum[0], upload_checksum[1], chunk_bytes): return Response('Checksum Mismatch.', status=460) # Run chunk validator chunk_bytes = self.validate_chunk(upload_offset, chunk_bytes) # Check for data if not chunk_bytes: return Response( 'No data. Make sure "validate_chunk" returns data.', status=status.HTTP_400_BAD_REQUEST) # Write file chunk_size = int(request.META.get('CONTENT_LENGTH', 102400)) try: upload.write_data(chunk_bytes, chunk_size) except Exception as e: return Response(str(e), status=status.HTTP_400_BAD_REQUEST) headers = { 'Upload-Offset': upload.upload_offset, } response_data = None if upload.upload_length == upload.upload_offset: # PATCHED: re-send request to our native upload() method from apps.project.api.v1 import UploadSessionViewSet file = UploadedFile(file=open(upload.temporary_file_path, 'rb'), name=upload.filename, size=upload.upload_length) request._files = MultiValueDict() request._files['file'] = file try: directory_path = os.path.dirname( json.loads(upload.upload_metadata)['relativePath']) except (KeyError, TypeError, json.JSONDecodeError): directory_path = None response = UploadSessionViewSet(request=request, format_kwarg=upload_session_id, action='upload', kwargs={ 'pk': upload_session_id }).upload( request=request, pk=upload_session_id, review_file=False, directory_path=directory_path) if response.status_code != 200: return response response_data = response.data # Trigger signal signals.received.send(sender=upload.__class__, instance=upload) # Add upload expiry to headers add_expiry_header(upload, headers) return Response(data=response_data, headers=headers, status=status.HTTP_204_NO_CONTENT)
def test_metadata_on_content_file_delete(self): # test that some of the metadata is not deleted on content file deletion files = [ UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name) ] utils.resource_file_add_pre_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=True) # there should be 2 content files self.assertEqual(self.resRaster.files.all().count(), 2) # there should be 2 format elements self.assertEqual(self.resRaster.metadata.formats.all().count(), 2) self.assertEqual( self.resRaster.metadata.formats.all().filter( value='application/vrt').count(), 1) self.assertEqual( self.resRaster.metadata.formats.all().filter( value='image/tiff').count(), 1) # delete content file that we added above hydroshare.delete_resource_file(self.resRaster.short_id, self.raster_tif_file_name, self.user) # there should no content file self.assertEqual(self.resRaster.files.all().count(), 0) # there should be a title element self.assertNotEqual(self.resRaster.metadata.title, None) # there should be no abstract element self.assertEqual(self.resRaster.metadata.description, None) # there should be 1 creator element self.assertEqual(self.resRaster.metadata.creators.all().count(), 1) # there should be no contributor element self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0) # there should be no coverage element self.assertEqual(self.resRaster.metadata.coverages.all().count(), 0) # there should be no format element self.assertEqual(self.resRaster.metadata.formats.all().count(), 0) # there should be no subject element self.assertEqual(self.resRaster.metadata.subjects.all().count(), 0) # testing extended metadata elements - there should not be any resource specific metadata self.assertEqual(self.resRaster.metadata.originalCoverage, None) self.assertEqual(self.resRaster.metadata.cellInformation, None) self.assertEqual(self.resRaster.metadata.bandInformations.count(), 0) self.resRaster.delete()
def netcdf_pre_add_files_to_resource(sender, **kwargs): nc_res = kwargs['resource'] files = kwargs['files'] validate_files_dict = kwargs['validate_files'] source_names = kwargs['source_names'] if __debug__: assert(isinstance(source_names, list)) if len(files) > 1: # file number validation validate_files_dict['are_files_valid'] = False validate_files_dict['message'] = 'Only one file can be uploaded.' file_selected = False in_file_name = '' nc_file_name = '' if files: file_selected = True in_file_name = files[0].file.name nc_file_name = os.path.splitext(files[0].name)[0] elif source_names: nc_file_name = os.path.splitext(os.path.basename(source_names[0]))[0] ref_tmpfiles = utils.get_fed_zone_files(source_names) if ref_tmpfiles: in_file_name = ref_tmpfiles[0] file_selected = True if file_selected and in_file_name: # file type validation and existing metadata update and create new ncdump text file nc_dataset = nc_utils.get_nc_dataset(in_file_name) if isinstance(nc_dataset, netCDF4.Dataset): # delete all existing resource files and metadata related for f in ResourceFile.objects.filter(object_id=nc_res.id): delete_resource_file_only(nc_res, f) # update resource modification info user = kwargs['user'] utils.resource_modified(nc_res, user, overwrite_bag=False) # extract metadata res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict(in_file_name) # update title info if res_dublin_core_meta.get('title'): if nc_res.metadata.title: nc_res.metadata.title.delete() nc_res.metadata.create_element('title', value=res_dublin_core_meta['title']) # update description info if res_dublin_core_meta.get('description'): if nc_res.metadata.description: nc_res.metadata.description.delete() nc_res.metadata.create_element('description', abstract=res_dublin_core_meta.get('description')) # update creator info if res_dublin_core_meta.get('creator_name'): name = res_dublin_core_meta.get('creator_name') email = res_dublin_core_meta.get('creator_email', '') url = res_dublin_core_meta.get('creator_url', '') arguments = dict(name=name, email=email, homepage=url) creator = nc_res.metadata.creators.all().filter(name=name).first() if creator: order = creator.order if order != 1: creator.delete() arguments['order'] = order nc_res.metadata.create_element('creator', **arguments) else: nc_res.metadata.create_element('creator', **arguments) # update contributor info if res_dublin_core_meta.get('contributor_name'): name_list = res_dublin_core_meta['contributor_name'].split(',') existing_contributor_names = [contributor.name for contributor in nc_res.metadata.contributors.all()] for name in name_list: if name not in existing_contributor_names: nc_res.metadata.create_element('contributor', name=name) # update subject info if res_dublin_core_meta.get('subject'): keywords = res_dublin_core_meta['subject'].split(',') existing_keywords = [subject.value for subject in nc_res.metadata.subjects.all()] for keyword in keywords: if keyword not in existing_keywords: nc_res.metadata.create_element('subject', value=keyword) # update source if res_dublin_core_meta.get('source'): for source in nc_res.metadata.sources.all(): source.delete() nc_res.metadata.create_element('source', derived_from=res_dublin_core_meta.get('source')) # update license element: if res_dublin_core_meta.get('rights'): raw_info = res_dublin_core_meta.get('rights') b = re.search("(?P<url>https?://[^\s]+)", raw_info) url = b.group('url') if b else '' statement = raw_info.replace(url, '') if url else raw_info if nc_res.metadata.rights: nc_res.metadata.rights.delete() nc_res.metadata.create_element('rights', statement=statement, url=url) # update relation if res_dublin_core_meta.get('references'): nc_res.metadata.relations.filter(type='cites').all().delete() nc_res.metadata.create_element('relation', type='cites', value=res_dublin_core_meta['references']) # update box info nc_res.metadata.coverages.all().delete() if res_dublin_core_meta.get('box'): nc_res.metadata.create_element('coverage', type='box', value=res_dublin_core_meta['box']) # update period info if res_dublin_core_meta.get('period'): nc_res.metadata.create_element('coverage', type='period', value=res_dublin_core_meta['period']) # update variable info nc_res.metadata.variables.all().delete() for var_info in res_type_specific_meta.values(): nc_res.metadata.create_element('variable', name=var_info['name'], unit=var_info['unit'], type=var_info['type'], shape=var_info['shape'], missing_value=var_info['missing_value'], descriptive_name=var_info['descriptive_name'], method=var_info['method']) # update the original spatial coverage meta nc_res.metadata.ori_coverage.all().delete() if res_dublin_core_meta.get('original-box'): if res_dublin_core_meta.get('projection-info'): nc_res.metadata.create_element( 'originalcoverage', value=res_dublin_core_meta['original-box'], projection_string_type=res_dublin_core_meta['projection-info']['type'], projection_string_text=res_dublin_core_meta['projection-info']['text'], datum=res_dublin_core_meta['projection-info']['datum']) else: nc_res.metadata.create_element('originalcoverage', value=res_dublin_core_meta['original-box']) # create the ncdump text file dump_file = create_header_info_txt_file(in_file_name, nc_file_name) dump_file_name = nc_file_name + '_header_info.txt' uploaded_file = UploadedFile(file=open(dump_file), name=dump_file_name) files.append(uploaded_file) else: validate_files_dict['are_files_valid'] = False validate_files_dict['message'] = 'Please check if the uploaded file is in ' \ 'valid NetCDF format.' if source_names and in_file_name: shutil.rmtree(os.path.dirname(in_file_name))
def test_copy_composite_resource(self): """Test that logical file type objects gets copied along with the metadata that each logical file type object contains. Here we are not testing resource level metadata copy as that has been tested in separate unit tests""" self.raster_obj = open(self.temp_raster_file, 'r') files = [UploadedFile(file=self.raster_obj, name='cea.tif')] self.composite_resource = hydroshare.create_resource( resource_type='CompositeResource', owner=self.owner, title='Test Composite Resource', files=files, auto_aggregate=False) # run the resource post creation signal utils.resource_post_create_actions( resource=self.composite_resource, user=self.owner, metadata=self.composite_resource.metadata) self.assertEqual(self.composite_resource.files.all().count(), 1) res_file = self.composite_resource.files.first() # check that the resource file is not associated with file type self.assertEqual(res_file.has_logical_file, False) # set the tif file to GeoRasterFile type GeoRasterLogicalFile.set_file_type(self.composite_resource, self.owner, res_file.id) # ensure a nonowner who does not have permission to view a resource cannot copy it with self.assertRaises(PermissionDenied): hydroshare.create_empty_resource(self.composite_resource.short_id, self.nonowner, action='copy') # give nonowner view privilege so nonowner can create a new copy of this resource self.owner.uaccess.share_resource_with_user(self.composite_resource, self.nonowner, PrivilegeCodes.VIEW) orig_res_file = self.composite_resource.files.first() orig_geo_raster_lfo = orig_res_file.logical_file # add some key value metadata orig_geo_raster_lfo.metadata.extra_metadata = { 'key-1': 'value-1', 'key-2': 'value-2' } # create a copy fo the composite resource new_composite_resource = hydroshare.create_empty_resource( self.composite_resource.short_id, self.nonowner, action='copy') new_composite_resource = hydroshare.copy_resource( self.composite_resource, new_composite_resource) # check that there is 2 GeoRasterLogicalFile objects self.assertEqual(GeoRasterLogicalFile.objects.count(), 2) # compare the 2 GeoRasterLogicalFile objects from the original resource and the new one orig_res_file = self.composite_resource.files.first() orig_geo_raster_lfo = orig_res_file.logical_file copy_res_file = new_composite_resource.files.first() copy_geo_raster_lfo = copy_res_file.logical_file # check that we put the 2 files in a new folder (cea) for res_file in self.composite_resource.files.all(): file_path, base_file_name = res_file.full_path, res_file.file_name expected_file_path = "{}/data/contents/cea/{}" expected_file_path = expected_file_path.format( self.composite_resource.root_path, base_file_name) self.assertEqual(file_path, expected_file_path) for res_file in new_composite_resource.files.all(): file_path, base_file_name = res_file.full_path, res_file.file_name expected_file_path = "{}/data/contents/cea/{}" expected_file_path = expected_file_path.format( new_composite_resource.root_path, base_file_name) self.assertEqual(file_path, expected_file_path) # both logical file objects should have 2 resource files self.assertEqual(orig_geo_raster_lfo.files.count(), copy_geo_raster_lfo.files.count()) self.assertEqual(orig_geo_raster_lfo.files.count(), 2) # both logical file objects should have same dataset_name self.assertEqual(orig_geo_raster_lfo.dataset_name, copy_geo_raster_lfo.dataset_name) # both should have same key/value metadata self.assertEqual(orig_geo_raster_lfo.metadata.extra_metadata, copy_geo_raster_lfo.metadata.extra_metadata) # both logical file objects should have same coverage metadata self.assertEqual(orig_geo_raster_lfo.metadata.coverages.count(), copy_geo_raster_lfo.metadata.coverages.count()) self.assertEqual(orig_geo_raster_lfo.metadata.coverages.count(), 1) org_spatial_coverage = orig_geo_raster_lfo.metadata.spatial_coverage copy_spatial_coverage = copy_geo_raster_lfo.metadata.spatial_coverage self.assertEqual(org_spatial_coverage.type, copy_spatial_coverage.type) self.assertEqual(org_spatial_coverage.type, 'box') self.assertEqual(org_spatial_coverage.value['projection'], copy_spatial_coverage.value['projection']) self.assertEqual(org_spatial_coverage.value['units'], copy_spatial_coverage.value['units']) self.assertEqual(org_spatial_coverage.value['northlimit'], copy_spatial_coverage.value['northlimit']) self.assertEqual(org_spatial_coverage.value['eastlimit'], copy_spatial_coverage.value['eastlimit']) self.assertEqual(org_spatial_coverage.value['southlimit'], copy_spatial_coverage.value['southlimit']) self.assertEqual(org_spatial_coverage.value['westlimit'], copy_spatial_coverage.value['westlimit']) # both logical file objects should have same original coverage org_orig_coverage = orig_geo_raster_lfo.metadata.originalCoverage copy_orig_coverage = copy_geo_raster_lfo.metadata.originalCoverage self.assertEqual(org_orig_coverage.value['projection'], copy_orig_coverage.value['projection']) self.assertEqual(org_orig_coverage.value['units'], copy_orig_coverage.value['units']) self.assertEqual(org_orig_coverage.value['northlimit'], copy_orig_coverage.value['northlimit']) self.assertEqual(org_orig_coverage.value['eastlimit'], copy_orig_coverage.value['eastlimit']) self.assertEqual(org_orig_coverage.value['southlimit'], copy_orig_coverage.value['southlimit']) self.assertEqual(org_orig_coverage.value['westlimit'], copy_orig_coverage.value['westlimit']) # both logical file objects should have same cell information metadata orig_cell_info = orig_geo_raster_lfo.metadata.cellInformation copy_cell_info = copy_geo_raster_lfo.metadata.cellInformation self.assertEqual(orig_cell_info.rows, copy_cell_info.rows) self.assertEqual(orig_cell_info.columns, copy_cell_info.columns) self.assertEqual(orig_cell_info.cellSizeXValue, copy_cell_info.cellSizeXValue) self.assertEqual(orig_cell_info.cellSizeYValue, copy_cell_info.cellSizeYValue) self.assertEqual(orig_cell_info.cellDataType, copy_cell_info.cellDataType) # both logical file objects should have same band information metadata self.assertEqual(orig_geo_raster_lfo.metadata.bandInformations.count(), 1) self.assertEqual(orig_geo_raster_lfo.metadata.bandInformations.count(), copy_geo_raster_lfo.metadata.bandInformations.count()) orig_band_info = orig_geo_raster_lfo.metadata.bandInformations.first() copy_band_info = copy_geo_raster_lfo.metadata.bandInformations.first() self.assertEqual(orig_band_info.noDataValue, copy_band_info.noDataValue) self.assertEqual(orig_band_info.maximumValue, copy_band_info.maximumValue) self.assertEqual(orig_band_info.minimumValue, copy_band_info.minimumValue) # make sure to clean up all created resources to clean up iRODS storage if self.composite_resource: self.composite_resource.delete() if new_composite_resource: new_composite_resource.delete()
def test_metadata_on_content_file_delete(self): files = [ UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name) ] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) self.resModelProgram.metadata.create_element('Description', abstract="test abstract") self.resModelProgram.metadata.create_element('Subject', value="test subject") release_date = '2016-10-24T21:05:00.315907+00:00' self.resModelProgram.metadata.create_element( 'MpMetadata', modelVersion='5.1.011', modelProgramLanguage='Fortran', modelOperatingSystem='Windows', modelReleaseDate=release_date, modelWebsite='http://www.hydroshare.org', modelCodeRepository='http://www.github.com', modelReleaseNotes='releaseNote.pdf', modelDocumentation='manual.pdf', modelSoftware='utilities.exe', modelEngine='sourceCode.zip') # there should one content file self.assertEquals(self.resModelProgram.files.all().count(), 1) # there should be one format element self.assertEquals(self.resModelProgram.metadata.formats.all().count(), 1) # the short path should just consist of the file name. self.assertEquals(self.resModelProgram.files.all()[0].short_path, self.file_name) # delete content file that we added above; note that file name is a short_path hydroshare.delete_resource_file(self.resModelProgram.short_id, self.file_name, self.user) # there should no content file self.assertEquals(self.resModelProgram.files.all().count(), 0) # test the core metadata at this point self.assertNotEquals(self.resModelProgram.metadata.title, None) # there should be an abstract element self.assertNotEquals(self.resModelProgram.metadata.description, None) # there should be one creator element self.assertEquals(self.resModelProgram.metadata.creators.all().count(), 1) # testing extended metadata elements self.assertNotEqual(self.resModelProgram.metadata.program, None)
def test_metadata_delete_on_resource_delete(self): # adding a valid netcdf file should generate some core metadata and all extended metadata files = [ UploadedFile(file=self.netcdf_file_obj, name=self.netcdf_file_name) ] utils.resource_file_add_pre_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resNetcdf, files=files, user=self.user, extract_metadata=True) # before resource delete # resource core metadata core_metadata_obj = self.resNetcdf.metadata self.assertEqual(CoreMetaData.objects.all().count(), 1) # there should be Creator metadata objects self.assertTrue( Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Contributor metadata objects self.assertTrue( Contributor.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be Identifier metadata objects self.assertTrue( Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Type metadata objects self.assertTrue( Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Source metadata objects self.assertTrue( Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Relation metadata objects self.assertTrue( Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse( Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Title metadata objects self.assertTrue( Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Description (Abstract) metadata objects self.assertTrue( Description.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be Date metadata objects self.assertTrue( Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Subject metadata objects self.assertTrue( Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Coverage metadata objects self.assertTrue( Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Format metadata objects self.assertTrue( Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Language metadata objects self.assertTrue( Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Rights metadata objects self.assertTrue( Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be original coverage metadata objects self.assertTrue( OriginalCoverage.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be Variable metadata objects self.assertTrue( Variable.objects.filter(object_id=core_metadata_obj.id).exists()) # delete resource hydroshare.delete_resource(self.resNetcdf.short_id) self.assertEqual(CoreMetaData.objects.all().count(), 0) # there should be no Creator metadata objects self.assertFalse( Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Contributor metadata objects self.assertFalse( Contributor.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be no Identifier metadata objects self.assertFalse( Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Type metadata objects self.assertFalse( Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse( Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse( Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse( Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Title metadata objects self.assertFalse( Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Description (Abstract) metadata objects self.assertFalse( Description.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be no Date metadata objects self.assertFalse( Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Subject metadata objects self.assertFalse( Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse( Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Format metadata objects self.assertFalse( Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Language metadata objects self.assertFalse( Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Rights metadata objects self.assertFalse( Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be original coverage metadata objects self.assertFalse( OriginalCoverage.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be Variable metadata objects self.assertFalse( Variable.objects.filter(object_id=core_metadata_obj.id).exists())
def test_metadata_delete_on_resource_delete(self): files = [ UploadedFile(file=self.text_file_obj, name=self.text_file_obj.name) ] utils.resource_file_add_pre_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) utils.resource_file_add_process(resource=self.resModelProgram, files=files, user=self.user, extract_metadata=False) self.resModelProgram.metadata.create_element('Description', abstract="test abstract") self.resModelProgram.metadata.create_element('Subject', value="test subject") release_date = '2016-10-24T21:05:00.315907+00:00' self.resModelProgram.metadata.create_element( 'MpMetadata', modelVersion='5.1.011', modelProgramLanguage='Fortran', modelOperatingSystem='Windows', modelReleaseDate=release_date, modelWebsite='http://www.hydroshare.org', modelCodeRepository='http://www.github.com', modelReleaseNotes='releaseNote.pdf', modelDocumentation='manual.pdf', modelSoftware='utilities.exe', modelEngine='sourceCode.zip') self.resModelProgram.metadata.create_element('Contributor', name="user2") # before resource delete core_metadata_obj = self.resModelProgram.metadata self.assertEqual(CoreMetaData.objects.all().count(), 1) # there should be Creator metadata objects self.assertTrue( Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Contributor metadata objects self.assertTrue( Contributor.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be Identifier metadata objects self.assertTrue( Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Type metadata objects self.assertTrue( Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse( Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse( Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse( Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Title metadata objects self.assertTrue( Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Description (Abstract) metadata objects self.assertTrue( Description.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be Date metadata objects self.assertTrue( Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Subject metadata objects self.assertTrue( Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse( Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Format metadata objects self.assertTrue( Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Language metadata objects self.assertTrue( Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be Rights metadata objects self.assertTrue( Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be Model Program metadata objects self.assertTrue( MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists()) # delete resource hydroshare.delete_resource(self.resModelProgram.short_id) self.assertEquals(CoreMetaData.objects.all().count(), 0) # there should be no Creator metadata objects self.assertFalse( Creator.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Contributor metadata objects self.assertFalse( Contributor.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be no Identifier metadata objects self.assertFalse( Identifier.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Type metadata objects self.assertFalse( Type.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Source metadata objects self.assertFalse( Source.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Relation metadata objects self.assertFalse( Relation.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Publisher metadata objects self.assertFalse( Publisher.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Title metadata objects self.assertFalse( Title.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Description (Abstract) metadata objects self.assertFalse( Description.objects.filter( object_id=core_metadata_obj.id).exists()) # there should be no Date metadata objects self.assertFalse( Date.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Subject metadata objects self.assertFalse( Subject.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Coverage metadata objects self.assertFalse( Coverage.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Format metadata objects self.assertFalse( Format.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Language metadata objects self.assertFalse( Language.objects.filter(object_id=core_metadata_obj.id).exists()) # there should be no Rights metadata objects self.assertFalse( Rights.objects.filter(object_id=core_metadata_obj.id).exists()) # resource specific metadata # there should be no Model Output metadata objects self.assertFalse( MpMetadata.objects.filter(object_id=core_metadata_obj.id).exists())
def setUp(self): self.files = [open(pwd + "cat1.jpg", "rb"), open(pwd + "cat2.jpg", "rb")] self.uploads = [UploadedFile(f, f.name, "image/jpeg", os.path.getsize(f.name)) for f in self.files]
def multiuploader(request): """ Main Multiuploader module. Parses data from jQuery plugin and makes database changes. """ if request.method == 'POST': log.info('received POST to main multiuploader view') if request.FILES == None: return HttpResponseBadRequest('Must have files attached!') #getting file data for farther manipulations file = request.FILES[u'files[]'] wrapped_file = UploadedFile(file) filename = wrapped_file.name file_size = wrapped_file.file.size log.info('Got file: "%s"' % str(filename)) log.info('Content type: "$s" % file.content_type') #writing file manually into model #because we don't need form of any type. image = MultiuploaderImage() image.filename = str(filename) image.image = file image.key_data = image.key_generate image.save() log.info('File saving done') #getting thumbnail url using sorl-thumbnail if 'image' in file.content_type.lower(): im = get_thumbnail(image, "80x80", quality=50) thumb_url = im.url else: thumb_url = '' #settings imports try: file_delete_url = settings.MULTI_FILE_DELETE_URL + '/' file_url = settings.MULTI_IMAGE_URL + '/' + image.key_data + '/' except AttributeError: file_delete_url = 'multi_delete/' file_url = 'multi_image/' + image.key_data + '/' #generating json response array result = [] result.append({ "name": filename, "size": file_size, "url": file_url, "thumbnail_url": thumb_url, "delete_url": file_delete_url + str(image.pk) + '/', "delete_type": "POST", }) response_data = simplejson.dumps(result) #checking for json data type #big thanks to Guy Shapiro if "application/json" in request.META['HTTP_ACCEPT_ENCODING']: mimetype = 'application/json' else: mimetype = 'text/plain' return HttpResponse(response_data, mimetype=mimetype) else: #GET return HttpResponse('Only POST accepted')
def test_unicode_uploadedfile_name(self): uf = UploadedFile(name='¿Cómo?', content_type='text') self.assertIs(type(repr(uf)), str)
def set_file_type(cls, resource, file_id, user): """ Sets a tif or zip raster resource file to GeoRasterFile type :param resource: an instance of resource type CompositeResource :param file_id: id of the resource file to be set as GeoRasterFile type :param user: user who is setting the file type :return: """ # had to import it here to avoid import loop from hs_core.views.utils import create_folder, remove_folder log = logging.getLogger() # get the file from irods res_file = utils.get_resource_file_by_id(resource, file_id) # base file name (no path included) file_name = utils.get_resource_file_name_and_extension(res_file)[1] # file name without the extension file_name = file_name[:-len(res_file.extension)] file_folder = res_file.file_folder upload_folder = '' if res_file is not None and res_file.has_generic_logical_file: # get the file from irods to temp dir temp_file = utils.get_file_from_irods(res_file) # validate the file error_info, files_to_add_to_resource = raster_file_validation( raster_file=temp_file) if not error_info: log.info("Geo raster file type file validation successful.") # extract metadata temp_dir = os.path.dirname(temp_file) temp_vrt_file_path = [ os.path.join(temp_dir, f) for f in os.listdir(temp_dir) if '.vrt' == os.path.splitext(f)[1] ].pop() metadata = extract_metadata(temp_vrt_file_path) log.info( "Geo raster file type metadata extraction was successful.") with transaction.atomic(): # create a geo raster logical file object to be associated with resource files logical_file = cls.create() # by default set the dataset_name attribute of the logical file to the # name of the file selected to set file type logical_file.dataset_name = file_name logical_file.save() try: # create a folder for the raster file type using the base file name as the # name for the new folder new_folder_path = cls.compute_file_type_folder( resource, file_folder, file_name) log.info("Folder created:{}".format(new_folder_path)) create_folder(resource.short_id, new_folder_path) new_folder_name = new_folder_path.split('/')[-1] if file_folder is None: upload_folder = new_folder_name else: upload_folder = os.path.join( file_folder, new_folder_name) # add all new files to the resource for f in files_to_add_to_resource: uploaded_file = UploadedFile( file=open(f, 'rb'), name=os.path.basename(f)) # the added resource file will be part of a new generic logical file # by default new_res_file = utils.add_file_to_resource( resource, uploaded_file, folder=upload_folder) # delete the generic logical file object if new_res_file.logical_file is not None: # deleting the file level metadata object will delete the associated # logical file object new_res_file.logical_file.metadata.delete() # make each resource file we added as part of the logical file logical_file.add_resource_file(new_res_file) log.info( "Geo raster file type - new files were added to the resource." ) # use the extracted metadata to populate file metadata for element in metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = element.items()[0] logical_file.metadata.create_element(k, **v) log.info( "Geo raster file type - metadata was saved to DB") # set resource to private if logical file is missing required metadata resource.update_public_and_discoverable() # delete the original resource file delete_resource_file(resource.short_id, res_file.id, user) log.info("Deleted original resource file.") except Exception as ex: msg = "Geo raster file type. Error when setting file type. Error:{}" msg = msg.format(ex.message) log.exception(msg) if upload_folder: # delete any new files uploaded as part of setting file type folder_to_remove = os.path.join( 'data', 'contents', upload_folder) remove_folder(user, resource.short_id, folder_to_remove) log.info("Deleted newly created file type folder") raise ValidationError(msg) finally: # remove temp dir if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) else: err_msg = "Geo raster file type file validation failed.{}".format( ' '.join(error_info)) log.info(err_msg) raise ValidationError(err_msg) else: if res_file is None: err_msg = "Failed to set Geo raster file type. " \ "Resource doesn't have the specified file." log.error(err_msg) raise ValidationError(err_msg) else: err_msg = "Failed to set Geo raster file type." \ "The specified file doesn't have a generic logical file type." log.error(err_msg) raise ValidationError(err_msg)
def test_update_responses(self): formxml = FormSubmissionBuilder( form_id='123', form_properties={ 'breakfast': 'toast', # Simple questions 'lunch': 'sandwich', 'cell': { # Simple group 'cytoplasm': 'squishy', 'organelles': 'grainy', }, 'shelves': [ # Simple repeat group {'position': 'top'}, {'position': 'middle'}, {'position': 'bottom'}, ], 'grandparent': [ # Repeat group with child group {'name': 'Haruki'}, {'name': 'Sugako'}, { 'name': 'Emma', 'parent': { 'name': 'Haruki', 'child': { 'name': 'Nao', }, } }, ], 'body': [ # Repeat group with child repeat group {'arm': [ {'elbow': '1'}, {'finger': '5'}, ]}, {'leg': [ {'knee': '1'}, {'toe': '5'}, ]}, ], } ).as_xml_string() pic = UploadedFile(BytesIO(b"fake"), 'pic.jpg', content_type='image/jpeg') xform = submit_form_locally(formxml, DOMAIN, attachments={ "image": pic }).xform updates = { 'breakfast': 'fruit', 'cell/organelles': 'bulbous', 'shelves[1]/position': 'third', 'shelves[3]/position': 'first', 'grandparent[1]/name': 'Haruki #1', 'grandparent[3]/name': 'Ema', 'grandparent[3]/parent/name': 'Haruki #2', 'grandparent[3]/parent/child/name': 'Nao-chan', 'body[1]/arm[1]/elbow': '2', 'body[2]/leg[2]/toe': '10', } errors = FormProcessorInterface(DOMAIN).update_responses( xform, updates, 'user1') form = XFormInstance.objects.get_form(xform.form_id) self.assertEqual(0, len(errors)) self.assertEqual('fruit', form.form_data['breakfast']) self.assertEqual('sandwich', form.form_data['lunch']) self.assertEqual('squishy', form.form_data['cell']['cytoplasm']) self.assertEqual('bulbous', form.form_data['cell']['organelles']) self.assertEqual('third', form.form_data['shelves'][0]['position']) self.assertEqual('middle', form.form_data['shelves'][1]['position']) self.assertEqual('first', form.form_data['shelves'][2]['position']) self.assertEqual('Haruki #1', form.form_data['grandparent'][0]['name']) self.assertEqual('Sugako', form.form_data['grandparent'][1]['name']) self.assertEqual('Ema', form.form_data['grandparent'][2]['name']) self.assertEqual('Haruki #2', form.form_data['grandparent'][2]['parent']['name']) self.assertEqual( 'Nao-chan', form.form_data['grandparent'][2]['parent']['child']['name']) self.assertEqual('2', form.form_data['body'][0]['arm'][0]['elbow']) self.assertEqual('5', form.form_data['body'][0]['arm'][1]['finger']) self.assertEqual('1', form.form_data['body'][1]['leg'][0]['knee']) self.assertEqual('10', form.form_data['body'][1]['leg'][1]['toe']) self.assertIn("image", form.attachments) self.assertEqual(form.get_attachment("image"), b"fake")
def get_uploaded_file(filename, content_type=None): return UploadedFile(open(filename, 'rb'), os.path.basename(filename), content_type or mimetypes.guess_type(filename)[0])
def set_file_type(cls, resource, user, file_id=None, folder_path=None): """ Creates a NetCDFLogicalFile (aggregation) from a netcdf file (.nc) resource file or a folder """ log = logging.getLogger() res_file, folder_path = cls._validate_set_file_type_inputs( resource, file_id, folder_path) # base file name (no path included) file_name = res_file.file_name # file name without the extension - needed for naming the new aggregation folder nc_file_name = file_name[:-len(res_file.extension)] resource_metadata = [] file_type_metadata = [] upload_folder = '' res_files_to_delete = [] # get the file from irods to temp dir temp_file = utils.get_file_from_irods(res_file) temp_dir = os.path.dirname(temp_file) # file validation and metadata extraction nc_dataset = nc_utils.get_nc_dataset(temp_file) if isinstance(nc_dataset, netCDF4.Dataset): msg = "NetCDF aggregation. Error when creating aggregation. Error:{}" file_type_success = False # extract the metadata from netcdf file res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict( temp_file) # populate resource_metadata and file_type_metadata lists with extracted metadata add_metadata_to_list(resource_metadata, res_dublin_core_meta, res_type_specific_meta, file_type_metadata, resource) # create the ncdump text file dump_file = create_header_info_txt_file(temp_file, nc_file_name) file_folder = res_file.file_folder aggregation_folder_created = False create_new_folder = cls._check_create_aggregation_folder( selected_res_file=res_file, selected_folder=folder_path, aggregation_file_count=1) with transaction.atomic(): # create a netcdf logical file object to be associated with # resource files dataset_title = res_dublin_core_meta.get('title', nc_file_name) logical_file = cls.initialize(dataset_title, resource) try: if folder_path is None: # we are here means aggregation is being created by selecting a file # create a folder for the netcdf file type using the base file # name as the name for the new folder if the file is not already in a folder if create_new_folder: upload_folder = cls._create_aggregation_folder( resource, file_folder, nc_file_name) aggregation_folder_created = True log.info( "NetCDF Aggregation creation - folder created:{}" .format(upload_folder)) else: # selected nc file is already in a folder upload_folder = file_folder # create logical file record in DB logical_file.save() if aggregation_folder_created: # copy the nc file to the new aggregation folder and make it part # of the logical file tgt_folder = upload_folder files_to_copy = [res_file] logical_file.copy_resource_files( resource, files_to_copy, tgt_folder) res_files_to_delete.append(res_file) else: # make the selected nc file as part of the aggregation/file type logical_file.add_resource_file(res_file) else: # logical file record gets created in DB logical_file.save() # folder has been selected to create aggregation upload_folder = folder_path # make the .nc file part of the aggregation logical_file.add_resource_file(res_file) # add the new dump txt file to the resource uploaded_file = UploadedFile( file=open(dump_file, 'rb'), name=os.path.basename(dump_file)) new_res_file = utils.add_file_to_resource( resource, uploaded_file, folder=upload_folder, add_to_aggregation=False) # make this new resource file we added part of the logical file logical_file.add_resource_file(new_res_file) log.info( "NetCDF aggregation creation - a new file was added to the resource." ) # use the extracted metadata to populate resource metadata for element in resource_metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = element.items()[0] if k == 'title': # update title element title_element = resource.metadata.title resource.metadata.update_element( 'title', title_element.id, **v) else: resource.metadata.create_element(k, **v) log.info( "NetCDF Aggregation creation - Resource metadata was saved to DB" ) # use the extracted metadata to populate file metadata for element in file_type_metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = element.items()[0] if k == 'subject': logical_file.metadata.keywords = v logical_file.metadata.save() # update resource level keywords resource_keywords = [ subject.value.lower() for subject in resource.metadata.subjects.all() ] for kw in logical_file.metadata.keywords: if kw.lower() not in resource_keywords: resource.metadata.create_element('subject', value=kw) else: logical_file.metadata.create_element(k, **v) log.info( "NetCDF aggregation - metadata was saved in aggregation" ) logical_file._finalize( user, resource, folder_created=aggregation_folder_created, res_files_to_delete=res_files_to_delete) file_type_success = True except Exception as ex: msg = msg.format(ex.message) log.exception(msg) finally: # remove temp dir if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) if not file_type_success: aggregation_from_folder = folder_path is not None cls._cleanup_on_fail_to_create_aggregation( user, resource, upload_folder, file_folder, aggregation_from_folder) raise ValidationError(msg) else: err_msg = "Not a valid NetCDF file. NetCDF aggregation validation failed." log.error(err_msg) # remove temp dir if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) raise ValidationError(err_msg)
def thumbnailify(filebit, sizebit): from PIL import Image from os.path import splitext from django.http import HttpResponseRedirect, HttpResponse from io import BytesIO from django.core.files.uploadedfile import UploadedFile import sys browser_kind = [ ".png",".jpg",".gif" ] jsc3d_kind = [ ".stl",".obj" ] text_kind = [".md",".txt"] # text_kind = [ ".txt" ] ##ext os the file extension, forced into lowercase becouse people are insane. ext = str(splitext(str(filebit.filename))[1].lower()) response = HttpResponse(mimetype="image/png") if ext in browser_kind: print("filebit.filename"+str(filebit.filename.name)) img = Image.open(filebit.filename) img.thumbnail(sizebit) print(img) # Create a file-like object to write thumb data (thumb data previously created # using PIL, and stored in variable 'img') # using PIL, and stored in variable 'thumb') # thumb_io = BytesIO() thumb_io = BytesIO() img.save( thumb_io, format='png') # Create a new Django file-like object to be used in models as ImageField using # InMemoryUploadedFile. If you look at the source in Django, a # SimpleUploadedFile is essentially instantiated similarly to what is shown here thumb_file = UploadedFile(thumb_io) thumb_file.name = str(sizebit)+"-"+str(filebit.filename)+".png" # Once you have a Django file-like object, you may assign it to your ImageField # and save. return(thumb_file, "browser") if ext in jsc3d_kind: from selenium import webdriver from django.conf import settings driver = webdriver.PhantomJS() driver.set_window_size(sizebit[0],sizebit[1]) # not optional driver.get(settings.URL+"/thumbs/jsc3d/"+str(filebit.pk)) imagedata = driver.get_screenshot_as_base64() # save a screenshot as base64 string, the only format phantom supports that isn't disk. import base64 from io import BytesIO #converts the base64 encoded image data into a python file object thumb_io = BytesIO(base64.b64decode(imagedata)) thumb_file = UploadedFile(thumb_io) thumb_file.name = str(sizebit)+"-"+str(filebit.filename)+".png" # thumb_file = False return(thumb_file, "jsc3d") if ext in text_kind: return(False, "text") # if ext in text_kind: # print("filebit.filename"+str(filebit.filename.name)) # from PIL import ImageFont, ImageDraw, Image # from django.conf import settings # from io import StringIO # img = Image.new("RGBA", (100,50), (255,255,255)) # draw = ImageDraw.Draw(img) # #print(settings.URL+"/static/DejaVuSerif-Bold.ttf") # #font = ImageFont.truetype(settings.URL+"/media/DejaVuSerif-Bold.ttf", 12) # f = filebit.filename.file.read(64) # draw.text((10, 0), (f.decode("utf-8")[:16]), (0,0,0) ) # draw.text((10, 10), (f.decode("utf-8")[16:32]), (0,0,0) ) # draw.text((10, 20), (f.decode("utf-8")[32:48]), (0,0,0) ) # draw.text((10, 30), (f.decode("utf-8")[48:64]), (0,0,0) ) # draw.text((10, 40), ("\t..."), (0,0,0) ) # img_resized = img.resize((sizebit), Image.ANTIALIAS) # #img = Image.open(filebit.filename) # #img.thumbnail(sizebit) # print(img) # # Create a file-like object to write thumb data (thumb data previously created # # using PIL, and stored in variable 'img') # # using PIL, and stored in variable 'thumb') # thumb_io = BytesIO() # img.save( thumb_io, format='png') # # # Create a new Django file-like object to be used in models as ImageField using # # InMemoryUploadedFile. If you look at the source in Django, a # # SimpleUploadedFile is essentially instantiated similarly to what is shown here # thumb_file = InMemoryUploadedFile(thumb_io, None, str(sizebit)+"-"+str(filebit.filename)+".png", 'image/jpeg', # 1, None) # # # Once you have a Django file-like object, you may assign it to your ImageField # # and save. # return(thumb_file, "browser") return(False, "norender")
def editOrCreateStuff(project, request, creating): ## Note: if creating == true this is a post being created. #Because there are so many similarities in creating a post vs editing a post we are using this method, and using creating when we need to do something different for editing vs creating. ## postmode! We are getting pretty post data from the user!!! if request.method == 'POST': ## get the forms and check that they are valid formValid=False if creating: form = createForm(request.POST, project) form2 = defaulttag(request.POST) if form.is_valid() and form2.is_valid() and request.user.is_authenticated(): formValid=True # If we are creating the post we need to set the author and title. project.author = request.user project.title = form.cleaned_data["title"] else: form = ProjectForm(request.POST, project) if form.is_valid() and str(project.author) == str(request.user): formValid=True ## if the form is valid make the changes to the project! if formValid: # Editing the Readme.md file stuff. if not creating: # Delete the old body text file... cause I'm a bad person and I don't know how to just open and write to the old one easily. readme = project.bodyFile try: readme = project.bodyFile readmename = path.split(str(readme.filename))[1] readme.delete() except: pass # Save body as file bodyText = fileobject() bodyText.parent = project from django.core.files.uploadedfile import UploadedFile import base64 from io import BytesIO from io import TextIOWrapper from io import StringIO #io = TextIOWrapper(TextIOBase(form.cleaned_data["body"])) io = StringIO(form.cleaned_data["body"]) txfl = UploadedFile(io) #editfield may be renaming your readme to readme.md every time. That's not good. try: bodyText.filename.save(readmename, txfl) except: bodyText.filename.save('README.md', txfl) txfl.close() io.close() bodyText.save() #### this did not appear to be happening in the create.... but I think it should have been? project.bodyFile = bodyText # Done with editing the README.md textfile. # list_to_tags(form.cleaned_data["tags"], project.tags) if creating: for i in form2.cleaned_data["categories"]: project.tags.add(i) # This may be redundant, but either way, this post is not a draft past this point. project.draft=False project.save() return HttpResponseRedirect('/project/'+str(project.pk)) #### If the form data was NOT valid else: if creating: return render_to_response('create.html', dict(user=request.user, form=form, form2=form2, project=project)) else: if str(project.author) == str(request.user): return render_to_response('edit.html', dict(project=project, user=request.user, form=form, )) else: return HttpResponse(status=403) #### Not POSTmode! We are setting up the form for the user to fill in. We are not getting form data from the user. ##### CREATE elif creating and request.user.is_authenticated(): form = createForm("",project) form2 = defaulttag() return render_to_response('create.html', dict(user=request.user, form=form, form2=form2, project=project)) ##### EDIT elif (not creating) and str(project.author) == str(request.user): if project.bodyFile: readme = project.bodyFile.filename.read() else: readme = project.body taglist = [] for i in project.tags.names(): taglist.append(i) taglist = ",".join(taglist) thumbnailstring = "/"+path.split(project.thumbnail.filename.url)[1] form = ProjectForm({'body': readme, 'thumbnail': thumbnailstring, 'tags' : str(taglist)}, project) return render_to_response('edit.html', dict(project=project, user=request.user, form=form,)) #return HttpResponse(response_data, mimetype="application/json") else: return HttpResponse(status=403)
def _read(file: UploadedFile, n=None): """Read the file, guessing encoding if needed""" binary_content = file.read(n) encoding = _get_encoding(file.encoding, binary_content) return binary_content.decode(encoding)
def test_bulk_metadata_update(self): # here we are testing the update() method of the RasterMetaData class # update of resource specific metadata should fail when the resource does not have content # files self.assertEqual(self.resRaster.files.all().count(), 0) self.assertEqual( self.resRaster.metadata.bandInformations.all().count(), 0) band_data = { 'original_band_name': 'bandinfo', 'name': 'Band_1', 'variableName': 'digital elevation', 'variableUnit': 'meter', 'method': 'this is method', 'comment': 'this is comment', 'maximumValue': 1000, 'minimumValue': 0, 'noDataValue': -9999 } metadata = [] metadata.append({'bandinformation': band_data}) with self.assertRaises(ValidationError): self.resRaster.metadata.update(metadata) self.assertEqual( self.resRaster.metadata.bandInformations.all().count(), 0) del metadata[:] # adding a valid tiff file should generate some core metadata and all extended metadata files = [ UploadedFile(file=self.raster_tif_file_obj, name=self.raster_tif_file_name) ] utils.resource_file_add_process(resource=self.resRaster, files=files, user=self.user, extract_metadata=False) # testing extended metadata element: band information self.assertEqual(self.resRaster.metadata.bandInformations.count(), 1) band_info = self.resRaster.metadata.bandInformations.first() self.assertEqual(band_info.noDataValue, '-3.40282346639e+38') self.assertEqual(band_info.maximumValue, '3031.44311523') self.assertEqual(band_info.minimumValue, '1358.33459473') self.assertEqual(band_info.name, 'Band_1') # updating of bandinformation using a name that does not exist (band-name) should fail band_data = { 'original_band_name': 'band-name', 'name': 'Band_1', 'variableName': 'digital elevation', 'variableUnit': 'meter', 'method': 'this is method', 'comment': 'this is comment', 'maximumValue': 1000, 'minimumValue': 0, 'noDataValue': -9999 } metadata.append({'bandinformation': band_data}) with self.assertRaises(ValidationError): self.resRaster.metadata.update(metadata) self.assertEqual( self.resRaster.metadata.bandInformations.all().count(), 1) # updating of bandinformation using a valid band lookup name (Band_1) should be successful band_data = { 'original_band_name': 'Band_1', 'name': 'Band_2', 'variableName': 'digital elevation', 'variableUnit': 'meter', 'method': 'this is method', 'comment': 'this is comment', 'maximumValue': 1000, 'minimumValue': 0, 'noDataValue': -9999 } del metadata[:] metadata.append({'bandinformation': band_data}) self.resRaster.metadata.update(metadata) self.assertEqual( self.resRaster.metadata.bandInformations.all().count(), 1) band_info = self.resRaster.metadata.bandInformations.first() self.assertEqual(band_info.name, 'Band_2') self.assertEqual(band_info.variableName, 'digital elevation') self.assertEqual(band_info.variableUnit, 'meter') self.assertEqual(band_info.method, 'this is method') self.assertEqual(band_info.comment, 'this is comment') self.assertEqual(band_info.maximumValue, '1000') self.assertEqual(band_info.minimumValue, '0') self.assertEqual(band_info.noDataValue, '-9999') # test updating only one attribute of bandinformation band_data = {'original_band_name': 'Band_2', 'name': 'Band_1'} del metadata[:] metadata.append({'bandinformation': band_data}) self.resRaster.metadata.update(metadata) self.assertEqual( self.resRaster.metadata.bandInformations.all().count(), 1) band_info = self.resRaster.metadata.bandInformations.first() self.assertEqual(band_info.name, 'Band_1') # test updating both core and resource specific metadata # there should be 1 creator self.assertEqual(self.resRaster.metadata.creators.all().count(), 1) # there should be no contributor self.assertEqual(self.resRaster.metadata.contributors.all().count(), 0) del metadata[:] metadata.append({'creator': {'name': 'creator one'}}) metadata.append({'creator': {'name': 'creator two'}}) metadata.append({'contributor': {'name': 'contributor one'}}) metadata.append({'contributor': {'name': 'contributor two'}}) band_data = {'original_band_name': 'Band_1', 'name': 'Band_3'} metadata.append({'bandinformation': band_data}) self.resRaster.metadata.update(metadata) band_info = self.resRaster.metadata.bandInformations.first() self.assertEqual(band_info.name, 'Band_3') # there should be 2 creators self.assertEqual(self.resRaster.metadata.creators.all().count(), 2) # there should be 2 contributor self.assertEqual(self.resRaster.metadata.contributors.all().count(), 2) self.resRaster.delete()
def migrate_tif_file(apps, schema_editor): log = logging.getLogger() istorage = IrodsStorage() copy_res_fail = [] vrt_update_fail = [] vrt_update_success = [] meta_update_fail = [] meta_update_success = [] # start migration for each raster resource that has raster files for res in RasterResource.objects.all(): if res.files.all(): # copy all the resource files to temp dir try: temp_dir = tempfile.mkdtemp() for res_file in res.files.all(): shutil.copy( res_file.resource_file.file.name, os.path.join( temp_dir, os.path.basename(res_file.resource_file.name))) vrt_file_path = [ os.path.join(temp_dir, f) for f in os.listdir(temp_dir) if '.vrt' == f[-4:] ].pop() except Exception as e: log.exception(str(e)) copy_res_fail.append('{}:{}'.format(res.short_id, res.metadata.title.value)) continue # update vrt file if the raster resource that has a single tif file try: if len(os.listdir(temp_dir)) == 2: # create new vrt file tif_file_path = [ os.path.join(temp_dir, f) for f in os.listdir(temp_dir) if '.tif' == f[-4:] ].pop() with open(os.devnull, 'w') as fp: subprocess.Popen( [ 'gdal_translate', '-of', 'VRT', tif_file_path, vrt_file_path ], stdout=fp, stderr=fp).wait() # remember to add .wait() # modify the vrt file contents tree = ET.parse(vrt_file_path) root = tree.getroot() for element in root.iter('SourceFilename'): element.attrib['relativeToVRT'] = '1' tree.write(vrt_file_path) # delete vrt res file for f in res.files.all(): if 'vrt' == f.resource_file.name[-3:]: f.resource_file.delete() f.delete() # add new vrt file to resource new_file = UploadedFile( file=open(vrt_file_path, 'r'), name=os.path.basename(vrt_file_path)) hydroshare.add_resource_files(res.short_id, new_file) # update the bag bag_name = 'bags/{res_id}.zip'.format(res_id=res.short_id) if istorage.exists(bag_name): # delete the resource bag as the old bag is not valid istorage.delete(bag_name) resource_modified(res, res.creator) vrt_update_success.append('{}:{}'.format( res.short_id, res.metadata.title.value)) except Exception as e: log.exception(str(e)) vrt_update_fail.append('{}:{}'.format( res.short_id, res.metadata.title.value)) # update the metadata for the band information of all the raster resources try: meta_updated = False # extract meta ori_dir = os.getcwd() os.chdir(temp_dir) res_md_dict = raster_meta_extract.get_raster_meta_dict( vrt_file_path) os.chdir(ori_dir) shutil.rmtree(temp_dir) # update band information metadata in django if res_md_dict['band_info']: for i, band_meta in list(res_md_dict['band_info'].items()): band_obj = res.metadata.bandInformation.filter( name='Band_{}'.format(i)).first() if band_obj: res.metadata.update_element( 'bandInformation', band_obj.id, maximumValue=band_meta['maximumValue'], minimumValue=band_meta['minimumValue'], noDataValue=band_meta['noDataValue'], ) meta_updated = True # update the bag if meta is updated if meta_updated: bag_name = 'bags/{res_id}.zip'.format(res_id=res.short_id) if istorage.exists(bag_name): # delete the resource bag as the old bag is not valid istorage.delete(bag_name) resource_modified(res, res.creator) meta_update_success.append('{}:{}'.format( res.short_id, res.metadata.title.value)) except Exception as e: log.exception(str(e)) meta_update_fail.append('{}:{}'.format( res.short_id, res.metadata.title.value)) # Print migration results print(('Copy resource to temp folder failure: Number: {} List: {}'.format( len(copy_res_fail), copy_res_fail))) print(('VRT file update success: Number: {} List{}'.format( len(vrt_update_success), vrt_update_success))) print(('VRT file update fail: Number: {} List{}'.format( len(vrt_update_fail), vrt_update_fail))) print(('Meta update success: Number: {} List {}'.format( len(meta_update_success), meta_update_success))) print( ('Meta update fail: Number: {} List {}'.format(len(meta_update_fail), meta_update_fail)))
def get_minecraft_avatar(minecraft_user, geometry_string, force=True): """ This method uses the sorl-thumbnail cache backend to prevent images from being downloaded every time. It requires an username. """ avatar_file = UploadedFile( file=StringIO(), name='%s/%s.png' % (settings.MEDIA_ROOT, tokey(minecraft_user))) try: # Check if the avatar is cached thumbnail = get_thumbnail( avatar_file, '100x100', quality=100, format='PNG') except IOError: download_thumbnail = True else: is_dummy = not hasattr(thumbnail, 'storage') if not is_dummy and not thumbnail.storage.exists(thumbnail.name): # It seems we have the avatar on cache (kvstore) # but it's not present on the storage download_thumbnail = True # Force remove thumbnail from kvstore sorl_default.kvstore.delete(thumbnail) # Log logger.warning('Avatar cache mismatch: %s (resetting)' % ( minecraft_user,)) else: logger.debug('Avatar fetched from cache: %s' % minecraft_user) download_thumbnail = False if download_thumbnail: logger.debug('Downloading avatar: %s' % minecraft_user) # Otherwise download avatar thumbnail = None try: skin_bin = requests.get( 'http://s3.amazonaws.com/MinecraftSkins/%s.png' % ( minecraft_user ) ).content except ConnectionError: return None try: skin = Image.open(StringIO(skin_bin)).convert('RGBA') except IOError: # Image not found or some permission error with S3 if minecraft_user != 'char': if not force: return None # Return default user avatar return settings.STATIC_URL + settings.DEFAULT_USER_AVATAR else: face = skin.crop((8, 8, 16, 16)) accessories = skin.crop((40, 8, 48, 16)) r, g, b, a = accessories.split() accessories = Image.merge('RGB', (r, g, b)) mask = Image.merge('L', (a,)) face.paste(accessories, (0, 0), mask) avatar = face.resize((135, 135)) avatar.save(avatar_file, 'PNG') avatar_file.seek(0) # Save through sorl backend thumbnail = get_thumbnail(avatar_file, '100x100', quality=100, format='PNG') # Use the cached file return get_thumbnail(thumbnail, geometry_string, quality=100, format='PNG').url
def handle_noargs(self, users=None, review_requests=None, diffs=None, reviews=None, diff_comments=None, password=None, verbosity=NORMAL, **options): num_of_requests = None num_of_diffs = None num_of_reviews = None num_of_diff_comments = None random.seed() if review_requests: num_of_requests = self.parseCommand("review_requests", review_requests) # Setup repository. repo_dir = os.path.abspath( os.path.join(sys.argv[0], "..", "scmtools", "testdata", "git_repo")) # Throw exception on error so transaction reverts. if not os.path.exists(repo_dir): raise CommandError("No path to the repository") self.repository = Repository.objects.create( name="Test Repository", path=repo_dir, tool=Tool.objects.get(name="Git")) if diffs: num_of_diffs = self.parseCommand("diffs", diffs) # Create the diff directory locations. diff_dir_tmp = os.path.abspath( os.path.join(sys.argv[0], "..", "reviews", "management", "commands", "diffs")) # Throw exception on error so transaction reverts. if not os.path.exists(diff_dir_tmp): raise CommandError("Diff dir does not exist") diff_dir = diff_dir_tmp + '/' # Add trailing slash. # Get a list of the appropriate files. files = [f for f in os.listdir(diff_dir) if f.endswith('.diff')] # Check for any diffs in the files. if len(files) == 0: raise CommandError("No diff files in this directory") if reviews: num_of_reviews = self.parseCommand("reviews", reviews) if diff_comments: num_of_diff_comments = self.parseCommand("diff-comments", diff_comments) # Users is required for any other operation. if not users: raise CommandError("At least one user must be added") # Start adding data to the database. for i in range(1, users + 1): new_user = User.objects.create( username=self.randUsername(), # Avoids having to flush db. first_name=random.choice(NAMES), last_name=random.choice(NAMES), email="*****@*****.**", is_staff=False, is_active=True, is_superuser=False) if password: new_user.set_password(password) new_user.save() else: new_user.set_password("test1") new_user.save() Profile.objects.create(user=new_user, first_time_setup_done=True, collapsed_diffs=True, wordwrapped_diffs=True, syntax_highlighting=True, show_submitted=True) # Review Requests. req_val = self.pickRandomValue(num_of_requests) if int(verbosity) > NORMAL: print "For user %s:%s" % (i, new_user.username) print "=============================" for j in range(0, req_val): if int(verbosity) > NORMAL: print "Request #%s:" % j review_request = ReviewRequest.objects.create(new_user, None) review_request.public = True review_request.summary = self.lorem_ipsum("summary") review_request.description = self.lorem_ipsum("description") review_request.shipit_count = 0 review_request.repository = self.repository # Set the targeted reviewer to superuser or 1st defined. if j == 0: review_request.target_people.add(User.objects.get(pk=1)) review_request.save() # Add the diffs if any to add. diff_val = self.pickRandomValue(num_of_diffs) # If adding diffs add history. if diff_val > 0: diffset_history = DiffSetHistory.objects.create( name='testDiffFile' + str(i)) diffset_history.save() # Won't execute if diff_val is 0, ie: no diffs requested. for k in range(0, diff_val): if int(verbosity) > NORMAL: print "%s:\tDiff #%s" % (i, k) random_number = random.randint(0, len(files) - 1) file_to_open = diff_dir + files[random_number] f = UploadedFile(open(file_to_open, 'r')) form = UploadDiffForm(review_request.repository, f) cur_diff = form.create(f, None, diffset_history) review_request.diffset_history = diffset_history review_request.save() review_request.publish(new_user) f.close() # Add the reviews if any. review_val = self.pickRandomValue(num_of_reviews) for l in range(0, review_val): if int(verbosity) > NORMAL: print "%s:%s:\t\tReview #%s:" % (i, j, l) reviews = Review.objects.create( review_request=review_request, user=new_user) reviews.publish(new_user) # Add comments if any. comment_val = self.pickRandomValue( num_of_diff_comments) for m in range(0, comment_val): if int(verbosity) > NORMAL: print "%s:%s:\t\t\tComments #%s" % (i, j, m) if m == 0: file_diff = cur_diff.files.order_by('id')[0] # Choose random lines to comment. # Max lines: should be mod'd in future to read # diff. max_lines = 220 first_line = random.randrange(1, max_lines - 1) remain_lines = max_lines - first_line num_lines = random.randrange(1, remain_lines) diff_comment = Comment.objects.create( filediff=file_diff, text="comment number %s" % (m + 1), first_line=first_line, num_lines=num_lines) review_request.publish(new_user) reviews.comments.add(diff_comment) reviews.save() reviews.publish(new_user) db.reset_queries() # No comments, so have previous layer clear queries. if comment_val == 0: db.reset_queries() if review_val == 0: db.reset_queries() if diff_val == 0: db.reset_queries() if req_val == 0: db.reset_queries() # Generate output as users & data is created. if req_val != 0: print "user %s created with %s requests" % (new_user.username, req_val) else: print "user %s created successfully" % new_user.username
def handle_noargs(self, users=None, review_requests=None, diffs=None, reviews=None, diff_comments=None, password=None, verbosity=NORMAL, **options): num_of_requests = None num_of_diffs = None num_of_reviews = None num_of_diff_comments = None random.seed() if review_requests: num_of_requests = self.parseCommand("review_requests", review_requests) # Setup repository. repo_dir = os.path.abspath( os.path.join(sys.argv[0], "..", "scmtools", "testdata", "git_repo")) # Throw exception on error so transaction reverts. if not os.path.exists(repo_dir): raise CommandError("No path to the repository") self.repository = Repository.objects.create( name="Test Repository", path=repo_dir, tool=Tool.objects.get(name="Git")) if diffs: num_of_diffs = self.parseCommand("diffs", diffs) # Create the diff directory locations. diff_dir_tmp = os.path.abspath( os.path.join(sys.argv[0], "..", "reviews", "management", "commands", "diffs")) # Throw exception on error so transaction reverts. if not os.path.exists(diff_dir_tmp): raise CommandError("Diff dir does not exist") diff_dir = diff_dir_tmp + '/' # Add trailing slash. # Get a list of the appropriate files. files = [f for f in os.listdir(diff_dir) if f.endswith('.diff')] # Check for any diffs in the files. if len(files) == 0: raise CommandError("No diff files in this directory") if reviews: num_of_reviews = self.parseCommand("reviews", reviews) if diff_comments: num_of_diff_comments = self.parseCommand("diff-comments", diff_comments) # Users is required for any other operation. if not users: raise CommandError("At least one user must be added") # Start adding data to the database. for i in range(1, users + 1): new_user = User.objects.create( username=self.randUsername(), # Avoids having to flush db. first_name=random.choice(NAMES), last_name=random.choice(NAMES), email="*****@*****.**", is_staff=False, is_active=True, is_superuser=False) if password: new_user.set_password(password) new_user.save() else: new_user.set_password("test1") new_user.save() Profile.objects.create( user=new_user, first_time_setup_done=True, collapsed_diffs=True, wordwrapped_diffs=True, syntax_highlighting=True, show_submitted=True) # Review Requests. req_val = self.pickRandomValue(num_of_requests) if int(verbosity) > NORMAL: print "For user %s:%s" % (i, new_user.username) print "=============================" for j in range(0, req_val): if int(verbosity) > NORMAL: print "Request #%s:" % j review_request = ReviewRequest.objects.create(new_user, None) review_request.public = True review_request.summary = self.lorem_ipsum("summary") review_request.description = self.lorem_ipsum("description") review_request.shipit_count = 0 review_request.repository = self.repository # Set the targeted reviewer to superuser or 1st defined. if j == 0: review_request.target_people.add(User.objects.get(pk=1)) review_request.save() # Add the diffs if any to add. diff_val = self.pickRandomValue(num_of_diffs) # If adding diffs add history. if diff_val > 0: diffset_history = DiffSetHistory.objects.create( name='testDiffFile' + str(i)) diffset_history.save() # Won't execute if diff_val is 0, ie: no diffs requested. for k in range(0, diff_val): if int(verbosity) > NORMAL: print "%s:\tDiff #%s" % (i, k) random_number = random.randint(0, len(files) - 1) file_to_open = diff_dir + files[random_number] f = UploadedFile(open(file_to_open, 'r')) form = UploadDiffForm(review_request.repository, f) cur_diff = form.create(f, None, diffset_history) review_request.diffset_history = diffset_history review_request.save() review_request.publish(new_user) f.close() # Add the reviews if any. review_val = self.pickRandomValue(num_of_reviews) for l in range(0, review_val): if int(verbosity) > NORMAL: print "%s:%s:\t\tReview #%s:" % (i, j, l) reviews = Review.objects.create( review_request=review_request, user=new_user) reviews.publish(new_user) # Add comments if any. comment_val = self.pickRandomValue( num_of_diff_comments) for m in range(0, comment_val): if int(verbosity) > NORMAL: print "%s:%s:\t\t\tComments #%s" % (i, j, m) if m == 0: file_diff = cur_diff.files.order_by('id')[0] # Choose random lines to comment. # Max lines: should be mod'd in future to read # diff. max_lines = 220 first_line = random.randrange(1, max_lines - 1) remain_lines = max_lines - first_line num_lines = random.randrange(1, remain_lines) diff_comment = Comment.objects.create( filediff=file_diff, text="comment number %s" % (m + 1), first_line=first_line, num_lines=num_lines) review_request.publish(new_user) reviews.comments.add(diff_comment) reviews.save() reviews.publish(new_user) db.reset_queries() # No comments, so have previous layer clear queries. if comment_val == 0: db.reset_queries() if review_val == 0: db.reset_queries() if diff_val == 0: db.reset_queries() if req_val == 0: db.reset_queries() # Generate output as users & data is created. if req_val != 0: print "user %s created with %s requests" % ( new_user.username, req_val) else: print "user %s created successfully" % new_user.username
def multiuploader(request, patient_id=None): """ Main Multiuploader module. Parses data from jQuery plugin and makes database changes. """ if request.method == 'POST': log.info('received POST to main multiuploader view') if request.FILES == None: return HttpResponseBadRequest('Must have files attached!') #getting file data for farther manipulations file = request.FILES[u'files[]'] wrapped_file = UploadedFile(file) filename = wrapped_file.name file_size = wrapped_file.file.size log.info ('Got file: "%s"' % str(filename)) log.info('Content type: "$s" % file.content_type') #writing file manually into model #because we don't need form of any type. print request.POST if patient_id=='new': print wrapped_file.read() return HttpResponse(status=204) image = MultiuploaderImage() patient=Patient.objects.get(pk=patient_id) image.filename=str(filename) image.image=file image.key_data = image.key_generate patient.multiuploaderimage_set.add(image) log.info('File saving done') #getting thumbnail url using sorl-thumbnail if 'image' in file.content_type.lower(): im = get_thumbnail(image, "80x80", quality=50) thumb_url = im.url else: thumb_url = '' #settings imports try: file_delete_url = settings.MULTI_FILE_DELETE_URL+'/' file_url = settings.MULTI_IMAGE_URL+'/'+image.key_data+'/' except AttributeError: file_delete_url = 'multi_delete/' file_url = 'multi_image/'+image.key_data+'/' """ is actually: [{"name": "Screenshot from 2012-11-14 16:17:46.png", "url": "multi_image/95925526541943247735000327303075602114185579370918344597903504067450818566531/", "thumbnail_url": "/media/cache/f8/bd/f8bd83aadeba651ff9c040bb394ce117.jpg", "delete_type": "POST", "delete_url": "multi_delete/7/", "size": 38520}] should be: {"files":[{"url":"http://jquerey-file-upload.appspot.com/AMIfv9734HSTDGd3tIybbnKVru--IjhjULKvNcIGUL2lvfqA93RNCAizDbvP-RQJNbh-N9m8UXsk-90jFFYSp8TlbZYhEcNN6Vb9HzQVQtdmF83H6sE_XkdnlI2V8lHX5V3Y4AamdX6VMbAt9sNWNx2BVGzhTfAYkRLYmRE1VzzWSe9C8c8Fu8g/Screenshot%20from%202012-11-14%2016%3A17%3A46.png","thumbnail_url":"http://lh5.ggpht.com/fcjVNT6qUGoMDtqqaNDNtU4mghy34qlzfj2GujikLgC7Nj5Bs4LUT_DWG_Q8OWujqvYHsKbeQ9pkvoAW4WiaubmqQxobIPyt=s80","name":"Screenshot from 2012-11-14 16:17:46.png","type":"image/png","size":38520,"delete_url":"http://jquery-file-upload.appspot.com/AMIfv9734HSTDGd3tIybbnKVru--IjhjULKvNcIGUL2lvfqA93RNCAizDbvP-RQJNbh-N9m8UXsk-90jFFYSp8TlbZYhEcNN6Vb9HzQVQtdmF83H6sE_XkdnlI2V8lHX5V3Y4AamdX6VMbAt9sNWNx2BVGzhTfAYkRLYmRE1VzzWSe9C8c8Fu8g/Screenshot%20from%202012-11-14%2016%3A17%3A46.png?delete=true","delete_type":"DELETE"}]} """ #generating json response array result = { 'files': [ {"name":filename, "size":file_size, "url":file_url, "thumbnail_url":thumb_url, "delete_url":file_delete_url+str(image.pk)+'/', "delete_type":"POST",} ] } response_data = simplejson.dumps(result) #checking for json data type #big thanks to Guy Shapiro if "application/json" in request.META['HTTP_ACCEPT_ENCODING']: mimetype = 'application/json' else: mimetype = 'text/plain' return HttpResponse(status=204) #response_data, mimetype=mimetype) else: #GETim return HttpResponse('Only POST accepted')
def get_form(request, form_class, field_order, bound=False): """ Generic function. Used for all submission types. Specify the ``form_class`` that's given in ``forms.py``. The ``field_order`` is a list of strings that indicates the linear order of the fields in the form. A ``bound`` form is a function of the object assigned to ``bound`` (see below). An unbound form is simply an empty form. """ if bound: if isinstance(bound, models.Revision): tags = ','.join([str(tag) for tag in bound.tags.all()]) fields = {'item_url': bound.item_url, 'title': bound.title, 'description': bound.description, 'sub_tags': tags, 'snippet_code': bound.item_code, 'sub_type': 'snippet', 'sub_license': bound.sub_license_id, } if bound.entry.sub_type == 'link': fields['sub_type'] = 'link' elif bound.entry.sub_type == 'package': fields['sub_type'] = 'package' elif bound.entry.sub_type == 'code': fields['sub_type'] = 'snippet' form_output = form_class(fields) else: if request.POST['sub_type'] == 'package': # Create a fake "UploadedFile" object, so the user can resume # editing or finish their submission, without being told # they have to reenter this field. zip_hash = request.POST.get('package_hash', '') zip_file = models.ZipFile.objects.filter(zip_hash=zip_hash) if zip_file: zip_name = zip_file[0].raw_zip_file.name uploaded_file = UploadedFile(zip_name, name=zip_name, content_type='application/zip', size=zip_file[0].raw_zip_file.size) uploaded_file.skip_validation = True # see ``forms.py`` request.FILES['package_file'] = uploaded_file form_output = form_class(request.POST, request.FILES) if request.POST['sub_type'] == 'package' and zip_file: form_output.fields['package_file'].initial = uploaded_file else: form_output = form_class() # Rearrange the form order form_output.fields.keyOrder = field_order index = 1 for field_name, field in form_output.fields.iteritems(): field.widget.attrs['tabindex'] = str(index) index += 1 if request.user.is_authenticated(): # Email field not required for signed-in users form_output.fields.pop('email') return form_output
def upload_file_view(self, request): try: if not self.has_add_permission(request): raise PermissionDenied FILE_PARAM_NAME = 'qqfile' self.init_parent_folder(request) if request.method == 'POST': if request.is_ajax() and request.GET.get( FILE_PARAM_NAME, None): from django.core.files.base import ContentFile from django.core.files.uploadedfile import UploadedFile content_file = ContentFile(request.raw_post_data) uploaded_file = UploadedFile( content_file, request.GET.get(FILE_PARAM_NAME), None, content_file.size) form = UploadForm(request.POST, {'file': uploaded_file}) else: form = UploadForm(request.POST, request.FILES) if form.is_valid(): node = FileNode(file=form.cleaned_data['file'], node_type=FileNode.FILE) parent_folder = self.get_parent_folder(request) if not parent_folder.is_top_node(): node.parent = parent_folder self.save_model(request, node, None, False) # Respond with 'ok' for the client to verify that the upload was successful, since sometimes a failed # request would not result in a HTTP error and look like a successful upload. # For instance: When requesting the admin view without authentication, there is a redirect to the # login form, which to SWFUpload looks like a successful upload request. if request.is_ajax(): return HttpResponse('{"success": true}', mimetype="application/json") else: messages.info( request, _('Successfully uploaded file %s.') % node.name) return HttpResponseRedirect( reverse('admin:media_tree_filenode_changelist')) else: # invalid form data if request.is_ajax(): return HttpResponse('{"error": "%s"}' % ' '.join([ item for sublist in form.errors.values() for item in sublist ]), mimetype="application/json") # Form is rendered for troubleshooting SWFUpload. If this form works, the problem is not server-side. if not settings.DEBUG: raise ViewDoesNotExist if request.method == 'GET': form = UploadForm() return render_to_response( 'admin/media_tree/filenode/upload_form.html', {'form': form}) except Exception as e: if request.is_ajax(): return HttpResponse('{"error": "%s"}' % ugettext('Server Error'), mimetype="application/json") else: raise
def set_file_type(cls, resource, file_id, user): """ Sets a tif or zip raster resource file to GeoRasterFile type :param resource: an instance of resource type CompositeResource :param file_id: id of the resource file to be set as GeoRasterFile type :param user: user who is setting the file type :return: """ # had to import it here to avoid import loop from hs_core.views.utils import create_folder, remove_folder log = logging.getLogger() # get the file from irods res_file = utils.get_resource_file_by_id(resource, file_id) if res_file is None: raise ValidationError("File not found.") if res_file.extension != '.nc': raise ValidationError("Not a NetCDF file.") # base file name (no path included) file_name = res_file.file_name # file name without the extension nc_file_name = file_name[:-len(res_file.extension)] resource_metadata = [] file_type_metadata = [] files_to_add_to_resource = [] upload_folder = '' if res_file.has_generic_logical_file: # get the file from irods to temp dir temp_file = utils.get_file_from_irods(res_file) temp_dir = os.path.dirname(temp_file) files_to_add_to_resource.append(temp_file) # file validation and metadata extraction nc_dataset = nc_utils.get_nc_dataset(temp_file) if isinstance(nc_dataset, netCDF4.Dataset): # Extract the metadata from netcdf file res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict( temp_file) # populate resource_metadata and file_type_metadata lists with extracted metadata add_metadata_to_list(resource_metadata, res_dublin_core_meta, res_type_specific_meta, file_type_metadata, resource) # create the ncdump text file dump_file = create_header_info_txt_file( temp_file, nc_file_name) files_to_add_to_resource.append(dump_file) file_folder = res_file.file_folder with transaction.atomic(): # create a netcdf logical file object to be associated with # resource files logical_file = cls.create() # by default set the dataset_name attribute of the logical file to the # name of the file selected to set file type unless the extracted metadata # has a value for title dataset_title = res_dublin_core_meta.get('title', None) if dataset_title is not None: logical_file.dataset_name = dataset_title else: logical_file.dataset_name = nc_file_name logical_file.save() try: # create a folder for the netcdf file type using the base file # name as the name for the new folder new_folder_path = cls.compute_file_type_folder( resource, file_folder, nc_file_name) create_folder(resource.short_id, new_folder_path) log.info("Folder created:{}".format(new_folder_path)) new_folder_name = new_folder_path.split('/')[-1] if file_folder is None: upload_folder = new_folder_name else: upload_folder = os.path.join( file_folder, new_folder_name) # add all new files to the resource for f in files_to_add_to_resource: uploaded_file = UploadedFile( file=open(f, 'rb'), name=os.path.basename(f)) # the added resource file will be part of a new generic logical file # by default new_res_file = utils.add_file_to_resource( resource, uploaded_file, folder=upload_folder) # delete the generic logical file object if new_res_file.logical_file is not None: # deleting the file level metadata object will delete the associated # logical file object new_res_file.logical_file.metadata.delete() # make each resource file we added part of the logical file logical_file.add_resource_file(new_res_file) log.info( "NetCDF file type - new files were added to the resource." ) # use the extracted metadata to populate resource metadata for element in resource_metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = element.items()[0] if k == 'title': # update title element title_element = resource.metadata.title resource.metadata.update_element( 'title', title_element.id, **v) else: resource.metadata.create_element(k, **v) log.info("Resource - metadata was saved to DB") # use the extracted metadata to populate file metadata for element in file_type_metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = element.items()[0] if k == 'subject': logical_file.metadata.keywords = v logical_file.metadata.save() # update resource level keywords resource_keywords = [ subject.value.lower() for subject in resource.metadata.subjects.all() ] for kw in logical_file.metadata.keywords: if kw.lower() not in resource_keywords: resource.metadata.create_element( 'subject', value=kw) else: logical_file.metadata.create_element(k, **v) log.info("NetCDF file type - metadata was saved to DB") # set resource to private if logical file is missing required metadata resource.update_public_and_discoverable() # delete the original resource file delete_resource_file(resource.short_id, res_file.id, user) log.info("Deleted original resource file.") except Exception as ex: msg = "NetCDF file type. Error when setting file type. Error:{}" msg = msg.format(ex.message) log.exception(msg) if upload_folder: # delete any new files uploaded as part of setting file type folder_to_remove = os.path.join( 'data', 'contents', upload_folder) remove_folder(user, resource.short_id, folder_to_remove) log.info("Deleted newly created file type folder") raise ValidationError(msg) finally: # remove temp dir if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) else: err_msg = "Not a valid NetCDF file. File type file validation failed." log.error(err_msg) # remove temp dir if os.path.isdir(temp_dir): shutil.rmtree(temp_dir) raise ValidationError(err_msg)
def setUp(self): super(TestCopyResource, self).setUp() self.group, _ = Group.objects.get_or_create(name='Hydroshare Author') # create a user who is the owner of the resource to be copied self.owner = hydroshare.create_account('*****@*****.**', username='******', first_name='owner_firstname', last_name='owner_lastname', superuser=False, groups=[]) # create a user who is NOT the owner of the resource to be copied self.nonowner = hydroshare.create_account( '*****@*****.**', username='******', first_name='nonowner_firstname', last_name='nonowner_lastname', superuser=False, groups=[]) # create a generic resource self.res_generic = hydroshare.create_resource( resource_type='GenericResource', owner=self.owner, title='Test Generic Resource') test_file1 = open('test1.txt', 'w') test_file1.write("Test text file in test1.txt") test_file1.close() test_file2 = open('test2.txt', 'w') test_file2.write("Test text file in test2.txt") test_file2.close() self.test_file1 = open('test1.txt', 'r') self.test_file2 = open('test2.txt', 'r') hydroshare.add_resource_files(self.res_generic.short_id, self.test_file1, self.test_file2) # create a generic empty resource with one license that prohibits derivation statement = 'This resource is shared under the Creative Commons Attribution-NoDerivs CC ' \ 'BY-ND.' url = 'http://creativecommons.org/licenses/by-nd/4.0/' metadata = [] metadata.append({'rights': {'statement': statement, 'url': url}}) self.res_generic_lic_nd = hydroshare.create_resource( resource_type='GenericResource', owner=self.owner, title='Test Generic Resource', metadata=metadata) # create a generic empty resource with another license that prohibits derivation statement = 'This resource is shared under the Creative Commons ' \ 'Attribution-NoCommercial-NoDerivs CC BY-NC-ND.' url = 'http://creativecommons.org/licenses/by-nc-nd/4.0/' metadata = [] metadata.append({'rights': {'statement': statement, 'url': url}}) self.res_generic_lic_nc_nd = hydroshare.create_resource( resource_type='GenericResource', owner=self.owner, title='Test Generic Resource', metadata=metadata) # create a raster resource that represents a specific resource type raster_file = 'hs_core/tests/data/cea.tif' temp_dir = tempfile.mkdtemp() self.temp_raster_file = os.path.join(temp_dir, 'cea.tif') shutil.copy(raster_file, self.temp_raster_file) self.raster_obj = open(self.temp_raster_file, 'r') files = [UploadedFile(file=self.raster_obj, name='cea.tif')] self.res_raster = hydroshare.create_resource( resource_type='RasterResource', owner=self.owner, title='Test Raster Resource', files=files, metadata=[]) # call the post creation process here for the metadata to be # extracted utils.resource_post_create_actions(resource=self.res_raster, user=self.owner, metadata=[])