def lookup_ref(format): deriv = None if "deriv" in request.args: deriv = request.args["deriv"] elif "size" in request.args: deriv = request.args["size"] params = {} param_map = (("filereference", "url"), ("type", "type"), ("prefix", "prefix"), ("user", "owner"), ("mime_type", "mime")) for ak, pk in param_map: if ak in request.args: if isinstance(request.args[ak], list): params[pk] = request.args[ak][0] else: params[pk] = request.args[ak] if "url" in params: mo = MediaObject.fromurl(params['url'], idbmodel=idbmodel) return respond_to_record(mo, deriv=deriv, format=format) where = [] for k in params: if k == "prefix": where.append("media.url LIKE %(prefix)s") params["prefix"] += "%" else: where.append("{0}=%({0})s".format(k)) results = MediaObject.query(conditions=where, params=params, idbmodel=idbmodel) logger.debug("Formatting %d results", len(results)) files = [get_json_for_record(r, deriv) for r in results] return jsonify({"files": files, "count": len(files)})
def test_mobj_full_run(testidbmodel, jpgpath): url = "http://test.idigbio.org/idigbio_logo.jpg" mobj = MediaObject.fromobj(jpgpath.open('rb'), url=url) assert mobj.url == "http://test.idigbio.org/idigbio_logo.jpg" assert mobj.etag etag = mobj.etag mobj.derivatives = False mobj.insert_object(testidbmodel) mobj.last_status = 200 mobj.last_check = datetime.now() mobj.modified = datetime.now() mobj.ensure_media(testidbmodel) mobj.ensure_media_object(testidbmodel) mfe = MediaObject.frometag(etag, idbmodel=testidbmodel) assert mfe for s in mobj.__slots__: if s != 'modified': assert (getattr(mobj, s) == getattr(mfe, s)) mfu = MediaObject.fromurl(url, idbmodel=testidbmodel) assert mfu assert mfu.url == url for s in mobj.__slots__: if s != 'modified': assert (getattr(mobj, s) == getattr(mfu, s))
def test_mobj_apimedia_idigibio_patch(testidbmodel): "URLs that are api.idigbio.org/v2/media contain etags and there is special handling for them" etag = "924709c6ebbd34030468185a323a437" url = "https://api.idigbio.org/v2/media/" + etag mo = MediaObject(etag=etag, bucket="images", detected_mime='image/jpeg') mo.ensure_object(idbmodel=testidbmodel) mfu = MediaObject.fromurl(url, idbmodel=testidbmodel) assert mfu.etag == etag assert mfu.detected_mime == 'image/jpeg' assert mfu.url is None
def get_media(self): try: self.etag = MediaObject.oldmediaapietag(self.url) self.media_object = MediaObject(url=self.url, etag=self.etag, type=self.type, bucket=self.type, mime=self.mime, detected_mime=self.mime) self.status_code = Status.OK except IndexError: logger.exception("What?") self.status_code = Status.UNHANDLED_FAILURE return self
def test_render_svg(client, mock): mock.patch.object( MediaObject, "fromuuid", return_value=MediaObject( url=u'http://collections.nmnh.si.edu/media/index.php?irn=7002478', mime=u'application/pdf', owner='872733a2-67a3-4c54-aa76-862735a5f334')) assert MediaObject.fromuuid("foobar") url = url_for('idb.data_api.v2_media.lookup_uuid', u="872733a2-67a3-4c54-aa76-862735a5f334", deriv="thumbnail") r = client.get(url) assert r.status_code == 200 assert r.content_type == "image/svg+xml; charset=utf-8"
def test_given_type_validation(jpgpath): mobj = MediaObject.fromobj(jpgpath.open('rb'), url='foo.png', type='images') assert mobj assert mobj.type == mobj.bucket == 'images' with pytest.raises(MediaValidationError): mobj = MediaObject.fromobj(jpgpath.open('rb'), url='foo.png', type='datasets') with pytest.raises(MediaValidationError): mobj = MediaObject.fromobj(jpgpath.open('rb'), url='foo.png', type='foobar')
def lookup_uuid(u, format): deriv = None if "deriv" in request.args: deriv = request.args["deriv"] elif "size" in request.args: deriv = request.args["size"] r = MediaObject.fromuuid(u, idbmodel=idbmodel) return respond_to_record(r, deriv=deriv, format=format)
def test_mobj_fromurl(testidbmodel, testdata): url = 'http://hasbrouck.asu.edu/imglib/pollen/Seeds/Cornus-stolonifera-272.jpg' mobj = MediaObject.fromurl(url, idbmodel=testidbmodel) assert mobj assert mobj.url == url assert mobj.type == 'images' == mobj.bucket assert mobj.mime == 'image/jpeg' == mobj.detected_mime assert mobj.owner assert mobj.etag == "3e17584bc43cf36617b6793515089656"
def test_mobj_bad_validation(pngpath): with pytest.raises(MediaValidationError): MediaObject.fromobj(pngpath.open('rb'), url='foo.png') with pytest.raises(MediaValidationError): MediaObject.fromobj(pngpath.open('rb'), url='foo.png', type="adsf") with pytest.raises(MediaValidationError): MediaObject.fromobj(pngpath.open('rb'), url='foo.png', type="images")
def upload_recordset(rsid, fname, idbmodel): filereference = "http://api.idigbio.org/v1/recordsets/" + rsid logger.debug("Starting Upload of %r", rsid) stor = IDigBioStorage() with open(fname, 'rb') as fobj: mo = MediaObject.fromobj(fobj, url=filereference, type='datasets', owner=config.IDB_UUID) k = mo.get_key(stor) if k.exists(): logger.debug("ETAG %s already present in Storage.", mo.etag) else: mo.upload(stor, fobj) logger.debug("ETAG %s uploading from %r", mo.etag, fname) mo.ensure_media(idbmodel) mo.ensure_object(idbmodel) mo.ensure_media_object(idbmodel) logger.debug("Finished Upload of %r, etag = %s", rsid, mo.etag) return mo.etag
def validate(self): if not self.ok: return self try: self.media_object = MediaObject.frombuff(self.content, url=self.url, type=self.type, mime=self.mime) logger.debug("Validated %s %s %s", self.url, self.type, self.mime) return self except MediaValidationError as mve: self.status_code = Status.VALIDATION_FAILURE self.reason = str(mve) detected_mime = sniff_mime(self.content) logger.error("Invalid! %s %s", self.url, self.reason) if detected_mime in ('text/html', 'text/plain'): sc = inspect_html_response(self.content) if sc: self.status_code = sc logger.error("HtmlResp %s %r", self.url, sc) return self
def test_mobj_fromobj_jpg(jpgpath): mobj = MediaObject.fromobj(jpgpath.open('rb'), url='foo.jpg') assert mobj.detected_mime == mobj.mime == 'image/jpeg' assert mobj.bucket == mobj.type == 'images' assert mobj.etag == jpgpath.computehash()
def test_mobj_from_etag_None(testidbmodel): mobj = MediaObject.frometag("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", idbmodel=testidbmodel) assert mobj is None
def test_mobj_from_url_None(testidbmodel): mobj = MediaObject.fromurl('a not found url should not be found', idbmodel=testidbmodel) assert mobj is None
def upload(): vals = {} j = request.get_json() if j is not None: vals.update(j) for k, v in request.values.iteritems(): vals[k] = v filereference = vals.get("filereference") if not filereference: logger.warning("No filereference specified") return json_error(400, "Missing filereference") obj = request.files.get('file') etag = vals.get('etag') media_type = vals.get("media_type") mime = vals.get("mime") try: mime, media_type = validate_mime_for_type(mime, media_type) except MediaValidationError as mve: logger.warning("Bad mime/media_type combo: %r/%r", mime, media_type) return json_error(400, str(mve)) r = MediaObject.fromurl(filereference, idbmodel=idbmodel) if r: logger.warning("Found existing object for %r", r.url) if r.owner != request.authorization.username: return json_error(403) if obj: # if either type or mime are null it will be ignored, if # present they change the behavior of fromobj try: mo = MediaObject.fromobj(obj, type=media_type, mime=mime, url=filereference, etag=etag) except MediaValidationError as mve: logger.warning("Validation failure, %r", mve) return json_error(400, str(mve)) mo.upload(IDigBioStorage(), obj) mo.insert_object(idbmodel) elif etag: mo = MediaObject.frometag(etag, idbmodel) if not mo or not mo.get_key(IDigBioStorage()).exists(): return json_error(404, "Unknown etag {0!r}".format(etag)) mo.last_status = 200 mo.last_check = datetime.now() mo.mime = mime or mo.detected_mime mo.type = media_type or mo.bucket else: mo = r or MediaObject() mo.last_status = None mo.last_check = None try: mo.mime, mo.type = validate_mime_for_type(mo.mime or mime, mo.type or media_type) if not (mo.mime and mo.type): logger.warning("Missing either mime(%r) or type(%r)", mo.mime, mo.type) return json_error(400, "Incomplete request") except MediaValidationError as mve: logger.warning("Validation Failure, %r", mve) return json_error(400, str(mve)) mo.url = filereference mo.owner = request.authorization.username if r: mo.update_media(idbmodel) else: mo.insert_media(idbmodel) if mo.etag: mo.ensure_media_object(idbmodel) idbmodel.commit() return respond_to_record(mo, format='json')
def test_mobj_fromobj_mp3(mp3path): mobj = MediaObject.fromobj(mp3path.open('rb'), url='foo.mp3') assert mobj.detected_mime == mobj.mime == 'audio/mpeg' assert mobj.etag == mp3path.computehash()
def get_key_by_url(self, url, idbmodel=None): mo = MediaObject.fromurl(url, idbmodel) if mo is None: raise Exception("No media with url {0!r}".format(url)) return self.get_key(mo.keyname, mo.bucketname)
def get_key_by_etag(self, etag, idbmodel=None): mo = MediaObject.frometag(etag, idbmodel=idbmodel) return self.get_key(mo.keyname, mo.bucketname)