def lookup_ref(format):
deriv = None
if "deriv" in request.args:
deriv = request.args["deriv"]
elif "size" in request.args:
deriv = request.args["size"]
params = {}
param_map = (("filereference", "url"), ("type", "type"),
("prefix", "prefix"), ("user", "owner"), ("mime_type", "mime"))
for ak, pk in param_map:
if ak in request.args:
if isinstance(request.args[ak], list):
params[pk] = request.args[ak][0]
else:
params[pk] = request.args[ak]
if "url" in params:
mo = MediaObject.fromurl(params['url'], idbmodel=idbmodel)
return respond_to_record(mo, deriv=deriv, format=format)
where = []
for k in params:
if k == "prefix":
where.append("media.url LIKE %(prefix)s")
params["prefix"] += "%"
else:
where.append("{0}=%({0})s".format(k))
results = MediaObject.query(conditions=where, params=params, idbmodel=idbmodel)
logger.debug("Formatting %d results", len(results))
files = [get_json_for_record(r, deriv) for r in results]
return jsonify({"files": files, "count": len(files)})
def test_mobj_full_run(testidbmodel, jpgpath):
url = "http://test.idigbio.org/idigbio_logo.jpg"
mobj = MediaObject.fromobj(jpgpath.open('rb'), url=url)
assert mobj.url == "http://test.idigbio.org/idigbio_logo.jpg"
assert mobj.etag
etag = mobj.etag
mobj.derivatives = False
mobj.insert_object(testidbmodel)
mobj.last_status = 200
mobj.last_check = datetime.now()
mobj.modified = datetime.now()
mobj.ensure_media(testidbmodel)
mobj.ensure_media_object(testidbmodel)
mfe = MediaObject.frometag(etag, idbmodel=testidbmodel)
assert mfe
for s in mobj.__slots__:
if s != 'modified':
assert (getattr(mobj, s) == getattr(mfe, s))
mfu = MediaObject.fromurl(url, idbmodel=testidbmodel)
assert mfu
assert mfu.url == url
for s in mobj.__slots__:
if s != 'modified':
assert (getattr(mobj, s) == getattr(mfu, s))
def test_mobj_apimedia_idigibio_patch(testidbmodel):
"URLs that are api.idigbio.org/v2/media contain etags and there is special handling for them"
etag = "924709c6ebbd34030468185a323a437"
url = "https://api.idigbio.org/v2/media/" + etag
mo = MediaObject(etag=etag, bucket="images", detected_mime='image/jpeg')
mo.ensure_object(idbmodel=testidbmodel)
mfu = MediaObject.fromurl(url, idbmodel=testidbmodel)
assert mfu.etag == etag
assert mfu.detected_mime == 'image/jpeg'
assert mfu.url is None
def get_media(self):
try:
self.etag = MediaObject.oldmediaapietag(self.url)
self.media_object = MediaObject(url=self.url,
etag=self.etag,
type=self.type,
bucket=self.type,
mime=self.mime,
detected_mime=self.mime)
self.status_code = Status.OK
except IndexError:
logger.exception("What?")
self.status_code = Status.UNHANDLED_FAILURE
return self
def test_render_svg(client, mock):
mock.patch.object(
MediaObject,
"fromuuid",
return_value=MediaObject(
url=u'http://collections.nmnh.si.edu/media/index.php?irn=7002478',
mime=u'application/pdf',
owner='872733a2-67a3-4c54-aa76-862735a5f334'))
assert MediaObject.fromuuid("foobar")
url = url_for('idb.data_api.v2_media.lookup_uuid',
u="872733a2-67a3-4c54-aa76-862735a5f334",
deriv="thumbnail")
r = client.get(url)
assert r.status_code == 200
assert r.content_type == "image/svg+xml; charset=utf-8"
def test_given_type_validation(jpgpath):
mobj = MediaObject.fromobj(jpgpath.open('rb'),
url='foo.png',
type='images')
assert mobj
assert mobj.type == mobj.bucket == 'images'
with pytest.raises(MediaValidationError):
mobj = MediaObject.fromobj(jpgpath.open('rb'),
url='foo.png',
type='datasets')
with pytest.raises(MediaValidationError):
mobj = MediaObject.fromobj(jpgpath.open('rb'),
url='foo.png',
type='foobar')
def lookup_uuid(u, format):
deriv = None
if "deriv" in request.args:
deriv = request.args["deriv"]
elif "size" in request.args:
deriv = request.args["size"]
r = MediaObject.fromuuid(u, idbmodel=idbmodel)
return respond_to_record(r, deriv=deriv, format=format)
def test_mobj_fromurl(testidbmodel, testdata):
url = 'http://hasbrouck.asu.edu/imglib/pollen/Seeds/Cornus-stolonifera-272.jpg'
mobj = MediaObject.fromurl(url, idbmodel=testidbmodel)
assert mobj
assert mobj.url == url
assert mobj.type == 'images' == mobj.bucket
assert mobj.mime == 'image/jpeg' == mobj.detected_mime
assert mobj.owner
assert mobj.etag == "3e17584bc43cf36617b6793515089656"
def test_mobj_bad_validation(pngpath):
with pytest.raises(MediaValidationError):
MediaObject.fromobj(pngpath.open('rb'), url='foo.png')
with pytest.raises(MediaValidationError):
MediaObject.fromobj(pngpath.open('rb'), url='foo.png', type="adsf")
with pytest.raises(MediaValidationError):
MediaObject.fromobj(pngpath.open('rb'), url='foo.png', type="images")
def upload_recordset(rsid, fname, idbmodel):
filereference = "http://api.idigbio.org/v1/recordsets/" + rsid
logger.debug("Starting Upload of %r", rsid)
stor = IDigBioStorage()
with open(fname, 'rb') as fobj:
mo = MediaObject.fromobj(fobj,
url=filereference,
type='datasets',
owner=config.IDB_UUID)
k = mo.get_key(stor)
if k.exists():
logger.debug("ETAG %s already present in Storage.", mo.etag)
else:
mo.upload(stor, fobj)
logger.debug("ETAG %s uploading from %r", mo.etag, fname)
mo.ensure_media(idbmodel)
mo.ensure_object(idbmodel)
mo.ensure_media_object(idbmodel)
logger.debug("Finished Upload of %r, etag = %s", rsid, mo.etag)
return mo.etag
def validate(self):
if not self.ok:
return self
try:
self.media_object = MediaObject.frombuff(self.content,
url=self.url,
type=self.type,
mime=self.mime)
logger.debug("Validated %s %s %s", self.url, self.type, self.mime)
return self
except MediaValidationError as mve:
self.status_code = Status.VALIDATION_FAILURE
self.reason = str(mve)
detected_mime = sniff_mime(self.content)
logger.error("Invalid! %s %s", self.url, self.reason)
if detected_mime in ('text/html', 'text/plain'):
sc = inspect_html_response(self.content)
if sc:
self.status_code = sc
logger.error("HtmlResp %s %r", self.url, sc)
return self
def test_mobj_fromobj_jpg(jpgpath):
mobj = MediaObject.fromobj(jpgpath.open('rb'), url='foo.jpg')
assert mobj.detected_mime == mobj.mime == 'image/jpeg'
assert mobj.bucket == mobj.type == 'images'
assert mobj.etag == jpgpath.computehash()
def test_mobj_from_etag_None(testidbmodel):
mobj = MediaObject.frometag("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
idbmodel=testidbmodel)
assert mobj is None
def test_mobj_from_url_None(testidbmodel):
mobj = MediaObject.fromurl('a not found url should not be found',
idbmodel=testidbmodel)
assert mobj is None
def upload():
vals = {}
j = request.get_json()
if j is not None:
vals.update(j)
for k, v in request.values.iteritems():
vals[k] = v
filereference = vals.get("filereference")
if not filereference:
logger.warning("No filereference specified")
return json_error(400, "Missing filereference")
obj = request.files.get('file')
etag = vals.get('etag')
media_type = vals.get("media_type")
mime = vals.get("mime")
try:
mime, media_type = validate_mime_for_type(mime, media_type)
except MediaValidationError as mve:
logger.warning("Bad mime/media_type combo: %r/%r", mime, media_type)
return json_error(400, str(mve))
r = MediaObject.fromurl(filereference, idbmodel=idbmodel)
if r:
logger.warning("Found existing object for %r", r.url)
if r.owner != request.authorization.username:
return json_error(403)
if obj:
# if either type or mime are null it will be ignored, if
# present they change the behavior of fromobj
try:
mo = MediaObject.fromobj(obj, type=media_type, mime=mime, url=filereference, etag=etag)
except MediaValidationError as mve:
logger.warning("Validation failure, %r", mve)
return json_error(400, str(mve))
mo.upload(IDigBioStorage(), obj)
mo.insert_object(idbmodel)
elif etag:
mo = MediaObject.frometag(etag, idbmodel)
if not mo or not mo.get_key(IDigBioStorage()).exists():
return json_error(404, "Unknown etag {0!r}".format(etag))
mo.last_status = 200
mo.last_check = datetime.now()
mo.mime = mime or mo.detected_mime
mo.type = media_type or mo.bucket
else:
mo = r or MediaObject()
mo.last_status = None
mo.last_check = None
try:
mo.mime, mo.type = validate_mime_for_type(mo.mime or mime, mo.type or media_type)
if not (mo.mime and mo.type):
logger.warning("Missing either mime(%r) or type(%r)", mo.mime, mo.type)
return json_error(400, "Incomplete request")
except MediaValidationError as mve:
logger.warning("Validation Failure, %r", mve)
return json_error(400, str(mve))
mo.url = filereference
mo.owner = request.authorization.username
if r:
mo.update_media(idbmodel)
else:
mo.insert_media(idbmodel)
if mo.etag:
mo.ensure_media_object(idbmodel)
idbmodel.commit()
return respond_to_record(mo, format='json')
def test_mobj_fromobj_mp3(mp3path):
mobj = MediaObject.fromobj(mp3path.open('rb'), url='foo.mp3')
assert mobj.detected_mime == mobj.mime == 'audio/mpeg'
assert mobj.etag == mp3path.computehash()
def get_key_by_url(self, url, idbmodel=None):
mo = MediaObject.fromurl(url, idbmodel)
if mo is None:
raise Exception("No media with url {0!r}".format(url))
return self.get_key(mo.keyname, mo.bucketname)
def get_key_by_etag(self, etag, idbmodel=None):
mo = MediaObject.frometag(etag, idbmodel=idbmodel)
return self.get_key(mo.keyname, mo.bucketname)