def create(self, docs): """Apply transformation requested in 'edit'""" ids = [] archive = get_resource_service("archive") for doc in docs: # first we get item and requested edit operations item = doc.pop("item", None) if item is None: try: item_id = doc.pop("item_id") except KeyError: raise errors.SuperdeskApiError.badRequestError( "either item or item_id must be specified") else: item_id = item[config.ID_FIELD] if item is None and item_id: item = next(archive.find({"_id": item_id})) edit = doc.pop("edit") # now we retrieve and load current original media rendition = item["renditions"]["original"] media_id = rendition["media"] media = current_app.media.get(media_id) out = im = Image.open(media) # we apply all requested operations on original media for operation, param in edit.items(): try: out = self.transform(out, operation, param) except ValueError: # if the operation can't be applied just ignore it logger.warning( "failed to apply operation: {operation} {param} for media {id}" .format(operation=operation, param=param, id=media_id)) buf = BytesIO() out.save(buf, format=im.format) # we set metadata buf.seek(0) content_type = rendition["mimetype"] ext = os.path.splitext(rendition["href"])[1] filename = str(uuid.uuid4()) + ext # and save transformed media in database media_id = current_app.media.put(buf, filename=filename, content_type=content_type) # now we recreate other renditions based on transformed original media buf.seek(0) renditions = generate_renditions(buf, media_id, [], "image", content_type, get_renditions_spec(), current_app.media.url_for_media) ids.append(item_id) doc["renditions"] = renditions return [ids]
def create(self, docs, **kwargs): ids = [] for doc in docs: item = doc.pop('item') orig = item['renditions']['original'] orig_file = get_file(orig, item) no_custom_crops = doc.get('no_custom_crops', False) rendition_spec = get_renditions_spec( no_custom_crops=no_custom_crops) inserted = [] mimetype = item.get('mimetype', orig.get('mimetype', '/')) media_type, content_type = mimetype.split('/') renditions = generate_renditions(orig_file, orig['media'], inserted, media_type, mimetype, rendition_spec, app.media.url_for_media) doc['renditions'] = renditions ids.append(item['_id']) updates = {'renditions': item['renditions']} try: get_resource_service('archive').update(item['_id'], updates, item) except Exception as ex: logger.warning( 'failed to update the renditions for original item in archive' ) return ids
def test_generate_renditions_custom_crop(self): inserted = [] renditions = get_renditions_spec() with open(IMG_PATH, "rb") as original: generated = generate_renditions( original, "id", inserted, "image", "image/jpeg", renditions, self.app.media.url_for_media ) # thumbnail should keep 4:3 ration self.assertIn("thumbnail", generated) self.assertEqual(160, generated["thumbnail"]["width"]) self.assertEqual(120, generated["thumbnail"]["height"]) # landscape should be cropped to 200x100 self.assertIn("landscape", generated) landscape = generated["landscape"] self.assertEqual(200, landscape["width"]) self.assertEqual(100, landscape["height"]) self.assertEqual(0, landscape["CropLeft"]) self.assertEqual(400, landscape["CropRight"]) self.assertEqual(50, landscape["CropTop"]) self.assertEqual(250, landscape["CropBottom"]) # portrait should be cropped to 150x200 self.assertIn("portrait", generated) portrait = generated["portrait"] self.assertEqual(150, portrait["width"]) self.assertEqual(200, portrait["height"]) self.assertEqual(0, portrait["CropTop"]) self.assertEqual(300, portrait["CropBottom"]) self.assertEqual(87, portrait["CropLeft"]) self.assertEqual(312, portrait["CropRight"])
def parse_item(self, image_path): filename = os.path.basename(image_path) content_type = mimetypes.guess_type(image_path)[0] guid = utils.generate_guid(type=GUID_TAG) item = {'guid': guid, 'uri': guid, config.VERSION: 1, ITEM_TYPE: CONTENT_TYPE.PICTURE, 'mimetype': content_type, 'versioncreated': utcnow(), } with open(image_path, 'rb') as f: _, content_type, file_metadata = process_file_from_stream(f, content_type=content_type) f.seek(0) file_id = app.media.put(f, filename=filename, content_type=content_type, metadata=file_metadata) filemeta.set_filemeta(item, file_metadata) f.seek(0) metadata = get_meta_iptc(f) f.seek(0) self.parse_meta(item, metadata) rendition_spec = get_renditions_spec(no_custom_crops=True) renditions = generate_renditions(f, file_id, [file_id], 'image', content_type, rendition_spec, url_for_media) item['renditions'] = renditions return item
def test_generate_renditions_custom_crop(self): inserted = [] renditions = get_renditions_spec() with open(IMG_PATH, 'rb') as original: generated = generate_renditions(original, 'id', inserted, 'image', 'image/jpeg', renditions, self.app.media.url_for_media) # thumbnail should keep 4:3 ration self.assertIn('thumbnail', generated) self.assertEqual(160, generated['thumbnail']['width']) self.assertEqual(120, generated['thumbnail']['height']) # landscape should be cropped to 200x100 self.assertIn('landscape', generated) landscape = generated['landscape'] self.assertEqual(200, landscape['width']) self.assertEqual(100, landscape['height']) self.assertEqual(0, landscape['CropLeft']) self.assertEqual(400, landscape['CropRight']) self.assertEqual(50, landscape['CropTop']) self.assertEqual(250, landscape['CropBottom']) # portrait should be cropped to 150x200 self.assertIn('portrait', generated) portrait = generated['portrait'] self.assertEqual(150, portrait['width']) self.assertEqual(200, portrait['height']) self.assertEqual(0, portrait['CropTop']) self.assertEqual(300, portrait['CropBottom']) self.assertEqual(87, portrait['CropLeft']) self.assertEqual(312, portrait['CropRight'])
def setUp(self): super().setUp() dirname = os.path.dirname(os.path.realpath(__file__)) image_path = os.path.normpath( os.path.join(dirname, "fixtures", self.filename)) content_type = mimetypes.guess_type(image_path)[0] guid = utils.generate_guid(type=GUID_TAG) self.item = { "guid": guid, "version": 1, "_id": guid, ITEM_TYPE: CONTENT_TYPE.PICTURE, "mimetype": content_type, "versioncreated": datetime.now(), } with open(image_path, "rb") as f: _, content_type, file_metadata = process_file_from_stream( f, content_type=content_type) f.seek(0) file_id = app.media.put(f, filename=self.filename, content_type=content_type, metadata=file_metadata) filemeta.set_filemeta(self.item, file_metadata) f.seek(0) rendition_spec = get_renditions_spec() renditions = generate_renditions(f, file_id, [file_id], "image", content_type, rendition_spec, url_for_media) self.item["renditions"] = renditions archive = get_resource_service("archive") archive.post([self.item])
def setUp(self): super().setUp() dirname = os.path.dirname(os.path.realpath(__file__)) image_path = os.path.normpath( os.path.join(dirname, 'fixtures', self.filename)) content_type = mimetypes.guess_type(image_path)[0] guid = utils.generate_guid(type=GUID_TAG) self.item = { 'guid': guid, 'version': 1, '_id': guid, ITEM_TYPE: CONTENT_TYPE.PICTURE, 'mimetype': content_type, 'versioncreated': datetime.now() } with open(image_path, 'rb') as f: _, content_type, file_metadata = process_file_from_stream( f, content_type=content_type) f.seek(0) file_id = app.media.put(f, filename=self.filename, content_type=content_type, metadata=file_metadata) filemeta.set_filemeta(self.item, file_metadata) f.seek(0) rendition_spec = get_renditions_spec() renditions = generate_renditions(f, file_id, [file_id], 'image', content_type, rendition_spec, url_for_media) self.item['renditions'] = renditions archive = get_resource_service('archive') archive.post([self.item])
def on_create(self, docs): """Create corresponding item on file upload.""" for doc in docs: if 'media' not in doc or doc['media'] is None: abort(400, description="No media found") file, content_type, metadata = self.get_file_from_document(doc) inserted = [doc['media']] file_type = content_type.split('/')[0] self._set_metadata(doc) try: doc[ITEM_TYPE] = self.type_av.get(file_type) rendition_spec = get_renditions_spec() renditions = generate_renditions(file, doc['media'], inserted, file_type, content_type, rendition_spec, url_for_media) doc['renditions'] = renditions doc['mimetype'] = content_type set_filemeta(doc, metadata) add_activity('upload', 'uploaded media {{ name }}', 'archive', item=doc, name=doc.get('headline', doc.get('mimetype')), renditions=doc.get('renditions')) except Exception as io: logger.exception(io) for file_id in inserted: delete_file_on_error(doc, file_id) abort(500)
def setUp(self): super().setUp() dirname = os.path.dirname(os.path.realpath(__file__)) image_path = os.path.normpath(os.path.join(dirname, 'fixtures', self.filename)) content_type = mimetypes.guess_type(image_path)[0] guid = utils.generate_guid(type=GUID_TAG) self.item = {'guid': guid, 'version': 1, '_id': guid, ITEM_TYPE: CONTENT_TYPE.PICTURE, 'mimetype': content_type, 'versioncreated': datetime.now() } with open(image_path, 'rb') as f: _, content_type, file_metadata = process_file_from_stream(f, content_type=content_type) f.seek(0) file_id = app.media.put(f, filename=self.filename, content_type=content_type, metadata=file_metadata) filemeta.set_filemeta(self.item, file_metadata) f.seek(0) rendition_spec = get_renditions_spec() renditions = generate_renditions(f, file_id, [file_id], 'image', content_type, rendition_spec, url_for_media) self.item['renditions'] = renditions archive = get_resource_service('archive') archive.post([self.item])
def find_one_raw(self, resource, _id): # XXX: preview is used here instead of paid download # see SDNTB-15 data = {} url = self._app.config['SCANPIX_SEARCH_URL'] + '/search' data['refPtrs'] = [_id] r = self._request(url, data) doc = r.json()['data'][0] self._parse_doc(doc) url = doc['renditions']['baseImage']['href'] # if MIME type can't be guessed, we default to jpeg mime_type = mimetypes.guess_type(url)[0] or 'image/jpeg' r = self._request(url, data) out = BytesIO(r.content) file_name, content_type, metadata = process_file_from_stream( out, mime_type) logger.debug('Going to save media file with %s ' % file_name) out.seek(0) try: file_id = self._app.media.put(out, filename=file_name, content_type=content_type, metadata=None) except Exception as e: logger.exception(e) raise SuperdeskApiError.internalError('Media saving failed') else: try: inserted = [file_id] doc['mimetype'] = content_type doc['filemeta'] = decode_metadata(metadata) # set the version created to now to bring it to the top of the desk, images can be quite old doc['versioncreated'] = utcnow() file_type = content_type.split('/')[0] rendition_spec = get_renditions_spec() renditions = generate_renditions(out, file_id, inserted, file_type, content_type, rendition_spec, url_for_media, insert_metadata=False) doc['renditions'] = renditions except (IndexError, KeyError, json.JSONDecodeError) as e: logger.exception("Internal error: {}".format(e)) delete_file_on_error(doc, file_id) raise SuperdeskApiError.internalError( 'Generating renditions failed') return doc
def on_create(self, docs): """Create corresponding item on file upload.""" for doc in docs: if 'media' not in doc or doc['media'] is None: abort(400, description="No media found") # check content type of video by python-magic content_type = magic.from_buffer(doc['media'].read(1024), mime=True) doc['media'].seek(0) file_type = content_type.split('/')[0] if file_type == 'video' and app.config.get("VIDEO_SERVER_ENABLE"): if not self.videoEditor.check_video_server(): raise SuperdeskApiError( message="Cannot connect to videoserver", status_code=500) # upload media to video server res, renditions, metadata = self.upload_file_to_video_server( doc) # get thumbnails for timeline bar self.videoEditor.get_timeline_thumbnails(doc.get('media'), 40) else: file, content_type, metadata = self.get_file_from_document(doc) inserted = [doc['media']] # if no_custom_crops is set to False the custom crops are generated automatically on media upload # see (SDESK-4742) rendition_spec = get_renditions_spec( no_custom_crops=app.config.get("NO_CUSTOM_CROPS")) with timer('archive:renditions'): renditions = generate_renditions(file, doc['media'], inserted, file_type, content_type, rendition_spec, url_for_media) try: self._set_metadata(doc) doc[ITEM_TYPE] = self.type_av.get(file_type) doc[ITEM_STATE] = CONTENT_STATE.PROGRESS doc['renditions'] = renditions doc['mimetype'] = content_type set_filemeta(doc, metadata) add_activity('upload', 'uploaded media {{ name }}', 'archive', item=doc, name=doc.get('headline', doc.get('mimetype')), renditions=doc.get('renditions')) except Exception as io: logger.exception(io) for file_id in inserted: delete_file_on_error(doc, file_id) if res: self.videoEditor.delete(res.get('_id')) abort(500)
def on_create(self, docs): """Create corresponding item on file upload.""" for doc in docs: if "media" not in doc or doc["media"] is None: abort(400, description="No media found") # check content type of video by python-magic content_type = app.media._get_mimetype(doc["media"]) doc["media"].seek(0) file_type = content_type.split("/")[0] if file_type == "video" and app.config.get("VIDEO_SERVER_ENABLED"): # upload media to video server res, renditions, metadata = self.upload_file_to_video_server( doc) # get thumbnails for timeline bar self.video_editor.create_timeline_thumbnails( doc.get("media"), 60) else: file, content_type, metadata = self.get_file_from_document(doc) inserted = [doc["media"]] # if no_custom_crops is set to False the custom crops are generated automatically on media upload # see (SDESK-4742) rendition_spec = get_renditions_spec( no_custom_crops=app.config.get("NO_CUSTOM_CROPS")) with timer("archive:renditions"): renditions = generate_renditions(file, doc["media"], inserted, file_type, content_type, rendition_spec, url_for_media) try: self._set_metadata(doc) doc[ITEM_TYPE] = self.type_av.get(file_type) doc[ITEM_STATE] = CONTENT_STATE.PROGRESS doc["renditions"] = renditions doc["mimetype"] = content_type set_filemeta(doc, metadata) add_activity( "upload", "uploaded media {{ name }}", "archive", item=doc, name=doc.get("headline", doc.get("mimetype")), renditions=doc.get("renditions"), ) except Exception as io: logger.exception(io) for file_id in inserted: delete_file_on_error(doc, file_id) if res: self.video_editor.delete(res.get("_id")) abort(500)
def create(self, docs, **kwargs): ids = [] for doc in docs: item = doc.pop('item') orig = item['renditions']['original'] orig_file = get_file(orig, item) rendition_spec = get_renditions_spec() inserted = [] media_type, content_type = item['mimetype'].split('/') renditions = generate_renditions(orig_file, orig['media'], inserted, media_type, item['mimetype'], rendition_spec, app.media.url_for_media) doc['renditions'] = renditions ids.append(item['_id']) return ids
def parse_item(self, image_path): filename = os.path.basename(image_path) content_type = mimetypes.guess_type(image_path)[0] guid = utils.generate_guid(type=GUID_TAG) item = { 'guid': guid, config.VERSION: 1, config.ID_FIELD: guid, ITEM_TYPE: CONTENT_TYPE.PICTURE, 'mimetype': content_type, 'versioncreated': datetime.now() } with open(image_path, 'rb') as f: _, content_type, file_metadata = process_file_from_stream( f, content_type=content_type) f.seek(0) file_id = app.media.put(f, filename=filename, content_type=content_type, metadata=file_metadata) filemeta.set_filemeta(item, file_metadata) f.seek(0) metadata = get_meta_iptc(f) f.seek(0) rendition_spec = get_renditions_spec(no_custom_crops=True) renditions = generate_renditions(f, file_id, [file_id], 'image', content_type, rendition_spec, url_for_media) item['renditions'] = renditions try: date_created, time_created = metadata[TAG.DATE_CREATED], metadata[ TAG.TIME_CREATED] except KeyError: pass else: # we format proper ISO 8601 date so we can parse it with dateutil datetime_created = '{}-{}-{}T{}:{}:{}{}{}:{}'.format( date_created[0:4], date_created[4:6], date_created[6:8], time_created[0:2], time_created[2:4], time_created[4:6], time_created[6], time_created[7:9], time_created[9:]) item['firstcreated'] = dateutil.parser.parse(datetime_created) # now we map IPTC metadata to superdesk metadata for source_key, dest_key in IPTC_MAPPING.items(): try: item[dest_key] = metadata[source_key] except KeyError: continue return item
def _get_renditions(self, article): """Get renditions for article.""" # get the actual article's renditions actual_renditions = article.get('renditions', {}) # renditions list that we want to publish if article['type'] is 'picture': renditions_to_publish = ['original'] + list(get_renditions_spec(without_internal_renditions=True).keys()) # filter renditions and keep only the ones we want to publish actual_renditions = {name: actual_renditions[name] for name in renditions_to_publish if name in actual_renditions} # format renditions to Ninjs renditions = {} for name, rendition in actual_renditions.items(): renditions[name] = self._format_rendition(rendition) return renditions
def _get_renditions(self, article): """Get renditions for article.""" # get the actual article's renditions actual_renditions = article.get('renditions', {}) # renditions list that we want to publish if article['type'] == 'picture': renditions_to_publish = ['original'] + list(get_renditions_spec(without_internal_renditions=True).keys()) # filter renditions and keep only the ones we want to publish actual_renditions = {name: actual_renditions[name] for name in renditions_to_publish if name in actual_renditions} # format renditions to Ninjs renditions = {} for name, rendition in actual_renditions.items(): renditions[name] = self._format_rendition(rendition) return renditions
def test_generate_renditions_base_image(self): inserted = [] renditions = get_renditions_spec() with open(BIG_IMG_PATH, 'rb') as original: generated = generate_renditions(original, 'id', inserted, 'image', 'image/jpeg', renditions, self.app.media.url_for_media) self.assertIn('landscape', generated) landscape = generated['landscape'] self.assertEqual(200, landscape['width']) self.assertEqual(100, landscape['height']) self.assertEqual(0, landscape['CropLeft']) self.assertEqual(1200, landscape['CropRight']) self.assertEqual(500, landscape['CropTop']) self.assertEqual(1100, landscape['CropBottom'])
def create(self, docs, **kwargs): ids = [] for doc in docs: item = doc.pop('item') orig = item['renditions']['original'] orig_file = get_file(orig, item) rendition_spec = get_renditions_spec() inserted = [] mimetype = item.get('mimetype', orig.get('mimetype', '/')) media_type, content_type = mimetype.split('/') renditions = generate_renditions(orig_file, orig['media'], inserted, media_type, mimetype, rendition_spec, app.media.url_for_media) doc['renditions'] = renditions ids.append(item['_id']) return ids
def parse_item(self, image_path): filename = os.path.basename(image_path) content_type = mimetypes.guess_type(image_path)[0] guid = utils.generate_guid(type=GUID_TAG) item = {'guid': guid, config.VERSION: 1, config.ID_FIELD: guid, ITEM_TYPE: CONTENT_TYPE.PICTURE, 'mimetype': content_type, 'versioncreated': datetime.now() } with open(image_path, 'rb') as f: _, content_type, file_metadata = process_file_from_stream(f, content_type=content_type) f.seek(0) file_id = app.media.put(f, filename=filename, content_type=content_type, metadata=file_metadata) filemeta.set_filemeta(item, file_metadata) f.seek(0) metadata = get_meta_iptc(f) f.seek(0) rendition_spec = get_renditions_spec(no_custom_crops=True) renditions = generate_renditions(f, file_id, [file_id], 'image', content_type, rendition_spec, url_for_media) item['renditions'] = renditions try: date_created, time_created = metadata[TAG.DATE_CREATED], metadata[TAG.TIME_CREATED] except KeyError: pass else: # we format proper ISO 8601 date so we can parse it with dateutil datetime_created = '{}-{}-{}T{}:{}:{}{}{}:{}'.format(date_created[0:4], date_created[4:6], date_created[6:8], time_created[0:2], time_created[2:4], time_created[4:6], time_created[6], time_created[7:9], time_created[9:]) item['firstcreated'] = dateutil.parser.parse(datetime_created) # now we map IPTC metadata to superdesk metadata for source_key, dest_key in IPTC_MAPPING.items(): try: item[dest_key] = metadata[source_key] except KeyError: continue return item
def test_generate_renditions_base_image(self): inserted = [] renditions = get_renditions_spec() with open(BIG_IMG_PATH, "rb") as original: generated = generate_renditions( original, "id", inserted, "image", "image/jpeg", renditions, self.app.media.url_for_media ) self.assertIn("landscape", generated) landscape = generated["landscape"] self.assertEqual(200, landscape["width"]) self.assertEqual(100, landscape["height"]) self.assertEqual(0, landscape["CropLeft"]) self.assertEqual(1200, landscape["CropRight"]) self.assertEqual(500, landscape["CropTop"]) self.assertEqual(1100, landscape["CropBottom"])
def create(self, docs): """Apply transformation requested in "edit" an item with updated rendition will be returned. All created files are temporary, meaning they will be deleted after a delay or a cleaning. """ try: item = request.json['item'] except KeyError: try: item_id = request.json['item_id'] except KeyError: raise errors.SuperdeskApiError.badRequestError( 'either item or item_id must be specified') item = next(get_resource_service('archive').find({'_id': item_id})) edit = request.json['edit'] rendition = item['renditions']['original'] media_id = rendition['media'] media = current_app.media.get(media_id) out = im = Image.open(media) for operation, param in edit.items(): try: out = self.transform(out, operation, param) except ValueError as e: raise errors.SuperdeskApiError.badRequestError( 'invalid edit instructions: {msg}'.format(msg=e)) buf = BytesIO() out.save(buf, format=im.format) buf.seek(0) content_type = rendition['mimetype'] ext = os.path.splitext(rendition['href'])[1] filename = str(uuid.uuid4()) + ext media_id = current_app.media.put(buf, filename=filename, content_type=content_type, folder='temp') buf.seek(0) renditions = generate_renditions(buf, media_id, [], 'image', content_type, get_renditions_spec(), current_app.media.url_for_media, temporary=True) item['renditions'] = renditions docs[:] = [item] return [item['_id']]
def update_renditions(item, href, old_item): """Update renditions for an item. If the old_item has renditions uploaded in to media then the old rendition details are assigned to the item, this avoids repeatedly downloading the same image and leaving the media entries orphaned. If there is no old_item the original is downloaded and renditions are generated. :param item: parsed item from source :param href: reference to original :param old_item: the item that we have already ingested, if it exists :return: item with renditions """ inserted = [] try: # If there is an existing set of renditions we keep those if old_item: media = old_item.get('renditions', {}).get('original', {}).get('media', {}) if media: item['renditions'] = old_item['renditions'] item['mimetype'] = old_item.get('mimetype') item['filemeta'] = old_item.get('filemeta') item['filemeta_json'] = old_item.get('filemeta_json') return content, filename, content_type = download_file_from_url(href) file_type, ext = content_type.split('/') metadata = process_file(content, file_type) file_guid = app.media.put(content, filename, content_type, metadata) inserted.append(file_guid) rendition_spec = get_renditions_spec() renditions = generate_renditions(content, file_guid, inserted, file_type, content_type, rendition_spec, url_for_media) item['renditions'] = renditions item['mimetype'] = content_type set_filemeta(item, metadata) except Exception as e: logger.exception(e) for file_id in inserted: app.media.delete(file_id) raise
def on_create(self, docs): """Create corresponding item on file upload.""" for doc in docs: if 'media' not in doc or doc['media'] is None: abort(400, description="No media found") file, content_type, metadata = self.get_file_from_document(doc) inserted = [doc['media']] file_type = content_type.split('/')[0] self._set_metadata(doc) try: doc[ITEM_TYPE] = self.type_av.get(file_type) doc[ITEM_STATE] = CONTENT_STATE.PROGRESS # if no_custom_crops is set to False the custom crops are generated automatically on media upload # see (SDESK-4742) rendition_spec = get_renditions_spec( no_custom_crops=app.config.get("NO_CUSTOM_CROPS")) with timer('archive:renditions'): renditions = generate_renditions(file, doc['media'], inserted, file_type, content_type, rendition_spec, url_for_media) doc['renditions'] = renditions doc['mimetype'] = content_type set_filemeta(doc, metadata) add_activity('upload', 'uploaded media {{ name }}', 'archive', item=doc, name=doc.get('headline', doc.get('mimetype')), renditions=doc.get('renditions')) except Exception as io: logger.exception(io) for file_id in inserted: delete_file_on_error(doc, file_id) abort(500)
def create(self, docs, **kwargs): ids = [] for doc in docs: item = doc.pop("item") orig = item["renditions"]["original"] orig_file = get_file(orig, item) no_custom_crops = doc.get("no_custom_crops", False) rendition_spec = get_renditions_spec( no_custom_crops=no_custom_crops) inserted = [] mimetype = item.get("mimetype", orig.get("mimetype", "/")) media_type, content_type = mimetype.split("/") renditions = generate_renditions(orig_file, orig["media"], inserted, media_type, mimetype, rendition_spec, app.media.url_for_media) doc["renditions"] = renditions ids.append(item["_id"]) return ids
def update_renditions(item, href, old_item): """Update renditions for an item. If the old_item has renditions uploaded in to media then the old rendition details are assigned to the item, this avoids repeatedly downloading the same image and leaving the media entries orphaned. If there is no old_item the original is downloaded and renditions are generated. :param item: parsed item from source :param href: reference to original :param old_item: the item that we have already ingested, if it exists :return: item with renditions """ inserted = [] try: # If there is an existing set of renditions we keep those if old_item: media = old_item.get('renditions', {}).get('original', {}).get('media', {}) if media: item['renditions'] = old_item['renditions'] item['mimetype'] = old_item.get('mimetype') item['filemeta'] = old_item.get('filemeta') item['filemeta_json'] = old_item.get('filemeta_json') return content, filename, content_type = download_file_from_url(href) file_type, ext = content_type.split('/') metadata = process_file(content, file_type) file_guid = app.media.put(content, filename, content_type, metadata) inserted.append(file_guid) rendition_spec = get_renditions_spec() renditions = generate_renditions(content, file_guid, inserted, file_type, content_type, rendition_spec, url_for_media) item['renditions'] = renditions item['mimetype'] = content_type set_filemeta(item, metadata) except Exception: for file_id in inserted: app.media.delete(file_id) raise
def create(self, docs): """Apply transformation requested in 'edit'""" ids = [] archive = get_resource_service('archive') for idx, doc in enumerate(docs): # first we get item and requested edit operations try: item = doc.pop('item') except KeyError: try: item_id = doc.pop('item_id') except KeyError: raise errors.SuperdeskApiError.badRequestError('either item or item_id must be specified') else: item_id = item[config.ID_FIELD] item = next(archive.find({'_id': item_id})) edit = doc.pop('edit') # new we retrieve and loag current original media rendition = item['renditions']['original'] media_id = rendition['media'] media = current_app.media.get(media_id) out = im = Image.open(media) # we apply all requested operations on original media for operation, param in edit.items(): try: out = self.transform(out, operation, param) except ValueError as e: raise errors.SuperdeskApiError.badRequestError('invalid edit instructions: {msg}'.format(msg=e)) buf = BytesIO() out.save(buf, format=im.format) # we set metadata buf.seek(0) content_type = rendition['mimetype'] ext = os.path.splitext(rendition['href'])[1] filename = str(uuid.uuid4()) + ext # and save transformed media in database media_id = current_app.media.put(buf, filename=filename, content_type=content_type, folder='temp') # now we recreate other renditions based on transformed original media buf.seek(0) # we keep old renditions to later delete old files renditions = generate_renditions(buf, media_id, [], 'image', content_type, get_renditions_spec(), current_app.media.url_for_media, temporary=True) # we update item in db, and the item we'll return to client updates = {'renditions': renditions} ids.append(item_id) archive.update(item_id, updates, item) # we keep old renditions to later delete old files item.update(updates) docs[idx] = item return [ids]
def create(self, docs): """Apply transformation requested in 'edit'""" ids = [] archive = get_resource_service('archive') for idx, doc in enumerate(docs): # first we get item and requested edit operations try: item = doc.pop('item') except KeyError: try: item_id = doc.pop('item_id') except KeyError: raise errors.SuperdeskApiError.badRequestError( 'either item or item_id must be specified') else: item_id = item[config.ID_FIELD] item = next(archive.find({'_id': item_id})) edit = doc.pop('edit') # now we retrieve and load current original media rendition = item['renditions']['original'] media_id = rendition['media'] media = current_app.media.get(media_id) out = im = Image.open(media) # we apply all requested operations on original media for operation, param in edit.items(): try: out = self.transform(out, operation, param) except ValueError: # if the operation can't be applied just ignore it logger.warning( 'failed to apply operation: {operation} {param} for media {id}' .format(operation=operation, param=param, id=media_id)) buf = BytesIO() out.save(buf, format=im.format) # we set metadata buf.seek(0) content_type = rendition['mimetype'] ext = os.path.splitext(rendition['href'])[1] filename = str(uuid.uuid4()) + ext # and save transformed media in database media_id = current_app.media.put(buf, filename=filename, content_type=content_type) # now we recreate other renditions based on transformed original media buf.seek(0) renditions = generate_renditions(buf, media_id, [], 'image', content_type, get_renditions_spec(), current_app.media.url_for_media) # we update item in db, and the item we'll return to client # we keep old renditions in GridFS for history references updates = {'renditions': renditions} ids.append(item_id) archive.update(item_id, updates, item) item.update(updates) docs[idx] = item return [ids]
def create(self, docs): """Apply transformation requested in 'edit'""" ids = [] archive = get_resource_service('archive') for idx, doc in enumerate(docs): # first we get item and requested edit operations try: item = doc.pop('item') except KeyError: try: item_id = doc.pop('item_id') except KeyError: raise errors.SuperdeskApiError.badRequestError('either item or item_id must be specified') else: item_id = item[config.ID_FIELD] item = next(archive.find({'_id': item_id})) edit = doc.pop('edit') # now we retrieve and load current original media rendition = item['renditions']['original'] media_id = rendition['media'] media = current_app.media.get(media_id) out = im = Image.open(media) # we apply all requested operations on original media for operation, param in edit.items(): try: out = self.transform(out, operation, param) except ValueError: # if the operation can't be applied just ignore it logger.warning('failed to apply operation: {operation} {param} for media {id}'.format( operation=operation, param=param, id=media_id)) buf = BytesIO() out.save(buf, format=im.format) # we set metadata buf.seek(0) content_type = rendition['mimetype'] ext = os.path.splitext(rendition['href'])[1] filename = str(uuid.uuid4()) + ext # and save transformed media in database media_id = current_app.media.put(buf, filename=filename, content_type=content_type) # now we recreate other renditions based on transformed original media buf.seek(0) renditions = generate_renditions(buf, media_id, [], 'image', content_type, get_renditions_spec(), current_app.media.url_for_media) # we update item in db, and the item we'll return to client # we keep old renditions in GridFS for history references updates = {'renditions': renditions} ids.append(item_id) archive.update(item_id, updates, item) item.update(updates) docs[idx] = item return [ids]
def get_renditions_filter(): renditions = set( get_renditions_spec(without_internal_renditions=True).keys()) renditions.add("original") return renditions
def test_get_rendition_spec_with_custom_crop(self): renditions = get_renditions_spec() for crop in self.crop_sizes.get('items'): self.assertIn(crop['name'], renditions)
def test_get_rendition_spec_no_custom_crop(self): renditions = get_renditions_spec(no_custom_crops=True) for crop in self.crop_sizes.get('items'): self.assertNotIn(crop['name'], renditions)
def find_one_raw(self, resource, _id): if self._headers is None: self.__set_auth_cookie(self._app) url = self._app.config['AAP_MM_SEARCH_URL'] + '/Assets/{}'.format(_id) r = self._http.request('GET', url, headers=self._headers) doc = json.loads(r.data.decode('UTF-8')) self._parse_doc(doc) if 'fetch_endpoint' in doc: del doc['fetch_endpoint'] # Only if we have credentials can we download the original if the account has that privilege if self._username is not None and self._password is not None: resolutions = self._get_resolutions(_id) if doc[ITEM_TYPE] == CONTENT_TYPE.PICTURE: if any(i['Name'] == 'Original' for i in resolutions['Image']): url = self._app.config['AAP_MM_SEARCH_URL'] + '/Assets/{}/Original/download'.format(_id) mime_type = 'image/jpeg' else: raise FileNotFoundError elif doc[ITEM_TYPE] == CONTENT_TYPE.VIDEO: if any(v['Name'] == 'Ipod' for v in resolutions['Video']): url = self._app.config['AAP_MM_SEARCH_URL'] + '/Assets/{}/Ipod/download'.format(_id) mime_type = doc.get('renditions').get('original').get('mimetype') else: raise FileNotFoundError else: raise NotImplementedError else: if doc[ITEM_TYPE] == CONTENT_TYPE.VIDEO: mime_type = doc.get('renditions').get('original').get('mimetype') else: mime_type = 'image/jpeg' url = doc['renditions']['original']['href'] r = self._http.request('GET', url, headers=self._headers) out = BytesIO(r.data) file_name, content_type, metadata = process_file_from_stream(out, mime_type) inserted = [] try: logger.debug('Going to save media file with %s ' % file_name) out.seek(0) file_id = self._app.media.put(out, filename=file_name, content_type=content_type, metadata=None) doc['mimetype'] = content_type doc['filemeta'] = decode_metadata(metadata) # set the version created to now to bring it to the top of the desk, images can be quite old doc['versioncreated'] = utcnow() inserted = [file_id] file_type = content_type.split('/')[0] rendition_spec = get_renditions_spec(no_custom_crops=True) renditions = generate_renditions(out, file_id, inserted, file_type, content_type, rendition_spec, self.url_for_media, insert_metadata=False) doc['renditions'] = renditions except Exception as io: logger.exception(io) for file_id in inserted: delete_file_on_error(doc, file_id) raise SuperdeskApiError.internalError('Generating renditions failed') return doc
def find_one_raw(self, resource, _id): if self._headers is None: self.__set_auth_cookie(self._app) url = self._app.config['AAP_MM_SEARCH_URL'] + '/Assets/{}'.format(_id) r = self._http.request('GET', url, headers=self._headers) doc = json.loads(r.data.decode('UTF-8')) self._parse_doc(doc) if 'fetch_endpoint' in doc: del doc['fetch_endpoint'] # Only if we have credentials can we download the original if the account has that privilege if self._username is not None and self._password is not None: resolutions = self._get_resolutions(_id) if doc[ITEM_TYPE] == CONTENT_TYPE.PICTURE: if any(i['Name'] == 'Original' for i in resolutions['Image']): url = self._app.config[ 'AAP_MM_SEARCH_URL'] + '/Assets/{}/Original/download'.format( _id) mime_type = 'image/jpeg' else: raise FileNotFoundError elif doc[ITEM_TYPE] == CONTENT_TYPE.VIDEO: if any(v['Name'] == 'Ipod' for v in resolutions['Video']): url = self._app.config[ 'AAP_MM_SEARCH_URL'] + '/Assets/{}/Ipod/download'.format( _id) mime_type = doc.get('renditions').get('original').get( 'mimetype') else: raise FileNotFoundError else: raise NotImplementedError else: if doc[ITEM_TYPE] == CONTENT_TYPE.VIDEO: mime_type = doc.get('renditions').get('original').get( 'mimetype') else: mime_type = 'image/jpeg' url = doc['renditions']['original']['href'] r = self._http.request('GET', url, headers=self._headers) out = BytesIO(r.data) file_name, content_type, metadata = process_file_from_stream( out, mime_type) inserted = [] try: logger.debug('Going to save media file with %s ' % file_name) out.seek(0) file_id = self._app.media.put(out, filename=file_name, content_type=content_type, metadata=None) doc['mimetype'] = content_type doc['filemeta'] = decode_metadata(metadata) # set the version created to now to bring it to the top of the desk, images can be quite old doc['versioncreated'] = utcnow() inserted = [file_id] file_type = content_type.split('/')[0] rendition_spec = get_renditions_spec(no_custom_crops=True) renditions = generate_renditions(out, file_id, inserted, file_type, content_type, rendition_spec, self.url_for_media, insert_metadata=False) doc['renditions'] = renditions except Exception as io: logger.exception(io) for file_id in inserted: delete_file_on_error(doc, file_id) raise SuperdeskApiError.internalError( 'Generating renditions failed') return doc