def get_keywords(self, text):
if not app.config['KEYWORDS_KEY_API']:
raise SuperdeskApiError.notFoundError(
_('AlchemyAPI key is not set'))
params = {
'apikey': app.config['KEYWORDS_KEY_API'],
'outputMode': 'json'
}
url = app.config['KEYWORDS_BASE_URL'] + '/text/TextGetRankedNamedEntities' + \
'?' + urllib.parse.urlencode(params)
values = {'text': text}
result = ""
try:
result = self._http.post(url, data=values)
except Exception as ex:
raise SuperdeskApiError.internalError(
_('Fail to connect to Alchemy service'), exception=ex)
try:
keywords = result.json()
return keywords.get('entities', [])
except Exception as ex:
raise SuperdeskApiError.internalError(
_('Fail to parse the response from Alchemy service'),
exception=ex)
def find_one_raw(self, resource, _id):
# XXX: preview is used here instead of paid download
# see SDNTB-15
data = {}
url = self._app.config['SCANPIX_SEARCH_URL'] + '/search'
data['refPtrs'] = [_id]
r = self._request(url, data)
doc = r.json()['data'][0]
self._parse_doc(doc)
url = doc['renditions']['baseImage']['href']
# if MIME type can't be guessed, we default to jpeg
mime_type = mimetypes.guess_type(url)[0] or 'image/jpeg'
r = self._request(url, data)
out = BytesIO(r.content)
file_name, content_type, metadata = process_file_from_stream(
out, mime_type)
logger.debug('Going to save media file with %s ' % file_name)
out.seek(0)
try:
file_id = self._app.media.put(out,
filename=file_name,
content_type=content_type,
metadata=None)
except Exception as e:
logger.exception(e)
raise SuperdeskApiError.internalError('Media saving failed')
else:
try:
inserted = [file_id]
doc['mimetype'] = content_type
doc['filemeta'] = decode_metadata(metadata)
# set the version created to now to bring it to the top of the desk, images can be quite old
doc['versioncreated'] = utcnow()
file_type = content_type.split('/')[0]
rendition_spec = get_renditions_spec()
renditions = generate_renditions(out,
file_id,
inserted,
file_type,
content_type,
rendition_spec,
url_for_media,
insert_metadata=False)
doc['renditions'] = renditions
except (IndexError, KeyError, json.JSONDecodeError) as e:
logger.exception("Internal error: {}".format(e))
delete_file_on_error(doc, file_id)
raise SuperdeskApiError.internalError(
'Generating renditions failed')
return doc
def create(self, docs, **kwargs):
if not self.provider:
raise SuperdeskApiError.internalError('Not set a keywords provider')
try:
ids = []
for doc in docs:
doc['keywords'] = self.provider.get_keywords(doc.get('text', ''))
ids.append(len(ids))
return ids
except Exception as ex:
raise SuperdeskApiError.internalError(str(ex))
def create(self, docs, **kwargs):
if not self.provider:
raise SuperdeskApiError.internalError(
_("Not set a keywords provider"))
try:
ids = []
for doc in docs:
doc["keywords"] = self.provider.get_keywords(
doc.get("text", ""))
ids.append(len(ids))
return ids
except Exception as ex:
raise SuperdeskApiError.internalError(str(ex), exception=ex)
def _save_cropped_image(self, file_stream, original, doc):
"""Saves the cropped image and returns the crop dictionary
:param file_stream: cropped image stream
:param original: original rendition
:param doc: crop data
:return dict: Crop values
:raises SuperdeskApiError.internalError
"""
crop = {}
try:
file_name, content_type, metadata = process_file_from_stream(
file_stream, content_type=original.get("mimetype")
)
file_stream.seek(0)
file_id = app.media.put(
file_stream, filename=file_name, content_type=content_type, resource="upload", metadata=metadata
)
crop["media"] = file_id
crop["mimetype"] = content_type
crop["href"] = url_for_media(file_id, content_type)
crop["CropTop"] = doc.get("CropTop", None)
crop["CropLeft"] = doc.get("CropLeft", None)
crop["CropRight"] = doc.get("CropRight", None)
crop["CropBottom"] = doc.get("CropBottom", None)
return crop
except Exception as ex:
try:
app.media.delete(file_id)
except Exception:
pass
raise SuperdeskApiError.internalError("Generating crop failed: {}".format(str(ex)), exception=ex)
def _check_server(self, text):
check_url = urljoin(self.base_url, PATH_CHECK)
r = requests.post(check_url, data={"text": text,
"options": json.dumps(self.grammalecte_config)})
if r.status_code != 200:
raise SuperdeskApiError.internalError("Unexpected return code from Grammalecte")
return self.grammalecte2superdesk(text, r.json())
def check(self, text, language=None):
check_url = API_URL.format(method="spellingchecker")
data = {
"key": self.api_key,
"input": text,
"startmarker": START_MARKER,
"endmarker": END_MARKER,
}
r = requests.post(check_url, data=data, timeout=self.CHECK_TIMEOUT)
if r.status_code != 200:
raise SuperdeskApiError.internalError(
"Unexpected return code from {}".format(self.name))
data = r.json()
err_list = []
check_data = {"errors": err_list}
len_end_marker = len(END_MARKER)
output = data["spellingchecker"]["output"]
marked = output["marked"].split(START_MARKER)
# the first item in "marked" is unmarked text, we start our index there
text_idx = len(marked.pop(0))
for marked_part in marked:
mistake = marked_part[:marked_part.find(END_MARKER)]
ercorr_data = {
"startOffset": text_idx,
"text": mistake,
"type": "spelling",
}
err_list.append(ercorr_data)
text_idx += len(marked_part) - len_end_marker
return check_data
def _save_cropped_image(self, file_stream, original, doc):
"""
Saves the cropped image and returns the crop dictionary
:param file_stream: cropped image stream
:param original: original rendition
:param doc: crop data
:return dict: Crop values
:raises SuperdeskApiError.internalError
"""
crop = {}
try:
file_name, content_type, metadata = process_file_from_stream(file_stream,
content_type=original.get('mimetype'))
file_stream.seek(0)
file_id = superdesk.app.media.put(file_stream, filename=file_name,
content_type=content_type,
resource='upload',
metadata=metadata)
crop['media'] = file_id
crop['mimetype'] = content_type
crop['href'] = url_for_media(file_id, content_type)
crop['CropTop'] = doc.get('CropTop', None)
crop['CropLeft'] = doc.get('CropLeft', None)
crop['CropRight'] = doc.get('CropRight', None)
crop['CropBottom'] = doc.get('CropBottom', None)
return crop
except Exception as ex:
try:
superdesk.app.media.delete(file_id)
except:
pass
raise SuperdeskApiError.internalError('Generating crop failed: {}'.format(str(ex)))
def update(self, provider, update):
"""
Clients consuming Ingest Services should invoke this to get items from the provider.
:param provider: Ingest Provider Details.
:type provider: dict :py:class: `superdesk.io.ingest_provider_model.IngestProviderResource`
:param update: Any update that is required on provider.
:type update: dict
:return: a list of articles which can be saved in Ingest Collection.
:raises SuperdeskApiError.internalError if Provider is closed
:raises SuperdeskIngestError if failed to get items from provider
"""
is_closed = provider.get('is_closed', False)
if isinstance(is_closed, datetime):
is_closed = False
if is_closed:
raise SuperdeskApiError.internalError('Ingest Provider is closed')
else:
try:
return self._update(provider, update) or []
except SuperdeskIngestError as error:
self.close_provider(provider, error)
raise error
def update(self, provider, update):
"""
Clients consuming Ingest Services should invoke this to get items from the provider.
:param provider: Ingest Provider Details.
:type provider: dict :py:class: `superdesk.io.ingest_provider_model.IngestProviderResource`
:param update: Any update that is required on provider.
:type update: dict
:return: a list of articles which can be saved in Ingest Collection.
:raises SuperdeskApiError.internalError if Provider is closed
:raises SuperdeskIngestError if failed to get items from provider
"""
if self._is_closed(provider):
raise SuperdeskApiError.internalError('Ingest Provider is closed')
else:
try:
self._provider = provider
self._log_msg("Start update execution.")
self._timer.start('update')
return self._update(provider, update) or []
except SuperdeskIngestError as error:
self.close_provider(provider, error)
raise error
finally:
self._log_msg(
"Stop update execution. Exec time: {:.4f} secs.".format(
self._timer.stop('update')))
# just in case stop all timers
self._timer.stop_all()
def factory(field):
if field not in FilterConditionFieldsEnum.__members__:
vocabulary = get_resource_service('vocabularies').find_one(req=None, _id=field)
if vocabulary:
if vocabulary['field_type'] == 'text':
return FilterConditionCustomTextField(field)
else:
return FilterConditionControlledVocabularyField(field)
raise SuperdeskApiError.internalError('Invalid filter conditions field %s' % field)
if FilterConditionFieldsEnum[field] == FilterConditionFieldsEnum.desk:
return FilterConditionDeskField(field)
elif FilterConditionFieldsEnum[field] == FilterConditionFieldsEnum.stage:
return FilterConditionStageField(field)
elif FilterConditionFieldsEnum[field] == FilterConditionFieldsEnum.anpa_category:
return FilterConditionCategoryField(field)
elif FilterConditionFieldsEnum[field] == FilterConditionFieldsEnum.genre:
return FilterConditionGenreField(field)
elif FilterConditionFieldsEnum[field] == FilterConditionFieldsEnum.sms:
return FilterConditionSmsField(field)
elif FilterConditionFieldsEnum[field] == FilterConditionFieldsEnum.subject:
return FilterConditionSubjectField(field)
elif FilterConditionFieldsEnum[field] == FilterConditionFieldsEnum.urgency:
return FilterConditionUrgencyField(field)
elif FilterConditionFieldsEnum[field] == FilterConditionFieldsEnum.priority:
return FilterConditionPriorityField(field)
elif FilterConditionFieldsEnum[field] == FilterConditionFieldsEnum.place:
return FilterConditionPlaceField(field)
elif FilterConditionFieldsEnum[field] == FilterConditionFieldsEnum.ingest_provider:
return FilterConditionIngestProviderField(field)
elif FilterConditionFieldsEnum[field] == FilterConditionFieldsEnum.embargo:
return FilterConditionEmbargoField(field)
else:
return FilterConditionField(field)
def server_error_handler(error):
"""Log server errors."""
if getattr(app, 'sentry'):
app.sentry.captureException()
logger.exception(error)
return_error = SuperdeskApiError.internalError()
return client_error_handler(return_error)
def update(self, id, updates, original):
archived_item = super().find_one(req=None, _id=id)
try:
if archived_item['type'] == 'composite':
self.__publish_package_items(archived_item, updates[config.LAST_UPDATED])
# document is saved to keep the initial changes
self.backend.update(self.datasource, id, updates, original)
original.update(updates)
if archived_item['type'] != 'composite':
# queue only text items
self.queue_transmission(original)
task = self.__send_to_publish_stage(original)
if task:
updates['task'] = task
# document is saved to change the status
updates[config.CONTENT_STATE] = 'published'
item = self.backend.update(self.datasource, id, updates, original)
original.update(updates)
user = get_user()
push_notification('item:publish', item=str(item.get('_id')), user=str(user))
original.update(super().find_one(req=None, _id=id))
except KeyError as e:
raise SuperdeskApiError.badRequestError(
message="Key is missing on article to be published: {}"
.format(str(e)))
except Exception as e:
logger.error("Something bad happened while publishing %s".format(id), e)
raise SuperdeskApiError.internalError(message="Failed to publish the item: {}"
.format(str(e)))
def crop_and_store_file(self, doc, content, filename, content_type):
# retrieve file name and metadata from file
file_name, content_type, metadata = process_file_from_stream(content, content_type=content_type)
# crop the file if needed, can change the image size
was_cropped, out = crop_image(content, filename, doc)
# the length in metadata could be updated if it was cropped
if was_cropped:
file_name, content_type, metadata_after_cropped = process_file_from_stream(out, content_type=content_type)
# when cropped, metadata are reseted. Then we update the previous metadata variable
metadata['length'] = metadata_after_cropped['length']
try:
logger.debug('Going to save media file with %s ' % file_name)
out.seek(0)
file_id = app.media.put(out, filename=file_name, content_type=content_type,
resource=self.datasource, metadata=metadata)
doc['media'] = file_id
doc['mimetype'] = content_type
doc['filemeta'] = decode_metadata(metadata)
inserted = [doc['media']]
file_type = content_type.split('/')[0]
rendition_spec = config.RENDITIONS['avatar']
renditions = generate_renditions(out, file_id, inserted, file_type,
content_type, rendition_spec, url_for_media)
doc['renditions'] = renditions
except Exception as io:
logger.exception(io)
for file_id in inserted:
delete_file_on_error(doc, file_id)
raise SuperdeskApiError.internalError('Generating renditions failed')
def crop_and_store_file(self, doc, content, filename, content_type):
# retrieve file name and metadata from file
file_name, content_type, metadata = process_file_from_stream(
content, content_type=content_type)
# crop the file if needed, can change the image size
was_cropped, out = crop_image(content, filename, doc)
# the length in metadata could be updated if it was cropped
if was_cropped:
file_name, content_type, metadata_after_cropped = process_file_from_stream(
out, content_type=content_type)
# when cropped, metadata are reseted. Then we update the previous metadata variable
metadata['length'] = metadata_after_cropped['length']
try:
logger.debug('Going to save media file with %s ' % file_name)
out.seek(0)
file_id = app.media.put(out,
filename=file_name,
content_type=content_type,
resource=self.datasource,
metadata=metadata)
doc['media'] = file_id
doc['mimetype'] = content_type
set_filemeta(doc, decode_metadata(metadata))
inserted = [doc['media']]
file_type = content_type.split('/')[0]
rendition_spec = config.RENDITIONS['avatar']
renditions = generate_renditions(out, file_id, inserted, file_type,
content_type, rendition_spec,
url_for_media)
doc['renditions'] = renditions
except Exception as io:
for file_id in inserted:
delete_file_on_error(doc, file_id)
raise SuperdeskApiError.internalError(
'Generating renditions failed', exception=io)
def _save_cropped_image(self, file_stream, original, doc):
"""
Saves the cropped image and returns the crop dictionary
:param file_stream: cropped image stream
:param original: original rendition
:param doc: crop data
:return dict: Crop values
:raises SuperdeskApiError.internalError
"""
crop = {}
try:
file_name, content_type, metadata = process_file_from_stream(
file_stream, content_type=original.get('mimetype'))
file_stream.seek(0)
file_id = superdesk.app.media.put(file_stream,
filename=file_name,
content_type=content_type,
resource='upload',
metadata=metadata)
crop['media'] = file_id
crop['mimetype'] = content_type
crop['href'] = url_for_media(file_id, content_type)
crop['CropTop'] = doc.get('CropTop', None)
crop['CropLeft'] = doc.get('CropLeft', None)
crop['CropRight'] = doc.get('CropRight', None)
crop['CropBottom'] = doc.get('CropBottom', None)
return crop
except Exception as ex:
try:
superdesk.app.media.delete(file_id)
except:
pass
raise SuperdeskApiError.internalError(
'Generating crop failed: {}'.format(str(ex)))
def put(self, content, filename=None, content_type=None, metadata=None):
""" Saves a new file using the storage system, preferably with the name
specified. If there already exists a file with this name name, the
storage system may modify the filename as necessary to get a unique
name. Depending on the storage system, a unique id or the actual name
of the stored file will be returned. The content type argument is used
to appropriately identify the file when it is retrieved.
"""
logger.debug('Going to save media file with %s ' % filename)
found, existing_file = self._check_exists(filename, raise_error=False)
if found:
return filename
try:
file_metadata = self.transform_metadata_to_amazon_format(metadata)
res = self.conn.upload(filename,
content,
self.container_name,
content_type=content_type,
headers=file_metadata)
if res.status_code not in (200, 201):
raise SuperdeskApiError.internalError(
'Uploading file to amazon S3 failed')
return filename
except Exception as ex:
logger.exception(ex)
raise
def download_file_from_url(url, request_kwargs=None):
"""Download file from given url.
In case url is relative it will prefix it with current host.
:param url: file url
"""
if not request_kwargs:
request_kwargs = {}
try:
rv = requests.get(url,
headers={"User-Agent": "Superdesk-1.0"},
timeout=(5, 25),
**request_kwargs)
except requests.exceptions.MissingSchema: # any route will do here, we only need host
rv = requests.get(urljoin(
url_for("static", filename="x", _external=True), url),
timeout=15,
**request_kwargs)
if rv.status_code not in (200, 201):
raise SuperdeskApiError.internalError(
"Failed to retrieve file from URL: %s" % url)
content = BytesIO(rv.content)
content_type = rv.headers.get("content-type", "image/jpeg").split(";")[0]
content_type = fix_content_type(content_type, content)
ext = str(content_type).split("/")[1]
name = str(ObjectId()) + ext
return content, name, content_type
def update(self, provider, update):
"""
Clients consuming Ingest Services should invoke this to get items from the provider.
:param provider: Ingest Provider Details.
:type provider: dict :py:class: `superdesk.io.ingest_provider_model.IngestProviderResource`
:param update: Any update that is required on provider.
:type update: dict
:return: a list of articles which can be saved in Ingest Collection.
:raises SuperdeskApiError.internalError if Provider is closed
:raises SuperdeskIngestError if failed to get items from provider
"""
is_closed = provider.get('is_closed', False)
if isinstance(is_closed, datetime):
is_closed = False
if is_closed:
raise SuperdeskApiError.internalError('Ingest Provider is closed')
else:
try:
return self._update(provider, update) or []
except SuperdeskIngestError as error:
self.close_provider(provider, error)
raise error
def update(self, id, updates, original):
"""
Handles workflow of each Publish, Corrected and Killed.
"""
try:
user = get_user()
last_updated = updates.get(config.LAST_UPDATED, utcnow())
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self._publish_package_items(original, updates)
queued_digital = False
package = None
if original[ITEM_TYPE] != CONTENT_TYPE.COMPOSITE:
# if target_for is set the we don't to digital client.
if not updates.get('targeted_for', original.get('targeted_for')):
# check if item is in a digital package
package = self.takes_package_service.get_take_package(original)
if package:
queued_digital = self._publish_takes_package(package, updates, original, last_updated)
else:
'''
If type of the item is text or preformatted then item need to be sent to digital subscribers.
So, package the item as a take.
'''
updated = copy(original)
updated.update(updates)
if original[ITEM_TYPE] in [CONTENT_TYPE.TEXT, CONTENT_TYPE.PREFORMATTED] and \
self.sending_to_digital_subscribers(updated):
# create a takes package
package_id = self.takes_package_service.package_story_as_a_take(updated, {}, None)
updates[LINKED_IN_PACKAGES] = updated[LINKED_IN_PACKAGES]
package = get_resource_service(ARCHIVE).find_one(req=None, _id=package_id)
queued_digital = self._publish_takes_package(package, updates, original, last_updated)
# queue only text items
queued_wire = \
self.publish(doc=original, updates=updates, target_media_type=WIRE if package else None)
queued = queued_digital or queued_wire
if not queued:
logger.exception('Nothing is saved to publish queue for story: {} for action: {}'.
format(original[config.ID_FIELD], self.publish_type))
self._update_archive(original=original, updates=updates, should_insert_into_versions=False)
push_notification('item:publish', item=str(id), unique_name=original['unique_name'],
desk=str(original.get('task', {}).get('desk', '')),
user=str(user.get(config.ID_FIELD, '')))
except SuperdeskApiError as e:
raise e
except KeyError as e:
raise SuperdeskApiError.badRequestError(
message="Key is missing on article to be published: {}".format(str(e)))
except Exception as e:
logger.exception("Something bad happened while publishing %s".format(id))
raise SuperdeskApiError.internalError(message="Failed to publish the item: {}".format(str(e)))
def update_metadata(self, key, metadata):
if not metadata:
return
metadata = self.transform_metadata_to_amazon_format(metadata)
res = self.conn.update_metadata(key, metadata, bucket=self.container_name)
if res.status_code not in (200, 201):
payload = "Updating metadata for file %s failed" % key
raise SuperdeskApiError.internalError(payload=payload)
def update(self, provider):
is_closed = provider.get('is_closed', False)
if isinstance(is_closed, datetime):
is_closed = False
if is_closed:
raise SuperdeskApiError.internalError('Ingest Provider is closed')
else:
return self._update(provider) or []
def update(self, provider):
is_closed = provider.get('is_closed', False)
if isinstance(is_closed, datetime):
is_closed = False
if is_closed:
raise SuperdeskApiError.internalError('Ingest Provider is closed')
else:
return self._update(provider) or []
def download_file_from_url(url):
rv = requests.get(url, timeout=15)
if rv.status_code not in (200, 201):
raise SuperdeskApiError.internalError('Failed to retrieve file from URL: %s' % url)
mime = magic.from_buffer(rv.content, mime=True).decode('UTF-8')
ext = mime.split('/')[1]
name = str(ObjectId()) + ext
return BytesIO(rv.content), name, mime
def download_file_from_url(url):
rv = requests.get(url)
if rv.status_code not in (200, 201):
raise SuperdeskApiError.internalError('Failed to retrieve file from URL: %s' % url)
mime = magic.from_buffer(rv.content, mime=True).decode('UTF-8')
ext = mime.split('/')[1]
name = 'stub.' + ext
return BytesIO(rv.content), name, mime
def download_file_from_url(url):
rv = requests.get(url, timeout=15)
if rv.status_code not in (200, 201):
raise SuperdeskApiError.internalError('Failed to retrieve file from URL: %s' % url)
mime = magic.from_buffer(rv.content, mime=True)
ext = str(mime).split('/')[1]
name = str(ObjectId()) + ext
return BytesIO(rv.content), name, str(mime)
def update(self, id, updates, original):
"""
Handles workflow of each Publish, Corrected, Killed and TakeDown.
"""
try:
user = get_user()
auto_publish = updates.get("auto_publish", False)
# unlock the item
set_unlock_updates(updates)
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self._publish_package_items(original, updates)
self._update_archive(original, updates, should_insert_into_versions=auto_publish)
else:
self._publish_associated_items(original, updates)
updated = deepcopy(original)
updated.update(deepcopy(updates))
if updates.get(ASSOCIATIONS):
self._refresh_associated_items(updated, skip_related=True) # updates got lost with update
if updated.get(ASSOCIATIONS):
self._fix_related_references(updated, updates)
signals.item_publish.send(self, item=updated)
self._update_archive(original, updates, should_insert_into_versions=auto_publish)
self.update_published_collection(published_item_id=original[config.ID_FIELD], updated=updated)
from apps.publish.enqueue import enqueue_published
enqueue_published.apply_async()
push_notification(
"item:publish",
item=str(id),
unique_name=original["unique_name"],
desk=str(original.get("task", {}).get("desk", "")),
user=str(user.get(config.ID_FIELD, "")),
)
if updates.get("previous_marked_user") and not updates.get("marked_for_user"):
# send notification so that marked for me list can be updated
get_resource_service("archive").handle_mark_user_notifications(updates, original, False)
except SuperdeskApiError:
raise
except KeyError as e:
logger.exception(e)
raise SuperdeskApiError.badRequestError(
message=_("Key is missing on article to be published: {exception}").format(exception=str(e))
)
except Exception as e:
logger.exception(e)
raise SuperdeskApiError.internalError(
message=_("Failed to publish the item: {id}").format(id=str(id)), exception=e
)
def generate_from_highcharts(options,
mimetype=MIME_TYPES.PNG,
base64=True,
scale=1,
width=None,
no_download=True):
try:
host = app.config.get('HIGHCHARTS_SERVER_HOST', 'localhost')
port = app.config.get('HIGHCHARTS_SERVER_PORT', '6060')
except RuntimeError:
# This can happen when working outside of the Flask context
# So default to host=localhost, port=6060
host = 'localhost'
port = '6060'
url = 'http://{}:{}'.format(host, port)
headers = {'Content-Type': 'application/json'}
# Set the width size of the image to generate
if 'exporting' not in options:
options['exporting'] = {}
options['exporting']['sourceWidth'] = width
payload = {
'options': options,
'type': mimetype,
'b64': base64,
'scale': scale,
'noDownload': no_download
}
try:
response = requests.post(url, headers=headers, json=payload)
except requests.exceptions.ConnectionError as e:
raise SuperdeskApiError.internalError(
'Socket connection error: {}'.format(e))
try:
response.raise_for_status()
except Exception as e:
logger.exception(e)
raise SuperdeskApiError.internalError(e)
return response.content
def update(self, id, updates, original):
"""
Handles workflow of each Publish, Corrected and Killed.
"""
try:
user = get_user()
auto_publish = updates.get('auto_publish', False)
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self._publish_package_items(original, updates)
self._update_archive(original,
updates,
should_insert_into_versions=auto_publish)
else:
self._refresh_associated_items(original)
updated = deepcopy(original)
updated.update(deepcopy(updates))
if updates.get(ASSOCIATIONS):
self._refresh_associated_items(
updated) # updates got lost with update
# process takes package for published or corrected items
# if no_takes is true but takes package exists then process takes package.
if self.published_state != CONTENT_STATE.KILLED and \
(not app.config.get('NO_TAKES', False) or
self.takes_package_service.get_take_package_id(updated)):
self._process_takes_package(original, updated, updates)
self._update_archive(original,
updates,
should_insert_into_versions=auto_publish)
self.update_published_collection(
published_item_id=original[config.ID_FIELD],
updated=updated)
from apps.publish.enqueue import enqueue_published
enqueue_published.apply_async()
push_notification('item:publish',
item=str(id),
unique_name=original['unique_name'],
desk=str(
original.get('task', {}).get('desk', '')),
user=str(user.get(config.ID_FIELD, '')))
except SuperdeskApiError as e:
raise
except KeyError as e:
logger.exception(e)
raise SuperdeskApiError.badRequestError(
message="Key is missing on article to be published: {}".format(
str(e)))
except Exception as e:
raise SuperdeskApiError.internalError(
message="Failed to publish the item: {}".format(str(id)),
exception=e)
def update_metadata(self, key, metadata):
if not metadata:
return
metadata = self.transform_metadata_to_amazon_format(metadata)
res = self.conn.update_metadata(key,
metadata,
bucket=self.container_name)
if res.status_code not in (200, 201):
payload = 'Updating metadata for file %s failed' % key
raise SuperdeskApiError.internalError(payload=payload)
def suggest(self, text, language=None):
check_url = API_URL.format(method="suggesties")
data = {
"key": self.api_key,
"input": text,
}
r = requests.post(check_url, data=data)
if r.status_code != 200:
raise SuperdeskApiError.internalError("Unexpected return code from {}".format(self.name))
return {'suggestions': self.list2suggestions(r.json()['suggesties']['output'].get('suggesties', []))}
def update(self, id, updates, original):
"""
Handles workflow of each Publish, Corrected and Killed.
"""
try:
user = get_user()
last_updated = updates.get(config.LAST_UPDATED, utcnow())
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self._publish_package_items(original, last_updated)
set_sign_off(updates, original)
queued_digital = False
package_id = None
if original[ITEM_TYPE] != CONTENT_TYPE.COMPOSITE:
# if target_for is set the we don't to digital client.
if not updates.get('targeted_for', original.get('targeted_for')):
# check if item is in a digital package
package_id = TakesPackageService().get_take_package_id(original)
if package_id:
queued_digital, takes_package = self._publish_takes_package(package_id, updates,
original, last_updated)
else:
# if item is going to be sent to digital subscribers, package it as a take
if self.sending_to_digital_subscribers(updates):
updated = copy(original)
updated.update(updates)
# create a takes package
package_id = TakesPackageService().package_story_as_a_take(updated, {}, None)
original = get_resource_service('archive').find_one(req=None, _id=original['_id'])
queued_digital, takes_package = self._publish_takes_package(package_id, updates,
original, last_updated)
# queue only text items
queued_wire = \
self.publish(doc=original, updates=updates, target_media_type=WIRE if package_id else None)
queued = queued_digital or queued_wire
if not queued:
raise PublishQueueError.item_not_queued_error(Exception('Nothing is saved to publish queue'), None)
self._set_version_last_modified_and_state(original, updates, last_updated)
self._update_archive(original=original, updates=updates, should_insert_into_versions=False)
push_notification('item:publish', item=str(id), unique_name=original['unique_name'],
desk=str(original.get('task', {}).get('desk', '')), user=str(user.get('_id', '')))
except SuperdeskApiError as e:
raise e
except KeyError as e:
raise SuperdeskApiError.badRequestError(
message="Key is missing on article to be published: {}".format(str(e)))
except Exception as e:
logger.exception("Something bad happened while publishing %s".format(id))
raise SuperdeskApiError.internalError(message="Failed to publish the item: {}".format(str(e)))
def _suggest_server(self, text):
if self.version_tuple < (1, 2):
logger.warning("Suggestions not available with this server version")
return {'suggestions': []}
check_url = urljoin(self.base_url, PATH_SUGGEST)
r = requests.post(check_url, data={"token": text})
if r.status_code != 200:
raise SuperdeskApiError.internalError("Unexpected return code from Grammalecte")
suggestions = r.json().get('suggestions', [])
return {'suggestions': self.list2suggestions(suggestions)}
def update(self, id, updates, original):
"""
Handles workflow of each Publish, Corrected and Killed.
"""
try:
user = get_user()
auto_publish = updates.pop('auto_publish', False)
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self._publish_package_items(original, updates)
self._update_archive(original,
updates,
should_insert_into_versions=auto_publish)
else:
self._refresh_associated_items(original)
self._publish_associations(original, id)
updated = deepcopy(original)
updated.update(updates)
if updates.get('associations'):
self._refresh_associated_items(
updated) # updates got lost with update
if self.published_state != CONTENT_STATE.KILLED:
self._process_takes_package(original, updated, updates)
self._update_archive(original,
updates,
should_insert_into_versions=auto_publish)
self.update_published_collection(
published_item_id=original[config.ID_FIELD],
updated=updated)
from apps.publish.enqueue import enqueue_published
enqueue_published.apply_async()
push_notification('item:publish',
item=str(id),
unique_name=original['unique_name'],
desk=str(
original.get('task', {}).get('desk', '')),
user=str(user.get(config.ID_FIELD, '')))
except SuperdeskApiError as e:
raise e
except KeyError as e:
raise SuperdeskApiError.badRequestError(
message="Key is missing on article to be published: {}".format(
str(e)))
except Exception as e:
logger.exception(
"Something bad happened while publishing %s".format(id))
raise SuperdeskApiError.internalError(
message="Failed to publish the item: {}".format(str(e)))
def update(self, id, updates, original):
"""
Handles workflow of each Publish, Corrected, Killed and TakeDown.
"""
try:
user = get_user()
auto_publish = updates.get('auto_publish', False)
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self._publish_package_items(original, updates)
self._update_archive(original,
updates,
should_insert_into_versions=auto_publish)
else:
self._publish_associated_items(original, updates)
updated = deepcopy(original)
updated.update(deepcopy(updates))
if updates.get(ASSOCIATIONS):
self._refresh_associated_items(
updated,
skip_related=True) # updates got lost with update
signals.item_publish.send(self, item=updated)
self._update_archive(original,
updates,
should_insert_into_versions=auto_publish)
self.update_published_collection(
published_item_id=original[config.ID_FIELD],
updated=updated)
from apps.publish.enqueue import enqueue_published
enqueue_published.apply_async()
push_notification('item:publish',
item=str(id),
unique_name=original['unique_name'],
desk=str(
original.get('task', {}).get('desk', '')),
user=str(user.get(config.ID_FIELD, '')))
except SuperdeskApiError:
raise
except KeyError as e:
logger.exception(e)
raise SuperdeskApiError.badRequestError(message=_(
"Key is missing on article to be published: {exception}").
format(exception=str(e)))
except Exception as e:
logger.exception(e)
raise SuperdeskApiError.internalError(
message=_("Failed to publish the item: {id}").format(
id=str(id)),
exception=e)
def update(self, provider):
is_closed = provider.get('is_closed', False)
if isinstance(is_closed, datetime):
is_closed = False
if is_closed:
raise SuperdeskApiError.internalError('Ingest Provider is closed')
else:
try:
return self._update(provider) or []
except SuperdeskIngestError as error:
self.close_provider(provider, error)
raise error
def _check_exists(self, id_or_filename, raise_error=True):
try:
obj = self.conn.get(id_or_filename, self.container_name)
if obj.status_code not in (200, 201) and raise_error:
message = "Retrieving file %s from amazon failed" % id_or_filename
raise SuperdeskApiError.internalError(message)
return (True, obj)
except Exception as ex:
if raise_error:
logger.exception(ex)
# File not found
return (False, None)
def get_bucket_objects(self, marker=None, bucket=None):
"""
Fetch the objects available in the specified bucket.
"""
bucket = bucket or self.container_name
params = {"marker": marker}
rv = self.conn.get_bucket_objects(bucket=bucket, extra_params=params)
if rv.status_code not in (200, 201):
message = "Retrieving the list of files from bucket %s failed" % bucket
raise SuperdeskApiError.internalError(message)
content = rv.content.decode("UTF-8")
return content
def _get_response(self, resp, *status):
json = json_util.loads(resp.text)
if resp.status_code not in status:
if resp.status_code == 400:
raise SuperdeskApiError.badRequestError(message=json)
if resp.status_code == 409:
raise SuperdeskApiError.conflictError(message=json)
if resp.status_code == 404:
raise SuperdeskApiError.notFoundError(message=json)
if resp.status_code == 500:
raise SuperdeskApiError.internalError(message=json)
return json
def _check_exists(self, id_or_filename, raise_error=True):
try:
obj = self.conn.get(id_or_filename, self.container_name)
if obj.status_code not in (200, 201) and raise_error:
message = 'Retrieving file %s from amazon failed' % id_or_filename
raise SuperdeskApiError.internalError(message)
return (True, obj)
except Exception as ex:
if raise_error:
logger.exception(ex)
# File not found
return (False, None)
def get_bucket_objects(self, marker=None, bucket=None):
'''
Fetch the objects available in the specified bucket.
'''
bucket = bucket or self.container_name
params = {'marker': marker}
rv = self.conn.get_bucket_objects(bucket=bucket, extra_params=params)
if rv.status_code not in (200, 201):
message = 'Retrieving the list of files from bucket %s failed' % bucket
raise SuperdeskApiError.internalError(message)
content = rv.content.decode('UTF-8')
return content
def update(self, provider):
is_closed = provider.get('is_closed', False)
if isinstance(is_closed, datetime):
is_closed = False
if is_closed:
raise SuperdeskApiError.internalError('Ingest Provider is closed')
else:
try:
return self._update(provider) or []
except SuperdeskIngestError as error:
self.close_provider(provider, error)
raise error
def on_created(self, docs):
"""Send email to user with reset password token."""
super().on_created(docs)
resetService = get_resource_service('reset_user_password')
activate_ttl = app.config['ACTIVATE_ACCOUNT_TOKEN_TIME_TO_LIVE']
for doc in docs:
if self.user_is_waiting_activation(doc):
tokenDoc = {'user': doc['_id'], 'email': doc['email']}
id = resetService.store_reset_password_token(tokenDoc, doc['email'], activate_ttl, doc['_id'])
if not id:
raise SuperdeskApiError.internalError('Failed to send account activation email.')
tokenDoc.update({'username': doc['username']})
send_activate_account_email(tokenDoc, activate_ttl)
def on_created(self, docs):
"""Send email to user with reset password token."""
super().on_created(docs)
resetService = get_resource_service('reset_user_password')
activate_ttl = app.config['ACTIVATE_ACCOUNT_TOKEN_TIME_TO_LIVE']
for doc in docs:
if self.user_is_waiting_activation(doc):
tokenDoc = {'user': doc['_id'], 'email': doc['email']}
id = resetService.store_reset_password_token(tokenDoc, doc['email'], activate_ttl, doc['_id'])
if not id:
raise SuperdeskApiError.internalError('Failed to send account activation email.')
tokenDoc.update({'username': doc['username']})
send_activate_account_email(tokenDoc)
def enqueue_item(self, item):
"""
Creates the corresponding entries in the publish queue for the given item
"""
try:
self._enqueue_item(item)
except SuperdeskApiError as e:
raise e
except KeyError as e:
raise SuperdeskApiError.badRequestError(
message="Key is missing on article to be published: {}".format(str(e)))
except Exception as e:
logger.exception("Something bad happened while publishing %s".format(id))
raise SuperdeskApiError.internalError(message="Failed to publish the item: {}".format(str(e)))
def create(self, docs, **kwargs):
try:
ids = []
for doc in docs:
doc['item'] = self.execute_macro(doc['item'], doc['macro'])
if doc.get('commit'):
item = superdesk.get_resource_service('archive').find_one(req=None, _id=doc['item']['_id'])
updates = doc['item'].copy()
updates.pop('_id')
superdesk.get_resource_service('archive').update(item['_id'], updates, item)
ids.append(doc['macro'])
return ids
except Exception as ex:
raise SuperdeskApiError.internalError(str(ex))
def get_keywords(self, text):
if not app.config['KEYWORDS_KEY_API']:
raise SuperdeskApiError.notFoundError('AlchemyAPI key is not set')
params = {'apikey': app.config['KEYWORDS_KEY_API'],
'outputMode': 'json'}
url = app.config['KEYWORDS_BASE_URL'] + '/text/TextGetRankedNamedEntities' + \
'?' + urllib.parse.urlencode(params)
values = {'text': text}
result = ""
try:
result = self._http.post(url, data=values)
except Exception as ex:
raise SuperdeskApiError.internalError('Fail to connect to Alchemy service', exception=ex)
try:
keywords = result.json()
return keywords.get('entities', [])
except Exception as ex:
raise SuperdeskApiError.internalError('Fail to parse the response from Alchemy service', exception=ex)
def store_file(self, doc, content, filename, content_type):
# retrieve file name and metadata from file
file_name, content_type, metadata = process_file_from_stream(content, content_type=content_type)
try:
content.seek(0)
file_id = doc['media_id']
existing = app.media.get(doc['media_id'], self.datasource)
if not existing:
file_id = app.media.put(content, filename=file_name, content_type=content_type,
resource=self.datasource, metadata=metadata, _id=ObjectId(doc['media_id']))
doc['media'] = file_id
doc['mime_type'] = content_type
doc['filemeta'] = decode_metadata(metadata)
except Exception as io:
raise SuperdeskApiError.internalError('Saving file failed', exception=io)
def update(self, id, updates, original):
archived_item = super().find_one(req=None, _id=id)
try:
if archived_item['type'] == 'composite':
self.__publish_package_items(archived_item, updates[config.LAST_UPDATED])
user = get_user()
updates[config.CONTENT_STATE] = 'published'
item = self.backend.update(self.datasource, id, updates, original)
push_notification('item:publish', item=str(item.get('_id')), user=str(user))
return item
except KeyError:
raise SuperdeskApiError.badRequestError(message="A non-existent content id is requested to publish")
except Exception as e:
logger.error("Something bad happened while publishing %s".format(id), e)
raise SuperdeskApiError.internalError(message="Failed to publish the item")
def process_file_from_stream(content, content_type=None):
content_type = content_type or content.content_type
content = BytesIO(content.read())
if 'application/' in content_type:
content_type = magic.from_buffer(content.getvalue(), mime=True).decode('UTF-8')
content.seek(0)
file_type, ext = content_type.split('/')
try:
metadata = process_file(content, file_type)
except OSError: # error from PIL when image is supposed to be an image but is not.
raise SuperdeskApiError.internalError('Failed to process file')
file_name = get_file_name(content)
content.seek(0)
metadata = encode_metadata(metadata)
metadata.update({'length': json.dumps(len(content.getvalue()))})
return file_name, content_type, metadata
def enqueue_item(self, item, content_type=None):
"""Creates the corresponding entries in the publish queue for the given item
:param item: Item to enqueue
:param content_type: item content type
:return bool: True if item is queued else false.
"""
try:
return self._enqueue_item(item, content_type)
except SuperdeskApiError as e:
raise e
except KeyError as e:
raise SuperdeskApiError.badRequestError(
message="Key is missing on article to be published: {}".format(str(e)))
except Exception as e:
logger.exception("Something bad happened while publishing %s".format(id))
raise SuperdeskApiError.internalError(message="Failed to publish the item: {}".format(str(e)), exception=e)
def update(self, id, updates, original):
archived_item = super().find_one(req=None, _id=id)
try:
any_channel_closed = False
if archived_item['type'] == 'composite':
self.__publish_package_items(archived_item, updates[config.LAST_UPDATED])
# document is saved to keep the initial changes
set_sign_off(updates, original)
self.backend.update(self.datasource, id, updates, original)
# document is saved to change the status
if (original.get('publish_schedule') or updates.get('publish_schedule')) \
and original[config.CONTENT_STATE] not in ['published', 'killed', 'scheduled']:
updates[config.CONTENT_STATE] = 'scheduled'
else:
updates[config.CONTENT_STATE] = self.published_state
original.update(updates)
get_component(ItemAutosave).clear(original['_id'])
if archived_item['type'] != 'composite':
# queue only text items
any_channel_closed = self.queue_transmission(original)
task = self.__send_to_publish_stage(original)
if task:
updates['task'] = task
self.backend.update(self.datasource, id, updates, original)
user = get_user()
push_notification('item:publish:closed:channels' if any_channel_closed else 'item:publish',
item=str(id), unique_name=archived_item['unique_name'],
desk=str(archived_item['task']['desk']), user=str(user.get('_id', '')))
original.update(super().find_one(req=None, _id=id))
except SuperdeskApiError as e:
raise e
except KeyError as e:
raise SuperdeskApiError.badRequestError(
message="Key is missing on article to be published: {}"
.format(str(e)))
except Exception as e:
logger.error("Something bad happened while publishing %s".format(id), e)
raise SuperdeskApiError.internalError(message="Failed to publish the item: {}"
.format(str(e)))
def download_file_from_url(url):
"""Download file from given url.
In case url is relative it will prefix it with current host.
:param url: file url
"""
try:
rv = requests.get(url, timeout=15)
except requests.exceptions.MissingSchema: # any route will do here, we only need host
rv = requests.get(urljoin(url_for('static', filename='x', _external=True), url), timeout=15)
if rv.status_code not in (200, 201):
raise SuperdeskApiError.internalError('Failed to retrieve file from URL: %s' % url)
mime = magic.from_buffer(rv.content, mime=True)
ext = str(mime).split('/')[1]
name = str(ObjectId()) + ext
return BytesIO(rv.content), name, str(mime)
def update(self, id, updates, original):
"""
Handles workflow of each Publish, Corrected and Killed.
"""
try:
user = get_user()
auto_publish = updates.get('auto_publish', False)
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self._publish_package_items(original, updates)
self._update_archive(original, updates, should_insert_into_versions=auto_publish)
else:
self._refresh_associated_items(original)
self._publish_associations(original, id)
updated = deepcopy(original)
updated.update(updates)
if updates.get('associations'):
self._refresh_associated_items(updated) # updates got lost with update
if self.published_state != CONTENT_STATE.KILLED and not app.config.get('NO_TAKES', False):
self._process_takes_package(original, updated, updates)
self._update_archive(original, updates, should_insert_into_versions=auto_publish)
self.update_published_collection(published_item_id=original[config.ID_FIELD], updated=updated)
from apps.publish.enqueue import enqueue_published
enqueue_published.apply_async()
push_notification('item:publish', item=str(id),
unique_name=original['unique_name'],
desk=str(original.get('task', {}).get('desk', '')),
user=str(user.get(config.ID_FIELD, '')))
except SuperdeskApiError as e:
raise e
except KeyError as e:
raise SuperdeskApiError.badRequestError(
message="Key is missing on article to be published: {}".format(str(e)))
except Exception as e:
logger.exception("Something bad happened while publishing %s".format(id))
raise SuperdeskApiError.internalError(message="Failed to publish the item: {}".format(str(e)))
def put(self, content, filename=None, content_type=None, metadata=None):
""" Saves a new file using the storage system, preferably with the name
specified. If there already exists a file with this name name, the
storage system may modify the filename as necessary to get a unique
name. Depending on the storage system, a unique id or the actual name
of the stored file will be returned. The content type argument is used
to appropriately identify the file when it is retrieved.
"""
logger.debug('Going to save media file with %s ' % filename)
found, existing_file = self._check_exists(filename, raise_error=False)
if found:
return filename
try:
file_metadata = self.transform_metadata_to_amazon_format(metadata)
res = self.conn.upload(filename, content, self.container_name, content_type=content_type,
headers=file_metadata)
if res.status_code not in (200, 201):
raise SuperdeskApiError.internalError('Uploading file to amazon S3 failed')
return filename
except Exception as ex:
logger.exception(ex)
raise
def update(self, id, updates, original):
"""
Handles workflow of each Publish, Corrected, Killed and TakeDown.
"""
try:
user = get_user()
auto_publish = updates.get('auto_publish', False)
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self._publish_package_items(original, updates)
self._update_archive(original, updates, should_insert_into_versions=auto_publish)
else:
self._publish_associated_items(original, updates, publish=True)
updated = deepcopy(original)
updated.update(deepcopy(updates))
self._publish_associated_items(updated, updates) # updates got lost with update
self._update_archive(original, updates, should_insert_into_versions=auto_publish)
self.update_published_collection(published_item_id=original[config.ID_FIELD], updated=updated)
from apps.publish.enqueue import enqueue_published
enqueue_published.apply_async()
push_notification('item:publish', item=str(id),
unique_name=original['unique_name'],
desk=str(original.get('task', {}).get('desk', '')),
user=str(user.get(config.ID_FIELD, '')))
except SuperdeskApiError:
raise
except KeyError as e:
logger.exception(e)
raise SuperdeskApiError.badRequestError(
message="Key is missing on article to be published: {}".format(str(e))
)
except Exception as e:
raise SuperdeskApiError.internalError(message="Failed to publish the item: {}".format(str(id)), exception=e)
def update(self, id, updates, original):
archived_item = super().find_one(req=None, _id=id)
try:
any_channel_closed = False
if archived_item['type'] == 'composite':
self.__publish_package_items(archived_item, updates[config.LAST_UPDATED])
# document is saved to keep the initial changes
set_sign_off(updates, original)
self.backend.update(self.datasource, id, updates, original)
# document is saved to change the status
if (original.get('publish_schedule') or updates.get('publish_schedule')) \
and original[config.CONTENT_STATE] not in PUBLISH_STATES:
updates[config.CONTENT_STATE] = 'scheduled'
else:
updates['publish_schedule'] = None
updates[config.CONTENT_STATE] = self.published_state
original.update(updates)
get_component(ItemAutosave).clear(original['_id'])
if archived_item['type'] != 'composite':
# check if item is in a digital package
package_id = TakesPackageService().get_take_package_id(original)
if package_id:
# process the takes to form digital master file content
package, package_updates = self.process_takes(take=original, package_id=package_id)
package_updates[config.CONTENT_STATE] = self.published_state
resolve_document_version(document=package_updates,
resource=ARCHIVE, method='PATCH',
latest_doc=package)
self.backend.update(self.datasource, package['_id'], package_updates, package)
package.update(package_updates)
insert_into_versions(doc=package)
# send it to the digital channels
any_channel_closed_digital, queued_digital = \
self.publish(doc=package, updates=None, target_output_channels=DIGITAL)
self.update_published_collection(published_item=package)
else:
any_channel_closed_digital = False
queued_digital = False
# queue only text items
any_channel_closed_wire, queued_wire = \
self.publish(doc=original, updates=updates, target_output_channels=WIRE if package_id else None)
any_channel_closed = any_channel_closed_digital or any_channel_closed_wire
queued = queued_digital or queued_wire
if not queued:
raise PublishQueueError.item_not_queued_error(Exception('Nothing is saved to publish queue'), None)
self.backend.update(self.datasource, id, updates, original)
user = get_user()
push_notification('item:publish:closed:channels' if any_channel_closed else 'item:publish',
item=str(id), unique_name=archived_item['unique_name'],
desk=str(archived_item.get('task', {}).get('desk', '')),
user=str(user.get('_id', '')))
original.update(super().find_one(req=None, _id=id))
except SuperdeskApiError as e:
raise e
except KeyError as e:
raise SuperdeskApiError.badRequestError(
message="Key is missing on article to be published: {}"
.format(str(e)))
except Exception as e:
logger.error("Something bad happened while publishing %s".format(id), e)
raise SuperdeskApiError.internalError(message="Failed to publish the item: {}"
.format(str(e)))