# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license
from superdesk.publish.publish_service import PublishService
from superdesk.publish import register_transmitter
from superdesk.errors import PublishFileError
from os import path
errors = [PublishFileError.fileSaveError().get_error_description()]
class FilePublishService(PublishService):
def _transmit(self, queue_item, subscriber):
try:
config = queue_item['destination']['config']
file_path = config['file_path']
if not path.isabs(file_path):
file_path = "/" + file_path
with open(path.join(file_path, PublishService.get_filename(queue_item)), 'wb') as f:
f.write(queue_item['encoded_item'])
except Exception as ex:
raise PublishFileError.fileSaveError(ex, config)
register_transmitter('File', FilePublishService(), errors)
try:
response = requests.post(resource_url, data=agenda_entry, headers=headers)
response.raise_for_status()
except Exception as ex:
logger.exception(ex)
message = 'Error pushing item %s: %s' % (response.status_code, response.text)
self._raise_publish_error(response.status_code, Exception(message), destination)
# need to rethrow exception as a superdesk exception for now for notifiers.
try:
# The id from agenda is returned as part of the location header when the Event is created in Agenda
location = response.headers.get('Location', None)
if location:
self._save_agenda_id(id, location, type)
response.raise_for_status()
except Exception as ex:
logger.exception(ex)
message = 'Error pushing item %s: %s' % (response.status_code, response.text)
self._raise_publish_error(response.status_code, Exception(message), destination)
self.user_api_key = None
def _save_agenda_id(self, id, location, type):
agendaId = location.split('/')[-1]
service = get_resource_service('events') if type == 'event' else get_resource_service('planning')
original = service.find_one(req=None, _id=id)
if original:
service.system_update(id, {'unique_id': agendaId}, original)
register_transmitter('http_agenda_push', HTTPAgendaPush(), errors)
class FTPPublishService(PublishService):
"""FTP Publish Service."""
def config_from_url(self, url):
"""Parse given url into ftp config. Used for tests.
:param url: url in form `ftp://username:password@host:port/dir`
"""
url_parts = urlparse(url)
return {
'username': url_parts.username,
'password': url_parts.password,
'host': url_parts.hostname,
'path': url_parts.path.lstrip('/'),
}
def _transmit(self, queue_item, subscriber):
config = queue_item.get('destination', {}).get('config', {})
try:
with ftp_connect(config) as ftp:
filename = PublishService.get_filename(queue_item)
b = BytesIO(queue_item['encoded_item'])
ftp.storbinary("STOR " + filename, b)
except PublishFtpError:
raise
except Exception as ex:
raise PublishFtpError.ftpError(ex, config)
register_transmitter('ftp', FTPPublishService(), errors)
class EmailPublishService(PublishService):
"""Email Publish Service."""
def _transmit(self, queue_item, subscriber):
config = queue_item.get('destination', {}).get('config', {})
try:
if not config.get('recipients'):
raise PublishEmailError.recipientNotFoundError(LookupError('recipient field not found!'))
admins = app.config['ADMINS']
recipients = config.get('recipients').rstrip(';').split(';')
subject = "Story: {}".format(queue_item['item_id'])
text_body = queue_item['formatted_item']
# sending email synchronously
send_email(subject=subject,
sender=admins[0],
recipients=recipients,
text_body=text_body,
html_body=None)
except PublishEmailError:
raise
except Exception as ex:
raise PublishEmailError.emailError(ex, queue_item.get('destination'))
register_transmitter('email', EmailPublishService(), errors)
"""Get the canonical request"""
return method.encode() + url.encode() + date.encode(
) + content_type.encode() + body
def _get_channel(self, destination):
"""Get the channel"""
return destination.get('config', {}).get('channel')
def _get_api_key(self, destination):
"""Get the api key"""
return destination.get('config', {}).get('api_key')
def _get_api_secret(self, destination):
"""Get the api secret"""
return destination.get('config', {}).get('api_secret')
def _get_header_image_rendition(self, destination):
return destination.get('config',
{}).get('header_image_rendition') or '16-9'
def _raise_publish_error(self, status_code, e, destination=None):
if status_code >= 400 and status_code < 500:
raise PublishHTTPPushClientError.httpPushError(e, destination)
elif status_code >= 500 and status_code < 600:
raise PublishHTTPPushServerError.httpPushError(e, destination)
else:
raise PublishHTTPPushError.httpPushError(e, destination)
register_transmitter('http_push_apple_news', HTTPAppleNewsPush(), errors)
@type item: dict
@param assets_url: the url where the media can be uploaded
@type assets_url: string
"""
for name, rendition in item.get("renditions", {}).items():
del item["renditions"][name]["href"]
if not self._media_exists(rendition["media"], assets_url):
media = app.media.get(rendition["media"], resource="upload")
files = {"media": (rendition["media"], media, rendition.get("mimetype") or rendition["mime_type"])}
response = requests.post(assets_url, files=files, data={"media_id": rendition["media"]})
if response.status_code != requests.codes.created: # @UndefinedVariable
raise Exception("Error pushing item %s media file %s" % (item._id, rendition["media"]))
def _media_exists(self, media_id, assets_url):
"""Returns true if the media with the given id exists at the service identified by assets_url.
Returns false otherwise. Raises Exception if the error code was not 200 or 404
@param media_id: the media identifier
@type media_id: string
@param assets_url: the url of the assest service
@type assets_url: string
@return: bool
"""
response = requests.get("%s/%s" % (assets_url, media_id))
if response.status_code not in (requests.codes.ok, requests.codes.not_found): # @UndefinedVariable
raise Exception("Error querying the assets service %s" % assets_url)
return response.status_code == requests.codes.ok # @UndefinedVariable
register_transmitter("http_push", HTTPPushService(), errors)
config = destination.get("config") or {}
try:
sqs = boto3.resource(
"sqs",
aws_access_key_id=config.get("access_key_id"),
aws_secret_access_key=config.get("secret_access_key"),
region_name=config.get("region"),
endpoint_url=config.get("endpoint_url"),
)
queue = sqs.get_queue_by_name(QueueName=config.get("queue_name"))
queue.send_message(
MessageBody=queue_item["formatted_item"],
MessageGroupId=config.get("message_group_id"),
)
except (EndpointConnectionError, ConnectionClosedError,
NewConnectionError) as error:
raise PublishAmazonSQSError.connectionError(error, destination)
except ClientError as error:
raise PublishAmazonSQSError.clientError(error, destination)
except Exception as error:
raise PublishAmazonSQSError.sendMessageError(error, destination)
def _transmit_media(self, media, destination):
# Not supported
pass
register_transmitter("amazon_sqs_fifo", AmazonSQSFIFOPublishService(), errors)
errors = [PublishPublicAPIError.publicAPIError().get_error_description()]
class PublicAPIPublishService(PublishService):
"""Public API Publish Service."""
def _transmit(self, formatted_item, subscriber, destination):
item = json.loads(formatted_item['formatted_item'])
self._fix_dates(item)
if item['type'] == ITEM_TYPE_COMPOSITE:
publicapiService = get_resource_service('publish_packages')
else:
publicapiService = get_resource_service('publish_items')
try:
public_item = publicapiService.find_one(req=None, _id=item['_id'])
if public_item:
publicapiService.patch(item['_id'], item)
else:
publicapiService.post([item])
except Exception as ex:
raise PublishPublicAPIError.publicAPIError(ex, destination)
def _fix_dates(self, item):
for field, value in item.items():
if isinstance(value, dict) and '$date' in value:
item[field] = datetime.utcfromtimestamp(value['$date'] / 1000)
register_transmitter('PublicArchive', PublicAPIPublishService(), errors)
with pyodbc.connect(config['connection_string']) as conn:
item = json.loads(queue_item['formatted_item'])
ret = self._CallStoredProc(conn,
procName=config['stored_procedure'],
paramDict=item)
conn.commit()
return ret
except Exception as ex:
raise PublishODBCError.odbcError(ex, config)
def _CallStoredProc(self, conn, procName, paramDict):
params = ''
for p in paramDict:
if paramDict[p]:
params += ('@{}=N\'{}\', '.format(p, paramDict[p]))
params = params[:-2]
sql = """SET NOCOUNT ON;
DECLARE @ret int
EXEC @ret = %s %s
SELECT @ret""" % (procName, params)
resp = conn.execute(sql).fetchone()
if resp is not None:
return resp[0]
else:
return 1
if pyodbc_available:
register_transmitter('ODBC', ODBCPublishService(), errors)
class FTPPublishService(PublishService):
"""FTP Publish Service."""
def config_from_url(self, url):
"""Parse given url into ftp config. Used for tests.
:param url: url in form `ftp://username:password@host:port/dir`
"""
url_parts = urlparse(url)
return {
'username': url_parts.username,
'password': url_parts.password,
'host': url_parts.hostname,
'path': url_parts.path.lstrip('/'),
}
def _transmit(self, queue_item, subscriber):
config = queue_item.get('destination', {}).get('config', {})
try:
with ftp_connect(config) as ftp:
filename = PublishService.get_filename(queue_item)
b = BytesIO(queue_item['encoded_item'])
ftp.storbinary("STOR " + filename, b)
except PublishFtpError:
raise
except Exception as ex:
raise PublishFtpError.ftpError(ex, config)
register_transmitter('ftp', FTPPublishService(), errors)
"""
def config_from_url(self, url):
"""Parse given url into ftp config. Used for tests.
:param url: url in form `ftp://username:password@host:port/dir`
"""
url_parts = urlparse(url)
return {
"username": url_parts.username,
"password": url_parts.password,
"host": url_parts.hostname,
"path": url_parts.path.lstrip("/"),
}
def _transmit(self, queue_item, subscriber):
config = queue_item.get("destination", {}).get("config", {})
try:
with ftp_connect(config) as ftp:
filename = get_publish_service().get_filename(queue_item)
b = BytesIO(queue_item["encoded_item"])
ftp.storbinary("STOR " + filename, b)
except PublishFtpError:
raise
except Exception as ex:
raise PublishFtpError.ftpError(ex, config)
register_transmitter("ftp", FTPPublishService(), errors)
from os import path
errors = [PublishFileError.fileSaveError().get_error_description()]
class FilePublishService(publish_service.PublishService):
"""Superdesk file transmitter.
It creates files on superdesk server in configured folder.
"""
NAME = 'File'
def _transmit(self, queue_item, subscriber):
try:
config = queue_item['destination']['config']
file_path = config['file_path']
if not path.isabs(file_path):
file_path = "/" + file_path
with open(
path.join(
file_path,
publish_service.get_publish_service().get_filename(
queue_item)), 'wb') as f:
f.write(queue_item['encoded_item'])
except Exception as ex:
raise PublishFileError.fileSaveError(ex, config)
register_transmitter('File', FilePublishService(), errors)
if assoc is None:
continue
media.update(parse_media(assoc))
for assoc2 in assoc.get("associations", {}).values():
if assoc2 is None:
continue
media.update(parse_media(assoc2))
# Retrieve the list of files that currently exist in the FTP server
remote_items = []
ftp.retrlines("LIST", remote_items.append)
for media_id, rendition in media.items():
if not self._media_exists(rendition, remote_items):
binary = app.media.get(media_id,
resource=rendition.get(
"resource", "upload"))
self._transmit_media(binary, rendition, ftp)
def _media_exists(self, rendition, items):
for file in items:
if get_rendition_file_name(rendition) in file:
return True
return False
def _transmit_media(self, binary, rendition, ftp):
ftp.storbinary("STOR " + get_rendition_file_name(rendition), binary)
register_transmitter("ftp", FTPPublishService(), errors)
from superdesk.publish import publish_service
from superdesk.publish import register_transmitter
from superdesk.errors import PublishFileError
from os import path
errors = [PublishFileError.fileSaveError().get_error_description()]
class FilePublishService(publish_service.PublishService):
"""Superdesk file transmitter.
It creates files on superdesk server in configured folder.
"""
NAME = "File"
def _transmit(self, queue_item, subscriber):
try:
config = queue_item["destination"]["config"]
file_path = config["file_path"]
if not path.isabs(file_path):
file_path = "/" + file_path
with open(path.join(file_path, publish_service.get_publish_service().get_filename(queue_item)), "wb") as f:
f.write(queue_item["encoded_item"])
except Exception as ex:
raise PublishFileError.fileSaveError(ex, config)
register_transmitter("File", FilePublishService(), errors)
config = queue_item.get('destination', {}).get('config', {})
try:
with pyodbc.connect(config['connection_string']) as conn:
item = json.loads(queue_item['formatted_item'])
ret = self._CallStoredProc(conn, procName=config['stored_procedure'], paramDict=item)
conn.commit()
return ret
except Exception as ex:
raise PublishODBCError.odbcError(ex, config)
def _CallStoredProc(self, conn, procName, paramDict):
params = ''
for p in paramDict:
if paramDict[p]:
params += ('@{}=N\'{}\', '.format(p, paramDict[p]))
params = params[:-2]
sql = """SET NOCOUNT ON;
DECLARE @ret int
EXEC @ret = %s %s
SELECT @ret""" % (procName, params)
resp = conn.execute(sql).fetchone()
if resp is not None:
return resp[0]
else:
return 1
if pyodbc_available:
register_transmitter('ODBC', ODBCPublishService(), errors)
ret = self._CallStoredProc(conn,
procName=config["stored_procedure"],
paramDict=item)
conn.commit()
return ret
except Exception as ex:
raise PublishODBCError.odbcError(ex, config)
def _CallStoredProc(self, conn, procName, paramDict):
params = ""
for p in paramDict:
if paramDict[p]:
params += "@{}=N'{}', ".format(p, paramDict[p])
params = params[:-2]
sql = """SET NOCOUNT ON;
DECLARE @ret int
EXEC @ret = %s %s
SELECT @ret""" % (
procName,
params,
)
resp = conn.execute(sql).fetchone()
if resp is not None:
return resp[0]
else:
return 1
if pyodbc_available:
register_transmitter("ODBC", ODBCPublishService(), errors)
encoded_data = bytes(data, "utf-8")
else:
encoded_data = data
mac = hmac.new(str.encode(secret_token),
msg=encoded_data,
digestmod="sha1")
return "sha1=" + str(mac.hexdigest())
def _get_secret_token(self, destination):
return destination.get("config", {}).get("secret_token", None)
def _get_assets_url(self, destination, media_id=None):
url = destination.get("config", {}).get("assets_url", None)
if media_id is not None:
return "/".join([url, str(media_id)])
return url
def _get_resource_url(self, destination):
return destination.get("config", {}).get("resource_url")
def _raise_publish_error(self, status_code, e, destination=None):
if status_code >= 400 and status_code < 500:
raise PublishHTTPPushClientError.httpPushError(e, destination)
elif status_code >= 500 and status_code < 600:
raise PublishHTTPPushServerError.httpPushError(e, destination)
else:
raise PublishHTTPPushError.httpPushError(e, destination)
register_transmitter("http_push", HTTPPushService(), errors)
from flask import current_app, json
from superdesk.publish import register_transmitter
from superdesk.publish.publish_service import PublishService
from superdesk.text_checkers.ai.imatrics import IMatrics
class IMatricsTransmitter(PublishService):
def _transmit(self, queue_item, subscriber):
imatrics = IMatrics(current_app)
item = json.loads(queue_item["formatted_item"])
imatrics.publish(item)
register_transmitter("imatrics", IMatricsTransmitter(), [])
r.strip() for r in config.get('recipients', '').split(';')
if r.strip()
]
bcc = [
r.strip() for r in config.get('recipients_bcc', '').split(';')
if r.strip()
]
if not recipients and not bcc:
raise PublishEmailError.recipientNotFoundError(
LookupError('recipient and bcc fields are empty!'))
subject = item.get('message_subject',
'Story: {}'.format(queue_item['item_id']))
text_body = item.get('message_text', queue_item['formatted_item'])
html_body = item.get('message_html', queue_item['formatted_item'])
# sending email synchronously
send_email(subject=subject,
sender=admins[0],
recipients=recipients,
text_body=text_body,
html_body=html_body,
bcc=bcc)
except Exception as ex:
raise PublishEmailError.emailError(ex,
queue_item.get('destination'))
register_transmitter('email', EmailPublishService(), errors)
Item to the stream then disconnect.
"""
def _transmit(self, queue_item, subscriber):
destination = queue_item.get('destination', {})
config = destination.get('config', {})
address = config.get('address')
port = int(config.get('port'))
# Create a TCP/IP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Connect the socket to the port on the server given by the caller
server_address = (address, port)
try:
sock.connect(server_address)
except Exception as ex:
sock.close()
raise PublishSocketError.socketConnectionError(exception=ex, destination=destination)
# try to send the complete item
try:
sock.sendall(queue_item['encoded_item'])
except Exception as ex:
raise PublishSocketError.socketSendError(exception=ex, destination=destination)
finally:
sock.close()
register_transmitter('socket', SocketPublishService(), errors)
