def format(self, article, subscriber, codes=None): try: formatted_doc = {} formatted_doc['headline'] = get_text(article.get('headline', ''), content='html') formatted_doc['headline'] = formatted_doc['headline'].replace('\'', '\'\'').replace('\xA0', ' ') formatted_doc['keyword'] = article.get('slugline', '').replace('\'', '\'\'') # body formatting is_last_take = self.is_last_take(article) if article.get(FORMAT) == FORMATS.PRESERVED: body = get_text( self.append_body_footer(article) if is_last_take else article.get('body_html', ''), content='html') formatted_doc['article_text'] = body.replace('\'', '\'\'') elif article.get(FORMAT, FORMATS.HTML) == FORMATS.HTML: body = self.get_wrapped_text_content( to_ascii(self.append_body_footer(article) if is_last_take else article.get('body_html', ''))).replace('\'', '\'\'') formatted_doc['article_text'] = body self.refine_article_body(formatted_doc, article) # Frame the text output according to AAP requirement formatted_output = 'KEYWORD: ' + formatted_doc.get('keyword', '') + '\r\n' formatted_output += 'HEADLINE: ' + formatted_doc.get('headline', '') + '\r\n' formatted_output += ' ' + formatted_doc.get('article_text', '') return [(0, json.dumps({'article_text': formatted_output}))] except Exception as ex: raise FormatterError.AAPTextFormatterError(ex, subscriber)
def format(self, article, subscriber, codes=None): """ Constructs a dictionary that represents the parameters passed to the SMS InsertAlerts stored procedure :return: returns the sequence number of the subscriber and the constructed parameter dictionary """ try: pub_seq_num = superdesk.get_resource_service('subscribers').generate_sequence_number(subscriber) sms_message = article.get('sms_message', article.get('abstract', '')).replace('\'', '\'\'') # category = 1 is used to indicate a test message category = '1' if superdesk.app.config.get('TEST_SMS_OUTPUT', True) is True \ else article.get('anpa_category', [{}])[0].get('qcode').upper() odbc_item = {'Sequence': pub_seq_num, 'Category': category, 'Headline': get_text(sms_message, content='html'), 'Priority': map_priority(article.get('priority'))} body = self.append_body_footer(article) if article[ITEM_TYPE] == CONTENT_TYPE.TEXT: body = get_text(body, content='html') odbc_item['StoryText'] = body.replace('\'', '\'\'') # @article_text odbc_item['ident'] = '0' return [(pub_seq_num, json.dumps(odbc_item))] except Exception as ex: raise FormatterError.AAPSMSFormatterError(ex, subscriber)
def format_for_source(self, article, subscriber, source, codes=None): """Constructs a dictionary that represents the parameters passed to the IPNews InsertNews stored procedure :type article: object :return: returns the sequence number of the subscriber and the constructed parameter dictionary """ pass_through = article.get('auto_publish', False) try: docs = [] for category in self._get_category_list(article.get('anpa_category')): # All NZN sourced content is AAP content for the AAP output formatted article['source'] = source pub_seq_num, odbc_item = self.get_odbc_item(article, subscriber, category, codes, pass_through) # determine if this is the last take is_last_take = self.is_last_take(article) if article.get(FORMAT) == FORMATS.PRESERVED: # @article_text body = get_text( self.append_body_footer(article) if is_last_take else article.get('body_html', ''), content='html') odbc_item['article_text'] = body.replace('\'', '\'\'') odbc_item['texttab'] = 't' elif article.get(FORMAT, FORMATS.HTML) == FORMATS.HTML: body = self.get_wrapped_text_content( to_ascii(self.append_body_footer(article) if is_last_take else article.get('body_html', ''))).replace('\'', '\'\'') # if this is the first take and we have a dateline inject it if self.is_first_part(article) and 'dateline' in article and 'text' in article.get('dateline', {})\ and not pass_through: if body.startswith(' '): body = ' {} {}'.format(article.get('dateline') .get('text').replace('\'', '\'\''), body[3:]) odbc_item['article_text'] = body odbc_item['texttab'] = 'x' if self.is_first_part(article) and not pass_through: self.add_ednote(odbc_item, article) self.add_byline(odbc_item, article) if not is_last_take: odbc_item['article_text'] += '\r\nMORE' else: odbc_item['article_text'] += '\r\n' + article.get('source', '') sign_off = article.get('sign_off', '') or '' if len(sign_off) > 0: odbc_item['article_text'] += ' ' + sign_off odbc_item['service_level'] = get_service_level(category, article) # @service_level odbc_item['wordcount'] = article.get('word_count') or 0 # @wordcount odbc_item['priority'] = map_priority(article.get('priority')) # @priority docs.append((pub_seq_num, json.dumps(odbc_item))) return docs except Exception as ex: raise FormatterError.AAPIpNewsFormatterError(ex, subscriber)
def add_byline(self, odbc_item, article): """ Add the byline to the article text :param odbc_item: :param article: :return: """ if article.get('byline') and article.get('byline') != '': byline = get_text(article.get('byline', ''), content='html') if len(byline) >= 3 and byline[:2].upper() != 'BY': byline = 'By ' + byline byline = ' {}\r\n\r\n'.format(byline).replace('\'', '\'\'') odbc_item['article_text'] = byline + odbc_item['article_text']
def get_odbc_item(self, article, subscriber, category, codes, pass_through=False): """ Construct an odbc_item with the common key value pairs populated, if pass_through is true then the headline original headline is maintained. :param article: :param subscriber: :param category: :param codes: :param pass_through: :return: """ article['headline'] = get_text(article.get('headline', ''), content='html') pub_seq_num = superdesk.get_resource_service('subscribers').generate_sequence_number(subscriber) odbc_item = dict(originator=article.get('source', None), sequence=pub_seq_num, category=category.get('qcode').lower(), author=get_text(article.get('byline', '') or '', content='html').replace('\'', '\'\''), keyword=SluglineMapper().map(article=article, category=category.get('qcode').upper(), truncate=True).replace('\'', '\'\'') if not pass_through else (article.get('slugline', '') or '').replace('\'', '\'\''), subject_reference=set_subject(category, article), take_key=(article.get('anpa_take_key', '') or '').replace('\'', '\'\'')) if 'genre' in article and len(article['genre']) >= 1: odbc_item['genre'] = article['genre'][0].get('name', None) else: odbc_item['genre'] = 'Current' # @genre odbc_item['news_item_type'] = 'News' odbc_item['fullStory'] = 1 odbc_item['ident'] = '0' # @ident odbc_item['selector_codes'] = ' '.join(codes) if codes else ' ' headline = to_ascii(LocatorMapper().get_formatted_headline(article, category.get('qcode').upper())) odbc_item['headline'] = headline.replace('\'', '\'\'').replace('\xA0', ' ') self.expand_subject_codes(odbc_item) self.set_usn(odbc_item, article) return pub_seq_num, odbc_item
def populate(item, **kwargs): """Populate the abstract field with the first sentence of the body""" # get the list of sentences of the body if not item.get('body_html', None): item['abstract'] = 'No body found to use for abstract...' else: sentences = p.split(item['body_html']) # chop the first sentence to size for abstract (64) if sentences and len(sentences) > 0: item['abstract'] = get_text(sentences[0][:64]).strip() return item
def populate(item, **kwargs): """Populate the abstract field with the first sentence of the body""" # get the list of sentences of the body if not item.get('body_html', None): item['abstract'] = 'No body found to use for abstract...' else: sentences = p.split(item['body_html']) # chop the first sentence to size for abstract (64) if sentences and len(sentences) > 0: item['abstract'] = get_text(sentences[0][:64]).strip() return item
def _sanitize_fields(self, doc, validator): """If maxlength or minlength is specified in the validator then remove any markups from that field :param doc: Article to be validated :param validator: Validation rule :return: updated article """ fields_to_check = ['minlength', 'maxlength'] schema = validator.get('schema', {}) for field in schema: if doc.get(field) and schema.get(field) and any(k in schema[field] for k in fields_to_check): try: doc[field] = get_text(doc[field]) except (ValueError, TypeError): # fails for json fields like subject, genre pass
def _format_content(self, article, news_item, nitf): """Adds the content set to the xml :param dict article: :param Element newsItem: :param Element nitf: """ content_set = SubElement(news_item, 'contentSet') if article.get(FORMAT) == FORMATS.PRESERVED: inline_data = get_text(self.append_body_footer(article)) SubElement(content_set, 'inlineData', attrib={'contenttype': 'text/plain'}).text = inline_data elif article[ITEM_TYPE] in [CONTENT_TYPE.TEXT, CONTENT_TYPE.COMPOSITE]: inline = SubElement(content_set, 'inlineXML', attrib={'contenttype': 'application/nitf+xml'}) inline.append(nitf)
def _sanitize_fields(self, doc, validator): """If maxlength or minlength is specified in the validator then remove any markups from that field :param doc: Article to be validated :param validator: Validation rule :return: updated article """ fields_to_check = ['minlength', 'maxlength'] schema = validator.get('schema', {}) for field in schema: if doc.get(field) and schema.get(field) and any( k in schema[field] for k in fields_to_check): try: doc[field] = get_text(doc[field]) except (ValueError, TypeError): # fails for json fields like subject, genre pass
def map_html_to_xml(self, element, html): """ Map the html text tags to xml :param element: The xml element to populate :param html: the html to parse the text from :return: """ html = html.replace('<br>', '<br/>').replace('</br>', '') html = re.sub('[\x00-\x09\x0b\x0c\x0e-\x1f]', '', html) html = html.replace('\n', ' ') html = re.sub(r'\s\s+', ' ', html) parsed = parse_html(html, content='html') for tag in parsed.xpath('//*'): if tag.getparent() is not None and tag.getparent().tag == 'body': p = etree.Element('p') p.text = to_ascii(get_text(to_string(tag, method='html'), content='html')) element.append(p)
def format_for_source(self, article, subscriber, source, codes=None): try: pass_through = article.get('auto_publish', False) docs = [] for category in self._get_category_list(article.get('anpa_category')): article['source'] = source pub_seq_num, odbc_item = self.get_odbc_item(article, subscriber, category, codes, pass_through) is_last_take = self.is_last_take(article) if article.get(FORMAT) == FORMATS.PRESERVED: # @article_text body = get_text( self.append_body_footer(article) if is_last_take else article.get('body_html', ''), content='html') odbc_item['article_text'] = body.replace('\'', '\'\'') else: body = self.get_text_content( to_ascii(self.append_body_footer(article) if is_last_take else article.get('body_html', ''))) if self.is_first_part(article) and 'dateline' in article \ and 'text' in article.get('dateline', {}) and not pass_through: if body.startswith(' '): body = ' {} {}'.format(article.get('dateline').get('text'), body[3:]) odbc_item['article_text'] = body.replace('\'', '\'\'') if self.is_first_part(article) and not pass_through: self.add_ednote(odbc_item, article) self.add_byline(odbc_item, article) if not is_last_take: odbc_item['article_text'] += '\r\nMORE' else: odbc_item['article_text'] += '\r\n' + source sign_off = article.get('sign_off', '') or '' if len(sign_off) > 0: odbc_item['article_text'] += ' ' + sign_off odbc_item['category'] = odbc_item.get('category', '').upper() odbc_item['selector_codes'] = odbc_item.get('selector_codes', '').upper() docs.append((pub_seq_num, json.dumps(odbc_item))) return docs except Exception as ex: raise FormatterError.AAPNewscentreFormatterError(ex, subscriber)
def append_body_footer(self, article): """ Checks if the article has any Public Service Announcements and if available appends each of them to the body. :return: body with public service announcements. """ try: article['body_html'] = article['body_html'].replace( '<br>', '<br/>') except KeyError: pass body = '' if article[ITEM_TYPE] in [ CONTENT_TYPE.TEXT, CONTENT_TYPE.PREFORMATTED ]: body = article.get('body_html', '') elif article[ITEM_TYPE] in [ CONTENT_TYPE.AUDIO, CONTENT_TYPE.PICTURE, CONTENT_TYPE.VIDEO ]: body = article.get('description', '') if body and article.get(FORMAT, '') == FORMATS.PRESERVED: body = body.replace('\n', '\r\n').replace('\r\r', '\r') parsed = parse_html(body, content='html') for br in parsed.xpath('//br'): br.tail = '\r\n' + br.tail if br.tail else '\r\n' etree.strip_elements(parsed, 'br', with_tail=False) body = etree.tostring(parsed, encoding="unicode") if body and article.get('body_footer'): footer = article.get('body_footer') if article.get(FORMAT, '') == FORMATS.PRESERVED: body = '{}\r\n{}'.format(body, get_text(footer)) else: body = '{}{}'.format(body, footer) return body
def ap_weather_format(item, **kwargs): if not item.get('slugline', '').startswith('WEA--GlobalWeather-Ce') or not item.get('source', '') == 'AP': raise SuperdeskApiError.badRequestError("Article should be an AP sourced weather table") item['slugline'] = 'WORLD WEATHER' text = get_text(item['body_html'], content='html') lines = text.splitlines() if not lines[0] == 'BC-WEA--Global Weather-Celsius,<': raise SuperdeskApiError.badRequestError("Table should be in Celsius only") # tabular column max lengths are extracted into this list columns = [] # map of the columns to extract and the substitutions to apply to the column columnMap = ({'index': 0}, {'index': 1}, {'index': 2}, {'index': 3, 'substitute': [('COND', 'CONDITIONS'), ('pc', 'partly cloudy'), ('clr', 'clear'), ('cdy', 'cloudy'), ('rn', 'rain'), ('sn', 'snow')]}) # story preamble preamble = 'Temperatures and conditions in world centres:\r\n' output = StringIO() output.write(preamble) # story is always datelined News York city = 'New York City' cities = app.locators.find_cities() located = [c for c in cities if c['city'].lower() == city.lower()] if 'dateline' not in item: item['dateline'] = {} item['dateline']['located'] = located[0] if len(located) > 0 else {'city_code': city, 'city': city, 'tz': 'UTC', 'dateline': 'city'} item['dateline']['date'] = datetime.fromtimestamp(get_date(item['firstcreated']).timestamp(), tz=timezone(item['dateline']['located']['tz'])) item['dateline']['source'] = 'AP' item['dateline']['text'] = format_dateline_to_locmmmddsrc(item['dateline']['located'], get_date(item['firstcreated']), source=item.get('original_source', 'AP')) item['headline'] = 'World Weather for ' + item['dateline']['date'].strftime('%b %-d') item['subject'] = [{"name": "weather", "qcode": "17000000"}] locator_map = superdesk.get_resource_service('vocabularies').find_one(req=None, _id='locators') item['place'] = [x for x in locator_map.get('items', []) if x['qcode'] == 'US'] if lines: # scan all the lines in the file for potential collimated lines and calculate the length # of the column for line in lines: row = re.split('[;\<]+', line) # only consider it if there are more than two rows if len(row) > 2: index = 0 for col in row: # check if the column is mapped map = [me for me in columnMap if me['index'] == index] if len(map): for sub in map[0].get('substitute', ()): col = col.replace(sub[0], sub[1]) # if it's a new column if 0 <= index < len(columns): # check the length if len(col) > columns[index]: columns[index] = len(col) else: columns.append(len(col)) index += 1 for line in lines: row = re.split('[;\<]+', line) if len(row) > 2: index = 0 for col in row: map = [me for me in columnMap if me['index'] == index] if len(map) > 0: for sub in map[0].get('substitute', ()): col = col.replace(sub[0], sub[1]) output.write( '{}'.format(col.lstrip('\t').ljust(columns[map[0].get('index')] + 2)).rstrip('\r\n')) index += 1 output.write('\r\n') item['body_html'] = '<pre>' + output.getvalue() + '</pre>' return item
def _transform_to_ninjs(self, article, subscriber, recursive=True): ninjs = { 'guid': article.get(GUID_FIELD, article.get('uri')), 'version': str(article.get(config.VERSION, 1)), 'type': self._get_type(article) } if article.get('byline'): ninjs['byline'] = article['byline'] located = article.get('dateline', {}).get('located', {}) if located: ninjs['located'] = located.get('city', '') for copy_property in self.direct_copy_properties: if article.get(copy_property) is not None: ninjs[copy_property] = article[copy_property] if 'body_text' not in article and 'alt_text' in article: ninjs['body_text'] = article['alt_text'] if 'title' in article: ninjs['headline'] = article['title'] if article.get('body_html'): ninjs['body_html'] = self.append_body_footer(article) if article.get('description'): ninjs['description_html'] = self.append_body_footer(article) if article.get('place'): ninjs['place'] = self._format_qcodes(article['place']) if article.get('profile'): ninjs['profile'] = self._format_profile(article['profile']) if recursive: if article[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE: ninjs[ASSOCIATIONS] = self._get_associations( article, subscriber) if article.get(ASSOCIATIONS): ninjs[ASSOCIATIONS].update( self._format_related(article, subscriber)) elif article.get(ASSOCIATIONS): ninjs[ASSOCIATIONS] = self._format_related(article, subscriber) elif article.get(ASSOCIATIONS): ninjs[ASSOCIATIONS] = self._format_related(article, subscriber) if article.get(EMBARGO): ninjs['embargoed'] = get_utc_schedule(article, EMBARGO).isoformat() if article.get('priority'): ninjs['priority'] = article['priority'] else: ninjs['priority'] = 5 if article.get('subject'): ninjs['subject'] = self._get_subject(article) if article.get('anpa_category'): ninjs['service'] = self._get_service(article) if article.get('renditions'): ninjs['renditions'] = self._get_renditions(article) elif 'url' in article: ninjs['renditions'] = self._generate_renditions(article) # SDPA-317 if article.get('abstract'): abstract = article.get('abstract', '') ninjs['description_html'] = abstract ninjs['description_text'] = get_text(abstract) elif article.get('description_text'): ninjs['description_text'] = article.get('description_text') if article.get('company_codes'): ninjs['organisation'] = [{ 'name': c.get('name', ''), 'rel': 'Securities Identifier', 'symbols': [{ 'ticker': c.get('qcode', ''), 'exchange': c.get('security_exchange', '') }] } for c in article['company_codes']] elif 'company' in article: ninjs['organisation'] = [{'name': article['company']}] if article.get('rewrite_of'): ninjs['evolvedfrom'] = article['rewrite_of'] if not ninjs.get('copyrightholder') and not ninjs.get( 'copyrightnotice') and not ninjs.get('usageterms'): ninjs.update( superdesk.get_resource_service('vocabularies').get_rightsinfo( article)) if article.get('genre'): ninjs['genre'] = self._format_qcodes(article['genre']) if article.get('flags', {}).get('marked_for_legal'): ninjs['signal'] = self._format_signal_cwarn() return ninjs
def get_value(self, article): try: return get_text(article[self.field.name]).replace('\n', ' ') except (etree.XMLSyntaxError, ValueError): return article[self.field.name]
def plaintext_filter(value): """Filter out html from value.""" return get_text(value).replace('\n', ' ').strip()
def get_value(self, article): try: return get_text(article[self.field.name]).replace('\n', ' ') except (etree.XMLSyntaxError, ValueError): return article[self.field.name]
def _transform_to_ninjs(self, article, subscriber, recursive=True): ninjs = { 'guid': article.get(GUID_FIELD, article.get('uri')), 'version': str(article.get(config.VERSION, 1)), 'type': self._get_type(article) } if article.get('byline'): ninjs['byline'] = article['byline'] located = article.get('dateline', {}).get('located', {}) if located: ninjs['located'] = located.get('city', '') for copy_property in self.direct_copy_properties: if article.get(copy_property) is not None: ninjs[copy_property] = article[copy_property] if 'body_text' not in article and 'alt_text' in article: ninjs['body_text'] = article['alt_text'] if 'title' in article: ninjs['headline'] = article['title'] if article.get('body_html'): ninjs['body_html'] = self.append_body_footer(article) if article.get('description'): ninjs['description_html'] = self.append_body_footer(article) if article.get('place'): ninjs['place'] = self._format_qcodes(article['place']) if article.get('profile'): ninjs['profile'] = self._format_profile(article['profile']) if recursive: if article[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE: ninjs[ASSOCIATIONS] = self._get_associations(article, subscriber) if article.get(ASSOCIATIONS): ninjs[ASSOCIATIONS].update(self._format_related(article, subscriber)) elif article.get(ASSOCIATIONS): ninjs[ASSOCIATIONS] = self._format_related(article, subscriber) elif article.get(ASSOCIATIONS): ninjs[ASSOCIATIONS] = self._format_related(article, subscriber) if article.get(EMBARGO): ninjs['embargoed'] = get_utc_schedule(article, EMBARGO).isoformat() if article.get('priority'): ninjs['priority'] = article['priority'] else: ninjs['priority'] = 5 if article.get('subject'): ninjs['subject'] = self._get_subject(article) if article.get('anpa_category'): ninjs['service'] = self._get_service(article) if article.get('renditions'): ninjs['renditions'] = self._get_renditions(article) elif 'url' in article: ninjs['renditions'] = self._generate_renditions(article) # SDPA-317 if article.get('abstract'): abstract = article.get('abstract', '') ninjs['description_html'] = abstract ninjs['description_text'] = get_text(abstract) elif article.get('description_text'): ninjs['description_text'] = article.get('description_text') if article.get('company_codes'): ninjs['organisation'] = [{'name': c.get('name', ''), 'rel': 'Securities Identifier', 'symbols': [{'ticker': c.get('qcode', ''), 'exchange': c.get('security_exchange', '')}]} for c in article['company_codes']] elif 'company' in article: ninjs['organisation'] = [{'name': article['company']}] if article.get('rewrite_of'): ninjs['evolvedfrom'] = article['rewrite_of'] if not ninjs.get('copyrightholder') and not ninjs.get('copyrightnotice') and not ninjs.get('usageterms'): ninjs.update(superdesk.get_resource_service('vocabularies').get_rightsinfo(article)) if article.get('genre'): ninjs['genre'] = self._format_qcodes(article['genre']) if article.get('flags', {}).get('marked_for_legal'): ninjs['signal'] = self._format_signal_cwarn() if article.get('attachments'): ninjs['attachments'] = self._format_attachments(article) return ninjs
def plaintext_filter(value): """Filter out html from value.""" return get_text(value).replace('\n', ' ').strip()
def _format_body_content(self, article, body_content): if article.get(FORMAT) == FORMATS.PRESERVED: pre = get_text(self.append_body_footer(article)) SubElement(body_content, 'pre').text = pre else: self.map_html_to_xml(body_content, self.append_body_footer(article))