def get_contents_to_be_deleted(token, config, api):
url = api + '/domains/' + config['domain']['_id'] + '/contents/_query'
data = {
'where': {
'path': config['path'] + '/',
'status': 'draft',
'sys.created_by': config['cms_username'],
'sys.published_version': {
'$exists': False
}
},
'select': {
'_id': 1
}
}
logger.info(data)
headers = {
'Authorization': 'Bearer {}'.format(token),
'Content-Type': 'application/json'
}
response = requests.post(url, data=json.dumps(data), headers=headers)
if response.status_code != 200:
raise BlupointError(
err_code="",
err_msg="",
status_code=response.status_code
)
response_json = json.loads(response.text)
logger.info('<{}> contents found for deleting.'.format(response_json['data']['count']))
return response_json['data']['items']
def get_contents_from_ap(agency, agency_config):
url = agency_config['input_url'] + '/AP.Distro.Feed/GetFeed.aspx?idList=31896&idListType=products&maxItems=20'
response = requests.get(url, auth=HTTPBasicAuth(agency_config['username'], agency_config['password']))
logger.warning(response.status_code)
if response.status_code != 200:
raise BlupointError(
err_msg="Agency Rss didnt return 200",
err_code="errors.InvalidUsage",
status_code=response.status_code
)
feeds_json = json.loads(json.dumps(xmltodict.parse(response.text)))
items = feeds_json['feed'].get('entry', [])
news = []
for item in items:
images = item.get('link')
r = {
'item_id': item['id'],
'title': item['title'],
'text': item['content']['#text'],
'updated': item['updated'],
'published': item['published'],
'byline': item['apcm:ContentMetadata']['apcm:ByLine'][0]['#text'],
'keywords': item.get('Keywords'),
'images': images
}
news.append(r)
logger.info("total content count from ap: <{}>".format(len(news)))
return news
def make_aa_request(agency, body):
url = body['input_url'] + '/abone/search'
data = {
'end_data': 'NOW',
'filter_language': 1
}
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
}
response = requests.post(url, data=data, headers=headers, auth=HTTPBasicAuth(body['username'], body['password']))
if response.status_code != 200:
raise BlupointError(
err_msg="Agency rss did not return 200",
err_code="errors.InvalidUsage",
status_code=response.status_code
)
feeds_json = json.loads(response.text)
items = feeds_json['data']['result']
text_news = []
for item in items:
if item['type'] == 'text':
text_news.append(item)
for i in range(0, 10):
detail_url = body['input_url'] + '/abone/document/' + text_news[10 - i]['id'] + '/newsml29'
response = requests.get(detail_url, headers=headers, auth=HTTPBasicAuth(body['username'], body['password']))
if response.status_code == 200:
break
response_json = parse_aa_response(response.text)
images = []
response_json = dict(response_json)
if 'images' in response_json and response_json['images']:
if isinstance(response_json['images'], dict):
image = {
"image_id": response_json['images']['@residref'],
"title": response_json['images']['title']
}
images.append(image)
if isinstance(response_json['images'], list):
for image in response_json['images']:
image = {
"image_id": image['@residref'],
"title": image['title']
}
images.append(image)
response_json['images'] = images
return json.dumps(response_json)
def make_dha_request(agency, body):
import urllib.request
req = urllib.request.Request(body['input_url'])
with urllib.request.urlopen(req) as response:
if response.status != 200:
raise BlupointError(err_code="errors.InvalidUsage",
err_msg="Agency news response is not 200",
status_code=response.status_code)
response_json = parse_dha_response(response.read().decode())
return json.dumps(response_json)
def make_reuters_request(agency, body):
url = body['input_url'] + '&limit=10&maxAge=2h'
response = requests.post(url,
auth=HTTPDigestAuth(body['username'],
body['password']))
if response.status_code != 200:
raise BlupointError(err_code="errors.InvalidUsage",
err_msg="Agency news response is not 200",
status_code=response.status_code)
response_json = parse_reuters_response(response.text)
return response_json
def make_iha_request(agency, body):
url = body['input_url'] + '&{}={}&{}={}'.format(
agency['auth_credential_parameters']['username'], body['username'],
agency['auth_credential_parameters']['password'], body['password'])
response = requests.get(url)
if response.status_code != 200:
raise BlupointError(err_msg="Agency Rss did not return 200",
err_code="errors.InvalidUsage",
status_code=response.status_code)
response_json = parse_iha_response(response.text)
return json.dumps(response_json)
def get_user_domains(token, user, settings):
url = settings['management_api'] + '/domains/_query'
headers = {'Authorization': 'Bearer {}'.format(token)}
body = {'where': {'membership_id': user['membership']['_id']}}
response = requests.post(url, headers=headers, json=body)
if response.status_code != 200:
raise BlupointError(err_code="errors.requestError",
err_msg=json.loads(response.text),
status_code=response.status_code)
domains = json.loads(response.text)
return domains['data']['items']
def get_user_by_token(token, settings):
url = settings['management_api'] + '/me'
headers = {'Authorization': 'Bearer {}'.format(token)}
response = requests.get(url, headers=headers)
if response.status_code != 201:
raise BlupointError(err_msg="Me Api did not return 201",
err_code="errors.providedTokenIsExpired",
status_code="401")
user = json.loads(response.text)
return user['user']
def get_contents_from_dha(agency, agency_config):
import urllib.request
url = agency_config['input_url']
req = urllib.request.Request(url)
with urllib.request.urlopen(req) as response:
if response.status != 200:
raise BlupointError(err_code="errors.InvalidUsage",
err_msg="Agency news response is not 200",
status_code=response.status_code)
o = xmltodict.parse(response.read().decode())
items = o['rss']['channel']['item']
logger.info("total content count from dha: <{}>".format(len(items)))
return items
def get_token(username, password, token_api):
data = {'username': username, 'password': password}
response = requests.post(token_api, data=json.dumps(data))
if response.status_code != 201:
logger.info(json.loads(response.text))
raise BlupointError(err_code="errors.errorOccurredWhileGetToken",
err_msg="Internal Server Error",
status_code=response.status_code,
context={'message': response.text})
response_json = json.loads(response.text)
return response_json['token']
def authenticate_user(credentials, settings):
url = settings['management_api'] + '/tokens'
response = requests.post(url, json=credentials)
if response.status_code != 201:
raise BlupointError(err_msg="Username or password is invalid",
err_code="errors.usernameOrPasswordIsInvalid",
status_code="401")
token = json.loads(response.text)
user = get_user_by_token(token['token'], settings)
return user, token['token']
def make_ap_request(agency, body):
url = body['input_url'] + '/AP.Distro.Feed/GetFeed.aspx?idList=31896&idListType=products&maxItems=1'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
}
response = requests.get(url, headers=headers, auth=HTTPBasicAuth(body['username'], body['password']))
if response.status_code != 200:
raise BlupointError(
err_code="errors.InvalidUsage",
err_msg="Agency news response is not 200",
status_code=response.status_code
)
response_json = parse_ap_response(response.text)
return json.dumps(response_json)
def get_contents_from_iha(agency, agency_config):
url = agency_config['input_url'] + '&{}={}&{}={}'.format(
agency['auth_credential_parameters']['username'],
agency_config['username'],
agency['auth_credential_parameters']['password'],
agency_config['password'])
response = requests.get(url)
if response.status_code != 200:
raise BlupointError(err_msg="Agency Rss did not return 200",
err_code="errors.InvalidUsage",
status_code=response.status_code)
o = json.loads(json.dumps(xmltodict.parse(response.text)))
items = o['rss']['channel']['item']
logger.info("total content count from iha: <{}>".format(len(items)))
return items
def get_content_types_field_definitions(settings, domain_id, content_type_id):
url = settings['management_api'] + '/domains/' + domain_id + '/content-types/' + content_type_id
headers = {
'Authorization': 'Bearer {}'.format(session['token']),
'Content-Type': 'application/json'
}
response = requests.get(url, headers=headers)
if response.status_code != 200:
raise BlupointError(
err_msg="Error occurred while getting field_definitions for content_type: <{}> in domain: <{}>".format(
domain_id, content_type_id),
err_code="errors.internalError",
status_code=response.status_code
)
content_type = json.loads(response.text)
return content_type.get('field_definitions', None)
def get_contents_from_reuters(agency, agency_config):
url = agency_config['input_url'] + '&limit=10&maxAge=2h'
response = requests.post(url,
auth=HTTPDigestAuth(agency_config['username'],
agency_config['password']))
if response.status_code != 200:
raise BlupointError(err_code="errors.InvalidUsage",
err_msg="Agency news response is not 200",
status_code=response.status_code)
o = json.loads(json.dumps(xmltodict.parse(response.text)))
news = o['rss']['channel']['item']
for new in news:
new['images'] = new.get('media:group', {}).get('media:content', [])
logger.info("total content count from reuters: <{}>".format(len(news)))
return news
def get_contents_from_aa(agency, agency_config):
url = agency_config['input_url'] + '/abone/search'
data = {'end_data': 'NOW', 'filter_language': 1}
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
}
response = requests.post(url,
data=data,
headers=headers,
auth=HTTPBasicAuth(agency_config['username'],
agency_config['password']))
if response.status_code != 200:
raise BlupointError(err_msg="Agency rss did not return 200",
err_code="errors.InvalidUsage",
status_code=response.status_code)
feeds_json = json.loads(response.text)
items = feeds_json['data']['result']
text_news = []
for item in items:
if item['type'] == 'text':
text_news.append(item)
news = []
i = 0
for item in text_news:
detail_url = agency_config['input_url'] + '/abone/document/' + item[
'id'] + '/newsml29'
response = requests.get(detail_url,
headers=headers,
auth=HTTPBasicAuth(agency_config['username'],
agency_config['password']))
if response.status_code != 200:
i += 1
continue
logger.info('response item id: <{}>'.format(item['id']))
try:
o = json.loads(json.dumps(xmltodict.parse(response.text)))
o = o['newsMessage']['itemSet']['newsItem']
r = {
'item_id':
item['id'],
'headline':
o['contentSet']['inlineXML']['nitf']['body']['body.head']
['headline']['hl1'],
'byline':
o['contentSet']['inlineXML']['nitf']['body']['body.head']
['byline']['byttl'],
'abstract':
o['contentSet']['inlineXML']['nitf']['body']['body.head']
['abstract'],
'content':
o['contentSet']['inlineXML']['nitf']['body']['body.content'],
'images':
o['itemMeta'].get('link', [])
}
except Exception as e:
logger.warning(str(e))
news.append(r)
logger.info("total content count from aa: <{}> for domain: <{}>".format(
len(text_news), agency_config['domain']['_id']))
logger.info("received text news: <{}> for domain: <{}>".format(
len(text_news) - i, agency_config['domain']['_id']))
return news