def shares_from_url(url, request_start_date):
params = {
'count': 1000,
# 'includeHistory': 'true',
'link': url,
'platforms': 'facebook',
'sortBy': 'total_interactions',
'startDate': request_start_date,
'token': CROWDTANGLE_API_KEY
}
api_endpoint = 'links'
response = requests.get('{}/{}'.format(CROWDTANGLE_API_URL, api_endpoint),
params).json()
shares = []
if response['status'] == 200:
if not response.get('result'):
logger.warning('Crowdtangle data returned is empty.')
return shares
for post in response['result']['posts']:
account = post['account']
shares.append({
'account': {
'crowdtangleIdentifier': str(account['id']),
'facebookIdentifier': str(account['platformId']),
'logoUrl': account['profileImage'],
'name': account['name'],
'url': account['url']
},
'post': {
'crowdtangleIdentifier':
str(post['id']),
'facebookIdentifier':
str(post['platformId']),
'publishedDate':
datetime.strptime(post['date'], '%Y-%m-%d %H:%M:%S'),
'url':
post['postUrl'],
}
})
else:
logger.error(
f'Error in fetching from Crowdtangle: {response.get("message", "Unknown exception.")}'
)
logger.warning('Returning empty interaction data')
sleep(30)
return shares
from flask import Flask
from sqlalchemy_api_handler import logger
from utils.jobs import get_all_jobs, write_jobs_to_file, remove_oldest_jobs_file
from utils.setup import setup
CLOCK_APP = Flask(__name__)
setup(CLOCK_APP, with_jobs=True)
if __name__ == '__main__':
# CLOCK_APP.async_scheduler.start()
CLOCK_APP.background_scheduler.start()
# atexit.register(lambda: CLOCK_APP.async_scheduler.shutdown())
atexit.register(CLOCK_APP.background_scheduler.shutdown)
print_jobs = True
try:
while True:
if print_jobs:
jobs = get_all_jobs(CLOCK_APP)
write_jobs_to_file(jobs)
remove_oldest_jobs_file()
time.sleep(60)
except (KeyboardInterrupt, SystemExit):
logger.warning('Scheduler interupted')
print_jobs = False
# CLOCK_APP.async_scheduler.shutdown()
CLOCK_APP.background_scheduler.shutdown()
import os
from datetime import datetime
import requests
from sqlalchemy_api_handler import logger
from urllib.parse import urlencode
from utils.date import strftime
from utils.config import IS_DEVELOPMENT
BUZZSUMO_API_KEY = os.environ.get('BUZZSUMO_API_KEY')
BUZZSUMO_API_URL = "http://api.buzzsumo.com/search"
if BUZZSUMO_API_KEY is None:
logger.warning('BUZZSUMO_API_KEY is not defined in the env')
DEVELOPMENT_TRENDINGS = [{
'id':
123,
'published_date':
1585956625,
'subdomain':
'www.cnn.com',
'tags':
'climate',
'thumbnail':
'https://cdn.cnn.com/cnnnext/dam/assets/170708175538-05-trump-abroad-0708-super-tease.jpg',
'title':
"Donald Trump buried a climate change report because 'I don't believe it'",
'total_facebook_shares':
1000,
'total_shares':
def sync_for(
name,
formula=None,
max_records=None,
session=None,
sync_to_airtable=False
):
if session is None:
session = requests.Session()
rows = request_airtable_rows(
SCIENCE_FEEDBACK_AIRTABLE_BASE_ID,
NAME_TO_AIRTABLE[name],
filter_by_formula=formula,
max_records=max_records,
session=session
)
entities = []
if rows:
logger.info(f'syncing table {NAME_TO_AIRTABLE[name]}')
else:
logger.info(f'nothing to sync for table {NAME_TO_AIRTABLE[name]}')
for (index, row) in enumerate(rows):
try:
entity = entity_from_row_for(name, row, index)
if entity:
entities.append(entity)
row['Synced time input'] = datetime.now().isoformat()
else:
row['Synced time input'] = 'ERROR'
except KeyError as exception:
logger.warning(f'Error while trying to create entity from row at table {NAME_TO_AIRTABLE[name]}')
logger.error(f'KeyError {exception}: {row}')
row['Synced time input'] = 'ERROR'
except Exception as exception:
logger.warning(f'Error while trying to create entity from row at table {NAME_TO_AIRTABLE[name]}')
logger.error(f'Unexpected error: {exception} - {sys.exc_info()[0]} at {row}')
row['Synced time input'] = 'ERROR'
def _update_10_rows_from_index(i):
records = [{'id': row['airtableId'], 'fields': {'Synced time input': row['Synced time input']}} for row in rows[i: i + 10]]
res = update_airtable_rows(
SCIENCE_FEEDBACK_AIRTABLE_BASE_ID,
NAME_TO_AIRTABLE[name],
{'records': records},
session=session
)
if res.status_code != 200:
logger.error(f'code: {res.status_code}, error: {res.content}')
try:
# Sync verdict status from wordpress
if name == 'verdict' and formula is not None:
entities = claim_verdicts_from_airtable(verdicts_to_sync=entities)
# Sync related contents for appearances
if name == 'appearance' and formula is not None:
for entity in entities:
sync_content(entity.quotingContent)
# Set the time synced so that the status in airtable is "Synced"
if sync_to_airtable:
for i in range(0, len(rows), 10):
try:
ApiHandler.save(*entities[i:i + 10])
_update_10_rows_from_index(i)
except Exception as exception:
logger.warning(f'Error while trying to save 10 entities at table {NAME_TO_AIRTABLE[name]}')
logger.error(f'Unexpected error: {exception} - {sys.exc_info()[0]}')
for index in range(i, i + 10):
rows[index]['Synced time input'] = 'BATCH ERROR'
_update_10_rows_from_index(i)
except Exception as exception:
logger.warning(f'Error while trying to save entities at table {NAME_TO_AIRTABLE[name]}')
logger.error(f'Unexpected error: {exception} - {sys.exc_info()[0]}')
import os
from time import sleep
from datetime import datetime
import requests
from sqlalchemy_api_handler import logger
CROWDTANGLE_API_URL = 'https://api.crowdtangle.com'
CROWDTANGLE_API_KEY = os.environ.get('CROWDTANGLE_API_KEY')
if CROWDTANGLE_API_KEY is None:
logger.warning('CROWDTANGLE_API_KEY is not defined in the env!')
def shares_from_url(url, request_start_date):
params = {
'count': 1000,
# 'includeHistory': 'true',
'link': url,
'platforms': 'facebook',
'sortBy': 'total_interactions',
'startDate': request_start_date,
'token': CROWDTANGLE_API_KEY
}
api_endpoint = 'links'
response = requests.get('{}/{}'.format(CROWDTANGLE_API_URL, api_endpoint),
params).json()