def do_on_behalf_of(db, report_id, user_id=None):
'''
un/sets report as on behalf of user
'''
if not db:
return (False, 'inner application error')
report_id = _get_id_value(report_id)
report_get = get_report_by_id(db, report_id)
if not report_get[0]:
return (False, 'report not found')
report = report_get[1]
if user_id:
user_id = _get_id_value(user_id)
timepoint = datetime.datetime.utcnow()
properties_set = {
FIELD_UPDATED: timepoint,
'proto': False,
}
removal_set = None
if user_id:
properties_set['on_behalf_id'] = user_id
else:
removal_set = {'on_behalf_id': ''}
update_report_set(db, report_id, properties_set, removal_set)
return (True, {'_id': report_id})
def do_on_behalf_of(db, report_id, user_id=None):
'''
un/sets report as on behalf of user
'''
if not db:
return (False, 'inner application error')
report_id = _get_id_value(report_id)
report_get = get_report_by_id(db, report_id)
if not report_get[0]:
return (False, 'report not found')
report = report_get[1]
if user_id:
user_id = _get_id_value(user_id)
timepoint = datetime.datetime.utcnow()
properties_set = {
FIELD_UPDATED: timepoint,
'proto': False,
}
removal_set = None
if user_id:
properties_set['on_behalf_id'] = user_id
else:
removal_set = {'on_behalf_id': ''}
update_report_set(db, report_id, properties_set, removal_set)
return (True, {'_id': report_id})
def get_coverage_info(db, coverage_id):
coverage_id = _get_id_value(coverage_id)
coll = db[COLL_COVERAGES]
if type(coverage_id) is ObjectId:
return (False, 'old form of coverage info retrieval')
else:
coverage = coll.find_one({'uuid': coverage_id})
if not coverage:
return (False, 'coverage not found')
base_url = get_conf('reports_url')
reports_url = base_url.replace(PUBLISHED_REPORTS_PLACEHOLDER, urllib.quote_plus(str(coverage_id)))
title = ''
if ('title' in coverage) and coverage['title']:
title = coverage['title']
description = ''
if ('description' in coverage) and coverage['description']:
description = coverage['description']
active = False
if (FIELD_ACTIVE_COVERAGE in coverage) and coverage[FIELD_ACTIVE_COVERAGE]:
active = coverage[FIELD_ACTIVE_COVERAGE]
info = {
'IsLive': active,
'Title': title,
'Description': description,
'PostPublished': {'href': reports_url,},
}
return (True, info)
def do_set_active_one(db, coverage_id, set_active):
'''
de/activate a coverage
'''
if not db:
return (False, 'inner application error')
coverage_id = _get_id_value(coverage_id)
coverage_get = get_coverage_by_id(db, coverage_id)
if not coverage_get[0]:
return (False, 'coverage not found')
coverage = coverage_get[1]
if set_active:
if (FIELD_DECAYED in coverage) and coverage[FIELD_DECAYED]:
return (False, 'can not activate decayed coverage')
timepoint = datetime.datetime.utcnow()
if type(coverage_id) is ObjectId:
sel_part = {'_id': coverage_id}
else:
sel_part = {'uuid': coverage_id}
update_coverage_set(db, sel_part, {FIELD_ACTIVE: set_active, '_updated': timepoint})
return (True, {'_id': coverage_id})
def do_get_session(db, session, offset=None, limit=None):
'''
returns data of a set of reports saved by the stream
'''
if not db:
return (False, 'inner application error')
session = _get_id_value(session)
list_spec = {'feed_type': FEED_TYPE, 'session': session}
coll = db[collection]
cursor = coll.find(list_spec).sort([('produced', 1)])
total = cursor.count()
if limit is None:
limit = DEFAULT_LIMIT
if offset:
cursor = cursor.skip(offset)
if limit:
cursor = cursor.limit(limit)
docs = []
for entry in cursor:
if not entry:
continue
docs.append(entry)
return (True, docs, {'total': total})
def do_get_session(db, session, offset=None, limit=None):
'''
returns data of a set of reports saved by the stream
'''
if not db:
return (False, 'inner application error')
session = _get_id_value(session)
list_spec = {'feed_type': FEED_TYPE, 'session': session}
coll = db[collection]
cursor = coll.find(list_spec).sort([('produced', 1)])
total = cursor.count()
if limit is None:
limit = DEFAULT_LIMIT
if offset:
cursor = cursor.skip(offset)
if limit:
cursor = cursor.limit(limit)
docs = []
for entry in cursor:
if not entry:
continue
docs.append(entry)
return (True, docs, {'total': total})
def get_report_author(db, report_id, author_form):
coll = db[COLL_REPORTS]
if author_form not in ['author', 'creator', 'icon']:
return (False, 'unknown author form')
report_id = _get_id_value(report_id)
report = coll.find_one({'_id': report_id})
if not report:
return (False, 'respective report not found')
if 'feed_type' not in report:
report['feed_type'] = DEFAULT_FIELD_TYPE
feed_type = report['feed_type']
if feed_type not in OUTPUT_FEED_TYPES:
return (False, 'unsupported report type')
author = None
user = None
if 'on_behalf_id' in report:
user_id = report['on_behalf_id']
user = load_local_user(db, user_id)
if not user:
user = None
if 'sms' == feed_type:
if 'author' == author_form:
author = get_sms_report_author(report_id, report, user)
if 'creator' == author_form:
author = get_sms_report_creator(report_id, report, user)
if 'icon' == author_form:
author = get_sms_report_icon(report_id, report, user)
if 'tweet' == feed_type:
if 'author' == author_form:
author = get_tweet_report_author(report_id, report, user)
if 'creator' == author_form:
author = get_tweet_report_creator(report_id, report, user)
if 'icon' == author_form:
author = get_tweet_report_icon(report_id, report, user)
if feed_type in ('plain', 'web_link'):
if 'author' == author_form:
author = get_plain_report_author(report_id, report, user)
if 'creator' == author_form:
author = get_plain_report_creator(report_id, report, user)
if 'icon' == author_form:
author = get_plain_report_icon(report_id, report, user)
return (True, author)
def do_get_one(db, coverage_id):
'''
returns data of a single coverage
'''
if not db:
return (False, 'inner application error')
coverage_id = _get_id_value(coverage_id)
res = get_coverage_by_id(db, coverage_id)
return res
def do_get_one(db, doc_id):
'''
returns data of a single sms report
'''
if not db:
return (False, 'inner application error')
doc_id = _get_id_value(doc_id)
res = do_get_one_by_id(db, doc_id)
return res
def do_get_one(db, service_id):
'''
returns data of a single service
'''
if not db:
return (False, 'inner application error')
service_id = _get_id_value(service_id)
res = get_service_by_id(db, service_id)
return res
def load_local_user(db, user_id):
user_id = _get_id_value(user_id)
coll = db[COLL_USERS]
user = coll.find_one({'_id': user_id})
if not user:
return None
return user
def load_local_user(db, user_id):
user_id = _get_id_value(user_id)
coll = db[COLL_USERS]
user = coll.find_one({'_id': user_id})
if not user:
return None
return user
def do_get_one(db, doc_id):
'''
returns data of a single sms report
'''
if not db:
return (False, 'inner application error')
doc_id = _get_id_value(doc_id)
res = do_get_one_by_id(db, doc_id)
return res
def get_report_author(db, report_id, author_form):
coll = db[COLL_REPORTS]
if author_form not in ['author', 'creator', 'icon']:
return (False, 'unknown author form')
report_id = _get_id_value(report_id)
report = coll.find_one({'_id': report_id})
if not report:
return (False, 'respective report not found')
if 'feed_type' not in report:
report['feed_type'] = DEFAULT_FIELD_TYPE
feed_type = report['feed_type']
if feed_type not in OUTPUT_FEED_TYPES:
return (False, 'unsupported report type')
author = None
user = None
if 'on_behalf_id' in report:
user_id = report['on_behalf_id']
user = load_local_user(db, user_id)
if not user:
user = None
if 'sms' == feed_type:
if 'author' == author_form:
author = get_sms_report_author(report_id, report, user)
if 'creator' == author_form:
author = get_sms_report_creator(report_id, report, user)
if 'icon' == author_form:
author = get_sms_report_icon(report_id, report, user)
if 'tweet' == feed_type:
if 'author' == author_form:
author = get_tweet_report_author(report_id, report, user)
if 'creator' == author_form:
author = get_tweet_report_creator(report_id, report, user)
if 'icon' == author_form:
author = get_tweet_report_icon(report_id, report, user)
if feed_type in ('plain', 'web_link'):
if 'author' == author_form:
author = get_plain_report_author(report_id, report, user)
if 'creator' == author_form:
author = get_plain_report_creator(report_id, report, user)
if 'icon' == author_form:
author = get_plain_report_icon(report_id, report, user)
return (True, author)
def do_get_one(db, service_id):
'''
returns data of a single service
'''
if not db:
return (False, 'inner application error')
service_id = _get_id_value(service_id)
res = get_service_by_id(db, service_id)
return res
def do_insert_one(db, coverage_data):
'''
creates a coverage
'''
if not db:
return (False, 'inner application error')
timepoint = datetime.datetime.utcnow()
uuid_value = None
if ('uuid' in coverage_data) and coverage_data['uuid']:
try:
uuid_value = str(uuid.UUID(str(coverage_data['uuid'])).hex)
except:
uuid_value = None
coverage_set = None
if uuid_value:
coverage_get = get_coverage_by_uuid(db, uuid_value)
if coverage_get[0]:
coverage_set = coverage_get[1]
if not coverage_set:
coverage_set = {}
if uuid_value:
coverage_set['uuid'] = uuid_value
else:
coverage_set['uuid'] = str(uuid.uuid4().hex)
try:
coverage_set['title'] = str(coverage_data['title'])
coverage_set['description'] = str(coverage_data['description'])
coverage_set['user_id'] = _get_id_value(coverage_data['user_id'])
coverage_set[FIELD_ACTIVE] = False
coverage_set[FIELD_DECAYED] = False
coverage_set['_created'] = timepoint
coverage_set['_updated'] = timepoint
#coverage_set['_etag'] = _get_etag()
except:
return (False, 'can not setup the coverage')
if ('active' in coverage_data) and (coverage_data['active']):
coverage_set[FIELD_ACTIVE] = True
coll = db[collection]
try:
coverage_id = coll.save(coverage_set)
except:
return (False, 'can not save the coverage data')
return (True, {'_id': coverage_id})
def do_delete_one(db, doc_id):
'''
deletes data of a single oauth info
'''
if not db:
return (False, 'inner application error')
doc_id = _get_id_value(doc_id)
coll = db[collection]
coll.remove({'_id': doc_id})
return (True, {'_id': doc_id})
def get_one(db, doc_id):
'''
returns data of a single oauth info
'''
if not db:
return (False, 'inner application error')
doc_id = _get_id_value(doc_id)
coll = db[collection]
doc = coll.find_one({'_id': doc_id})
if not doc:
return (False, 'oauth info not found')
return (True, doc)
def get_one(db, doc_id):
'''
returns data of a single filter info
'''
if not db:
return (False, 'inner application error')
doc_id = _get_id_value(doc_id)
coll = db[collection]
doc = coll.find_one({'_id': doc_id})
if not doc:
return (False, 'filter info not found')
return (True, doc)
def do_get_one(db, doc_id):
'''
returns data of a single stream info
'''
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
doc_id = _get_id_value(doc_id)
coll = db[collection]
doc = coll.find_one({'_id': doc_id})
if not doc:
return (False, 'stream info not found')
might_run = True
if ('control' not in doc) or (type(doc['control']) is not dict):
might_run = False
if might_run:
for key in schema['control']:
if (key not in doc['control']) or (not doc['control'][key]):
might_run = False
break
if might_run:
connector = controller.NewstwisterConnector(doc['control']['streamer_url'])
res = connector.request_status(doc['control']['process_id'])
# let None here when can not check that it runs
if res is not None:
doc['control']['switch_on'] = True if res else False
else:
doc['control']['switch_on'] = None
if res is False:
# update info in db when we know it has been stopped
timepoint = datetime.datetime.utcnow()
update_set = {'control.switch_on': False, 'control.process_id': None, 'logs.stopped': timepoint}
coll.update({'_id': doc_id}, {'$set': update_set}, upsert=False)
if ('spec' not in doc) or (type(doc['spec']) is not dict):
return (False, 'wrong stream info (spec) in db')
if ('control' not in doc) or (type(doc['control']) is not dict):
return (False, 'wrong stream info (control) in db')
return (True, doc)
def do_delete_one(db, service_id):
'''
remove one service
'''
if not db:
return (False, 'inner application error')
service_id = _get_id_value(service_id)
service_get = get_service_by_id(db, service_id)
if not service_get[0]:
return (False, 'service not found')
service = service_get[1]
coll = db[collection]
cursor = coll.remove({'_id': service_id})
return (True, {'_id': service_id})
def do_delete_one(db, service_id):
'''
remove one service
'''
if not db:
return (False, 'inner application error')
service_id = _get_id_value(service_id)
service_get = get_service_by_id(db, service_id)
if not service_get[0]:
return (False, 'service not found')
service = service_get[1]
coll = db[collection]
cursor = coll.remove({'_id': service_id})
return (True, {'_id': service_id})
def do_set_active_one(db, service_id, set_active):
'''
de/activate a service
'''
if not db:
return (False, 'inner application error')
service_id = _get_id_value(service_id)
service_get = get_service_by_id(db, service_id)
if not service_get[0]:
return (False, 'service not found')
service = service_get[1]
timepoint = datetime.datetime.utcnow()
update_service_set(db, service_id, {FIELD_ACTIVE: set_active, '_updated': timepoint})
return (True, {'_id': service_id})
def do_get_one(db, doc_id):
'''
returns data of a single stream control
'''
if not db:
return (False, 'inner application error')
doc_id = _get_id_value(doc_id)
spec_field = 'report_id'
if type(doc_id) is ObjectId:
spec_field = '_id'
coll = db[collection]
doc = coll.find_one({'feed_type': FEED_TYPE, spec_field: doc_id})
if not doc:
return (False, 'report not found')
return (True, doc)
def do_get_one(db, doc_id):
'''
returns data of a single stream control
'''
if not db:
return (False, 'inner application error')
doc_id = _get_id_value(doc_id)
spec_field = 'report_id'
if type(doc_id) is ObjectId:
spec_field = '_id'
coll = db[collection]
doc = coll.find_one({'feed_type': FEED_TYPE, spec_field: doc_id})
if not doc:
return (False, 'report not found')
return (True, doc)
def extract_annotation_status(db, report):
# taking annotation from status
is_summary = False
if (FIELD_SUMMARY_REPORT in report) and (report[FIELD_SUMMARY_REPORT]
is not None):
is_summary = _get_boolean(report[FIELD_SUMMARY_REPORT])
status_desc = None
status_filled = False
if (FIELD_STATUS_REPORT in report) and (report[FIELD_STATUS_REPORT]):
status_filled = True
status_ident = _get_id_value(report[FIELD_STATUS_REPORT])
if type(status_ident) is ObjectId:
status_desc = take_status_desc_by_id(db, status_ident)
else:
status_desc = take_status_desc_by_key(db, status_ident)
if not status_filled:
if (FIELD_STATUS_VERIFIED_REPORT
in report) and (report[FIELD_STATUS_VERIFIED_REPORT]
is not None):
verified_flag = _get_boolean(report[FIELD_STATUS_VERIFIED_REPORT])
if verified_flag:
status_desc = take_status_desc_by_key(db, STATUS_VERIFIED_KEY)
is_assigned = False
if status_desc is None:
if (FIELD_ASSIGNED_REPORT in report) and (type(
report[FIELD_ASSIGNED_REPORT]) in (list, tuple)):
for item in report[FIELD_ASSIGNED_REPORT]:
if item is not None:
is_assigned = True
break
if is_assigned:
status_desc = take_status_desc_by_key(db, STATUS_ASSIGNED_KEY)
if (status_desc is None) and (not is_assigned) and (not is_summary):
status_desc = take_status_desc_by_key(db, STATUS_NEW_KEY)
if not status_desc:
status_desc = None
return status_desc
def do_unpublish_one(db, coverage_id):
'''
unpublish all reports from a coverage
'''
if not db:
return (False, 'inner application error')
coverage_id = _get_id_value(coverage_id)
timepoint = datetime.datetime.utcnow()
update_set = {
FIELD_UPDATED_REPORT: timepoint,
#'_etag': _get_etag(),
}
excise_set = {FIELD_COVERAGES_PUBLISHED: [coverage_id]}
coll = db[collection_reports]
coll.update({FIELD_COVERAGES_PUBLISHED: coverage_id}, {'$pullAll': excise_set, '$set': update_set}, multi=True, upsert=False)
return (True, {'_id': coverage_id})
def do_patch_one(db, doc_id=None, data=None):
'''
sets the "proto" field value
'''
if not db:
return (False, 'inner application error')
if data is None:
return (False, 'data not provided')
if doc_id is not None:
doc_id = _get_id_value(doc_id)
if type(data) is not dict:
return (False, 'data should be a dict')
if ('proto' not in data) or (data['proto'] is None):
return (False, 'data should contain a "proto" field')
try:
proto = bool(_get_boolean(data['proto']))
except:
return (False, 'the "proto" field has to be a boolean value')
res = do_get_one(db, doc_id)
if not res[0]:
return res
coll = db[collection]
spec_field = 'report_id'
if type(doc_id) is ObjectId:
spec_field = '_id'
update_set = {}
update_set['proto'] = proto
update_set[FIELD_UPDATED] = datetime.datetime.utcnow()
#update_set['_etag'] = _get_etag()
coll.update({'feed_type': FEED_TYPE, spec_field: doc_id}, {'$set': update_set}, upsert=False)
return (True, {'report_id': doc_id})
def do_set_active_one(db, service_id, set_active):
'''
de/activate a service
'''
if not db:
return (False, 'inner application error')
service_id = _get_id_value(service_id)
service_get = get_service_by_id(db, service_id)
if not service_get[0]:
return (False, 'service not found')
service = service_get[1]
timepoint = datetime.datetime.utcnow()
update_service_set(db, service_id, {
FIELD_ACTIVE: set_active,
'_updated': timepoint
})
return (True, {'_id': service_id})
def extract_annotation_status(db, report):
# taking annotation from status
is_summary = False
if (FIELD_SUMMARY_REPORT in report) and (report[FIELD_SUMMARY_REPORT] is not None):
is_summary = _get_boolean(report[FIELD_SUMMARY_REPORT])
status_desc = None
status_filled = False
if (FIELD_STATUS_REPORT in report) and (report[FIELD_STATUS_REPORT]):
status_filled = True
status_ident = _get_id_value(report[FIELD_STATUS_REPORT])
if type(status_ident) is ObjectId:
status_desc = take_status_desc_by_id(db, status_ident)
else:
status_desc = take_status_desc_by_key(db, status_ident)
if not status_filled:
if (FIELD_STATUS_VERIFIED_REPORT in report) and (report[FIELD_STATUS_VERIFIED_REPORT] is not None):
verified_flag = _get_boolean(report[FIELD_STATUS_VERIFIED_REPORT])
if verified_flag:
status_desc = take_status_desc_by_key(db, STATUS_VERIFIED_KEY)
is_assigned = False
if status_desc is None:
if (FIELD_ASSIGNED_REPORT in report) and (type(report[FIELD_ASSIGNED_REPORT]) in (list, tuple)):
for item in report[FIELD_ASSIGNED_REPORT]:
if item is not None:
is_assigned = True
break
if is_assigned:
status_desc = take_status_desc_by_key(db, STATUS_ASSIGNED_KEY)
if (status_desc is None) and (not is_assigned) and (not is_summary):
status_desc = take_status_desc_by_key(db, STATUS_NEW_KEY)
if not status_desc:
status_desc = None
return status_desc
def get_coverage_info(db, coverage_id):
coverage_id = _get_id_value(coverage_id)
coll = db[COLL_COVERAGES]
if type(coverage_id) is ObjectId:
return (False, 'old form of coverage info retrieval')
else:
coverage = coll.find_one({'uuid': coverage_id})
if not coverage:
return (False, 'coverage not found')
base_url = get_conf('reports_url')
reports_url = base_url.replace(PUBLISHED_REPORTS_PLACEHOLDER,
urllib.quote_plus(str(coverage_id)))
title = ''
if ('title' in coverage) and coverage['title']:
title = coverage['title']
description = ''
if ('description' in coverage) and coverage['description']:
description = coverage['description']
active = False
if (FIELD_ACTIVE_COVERAGE in coverage) and coverage[FIELD_ACTIVE_COVERAGE]:
active = coverage[FIELD_ACTIVE_COVERAGE]
info = {
'IsLive': active,
'Title': title,
'Description': description,
'PostPublished': {
'href': reports_url,
},
}
return (True, info)
def do_get_one(db, alias_type, alias_value):
'''
returns data of a single citizen alias
'''
if not db:
return (False, 'inner application error')
res = None
if 'alias_id' == alias_type:
alias_value = _get_id_value(alias_value)
res = get_one_by_id(db, alias_value)
if 'phone_number' == alias_type:
try:
phone_number = str(alias_value)
except:
return (False, 'wrong phone number form')
res = get_one_by_phone_number(db, phone_number)
if not res:
return (False, 'unknown form of citizen alias')
return res
def do_get_one(db, alias_type, alias_value):
'''
returns data of a single citizen alias
'''
if not db:
return (False, 'inner application error')
res = None
if 'alias_id' == alias_type:
alias_value = _get_id_value(alias_value)
res = get_one_by_id(db, alias_value)
if 'phone_number' == alias_type:
try:
phone_number = str(alias_value)
except:
return (False, 'wrong phone number form')
res = get_one_by_phone_number(db, phone_number)
if not res:
return (False, 'unknown form of citizen alias')
return res
def do_delete_one(db, doc_id):
'''
deletes data of a single stream info
'''
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
doc_id = _get_id_value(doc_id)
coll = db[collection]
doc = coll.find_one({'_id': doc_id})
if not doc:
return (False, 'requested stream not found')
might_run = True
if ('control' not in doc) or (type(doc['control']) is not dict):
might_run = False
if might_run:
for key in schema['control']:
if (key not in doc['control']) or (not doc['control'][key]):
might_run = False
break
if might_run:
connector = controller.NewstwisterConnector(doc['control']['streamer_url'])
res = connector.request_stop(doc['control']['process_id'])
if not res:
return (False, 'can not stop the stream')
coll.remove({'_id': doc_id})
return (True, {'_id': doc_id})
def do_post(holder, tweet_id, tweet, channel_type, endpoint, request_id, feed_filter, client_ip):
try:
endpoint_id = endpoint['endpoint_id']
except:
return (False, 'endpoint[endpoint_id] not provided',)
endpoint_id = _get_id_value(endpoint_id)
request_id = _get_id_value(request_id)
feed_type = get_conf('feed_type')
# check if the tweet is a new tweet, already saved tweet, a retweet, or a retweet on an already saved tweet
# for a retweet, set endorsers (and may be comments if a new one)
retweeted_report = None
retweeted_id = None
retweeted_tweet = None
try:
current_sub = tweet
while ('retweeted_status' in current_sub) and current_sub['retweeted_status']:
current_sub = current_sub['retweeted_status']
retweeted_tweet = current_sub
retweeted_id = retweeted_tweet['id_str']
except:
retweeted_id = None
retweeted_tweet = None
# we need to have endorsers extracted early for extracting the reasons
report_endorsers = []
if retweeted_id:
try:
endorser_ids = [{'type':'user_id', 'value':tweet['user']['id_str']}, {'type':'user_name', 'value':tweet['user']['screen_name']}]
report_endorsers = [{'authority': 'twitter', 'identifiers': endorser_ids}]
except:
return (False, 'can not create endorsers',)
# if tweet already saved, add channels part
main_report_id = gen_id(feed_type, (retweeted_id if retweeted_id else tweet_id))
tweet_report = get_tweet(main_report_id)
if tweet_report:
main_report_id = tweet_report['report_id']
one_channel = {'type': channel_type, 'value': endpoint_id, 'request': request_id, 'filter': feed_filter, 'reasons': {}}
try:
expanded_text = _get_expanded_text(tweet_report['original'])
reasons = {}
if 'stream' == channel_type:
reasons = find_stream_reason(feed_filter, expanded_text, tweet_report['authors'], report_endorsers, tweet_report['recipients'])
if 'search' == channel_type:
reasons = find_search_reason(feed_filter, expanded_text, tweet_report['authors'], report_endorsers, tweet_report['recipients'])
one_channel['reasons'] = reasons
except Exception as exc:
sys.stderr.write(str(exc) + '\n')
return (False, 'can extract the reasons')
holder.add_channels(feed_type, main_report_id, [one_channel])
# for a retweet, set endorsers for the original tweet
if retweeted_id:
try:
holder.add_endorsers(feed_type, main_report_id, report_endorsers)
except:
return (False, 'can not add endorsers',)
else:
main_tweet_id = retweeted_id if retweeted_id else tweet_id
main_tweet = retweeted_tweet if retweeted_tweet else tweet
res = process_new_tweet(holder, main_tweet_id, main_tweet, channel_type, endpoint_id, request_id, feed_filter, report_endorsers, client_ip)
if not res[0]:
return (False, res[1])
return (True,)
def do_post_reply(db, sms_gateway_api, sms_gateway_url, sms_gateway_key, message, report_id, user_id, language, sensitive, client_ip):
try:
controller = importlib.import_module('citizendesk.feeds.sms.external.' + sms_gateway_api)
except:
controller = None
if not controller:
return (False, 'no sms gateway controller available')
if not db:
return (False, 'no database available')
if not sms_gateway_url:
return (False, 'no sms gateway url configured')
if not sms_gateway_key:
return (False, 'no sms gateway key configured')
if not message:
return (False, 'no message provided')
if not report_id:
return (False, 'report_id not provided')
from citizendesk.feeds.sms.report.storage import do_get_one_by_id as get_one_report_by_id
from citizendesk.feeds.sms.citizen_alias.storage import get_one_by_id as get_one_citizen_alias_by_id
from citizendesk.feeds.sms.citizen_alias.storage import get_one_by_phone_number as get_one_citizen_alias_by_phone_number
report_id = _get_id_value(report_id)
report_res = get_one_report_by_id(db, report_id)
if not report_res[0]:
return (False, 'report of given id not found')
report_source = report_res[1]
was_sent = False
was_received = False
channel_type = get_conf('channel_type')
channel_value_send = get_conf('channel_value_send')
channel_value_receive = get_conf('channel_value_receive')
alias_doctype = get_conf('alias_doctype')
authority = get_conf('authority')
if type(report_source) is not dict:
return (False, 'wrong structure of report')
if ('channels' in report_source) and (type(report_source['channels']) in (list, tuple)):
for one_channel in report_source['channels']:
if (type(one_channel) is dict) and ('type' in one_channel) and ('value' in one_channel):
if one_channel['type'] != channel_type:
continue
if one_channel['value'] == channel_value_send:
was_sent = True
if one_channel['value'] == channel_value_receive:
was_received = True
if (not was_sent) and (not was_received):
return (False, 'unknown form of report')
citizen_aliases = []
if was_sent and ('targets' in report_source) and (type(report_source['targets']) in (list, tuple)):
for one_target in report_source['targets']:
if (type(one_target) is dict) and ('type' in one_target) and ('value' in one_target):
if one_target['type'] == alias_doctype:
one_target_value = _get_id_value(one_target['value'])
one_citizen_alias_res = get_one_citizen_alias_by_id(db, one_target_value)
if one_citizen_alias_res[0]:
citizen_aliases.append(one_citizen_alias_res[1])
if was_received and ('authors' in report_source) and (type(report_source['authors']) in (list, tuple)):
for one_author in report_source['authors']:
one_phone_number = _get_phone_number(one_author)
if not one_phone_number:
continue
one_citizen_alias_res = get_one_citizen_alias_by_phone_number(db, one_phone_number)
if one_citizen_alias_res[0]:
citizen_aliases.append(one_citizen_alias_res[1])
if not citizen_aliases:
return (False, 'not target suitable citizen alias found')
use_targets = []
use_recipients = []
use_phone_numbers = []
for one_citizen_alias in citizen_aliases:
if (type(one_citizen_alias) is not dict):
continue
if ('_id' not in one_citizen_alias) or (not one_citizen_alias['_id']):
continue
one_target = {'type': alias_doctype, 'value': one_citizen_alias['_id']}
if ('identifiers' not in one_citizen_alias) or (not one_citizen_alias['identifiers']):
continue
one_recipient = {'authority': authority, 'identifiers': one_citizen_alias['identifiers']}
one_phone_number = _get_phone_number(one_citizen_alias)
if not one_phone_number:
continue
use_phone_numbers.append(one_phone_number)
use_recipients.append(one_recipient)
use_targets.append(one_target)
if not use_targets:
return (False, 'no valid recipient found')
# putting all the recipients into a single report; and thus using its session for next communication with all the recipients
try:
report = _prepare_sms_reply_report(report_source, use_targets, use_recipients, message, user_id, language, sensitive, client_ip)
except:
report = None
if not report:
return (False, 'report could not be prepared')
# we either save the report before sms sending, and deleting it if sending fails,
# or we first send sms, and if success on it, we save the report then;
# thus either transiently having a false report, or a possibility of not having the report
# on sent sms if the report saving fails at the end (should not hopefully though)
doc_id = report_holder.save_report(report)
if not doc_id:
return (False, 'report could not be saved')
connector = controller.SMSConnector(sms_gateway_url, sms_gateway_key)
res = connector.send_sms(message, {'phone_numbers': use_phone_numbers})
if not res[0]:
report_holder.delete_report(doc_id)
return (False, 'message could not be sent', res[1])
return (True, {'_id': doc_id})
def prepare_sms_send_report(targets,
recipients,
message,
user_id=None,
language=None,
sensitive=None,
client_ip=None):
''' preparing an sms-based report when the sms is sent not as a reply, i.e. starting a new session for it '''
channel_type = get_conf('channel_type')
channel_value = get_conf('channel_value_send')
feed_type = get_conf('feed_type')
if user_id:
user_id = _get_id_value(user_id)
channel = {
'type': channel_type,
'value': channel_value,
'filter': None,
'reasons': None,
'request': None
}
current_timestamp = datetime.datetime.utcnow()
doc = {
'report_id':
gen_id(feed_type, channel_type, channel_value,
current_timestamp), # to generate the report_id
'channels': [channel],
'recipients':
recipients,
'authors': [], # no citizen here
'endorsers': [], # no citizen here
'publisher':
get_conf('publisher'),
'feed_type':
feed_type,
'parent_id':
None, # not a reply here
'session':
None,
'user_id':
user_id, # who requested the sending, if not automatic
'pinned_id':
None,
'language':
language,
'produced':
current_timestamp,
'_created':
current_timestamp,
'_updated':
current_timestamp,
'assignments': [],
'original_id':
None,
'original': {
'message': message
},
'texts': [{
'original': message,
'transcript': None
}],
'tags': [], # we shall extract possible #tags, alike at sms receiving
'sensitive':
None,
'summary':
False,
'local':
True,
'targets':
targets, # alias_ids and/or group_ids for replies, on sent reports
'proto':
False,
'client_ip':
client_ip # for logging
}
if sensitive is not None:
doc['sensitive'] = _get_boolean(sensitive)
doc['session'] = doc['report_id']
doc['tags'] = _extract_tags(message)
return doc
def do_publish_one(db, report_id, coverage_id=None):
'''
sets report as published in a coverage; the coverage_id can be either _id or uuid
'''
if not db:
return (False, 'inner application error')
report_id = _get_id_value(report_id)
report_get = get_report_by_id(db, report_id)
if not report_get[0]:
return (False, 'report not found')
report = report_get[1]
if 'coverages' not in report:
coverages = {}
for cov_set in COVERAGE_SETS:
coverages[cov_set] = []
update_report_set(db, report_id, {'coverages': coverages})
report['coverages'] = coverages
coverages = report['coverages']
if type(coverages) is not dict:
return (False, 'report coverages not a dict')
for cov_set in COVERAGE_SETS:
if (cov_set not in coverages) or (type(coverages[cov_set])
is not list):
return (False, 'report coverage parts missing or wrong')
to_publish = coverages['targeting']
if coverage_id:
coverage_uuid = None
coverage_id = _get_id_value(coverage_id)
if type(coverage_id) is ObjectId:
coverage_get = get_coverage_by_id(db, coverage_id)
else:
coverage_uuid = coverage_id
coverage_get = get_coverage_by_uuid(db, coverage_uuid)
if not coverage_get[0]:
return (False, 'coverage not found')
if not coverage_uuid:
# here we know the coverage_id was _id of an existent coverage
coverage_got = coverage_get[1]
if ('uuid' in coverage_got) and coverage_got['uuid']:
coverage_uuid = coverage_got['uuid']
else:
coverage_uuid = str(uuid.uuid4().hex)
update_coverage_set({'_id': coverage_id},
{'uuid': coverage_uuid})
to_publish = [coverage_uuid]
cov_published = coverages['published']
cov_outgested = coverages['outgested']
set_published = False
set_outgested = False
if not to_publish:
return (False, 'no coverage to be published in')
for one_cov in to_publish:
if one_cov not in cov_published:
set_published = True
cov_published.append(one_cov)
if one_cov not in cov_outgested:
set_outgested = True
cov_outgested.append(one_cov)
if set_published:
update_report_set(db, report_id,
{'coverages.published': cov_published})
if set_outgested:
update_report_set(db, report_id,
{'coverages.outgested': cov_outgested})
timepoint = datetime.datetime.utcnow()
adjective_set = {
FIELD_UPDATED: timepoint,
'proto': False,
}
if FIELD_UUID not in report:
adjective_set[FIELD_UUID] = str(uuid.uuid4().hex)
update_report_set(db, report_id, adjective_set)
return (True, {'_id': report_id})
def do_unpublish_one(db, report_id, coverage_id=None):
'''
sets report as unpublished in a coverage; the coverage_id can be either _id or uuid
'''
if not db:
return (False, 'inner application error')
report_id = _get_id_value(report_id)
report_get = get_report_by_id(db, report_id)
if not report_get[0]:
return (False, 'report not found')
report = report_get[1]
if 'coverages' not in report:
coverages = {}
for cov_set in COVERAGE_SETS:
coverages[cov_set] = []
update_report_set(db, report_id, {'coverages': coverages})
report['coverages'] = coverages
coverages = report['coverages']
if type(coverages) is not dict:
return (False, 'report coverages not a dict')
if ('published' not in coverages) or (type(coverages['published'])
is not list):
return (False, 'published coverages missing or wrong in report')
cov_published = []
if coverage_id:
coverage_uuid = None
coverage_id = _get_id_value(coverage_id)
if type(coverage_id) is ObjectId:
coverage_get = get_coverage_by_id(db, coverage_id)
if coverage_get[0] and (
'uuid' in coverage_get[1]) and coverage_get[1]['uuid']:
coverage_uuid = coverage_get[1]['uuid']
else:
coverage_uuid = coverage_id
curr_specifiers = [coverage_id]
if coverage_uuid:
curr_specifiers.append(coverage_uuid)
was_published = False
for one_spec in curr_specifiers:
if (coverage_id in coverages['published']):
was_published = True
break
if not was_published:
return (False, 'not published into the coverage')
for one_cov in coverages['published']:
if one_cov and (one_cov != coverage_id) and (one_cov !=
coverage_uuid):
cov_published.append(one_cov)
update_report_set(db, report_id, {'coverages.published': cov_published})
timepoint = datetime.datetime.utcnow()
adjective_set = {
FIELD_UPDATED: timepoint,
'proto': False,
}
update_report_set(db, report_id, adjective_set)
return (True, {'_id': report_id})
def do_post_one(db, doc_id=None, data=None):
'''
sets data of a single oauth info
'''
if not db:
return (False, 'inner application error')
if data is None:
return (False, 'data not provided')
if ('spec' not in data) or (type(data['spec']) is not dict):
return (False, '"spec" part not provided')
spec = data['spec']
doc_id = _get_id_value(doc_id)
coll = db[collection]
timepoint = datetime.datetime.utcnow()
created = timepoint
updated = timepoint
if doc_id is not None:
entry = coll.find_one({'_id': doc_id})
if not entry:
return (False, '"oauth" of the provided _id not found')
try:
if ('_created' in entry) and entry['_created']:
created = entry['_created']
except:
created = timepoint
doc = {
'_created': created,
'_updated': updated,
'spec': {}
}
for key in schema['spec']:
doc['spec'][key] = None
if key in spec:
doc['spec'][key] = spec[key]
res = _check_schema(doc['spec'])
if not res[0]:
return res
if not doc_id:
try:
if USE_SEQUENCES:
entry = db['counters'].find_and_modify(query={'_id': collection}, update={'$inc': {'next':1}}, new=True, upsert=True, full_response=False)
doc_id = entry['next']
else:
doc_id = ObjectId()
except:
return (False, 'can not create document id')
doc['_id'] = doc_id
doc_id = coll.save(doc)
return (True, {'_id': doc_id})
def do_get_list(db, endpoint_type, endpoint_id, proto=None, offset=None, limit=None, sort=None, other=None):
'''
returns data of a set of reports saved by a stream or a search
'''
if not db:
return (False, 'inner application error')
endpoint_id = _get_id_value(endpoint_id)
list_spec = {'feed_type': FEED_TYPE, 'channels': {'$elemMatch': {'type': endpoint_type, 'value': endpoint_id}}}
if proto is not None:
try:
proto = bool(_get_boolean(proto))
except:
return (False, 'the "proto" specifier has to be a boolean value')
list_spec['proto'] = proto
sort_list = _get_sort(sort)
if not sort_list:
sort_list = [('produced', 1)]
text_only = False
if other and ('text_only' in other) and other['text_only']:
try:
text_only = bool(_get_boolean(other['text_only']))
except:
text_only = False
coll = db[collection]
cursor = coll.find(list_spec).sort(sort_list)
total = cursor.count()
if limit is None:
limit = DEFAULT_LIMIT
if offset:
cursor = cursor.skip(offset)
if limit:
cursor = cursor.limit(limit)
docs = []
for entry in cursor:
if not entry:
continue
if not text_only:
docs.append(entry)
else:
if (not 'texts' in entry):
continue
if (not entry['texts']):
continue
if (type(entry['texts']) not in [list]):
continue
for one_text in entry['texts']:
source = None
if ('sources' in entry) and (type(entry['sources']) is list):
if len(entry['sources']) and (entry['sources'][0]):
source = entry['sources'][0]
if source:
if (type(one_text) is dict):
one_text['source'] = source
docs.append(one_text)
return (True, docs, {'total': total})
def do_patch_one(db, doc_id=None, data=None, force=None):
'''
starts/stops the stream
'''
try:
from citizendesk.feeds.twt.filter.storage import get_one as filter_get_one
except:
return (False, 'filter processor not available')
try:
from citizendesk.feeds.twt.oauth.storage import get_one as oauth_get_one
except:
return (False, 'oauth processor not available')
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
if data is None:
return (False, 'data not provided')
if type(data) is not dict:
return (False, 'invalid data provided')
if ('control' not in data) or (type(data['control']) is not dict):
return (False, '"control" part not provided')
if doc_id is None:
return (False, 'stream _id not provided')
doc_id = _get_id_value(doc_id)
try:
force_val = bool(_get_boolean(force))
except:
force_val = None
coll = db[collection]
entry = coll.find_one({'_id': doc_id})
if not entry:
return (False, '"stream" of the provided _id not found: ' + str(doc_id))
if ('spec' not in entry) or (type(entry['spec']) is not dict):
return (False, '"stream" of the provided _id does not contain "spec" part')
doc = {'control': {}}
for key in schema['control']:
doc['control'][key] = None
if key in data['control']:
doc['control'][key] = data['control'][key]
res = _check_schema(None, doc['control'])
if not res[0]:
return res
should_run = False
if doc['control']['switch_on']:
if not doc['control']['streamer_url']:
return (False, 'can not start the stream without the "streamer_url" being set')
else:
should_run = True
filter_id = None
oauth_id = None
if should_run:
if ('spec' in entry) and (type(entry['spec']) is dict):
if 'filter_id' in entry['spec']:
filter_id = entry['spec']['filter_id']
if 'oauth_id' in entry['spec']:
oauth_id = entry['spec']['oauth_id']
if (not filter_id) or (not oauth_id):
return (False, 'Can not start the stream without assigned filter_id and oauth_id')
new_stream_url = None
connector = None
if should_run:
new_stream_url = doc['control']['streamer_url']
connector = controller.NewstwisterConnector(new_stream_url)
# load the supplemental data
filter_info = None
oauth_info = None
if should_run:
res = filter_get_one(db, filter_id)
if not res[0]:
return (False, 'filter info with _id equal to "spec.filter_id" not found')
filter_info = res[1]
if ('spec' not in filter_info) or (type(filter_info['spec']) is not dict):
return (False, 'set filter without "spec" part')
res = oauth_get_one(db, oauth_id)
if not res[0]:
return (False, 'oauth info with _id equal to "spec.oauth_id" not found')
oauth_info = res[1]
if ('spec' not in oauth_info) or (type(oauth_info['spec']) is not dict):
return (False, 'set oauth without "spec" part')
filter_spec = {}
filter_spec_original = {}
if should_run and filter_info and filter_info['spec']:
try:
prep_res = _prepare_filter(db, connector, filter_info['spec'])
if not prep_res[0]:
return (False, prep_res[1])
filter_spec = prep_res[1]
filter_spec_original = prep_res[2]
except Exception as exc:
return (False, 'can not prepare filter params: ' + str(exc))
if not filter_spec:
return (False, 'no filter spec was prepared')
if not filter_spec_original:
return (False, 'no filter spec original was prepared')
# check if the oauth_id is not used by any other running feed
if should_run:
running_count = coll.find({'spec.oauth_id': oauth_id, 'control.switch_on': True, '_id': {'$ne': doc_id}}).count()
if running_count:
return (False, 'the "oauth_id" is already used at a running stream')
# stop first in any case, since filter/oauth specs might have changed
might_run = True
if not entry:
might_run = False
if might_run:
if ('control' not in entry) or (type(entry['control']) is not dict):
might_run = False
if might_run:
for key in schema['control']:
if (key not in entry['control']) or (not entry['control'][key]):
might_run = False
break
if might_run:
cur_stream_url = entry['control']['streamer_url']
connector = controller.NewstwisterConnector(cur_stream_url)
res = connector.request_stop(entry['control']['process_id'])
if (not res) and (not force_val):
return (False, 'can not stop the stream')
# update info in db when we know it has been stopped
timepoint = datetime.datetime.utcnow()
update_set = {'control.switch_on': False, 'control.process_id': None, 'logs.stopped': timepoint}
coll.update({'_id': doc_id}, {'$set': update_set}, upsert=False)
if should_run:
res = connector.request_start(str(doc_id), oauth_info['spec'], filter_spec, filter_spec_original)
if not res:
return (False, 'can not start the stream')
try:
proc_id = int(res)
except:
proc_id = res
# update info in db when we know it has been started
timepoint = datetime.datetime.utcnow()
update_set = {'control.switch_on': True, 'control.process_id': proc_id, 'control.streamer_url': new_stream_url, 'logs.started': timepoint}
coll.update({'_id': doc_id}, {'$set': update_set}, upsert=False)
return (True, {'_id': doc_id})
def do_post_one(db, alias_id, alias_spec, user_id):
'''
sets a basic info on phone_number-based citizen alias
'''
if not db:
return (False, 'inner application error')
if type(alias_spec) is not dict:
return (False, 'citizen specification should be a dict')
if 'phone_number' not in alias_spec:
return (False, 'phone number specification not provided')
use_phone_number = alias_spec['phone_number']
try:
use_phone_number = str(use_phone_number)
except:
return (False, 'wrong specification of phone number')
if not use_phone_number:
return (False, 'empty phone number specification')
user_id = _get_id_value(user_id)
if not alias_id:
search_res = get_one_by_phone_number(db, use_phone_number)
if search_res[0]:
return (False, 'active citizen alias with given phone number already exists', search_res[1]['_id'])
alias_doc = None
if alias_id:
alias_id = _get_id_value(alias_id)
alias_doc = get_one_by_id(db, alias_id)
if not alias_doc:
return (False, 'citizen alias of provided id not found')
spec_keys = {
'description': [str, unicode],
'name_first': [str, unicode],
'name_last': [str, unicode],
'name_full': [str, unicode],
'languages': [list, tuple],
'places': [list, tuple],
'home_pages': [list, tuple],
'verified': [bool]
}
use_identifiers = _create_phone_number_identities(use_phone_number)
if not alias_doc:
alias_use = {
'authority': AUTHORITY,
'identifiers': use_identifiers,
'verified': False,
'local': True,
'created_by': user_id
}
for key in spec_keys:
if (key in alias_spec) and (type(alias_spec[key]) in spec_keys[key]):
alias_use[key] = alias_spec[key]
alias_id = citizen_holder.save_alias(alias_use)
if not alias_id:
return (False, 'can not save the citizen alias')
else:
alias_use = {}
alias_use['identifiers'] = use_identifiers
alias_use['updated_by'] = user_id
alias_use['local'] = True
for key in spec_keys:
if (key in alias_spec) and (type(alias_spec[key]) in spec_keys[key]):
alias_use[key] = alias_spec[key]
#alias_use['_etag'] = _get_etag()
coll = db[collection]
alias_id = coll.update({'_id': alias_doc['_id']}, {'$set': alias_use}, upsert=False)
if not alias_id:
return (False, 'can not update the citizen alias')
return (True, {'_id': alias_id})
def get_coverage_published_report_list(db, coverage_id, cid_last):
'''
SMS: original/transcribed text, can have two comments
both creator and author are sms-based user/citizen, of smsblog source type (source name is name of sms feed)
http://sourcefabric.superdesk.pro/resources/LiveDesk/Blog/1/Post/854
tweet: the original tweet (with some predefined adjusting), without any (other) changes, can have two comments
creator is a local user, author is just the twitter source
http://sourcefabric.superdesk.pro/resources/LiveDesk/Blog/1/Post/855
plain: a text
http://sourcefabric.superdesk.pro/resources/LiveDesk/Blog/1/Post/856
link: a web link
http://sourcefabric.superdesk.pro/resources/LiveDesk/Blog/38/Post/1070
other: ignoring by now
'''
try:
coverage_id = _get_id_value(coverage_id)
except:
pass
res_none = {
'total': 0,
'limit': 0,
'offset': 0,
'PostList': []
}
coll = db[COLL_COVERAGES]
if type(coverage_id) is ObjectId:
return (False, 'old form of coverage reports retrieval')
else:
coverage = coll.find_one({'uuid': coverage_id})
if not coverage:
return (False, 'coverage not found')
if (FIELD_ACTIVE_COVERAGE not in coverage) or (not coverage[FIELD_ACTIVE_COVERAGE]):
return (True, res_none)
author_url_template = get_conf('author_url')
creator_url_template = get_conf('creator_url')
coll = db[COLL_REPORTS]
reports = []
put_up_border = datetime.datetime.utcfromtimestamp(0)
search_spec = {
'coverages.outgested': {'$in': [coverage_id, coverage['_id']]},
FIELD_DECAYED_REPORT: {'$ne': True},
}
if cid_last:
update_last = update_from_cid(cid_last)
search_spec[FIELD_UPDATED_REPORT] = {'$gt': update_last}
put_up_border = update_last
else:
search_spec[FIELD_UPDATED_REPORT] = {'$exists': True}
# probably some adjusted dealing with: local, summary, unpublished/deleted reports
cursor = coll.find(search_spec).sort([(FIELD_UPDATED_REPORT, 1)])
for entry in cursor:
if 'feed_type' not in entry:
entry['feed_type'] = DEFAULT_FIELD_TYPE
with_fields = True
for key in [FIELD_UPDATED_REPORT, FIELD_UUID_REPORT, 'feed_type', 'texts', 'coverages']:
if (key not in entry) or (not entry[key]):
with_fields = False
break
if not with_fields:
continue
if type(entry['coverages']) is not dict:
continue
if ('published' not in entry['coverages']):
continue
if type(entry['coverages']['published']) not in (list, tuple):
continue
feed_type = entry['feed_type']
if feed_type not in OUTPUT_FEED_TYPES:
continue
content = None
meta = None
post_type_key = 'normal'
if 'sms' == feed_type:
adapted = adapt_sms_report(db, entry)
content = adapted['content']
meta = adapted['meta']
if 'tweet' == feed_type:
adapted = adapt_tweet_report(db, entry)
content = adapted['content']
meta = adapted['meta']
if 'plain' == feed_type:
adapted = adapt_plain_report(db, entry)
content = adapted['content']
meta = adapted['meta']
if 'web_link' == feed_type:
adapted = adapt_link_report(db, entry)
content = adapted['content']
meta = adapted['meta']
post_type_key = 'link'
use_cid = 0
if FIELD_UPDATED_REPORT in entry:
use_cid = cid_from_update(entry[FIELD_UPDATED_REPORT])
one_report = {
'CId': use_cid,
'IsPublished': 'True',
'Type': {'Key': post_type_key},
'Content': content,
'Meta': meta,
}
if FIELD_UUID_REPORT in entry:
one_report['Uuid'] = entry[FIELD_UUID_REPORT]
if (FIELD_PUTUP_REPORT in entry) and entry[FIELD_PUTUP_REPORT]:
try:
if entry[FIELD_PUTUP_REPORT] > put_up_border:
one_report['PutUp'] = 'True'
except:
pass
if coverage_id not in entry['coverages']['published']:
if coverage['_id'] not in entry['coverages']['published']:
one_report['DeletedOn'] = '01/01/70 12:01 AM'
link_id = urllib.quote_plus(str(entry['_id']))
author_url = author_url_template.replace(REPORT_LINK_ID_PLACEHOLDER, link_id)
creator_url = creator_url_template.replace(REPORT_LINK_ID_PLACEHOLDER, link_id)
one_report['Author'] = {'href': author_url}
one_report['Creator'] = {'href': creator_url}
reports.append(one_report)
res = {
'total': len(reports),
'limit': len(reports),
'offset': 0,
'PostList': reports
}
return (True, res)
def do_finalize_one(db, auther_url, doc_id, data):
'''
finalizes authorization initialization on info of one app data
'''
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
if data is None:
return (False, 'data not provided')
if ('spec' not in data) or (type(data['spec']) is not dict):
return (False, '"spec" part not provided')
spec = data['spec']
coll = db[collection]
if doc_id is None:
return (False, 'document id not provided')
doc_id = _get_id_value(doc_id)
entry = coll.find_one({'_id': doc_id})
if not entry:
return (False, 'document of the provided _id not found')
if ('spec' not in entry) or (type(entry['spec']) is not dict):
return (False, 'damaged document structure')
spec = entry['spec']
needed_keys = ['app_consumer_key', 'app_consumer_secret', 'temporary_access_token_key', 'temporary_access_token_secret']
for one_key in needed_keys:
if (one_key not in spec) or (not spec[one_key]):
return (False, 'document missing the "' + str(one_key) + '" part')
if ('spec' not in data) or (type(data['spec']) is not dict):
return (False, '"spec" part not provided')
spec_data = data['spec']
for one_key in ['verifier_pin']:
if (one_key not in spec_data) or (not spec_data[one_key]):
return (False, str(one_key) + ' not provided')
authfin_data = {
'oauth_info': {
'consumer_key': spec['app_consumer_key'],
'consumer_secret': spec['app_consumer_secret'],
'access_token_key': spec['temporary_access_token_key'],
'access_token_secret': spec['temporary_access_token_secret'],
},
'payload': {
'verifier': spec_data['verifier_pin'],
}
}
connector = controller.NewstwisterConnector(auther_url)
res = connector.request_authfin(authfin_data['oauth_info'], authfin_data['payload'])
if not res[0]:
err_msg = 'error during authfin request dispatching: ' + res[1]
return (False, err_msg)
ret_envelope = res[1]
if type(ret_envelope) is not dict:
return (False, 'unknown form of returned authfin data: ' + str(type(ret_envelope)))
if ('status' not in ret_envelope) or (not ret_envelope['status']):
err_msg = ''
if ('error' in ret_envelope) and (ret_envelope['error']):
err_msg = ': ' + str(ret_envelope['error'])
return (False, 'status not acknowledged in returned authfin data' + err_msg)
if ('data' not in ret_envelope) or (not ret_envelope['data']):
return (False, 'payload not provided in returned authfin data')
ret_data = ret_envelope['data']
if type(ret_data) is not dict:
return (False, 'unknown form of returned payload in authfin data: ' + str(type(ret_data)))
for part in ['oauth_token_key', 'oauth_token_secret', 'user_id', 'screen_name']:
if (part not in ret_data) or (not ret_data[part]):
return (False, 'returned authfin data missing the "' + str(part) + '" part')
try:
screen_name_search = ret_data['screen_name'].lower()
except:
screen_name_search = ret_data['screen_name']
timepoint = datetime.datetime.utcnow()
created = timepoint
updated = timepoint
try:
if ('_created' in entry) and entry['_created']:
created = entry['_created']
except:
created = timepoint
save_data = {
'_id': doc_id,
'spec': {
'app_consumer_key': spec['app_consumer_key'],
'app_consumer_secret': spec['app_consumer_secret'],
'temporary_access_token_key': None,
'temporary_access_token_secret': None,
'authorized_access_token_key': ret_data['oauth_token_key'],
'authorized_access_token_secret': ret_data['oauth_token_secret'],
'verifier_url': None,
'user_id': ret_data['user_id'],
'screen_name': ret_data['screen_name'],
'screen_name_search': screen_name_search,
},
'_created': created,
'_updated': updated,
}
coll = db[collection]
doc_id = coll.save(save_data)
if not doc_id:
return (False, 'can not save the authfin data')
return (True, {'_id': doc_id})
def do_post_send(db,
sender_url,
authorized_id,
user_id,
endpoint_id,
tweet_spec,
report_id=None):
'''
sends (a reply to) a tweet
after it is sent and received, local=True, and user_id=user_id are set in the report
'''
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
if not authorized_id:
return (False, 'authorized_id not specified')
if not user_id:
return (False, 'user_id not specified')
if not endpoint_id:
return (False, 'endpoint_id not specified')
if type(tweet_spec) is not dict:
return (False, 'unknown form of tweet spec')
if ('message' not in tweet_spec) or (not tweet_spec['message']):
return (False, 'message text not provided')
follow_part = []
tweet_data = {
'endpoint_id': endpoint_id,
'status': tweet_spec['message'],
'filter': {},
}
if ('sensitive' in tweet_spec) and (tweet_spec['sensitive'] is not None):
sensitive = _get_boolean(tweet_spec['sensitive'])
if sensitive:
tweet_data['possibly_sensitive'] = 'true'
if ('display_coordinates'
in tweet_spec) and (tweet_spec['display_coordinates'] is not None):
display_coordinates = _get_boolean(tweet_spec['display_coordinates'])
if display_coordinates:
tweet_data['display_coordinates'] = 'true'
else:
tweet_data['display_coordinates'] = 'false'
for key in ['lat', 'long', 'place_id']:
if (key in tweet_spec) and (tweet_spec[key]):
try:
tweet_data[key] = str(tweet_spec[key])
except:
return (False, 'wrong "' + str(key) + '" part in tweet spec')
if report_id is not None:
report_id = _get_id_value(report_id)
search_spec = {'feed_type': FEED_TYPE}
if type(report_id) is ObjectId:
search_spec['_id'] = report_id
else:
search_spec['report_id'] = report_id
coll = db[collection_reports]
report = coll.find_one(search_spec)
if not report:
return (False, 'specified report not found')
try:
orig_user_screen_name = str(
report['original']['user']['screen_name']).lower()
if orig_user_screen_name not in follow_part:
follow_part.append(orig_user_screen_name)
except:
return (False, 'can not find the original tweet sender')
check_inclusion = '@' + orig_user_screen_name.lower()
try:
if check_inclusion not in tweet_spec['message'].lower():
return (
False,
'mentioning the original tweet sender not found in the tweet text'
)
except:
return (False,
'can not check inclusion of the original tweet sender')
try:
tweet_data['in_reply_to_status_id'] = str(report['original_id'])
except:
return (False, 'can not find id_str of the original tweet')
coll = db[collection_authorized]
authorized_id = _get_id_value(authorized_id)
authorized_data = coll.find_one({'_id': authorized_id})
if not authorized_data:
return (False, 'saved report on the send-tweet not found')
try:
authorized_spec = {
'consumer_key':
authorized_data['spec']['app_consumer_key'],
'consumer_secret':
authorized_data['spec']['app_consumer_secret'],
'access_token_key':
authorized_data['spec']['authorized_access_token_key'],
'access_token_secret':
authorized_data['spec']['authorized_access_token_secret'],
}
sender_screen_name = str(authorized_data['spec']['screen_name_search'])
if sender_screen_name not in follow_part:
follow_part.append(sender_screen_name)
except Exception as exc:
return (False,
'authorized info does not contain all the required data: ' +
str(exc))
for key in authorized_spec:
if not authorized_spec[key]:
return (False,
'the "' + str(key) + '" part of authorized info is empty')
tweet_data['filter']['follow'] = follow_part
connector = controller.NewstwisterConnector(sender_url)
res = connector.send_tweet(authorized_spec, tweet_data)
if not res[0]:
err_msg = 'error during send-tweet request dispatching: ' + res[1]
return (False, err_msg)
ret_envelope = res[1]
if type(ret_envelope) is not dict:
return (False, 'unknown form of returned send-tweet data: ' +
str(type(ret_envelope)))
if ('status' not in ret_envelope) or (not ret_envelope['status']):
err_msg = ''
if ('error' in ret_envelope) and (ret_envelope['error']):
err_msg = ': ' + str(ret_envelope['error'])
return (False, 'status not acknowledged in returned send-tweet data' +
err_msg)
if ('data' not in ret_envelope) or (not ret_envelope['data']):
return (False, 'payload not provided in returned send-tweet data')
ret_data = ret_envelope['data']
if type(ret_data) is not dict:
return (False,
'unknown form of returned payload in send-tweet data: ' +
str(type(ret_data)))
if 'id_str' not in ret_data:
return (False, 'returned send-tweet data without tweet identifier')
coll = db[collection_reports]
saved_tweet = coll.find_one({'original_id': ret_data['id_str']})
if not saved_tweet:
return (False, 'saved report on the send-tweet not found')
doc_id = saved_tweet['_id']
saved_update = {
'local': True,
'user_id': _get_id_value(user_id),
'proto': False,
FIELD_UPDATED: datetime.datetime.utcnow(),
#'_etag': _get_etag(),
}
coll.update({'_id': doc_id}, {'$set': saved_update})
return (True, {'_id': doc_id})
def get_coverage_published_report_list(db, coverage_id, cid_last):
'''
SMS: original/transcribed text, can have two comments
both creator and author are sms-based user/citizen, of smsblog source type (source name is name of sms feed)
http://sourcefabric.superdesk.pro/resources/LiveDesk/Blog/1/Post/854
tweet: the original tweet (with some predefined adjusting), without any (other) changes, can have two comments
creator is a local user, author is just the twitter source
http://sourcefabric.superdesk.pro/resources/LiveDesk/Blog/1/Post/855
plain: a text
http://sourcefabric.superdesk.pro/resources/LiveDesk/Blog/1/Post/856
link: a web link
http://sourcefabric.superdesk.pro/resources/LiveDesk/Blog/38/Post/1070
other: ignoring by now
'''
try:
coverage_id = _get_id_value(coverage_id)
except:
pass
res_none = {'total': 0, 'limit': 0, 'offset': 0, 'PostList': []}
coll = db[COLL_COVERAGES]
if type(coverage_id) is ObjectId:
return (False, 'old form of coverage reports retrieval')
else:
coverage = coll.find_one({'uuid': coverage_id})
if not coverage:
return (False, 'coverage not found')
if (FIELD_ACTIVE_COVERAGE
not in coverage) or (not coverage[FIELD_ACTIVE_COVERAGE]):
return (True, res_none)
author_url_template = get_conf('author_url')
creator_url_template = get_conf('creator_url')
coll = db[COLL_REPORTS]
reports = []
put_up_border = datetime.datetime.utcfromtimestamp(0)
search_spec = {
'coverages.outgested': {
'$in': [coverage_id, coverage['_id']]
},
FIELD_DECAYED_REPORT: {
'$ne': True
},
}
if cid_last:
update_last = update_from_cid(cid_last)
search_spec[FIELD_UPDATED_REPORT] = {'$gt': update_last}
put_up_border = update_last
else:
search_spec[FIELD_UPDATED_REPORT] = {'$exists': True}
# probably some adjusted dealing with: local, summary, unpublished/deleted reports
cursor = coll.find(search_spec).sort([(FIELD_UPDATED_REPORT, 1)])
for entry in cursor:
if 'feed_type' not in entry:
entry['feed_type'] = DEFAULT_FIELD_TYPE
with_fields = True
for key in [
FIELD_UPDATED_REPORT, FIELD_UUID_REPORT, 'feed_type', 'texts',
'coverages'
]:
if (key not in entry) or (not entry[key]):
with_fields = False
break
if not with_fields:
continue
if type(entry['coverages']) is not dict:
continue
if ('published' not in entry['coverages']):
continue
if type(entry['coverages']['published']) not in (list, tuple):
continue
feed_type = entry['feed_type']
if feed_type not in OUTPUT_FEED_TYPES:
continue
content = None
meta = None
post_type_key = 'normal'
if 'sms' == feed_type:
adapted = adapt_sms_report(db, entry)
content = adapted['content']
meta = adapted['meta']
if 'tweet' == feed_type:
adapted = adapt_tweet_report(db, entry)
content = adapted['content']
meta = adapted['meta']
if 'plain' == feed_type:
adapted = adapt_plain_report(db, entry)
content = adapted['content']
meta = adapted['meta']
if 'web_link' == feed_type:
adapted = adapt_link_report(db, entry)
content = adapted['content']
meta = adapted['meta']
post_type_key = 'link'
use_cid = 0
if FIELD_UPDATED_REPORT in entry:
use_cid = cid_from_update(entry[FIELD_UPDATED_REPORT])
one_report = {
'CId': use_cid,
'IsPublished': 'True',
'Type': {
'Key': post_type_key
},
'Content': content,
'Meta': meta,
}
if FIELD_UUID_REPORT in entry:
one_report['Uuid'] = entry[FIELD_UUID_REPORT]
if (FIELD_PUTUP_REPORT in entry) and entry[FIELD_PUTUP_REPORT]:
try:
if entry[FIELD_PUTUP_REPORT] > put_up_border:
one_report['PutUp'] = 'True'
except:
pass
if coverage_id not in entry['coverages']['published']:
if coverage['_id'] not in entry['coverages']['published']:
one_report['DeletedOn'] = '01/01/70 12:01 AM'
link_id = urllib.quote_plus(str(entry['_id']))
author_url = author_url_template.replace(REPORT_LINK_ID_PLACEHOLDER,
link_id)
creator_url = creator_url_template.replace(REPORT_LINK_ID_PLACEHOLDER,
link_id)
one_report['Author'] = {'href': author_url}
one_report['Creator'] = {'href': creator_url}
reports.append(one_report)
res = {
'total': len(reports),
'limit': len(reports),
'offset': 0,
'PostList': reports
}
return (True, res)
def do_post(db, url, channel_type, request_id, client_ip):
'''
* assure citizen_alias exists (i.e. create/save if does not yet)
* create and save the report
* channel_type: frontend, bookmarklet, etc.
* request_id: commonly _id or uuid for filtering
'''
logger = get_logger()
timestamp = datetime.datetime.utcnow()
channel_type = _get_id_value(channel_type)
request_id = _get_id_value(request_id)
if not url:
return (False, 'emtpy url')
try:
url = url.strip()
url_parsed = urlparse(url, scheme='http')
except:
url_parsed = None
if not url_parsed:
return (False, 'can not parse the url')
if not url_parsed.netloc:
return (False, 'url without domain part')
# taking params
feed_type = get_conf('feed_type')
eff_tlds = get_conf('eff_tlds')
# taking info
page_info_got = get_page_info(url)
if not page_info_got[0]:
return (False, 'can not get page info: ' + page_info_got[1])
page_info = page_info_got[1]
report_id = report_holder.gen_id(feed_type)
session_url = ''
if ('url' in page_info) and page_info['url']:
try:
session_url = page_info['url'].split('#')[0].strip().lower()
except:
pass
if not session_url:
session_url = url.split('#')[0].strip().lower()
session = feed_type + '||' + session_url
source_url = ''
if ('url' in page_info) and page_info['url']:
try:
source_url = page_info['url'].strip()
except:
pass
if not source_url:
source_url = url.strip()
domain_effective = ''
if ('domain_name' in page_info) and page_info['domain_name']:
try:
domain_effective = str(page_info['domain_name']).split(':')[0].strip()
except:
pass
if not domain_effective:
domain_effective = url_parsed.netloc.split(':')[0].strip()
if eff_tlds:
citizen_id = take_specific_domain(eff_tlds, domain_effective)
else:
citizen_id = domain_effective
if not citizen_id:
citizen_id = source_url
parent_id = None
publisher = None
if ('feed_name' in page_info) and page_info['feed_name']:
feed_name = page_info['feed_name']
else:
feed_name = None
use_language = None
if ('language' in page_info) and page_info['language']:
use_language = page_info['language']
authority = get_conf('authority')
use_identifiers = {
'user_id': citizen_id,
'user_id_search': citizen_id.lower(),
'user_name': citizen_id,
'user_name_search': citizen_id.lower(),
}
channel_value = None
if ('type' in page_info) and page_info['type']:
try:
channel_value = page_info['type'].strip().lower()
if not channel_value:
channel_value = None
except:
channel_value = None
channels = [{'type': channel_type, 'value': channel_value, 'filter': None, 'request': request_id, 'reasons': None}]
authors = [{'authority': authority, 'identifiers': use_identifiers}]
author_name = ''
if 'author' in page_info:
author_name = page_info['author']
alias_res_got = assure_citizen_alias(db, authority, use_identifiers, author_name)
endorsers = []
original = page_info
original_id = url # notice that we group the (effective) URLs via session
tags = []
texts = []
text_parts = ['title', 'description']
for one_text_part in text_parts:
if (one_text_part in page_info) and page_info[one_text_part]:
texts.append({'original': page_info[one_text_part], 'transcript': None})
estimated = timestamp
if ('date' in page_info) and page_info['date'] and (type(page_info['date']) is datetime.datetime):
estimated = page_info['date']
# site_icon added at the end of the media list
media = []
if ('image' in page_info) and page_info['image'] and (type(page_info['image']) in (list, tuple)):
for one_image_link in page_info['image']:
try:
link_ssl = None
if one_image_link.startswith('https'):
link_ssl = one_image_link
media.append({
'link': one_image_link,
'link_ssl': link_ssl,
'data': None,
'name': None,
'width': None,
'height': None,
})
except:
pass
if ('site_icon' in page_info) and page_info['site_icon']:
try:
link_ssl = None
if page_info['site_icon'].startswith('https'):
link_ssl = page_info['site_icon']
media.append({
'link': page_info['site_icon'],
'link_ssl': link_ssl,
'data': None,
'name': None,
'width': None,
'height': None,
})
except:
pass
report = {}
report['report_id'] = report_id
report['parent_id'] = None
report['client_ip'] = client_ip
report['feed_type'] = feed_type
report['produced'] = estimated
report['session'] = session
report['publisher'] = publisher
report['channels'] = channels
report['authors'] = authors
report['original'] = original
report['texts'] = texts
report['tags'] = tags
report['language'] = use_language
report['sources'] = [source_url]
report['media'] = media
report['editable'] = False
report['proto'] = False
report_doc_id = report_holder.save_report(report)
if not report_doc_id:
return (False, 'can not save URL report')
return (True, {'_id': report_doc_id})
def do_post_send(db, sms_gateway_api, sms_gateway_url, sms_gateway_key, message, targets, user_id, language, sensitive, client_ip):
try:
controller = importlib.import_module('citizendesk.feeds.sms.external.' + sms_gateway_api)
except:
controller = None
if not controller:
return (False, 'no sms gateway controller available')
if not db:
return (False, 'no database available')
if not sms_gateway_url:
return (False, 'no sms gateway url configured')
if not sms_gateway_key:
return (False, 'no sms gateway key configured')
if not message:
return (False, 'no message provided')
if not targets:
return (False, 'no targets provided')
from citizendesk.feeds.sms.citizen_alias.storage import get_one_by_id as get_one_citizen_alias_by_id
use_targets = []
use_recipients = []
use_phone_numbers = []
authority = get_conf('authority')
alias_doctype = get_conf('alias_doctype')
if type(targets) not in [list, tuple]:
return (False, '')
for one_target in targets:
if type(one_target) is not dict:
continue
if 'type' not in one_target:
continue
if one_target['type'] != alias_doctype:
continue
if 'value' not in one_target:
continue
one_target_id = _get_id_value(one_target['value'])
alias_res = get_one_citizen_alias_by_id(db, one_target_id)
if (not alias_res) or (not alias_res[0]):
continue
alias = alias_res[1]
if (type(alias) is not dict) or (not alias):
continue
if ('identifiers' not in alias) or (not alias['identifiers']):
continue
one_phone_number = _get_phone_number(alias)
if not one_phone_number:
continue
one_recipient = {'authority': authority, 'identifiers': alias['identifiers']}
use_phone_numbers.append(one_phone_number)
use_recipients.append(one_recipient)
use_targets.append(one_target)
if not use_targets:
return (False, 'no valid recipient found')
# putting all the recipients into a single report; and thus using its session for next communication with all the recipients
try:
report = _prepare_sms_send_report(use_targets, use_recipients, message, user_id, language, sensitive, client_ip)
except:
report = None
if not report:
return (False, 'report could not be prepared')
# we either save the report before sms sending, and deleting it if sending fails,
# or we first send sms, and if success on it, we save the report then;
# thus either transiently having a false report, or a possibility of not having the report
# on sent sms if the report saving fails at the end (should not hopefully though)
doc_id = report_holder.save_report(report)
if not doc_id:
return (False, 'report could not be saved')
connector = controller.SMSConnector(sms_gateway_url, sms_gateway_key)
res = connector.send_sms(message, {'phone_numbers': use_phone_numbers})
if not res[0]:
report_holder.delete_report(doc_id)
return (False, 'message could not be sent', res[1])
return (True, {'_id': doc_id})
