def do_get_list(db, offset=None, limit=None, sort=None, other=None):
'''
returns data of a set of twt citizen aliases
'''
if not db:
return (False, 'inner application error')
list_spec = {'authority': AUTHORITY}
sort_list = _get_sort(sort)
if not sort_list:
sort_list = [('produced', 1)]
name_only = False
if other and ('name_only' in other) and other['name_only']:
try:
name_only = bool(_get_boolean(other['name_only']))
except:
name_only = False
coll = db[collection]
cursor = coll.find(list_spec).sort(sort_list)
total = cursor.count()
if limit is None:
limit = DEFAULT_LIMIT
if offset:
cursor = cursor.skip(offset)
if limit:
cursor = cursor.limit(limit)
docs = []
for entry in cursor:
if not entry:
continue
if not name_only:
docs.append(entry)
else:
if (not 'identifiers' in entry):
continue
if type(entry['identifiers']) not in (list, tuple):
continue
if not len(entry['identifiers']):
continue
if not entry['identifiers'][0]:
continue
one_name = entry['identifiers'][0]
if type(one_name) is not dict:
continue
source = None
if ('sources' in entry) and (type(entry['sources']) is list):
if len(entry['sources']) and (entry['sources'][0]):
source = entry['sources'][0]
if source:
one_name['source'] = source
docs.append(one_name)
return (True, docs, {'total': total})
def do_get_list(db, offset=None, limit=None, sort=None, other=None):
'''
returns data of a set of twt citizen aliases
'''
if not db:
return (False, 'inner application error')
list_spec = {'authority': AUTHORITY}
sort_list = _get_sort(sort)
if not sort_list:
sort_list = [('produced', 1)]
name_only = False
if other and ('name_only' in other) and other['name_only']:
try:
name_only = bool(_get_boolean(other['name_only']))
except:
name_only = False
coll = db[collection]
cursor = coll.find(list_spec).sort(sort_list)
total = cursor.count()
if limit is None:
limit = DEFAULT_LIMIT
if offset:
cursor = cursor.skip(offset)
if limit:
cursor = cursor.limit(limit)
docs = []
for entry in cursor:
if not entry:
continue
if not name_only:
docs.append(entry)
else:
if (not 'identifiers' in entry):
continue
if type(entry['identifiers']) is not dict:
continue
one_name = {}
if 'user_id' in entry['identifiers']:
one_name['user_id'] = entry['identifiers']['user_id']
if 'user_name' in entry['identifiers']:
one_name['user_name'] = entry['identifiers']['user_name']
source = None
if ('sources' in entry) and (type(entry['sources'])
in (list, tuple)):
if len(entry['sources']) and (entry['sources'][0]):
source = entry['sources'][0]
if source:
one_name['source'] = source
docs.append(one_name)
return (True, docs, {'total': total})
def feed_any_coverage_insert_one():
from citizendesk.feeds.any.coverage import process
logger = get_logger()
client_ip = get_client_ip()
coverage_data = {}
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {
'Content-Type': 'application/json'
})
try:
coverage_data['title'] = data['title']
coverage_data['description'] = data['description']
coverage_data['user_id'] = data['user_id']
except:
return (json.dumps(
'provided data should contain "title", "description", "user_id" parts'
), 404, {
'Content-Type': 'application/json'
})
if 'active' in data:
coverage_data['active'] = _get_boolean(data['active'])
try:
if ('uuid' in data) and data['uuid']:
coverage_data['uuid'] = str(data['uuid'])
except:
pass
res = process.do_insert_one(mongo_dbs.get_db().db, coverage_data)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 404, {
'Content-Type': 'application/json'
})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 200, {
'Content-Type': 'application/json'
})
def extract_annotation_status(db, report):
# taking annotation from status
is_summary = False
if (FIELD_SUMMARY_REPORT in report) and (report[FIELD_SUMMARY_REPORT]
is not None):
is_summary = _get_boolean(report[FIELD_SUMMARY_REPORT])
status_desc = None
status_filled = False
if (FIELD_STATUS_REPORT in report) and (report[FIELD_STATUS_REPORT]):
status_filled = True
status_ident = _get_id_value(report[FIELD_STATUS_REPORT])
if type(status_ident) is ObjectId:
status_desc = take_status_desc_by_id(db, status_ident)
else:
status_desc = take_status_desc_by_key(db, status_ident)
if not status_filled:
if (FIELD_STATUS_VERIFIED_REPORT
in report) and (report[FIELD_STATUS_VERIFIED_REPORT]
is not None):
verified_flag = _get_boolean(report[FIELD_STATUS_VERIFIED_REPORT])
if verified_flag:
status_desc = take_status_desc_by_key(db, STATUS_VERIFIED_KEY)
is_assigned = False
if status_desc is None:
if (FIELD_ASSIGNED_REPORT in report) and (type(
report[FIELD_ASSIGNED_REPORT]) in (list, tuple)):
for item in report[FIELD_ASSIGNED_REPORT]:
if item is not None:
is_assigned = True
break
if is_assigned:
status_desc = take_status_desc_by_key(db, STATUS_ASSIGNED_KEY)
if (status_desc is None) and (not is_assigned) and (not is_summary):
status_desc = take_status_desc_by_key(db, STATUS_NEW_KEY)
if not status_desc:
status_desc = None
return status_desc
def prepare_sms_send_report(targets, recipients, message, user_id=None, language=None, sensitive=None, client_ip=None):
''' preparing an sms-based report when the sms is sent not as a reply, i.e. starting a new session for it '''
channel_type = get_conf('channel_type')
channel_value = get_conf('channel_value_send')
feed_type = get_conf('feed_type')
channel = {
'type': channel_type,
'value': channel_value,
'filter': None,
'reasons': None,
'request': None
}
current_timestamp = datetime.datetime.now()
doc = {
'report_id': gen_id(feed_type, channel_type, channel_value, current_timestamp), # to generate the report_id
'channels': [channel],
'recipients': recipients,
'authors': [], # no citizen here
'endorsers': [], # no citizen here
'publisher': get_conf('publisher'),
'feed_type': feed_type,
'parent_id': None, # not a reply here
'session': None,
'user_id': user_id, # who requested the sending, if not automatic
'pinned_id': None,
'coverage_id': None,
'language': language,
'produced': current_timestamp,
'created': current_timestamp,
'assignments': [],
'original_id': None,
'original': {'message': message},
'texts': [{'original': message, 'transcript': None}],
'tags': [], # we shall extract possible #tags, alike at sms receiving
'is_published': False,
'sensitive': None,
'summary': False,
'local': True,
'targets': targets, # alias_ids and/or group_ids for replies, on sent reports
'proto': False,
'client_ip': client_ip # for logging
}
if sensitive is not None:
doc['sensitive'] = _get_boolean(sensitive)
doc['session'] = doc['report_id']
doc['tags'] = _extract_tags(message)
return doc
def do_get_list(db, offset, limit, sort, other):
'''
returns list of coverages
'''
if not db:
return (False, 'inner application error')
list_spec = {}
sort_list = _get_sort(sort)
if not sort_list:
sort_list = [('produced', 1)]
name_only = False
if other and ('name_only' in other) and other['name_only']:
try:
name_only = bool(_get_boolean(other['name_only']))
except:
name_only = False
coll = db[collection]
cursor = coll.find(list_spec).sort(sort_list)
total = cursor.count()
if limit is None:
limit = DEFAULT_LIMIT
if offset:
cursor = cursor.skip(offset)
if limit:
cursor = cursor.limit(limit)
docs = []
for entry in cursor:
if not entry:
continue
if not name_only:
docs.append(entry)
else:
if 'title' not in entry:
continue
one_name = {
'title': entry['title'],
FIELD_ACTIVE: None,
}
if FIELD_ACTIVE in entry:
one_name[FIELD_ACTIVE] = entry[FIELD_ACTIVE]
docs.append(one_name)
return (True, docs, {'total': total})
def feed_twt_citizen_alias_get_one(alias_type, alias_value):
from citizendesk.feeds.twt.citizen_alias import process
from citizendesk.feeds.config import get_config
logger = get_logger()
client_ip = get_client_ip()
other = {'force': None}
for key in other:
if key in request.args:
other[key] = request.args.get(key)
force = False
if other and ('force' in other):
force = _get_boolean(other['force'])
res = process.do_get_one(mongo_dbs.get_db().db, alias_type, alias_value)
err_reason = None
err_msg = None
if not res[0]:
err_msg = res[1]
if (not res[0]) and force:
err_msg = res[1]
searcher_url = get_config('newstwister_url')
res_aux = process.do_request_one(mongo_dbs.get_db().db, searcher_url,
alias_type, alias_value)
if not res_aux[0]:
err_msg = res_aux[1]
if 2 < len(res_aux):
err_reason = res_aux[2]
else:
res = process.do_get_one(mongo_dbs.get_db().db, alias_type,
alias_value)
if not res[0]:
err_msg = res_aux[1]
err_reason = None
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': err_msg}}
if err_reason:
ret_data['_meta']['reason'] = err_reason
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 404, {
'Content-Type': 'application/json'
})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 200, {
'Content-Type': 'application/json'
})
def extract_annotation_status(db, report):
# taking annotation from status
is_summary = False
if (FIELD_SUMMARY_REPORT in report) and (report[FIELD_SUMMARY_REPORT] is not None):
is_summary = _get_boolean(report[FIELD_SUMMARY_REPORT])
status_desc = None
status_filled = False
if (FIELD_STATUS_REPORT in report) and (report[FIELD_STATUS_REPORT]):
status_filled = True
status_ident = _get_id_value(report[FIELD_STATUS_REPORT])
if type(status_ident) is ObjectId:
status_desc = take_status_desc_by_id(db, status_ident)
else:
status_desc = take_status_desc_by_key(db, status_ident)
if not status_filled:
if (FIELD_STATUS_VERIFIED_REPORT in report) and (report[FIELD_STATUS_VERIFIED_REPORT] is not None):
verified_flag = _get_boolean(report[FIELD_STATUS_VERIFIED_REPORT])
if verified_flag:
status_desc = take_status_desc_by_key(db, STATUS_VERIFIED_KEY)
is_assigned = False
if status_desc is None:
if (FIELD_ASSIGNED_REPORT in report) and (type(report[FIELD_ASSIGNED_REPORT]) in (list, tuple)):
for item in report[FIELD_ASSIGNED_REPORT]:
if item is not None:
is_assigned = True
break
if is_assigned:
status_desc = take_status_desc_by_key(db, STATUS_ASSIGNED_KEY)
if (status_desc is None) and (not is_assigned) and (not is_summary):
status_desc = take_status_desc_by_key(db, STATUS_NEW_KEY)
if not status_desc:
status_desc = None
return status_desc
def setup_config(workingon_config_data):
if type(workingon_config_data) is not dict:
return False
if (CONFIG_USE_SETUP not in workingon_config_data) or (not workingon_config_data[CONFIG_USE_SETUP]):
return False
to_setup = _get_boolean(workingon_config_data[CONFIG_USE_SETUP])
if not to_setup:
return False
if CONFIG_USE_REPORTS in workingon_config_data:
if (workingon_config_data[CONFIG_USE_REPORTS]) and (workingon_config_data[CONFIG_USE_REPORTS] != 'assignments'):
set_conf('field_public_report', workingon_config_data[CONFIG_USE_REPORTS])
set_conf('condition_public_report', True)
return True
def feed_any_coverage_insert_one():
from citizendesk.feeds.any.coverage import process
logger = get_logger()
client_ip = get_client_ip()
coverage_data = {}
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {'Content-Type': 'application/json'})
try:
coverage_data['title'] = data['title']
coverage_data['description'] = data['description']
coverage_data['user_id'] = data['user_id']
except:
return (json.dumps('provided data should contain "title", "description", "user_id" parts'), 404, {'Content-Type': 'application/json'})
if 'active' in data:
coverage_data['active'] = _get_boolean(data['active'])
try:
if ('uuid' in data) and data['uuid']:
coverage_data['uuid'] = str(data['uuid'])
except:
pass
res = process.do_insert_one(mongo_dbs.get_db().db, coverage_data)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def feed_twt_citizen_alias_get_one(alias_type, alias_value):
from citizendesk.feeds.twt.citizen_alias import process
from citizendesk.feeds.config import get_config
logger = get_logger()
client_ip = get_client_ip()
other = {'force': None}
for key in other:
if key in request.args:
other[key] = request.args.get(key)
force = False
if other and ('force' in other):
force = _get_boolean(other['force'])
res = process.do_get_one(mongo_dbs.get_db().db, alias_type, alias_value)
err_reason = None
err_msg = None
if not res[0]:
err_msg = res[1]
if (not res[0]) and force:
err_msg = res[1]
searcher_url = get_config('newstwister_url')
res_aux = process.do_request_one(mongo_dbs.get_db().db, searcher_url, alias_type, alias_value)
if not res_aux[0]:
err_msg = res_aux[1]
if 2 < len(res_aux):
err_reason = res_aux[2]
else:
res = process.do_get_one(mongo_dbs.get_db().db, alias_type, alias_value)
if not res[0]:
err_msg = res_aux[1]
err_reason = None
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': err_msg}}
if err_reason:
ret_data['_meta']['reason'] = err_reason
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def setup_config(workingon_config_data):
if type(workingon_config_data) is not dict:
return False
if (CONFIG_USE_SETUP not in workingon_config_data) or (
not workingon_config_data[CONFIG_USE_SETUP]):
return False
to_setup = _get_boolean(workingon_config_data[CONFIG_USE_SETUP])
if not to_setup:
return False
if CONFIG_USE_REPORTS in workingon_config_data:
if (workingon_config_data[CONFIG_USE_REPORTS]) and (
workingon_config_data[CONFIG_USE_REPORTS] != 'assignments'):
set_conf('field_public_report',
workingon_config_data[CONFIG_USE_REPORTS])
set_conf('condition_public_report', True)
return True
def do_patch_one(db, doc_id=None, data=None):
'''
sets the "proto" field value
'''
if not db:
return (False, 'inner application error')
if data is None:
return (False, 'data not provided')
if doc_id is not None:
doc_id = _get_id_value(doc_id)
if type(data) is not dict:
return (False, 'data should be a dict')
if ('proto' not in data) or (data['proto'] is None):
return (False, 'data should contain a "proto" field')
try:
proto = bool(_get_boolean(data['proto']))
except:
return (False, 'the "proto" field has to be a boolean value')
res = do_get_one(db, doc_id)
if not res[0]:
return res
coll = db[collection]
spec_field = 'report_id'
if type(doc_id) is ObjectId:
spec_field = '_id'
update_set = {}
update_set['proto'] = proto
update_set[FIELD_UPDATED] = datetime.datetime.utcnow()
#update_set['_etag'] = _get_etag()
coll.update({'feed_type': FEED_TYPE, spec_field: doc_id}, {'$set': update_set}, upsert=False)
return (True, {'report_id': doc_id})
def do_post_send(db,
sender_url,
authorized_id,
user_id,
endpoint_id,
tweet_spec,
report_id=None):
'''
sends (a reply to) a tweet
after it is sent and received, local=True, and user_id=user_id are set in the report
'''
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
if not authorized_id:
return (False, 'authorized_id not specified')
if not user_id:
return (False, 'user_id not specified')
if not endpoint_id:
return (False, 'endpoint_id not specified')
if type(tweet_spec) is not dict:
return (False, 'unknown form of tweet spec')
if ('message' not in tweet_spec) or (not tweet_spec['message']):
return (False, 'message text not provided')
follow_part = []
tweet_data = {
'endpoint_id': endpoint_id,
'status': tweet_spec['message'],
'filter': {},
}
if ('sensitive' in tweet_spec) and (tweet_spec['sensitive'] is not None):
sensitive = _get_boolean(tweet_spec['sensitive'])
if sensitive:
tweet_data['possibly_sensitive'] = 'true'
if ('display_coordinates'
in tweet_spec) and (tweet_spec['display_coordinates'] is not None):
display_coordinates = _get_boolean(tweet_spec['display_coordinates'])
if display_coordinates:
tweet_data['display_coordinates'] = 'true'
else:
tweet_data['display_coordinates'] = 'false'
for key in ['lat', 'long', 'place_id']:
if (key in tweet_spec) and (tweet_spec[key]):
try:
tweet_data[key] = str(tweet_spec[key])
except:
return (False, 'wrong "' + str(key) + '" part in tweet spec')
if report_id is not None:
report_id = _get_id_value(report_id)
search_spec = {'feed_type': FEED_TYPE}
if type(report_id) is ObjectId:
search_spec['_id'] = report_id
else:
search_spec['report_id'] = report_id
coll = db[collection_reports]
report = coll.find_one(search_spec)
if not report:
return (False, 'specified report not found')
try:
orig_user_screen_name = str(
report['original']['user']['screen_name']).lower()
if orig_user_screen_name not in follow_part:
follow_part.append(orig_user_screen_name)
except:
return (False, 'can not find the original tweet sender')
check_inclusion = '@' + orig_user_screen_name.lower()
try:
if check_inclusion not in tweet_spec['message'].lower():
return (
False,
'mentioning the original tweet sender not found in the tweet text'
)
except:
return (False,
'can not check inclusion of the original tweet sender')
try:
tweet_data['in_reply_to_status_id'] = str(report['original_id'])
except:
return (False, 'can not find id_str of the original tweet')
coll = db[collection_authorized]
authorized_id = _get_id_value(authorized_id)
authorized_data = coll.find_one({'_id': authorized_id})
if not authorized_data:
return (False, 'saved report on the send-tweet not found')
try:
authorized_spec = {
'consumer_key':
authorized_data['spec']['app_consumer_key'],
'consumer_secret':
authorized_data['spec']['app_consumer_secret'],
'access_token_key':
authorized_data['spec']['authorized_access_token_key'],
'access_token_secret':
authorized_data['spec']['authorized_access_token_secret'],
}
sender_screen_name = str(authorized_data['spec']['screen_name_search'])
if sender_screen_name not in follow_part:
follow_part.append(sender_screen_name)
except Exception as exc:
return (False,
'authorized info does not contain all the required data: ' +
str(exc))
for key in authorized_spec:
if not authorized_spec[key]:
return (False,
'the "' + str(key) + '" part of authorized info is empty')
tweet_data['filter']['follow'] = follow_part
connector = controller.NewstwisterConnector(sender_url)
res = connector.send_tweet(authorized_spec, tweet_data)
if not res[0]:
err_msg = 'error during send-tweet request dispatching: ' + res[1]
return (False, err_msg)
ret_envelope = res[1]
if type(ret_envelope) is not dict:
return (False, 'unknown form of returned send-tweet data: ' +
str(type(ret_envelope)))
if ('status' not in ret_envelope) or (not ret_envelope['status']):
err_msg = ''
if ('error' in ret_envelope) and (ret_envelope['error']):
err_msg = ': ' + str(ret_envelope['error'])
return (False, 'status not acknowledged in returned send-tweet data' +
err_msg)
if ('data' not in ret_envelope) or (not ret_envelope['data']):
return (False, 'payload not provided in returned send-tweet data')
ret_data = ret_envelope['data']
if type(ret_data) is not dict:
return (False,
'unknown form of returned payload in send-tweet data: ' +
str(type(ret_data)))
if 'id_str' not in ret_data:
return (False, 'returned send-tweet data without tweet identifier')
coll = db[collection_reports]
saved_tweet = coll.find_one({'original_id': ret_data['id_str']})
if not saved_tweet:
return (False, 'saved report on the send-tweet not found')
doc_id = saved_tweet['_id']
saved_update = {
'local': True,
'user_id': _get_id_value(user_id),
'proto': False,
FIELD_UPDATED: datetime.datetime.utcnow(),
#'_etag': _get_etag(),
}
coll.update({'_id': doc_id}, {'$set': saved_update})
return (True, {'_id': doc_id})
def prepare_sms_send_report(targets,
recipients,
message,
user_id=None,
language=None,
sensitive=None,
client_ip=None):
''' preparing an sms-based report when the sms is sent not as a reply, i.e. starting a new session for it '''
channel_type = get_conf('channel_type')
channel_value = get_conf('channel_value_send')
feed_type = get_conf('feed_type')
if user_id:
user_id = _get_id_value(user_id)
channel = {
'type': channel_type,
'value': channel_value,
'filter': None,
'reasons': None,
'request': None
}
current_timestamp = datetime.datetime.utcnow()
doc = {
'report_id':
gen_id(feed_type, channel_type, channel_value,
current_timestamp), # to generate the report_id
'channels': [channel],
'recipients':
recipients,
'authors': [], # no citizen here
'endorsers': [], # no citizen here
'publisher':
get_conf('publisher'),
'feed_type':
feed_type,
'parent_id':
None, # not a reply here
'session':
None,
'user_id':
user_id, # who requested the sending, if not automatic
'pinned_id':
None,
'language':
language,
'produced':
current_timestamp,
'_created':
current_timestamp,
'_updated':
current_timestamp,
'assignments': [],
'original_id':
None,
'original': {
'message': message
},
'texts': [{
'original': message,
'transcript': None
}],
'tags': [], # we shall extract possible #tags, alike at sms receiving
'sensitive':
None,
'summary':
False,
'local':
True,
'targets':
targets, # alias_ids and/or group_ids for replies, on sent reports
'proto':
False,
'client_ip':
client_ip # for logging
}
if sensitive is not None:
doc['sensitive'] = _get_boolean(sensitive)
doc['session'] = doc['report_id']
doc['tags'] = _extract_tags(message)
return doc
def do_get_list(db,
spec_type,
spec_id,
offset=None,
limit=None,
sort=None,
other=None):
'''
returns data of a set of sms reports
'''
if not db:
return (False, 'inner application error')
list_spec = {'feed_type': FEED_TYPE}
if 'session_id' == spec_type:
list_spec['session'] = spec_id
if 'sent_to' == spec_type:
list_spec[
'recipients.identifiers.user_id_search'] = normalize_phone_number(
spec_id)
if 'received_from' == spec_type:
list_spec[
'authors.identifiers.user_id_search'] = normalize_phone_number(
spec_id)
sort_list = _get_sort(sort)
if not sort_list:
sort_list = [('produced', 1)]
text_only = False
if other and ('text_only' in other) and other['text_only']:
try:
text_only = bool(_get_boolean(other['text_only']))
except:
text_only = False
coll = db[collection]
cursor = coll.find(list_spec).sort(sort_list)
total = cursor.count()
if limit is None:
limit = DEFAULT_LIMIT
if offset:
cursor = cursor.skip(offset)
if limit:
cursor = cursor.limit(limit)
docs = []
for entry in cursor:
if not entry:
continue
if not text_only:
docs.append(entry)
else:
one_text = {'original': None, 'authors': None, 'recipients': None}
authors = []
recipients = []
if 'original' in entry:
one_text['original'] = entry['original']
if ('authors' in entry) and (type(entry['authors'])
in (list, tuple)):
for one_author in entry['authors']:
if (type(one_author) is dict) and (
'identifiers'
in one_author) and (one_author['identifiers']):
one_alias_set = one_author['identifiers']
if type(one_alias_set) is dict:
if ('user_id' in one_alias_set
) and one_alias_set['user_id']:
authors.append(one_alias_set['user_id'])
if ('recipients' in entry) and (type(entry['recipients'])
in (list, tuple)):
for one_recipient in entry['recipients']:
if (type(one_recipient) is
dict) and ('identifiers' in one_recipient) and (
one_recipient['identifiers']):
one_alias_set = one_recipient['identifiers']
if type(one_alias_set) is dict:
if ('user_id' in one_alias_set
) and one_alias_set['user_id']:
recipients.append(one_alias_set['user_id'])
if authors:
try:
one_text['authors'] = ', '.join(authors)
except:
pass
if recipients:
try:
one_text['recipients'] = ', '.join(recipients)
except:
pass
if (not one_text['original']) and (not one_text['authors']) and (
not one_text['recipients']):
continue
docs.append(one_text)
return (True, docs, {'total': total})
def do_post_send(db, sender_url, authorized_id, user_id, endpoint_id, tweet_spec, report_id=None):
'''
sends (a reply to) a tweet
after it is sent and received, local=True, and user_id=user_id are set in the report
'''
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
if not authorized_id:
return (False, 'authorized_id not specified')
if not user_id:
return (False, 'user_id not specified')
if not endpoint_id:
return (False, 'endpoint_id not specified')
if type(tweet_spec) is not dict:
return (False, 'unknown form of tweet spec')
if ('message' not in tweet_spec) or (not tweet_spec['message']):
return (False, 'message text not provided')
follow_part = []
tweet_data = {
'endpoint_id': endpoint_id,
'status': tweet_spec['message'],
'filter': {},
}
if ('sensitive' in tweet_spec) and (tweet_spec['sensitive'] is not None):
sensitive = _get_boolean(tweet_spec['sensitive'])
if sensitive:
tweet_data['possibly_sensitive'] = 'true'
if ('display_coordinates' in tweet_spec) and (tweet_spec['display_coordinates'] is not None):
display_coordinates = _get_boolean(tweet_spec['display_coordinates'])
if display_coordinates:
tweet_data['display_coordinates'] = 'true'
else:
tweet_data['display_coordinates'] = 'false'
for key in ['lat', 'long', 'place_id']:
if (key in tweet_spec) and (tweet_spec[key]):
try:
tweet_data[key] = str(tweet_spec[key])
except:
return (False, 'wrong "' + str(key) + '" part in tweet spec')
if report_id is not None:
report_id = _get_id_value(report_id)
search_spec = {'feed_type': FEED_TYPE}
if type(report_id) is ObjectId:
search_spec['_id'] = report_id
else:
search_spec['report_id'] = report_id
coll = db[collection_reports]
report = coll.find_one(search_spec)
if not report:
return (False, 'specified report not found')
try:
orig_user_screen_name = str(report['original']['user']['screen_name']).lower()
if orig_user_screen_name not in follow_part:
follow_part.append(orig_user_screen_name)
except:
return (False, 'can not find the original tweet sender')
check_inclusion = '@' + orig_user_screen_name.lower()
try:
if check_inclusion not in tweet_spec['message'].lower():
return (False, 'mentioning the original tweet sender not found in the tweet text')
except:
return (False, 'can not check inclusion of the original tweet sender')
try:
tweet_data['in_reply_to_status_id'] = str(report['original_id'])
except:
return (False, 'can not find id_str of the original tweet')
coll = db[collection_authorized]
authorized_id = _get_id_value(authorized_id)
authorized_data = coll.find_one({'_id': authorized_id})
if not authorized_data:
return (False, 'saved report on the send-tweet not found')
try:
authorized_spec = {
'consumer_key': authorized_data['spec']['app_consumer_key'],
'consumer_secret': authorized_data['spec']['app_consumer_secret'],
'access_token_key': authorized_data['spec']['authorized_access_token_key'],
'access_token_secret': authorized_data['spec']['authorized_access_token_secret'],
}
sender_screen_name = str(authorized_data['spec']['screen_name_search'])
if sender_screen_name not in follow_part:
follow_part.append(sender_screen_name)
except Exception as exc:
return (False, 'authorized info does not contain all the required data: ' + str(exc))
for key in authorized_spec:
if not authorized_spec[key]:
return (False, 'the "' + str(key) + '" part of authorized info is empty')
tweet_data['filter']['follow'] = follow_part
connector = controller.NewstwisterConnector(sender_url)
res = connector.send_tweet(authorized_spec, tweet_data)
if not res[0]:
err_msg = 'error during send-tweet request dispatching: ' + res[1]
return (False, err_msg)
ret_envelope = res[1]
if type(ret_envelope) is not dict:
return (False, 'unknown form of returned send-tweet data: ' + str(type(ret_envelope)))
if ('status' not in ret_envelope) or (not ret_envelope['status']):
err_msg = ''
if ('error' in ret_envelope) and (ret_envelope['error']):
err_msg = ': ' + str(ret_envelope['error'])
return (False, 'status not acknowledged in returned send-tweet data' + err_msg)
if ('data' not in ret_envelope) or (not ret_envelope['data']):
return (False, 'payload not provided in returned send-tweet data')
ret_data = ret_envelope['data']
if type(ret_data) is not dict:
return (False, 'unknown form of returned payload in send-tweet data: ' + str(type(ret_data)))
if 'id_str' not in ret_data:
return (False, 'returned send-tweet data without tweet identifier')
coll = db[collection_reports]
saved_tweet = coll.find_one({'original_id': ret_data['id_str']})
if not saved_tweet:
return (False, 'saved report on the send-tweet not found')
doc_id = saved_tweet['_id']
saved_update = {
'local': True,
'user_id': _get_id_value(user_id),
'proto': False,
FIELD_UPDATED: datetime.datetime.utcnow(),
#'_etag': _get_etag(),
}
coll.update({'_id': doc_id}, {'$set': saved_update})
return (True, {'_id': doc_id})
def do_get_list(db, offset=None, limit=None, sort=None, other=None):
'''
returns data of a set of sms citizen aliases
'''
if not db:
return (False, 'inner application error')
list_spec = {'authority': AUTHORITY}
sort_list = _get_sort(sort)
if not sort_list:
sort_list = [('produced', 1)]
name_only = False
if other and ('name_only' in other) and other['name_only']:
try:
name_only = bool(_get_boolean(other['name_only']))
except:
name_only = False
coll = db[collection]
cursor = coll.find(list_spec).sort(sort_list)
total = cursor.count()
if limit is None:
limit = DEFAULT_LIMIT
if offset:
cursor = cursor.skip(offset)
if limit:
cursor = cursor.limit(limit)
docs = []
for entry in cursor:
if not entry:
continue
if not name_only:
docs.append(entry)
else:
if (not 'identifiers' in entry):
continue
if type(entry['identifiers']) is not dict:
continue
one_name = {}
if 'user_id' not in entry['identifiers']:
continue
one_name['phone_number'] = entry['identifiers']['user_id']
if not one_name['phone_number']:
continue
description = None
if ('description' in entry) and entry['description']:
description = entry['description']
if description:
one_name['description'] = description
name_list = []
for name_key in ['name_first', 'name_last', 'name_full']:
if (name_key in entry) and entry[name_key]:
name_list.append(entry[name_key])
if name_list:
one_name['names'] = ' - '.join(name_list)
docs.append(one_name)
return (True, docs, {'total': total})
def do_get_list(db, offset=None, limit=None, sort=None, other=None):
'''
returns data of a set of sms citizen aliases
'''
if not db:
return (False, 'inner application error')
list_spec = {'authority': AUTHORITY}
sort_list = _get_sort(sort)
if not sort_list:
sort_list = [('produced', 1)]
name_only = False
if other and ('name_only' in other) and other['name_only']:
try:
name_only = bool(_get_boolean(other['name_only']))
except:
name_only = False
coll = db[collection]
cursor = coll.find(list_spec).sort(sort_list)
total = cursor.count()
if limit is None:
limit = DEFAULT_LIMIT
if offset:
cursor = cursor.skip(offset)
if limit:
cursor = cursor.limit(limit)
docs = []
for entry in cursor:
if not entry:
continue
if not name_only:
docs.append(entry)
else:
if (not 'identifiers' in entry):
continue
if type(entry['identifiers']) is not dict:
continue
one_name = {}
if 'user_id' not in entry['identifiers']:
continue
one_name['phone_number'] = entry['identifiers']['user_id']
if not one_name['phone_number']:
continue
description = None
if ('description' in entry) and entry['description']:
description = entry['description']
if description:
one_name['description'] = description
name_list = []
for name_key in ['name_first', 'name_last', 'name_full']:
if (name_key in entry) and entry[name_key]:
name_list.append(entry[name_key])
if name_list:
one_name['names'] = ' - '.join(name_list)
docs.append(one_name)
return (True, docs, {'total': total})
def do_get_list(db, endpoint_type, endpoint_id, proto=None, offset=None, limit=None, sort=None, other=None):
'''
returns data of a set of reports saved by a stream or a search
'''
if not db:
return (False, 'inner application error')
endpoint_id = _get_id_value(endpoint_id)
list_spec = {'feed_type': FEED_TYPE, 'channels': {'$elemMatch': {'type': endpoint_type, 'value': endpoint_id}}}
if proto is not None:
try:
proto = bool(_get_boolean(proto))
except:
return (False, 'the "proto" specifier has to be a boolean value')
list_spec['proto'] = proto
sort_list = _get_sort(sort)
if not sort_list:
sort_list = [('produced', 1)]
text_only = False
if other and ('text_only' in other) and other['text_only']:
try:
text_only = bool(_get_boolean(other['text_only']))
except:
text_only = False
coll = db[collection]
cursor = coll.find(list_spec).sort(sort_list)
total = cursor.count()
if limit is None:
limit = DEFAULT_LIMIT
if offset:
cursor = cursor.skip(offset)
if limit:
cursor = cursor.limit(limit)
docs = []
for entry in cursor:
if not entry:
continue
if not text_only:
docs.append(entry)
else:
if (not 'texts' in entry):
continue
if (not entry['texts']):
continue
if (type(entry['texts']) not in [list]):
continue
for one_text in entry['texts']:
source = None
if ('sources' in entry) and (type(entry['sources']) is list):
if len(entry['sources']) and (entry['sources'][0]):
source = entry['sources'][0]
if source:
if (type(one_text) is dict):
one_text['source'] = source
docs.append(one_text)
return (True, docs, {'total': total})
def do_get_list(db, spec_type, spec_id, offset=None, limit=None, sort=None, other=None):
'''
returns data of a set of sms reports
'''
if not db:
return (False, 'inner application error')
list_spec = {'feed_type': FEED_TYPE}
if 'session_id' == spec_type:
list_spec['session'] = spec_id
if 'sent_to' == spec_type:
list_spec['recipients.identifiers.user_id_search'] = normalize_phone_number(spec_id)
if 'received_from' == spec_type:
list_spec['authors.identifiers.user_id_search'] = normalize_phone_number(spec_id)
sort_list = _get_sort(sort)
if not sort_list:
sort_list = [('produced', 1)]
text_only = False
if other and ('text_only' in other) and other['text_only']:
try:
text_only = bool(_get_boolean(other['text_only']))
except:
text_only = False
coll = db[collection]
cursor = coll.find(list_spec).sort(sort_list)
total = cursor.count()
if limit is None:
limit = DEFAULT_LIMIT
if offset:
cursor = cursor.skip(offset)
if limit:
cursor = cursor.limit(limit)
docs = []
for entry in cursor:
if not entry:
continue
if not text_only:
docs.append(entry)
else:
one_text = {
'original': None,
'authors': None,
'recipients': None
}
authors = []
recipients = []
if 'original' in entry:
one_text['original'] = entry['original']
if ('authors' in entry) and (type(entry['authors']) in (list, tuple)):
for one_author in entry['authors']:
if (type(one_author) is dict) and ('identifiers' in one_author) and (one_author['identifiers']):
one_alias_set = one_author['identifiers']
if type(one_alias_set) is dict:
if ('user_id' in one_alias_set) and one_alias_set['user_id']:
authors.append(one_alias_set['user_id'])
if ('recipients' in entry) and (type(entry['recipients']) in (list, tuple)):
for one_recipient in entry['recipients']:
if (type(one_recipient) is dict) and ('identifiers' in one_recipient) and (one_recipient['identifiers']):
one_alias_set = one_recipient['identifiers']
if type(one_alias_set) is dict:
if ('user_id' in one_alias_set) and one_alias_set['user_id']:
recipients.append(one_alias_set['user_id'])
if authors:
try:
one_text['authors'] = ', '.join(authors)
except:
pass
if recipients:
try:
one_text['recipients'] = ', '.join(recipients)
except:
pass
if (not one_text['original']) and (not one_text['authors']) and (not one_text['recipients']):
continue
docs.append(one_text)
return (True, docs, {'total': total})
def do_get_list(db, endpoint_type, endpoint_id, proto=None, offset=None, limit=None, sort=None, other=None):
'''
returns data of a set of reports saved by a stream or a search
'''
if not db:
return (False, 'inner application error')
endpoint_id = _get_id_value(endpoint_id)
list_spec = {'feed_type': FEED_TYPE, 'channels': {'$elemMatch': {'type': endpoint_type, 'value': endpoint_id}}}
if proto is not None:
try:
proto = bool(_get_boolean(proto))
except:
return (False, 'the "proto" specifier has to be a boolean value')
list_spec['proto'] = proto
sort_list = _get_sort(sort)
if not sort_list:
sort_list = [('produced', 1)]
text_only = False
if other and ('text_only' in other) and other['text_only']:
try:
text_only = bool(_get_boolean(other['text_only']))
except:
text_only = False
coll = db[collection]
cursor = coll.find(list_spec).sort(sort_list)
total = cursor.count()
if limit is None:
limit = DEFAULT_LIMIT
if offset:
cursor = cursor.skip(offset)
if limit:
cursor = cursor.limit(limit)
docs = []
for entry in cursor:
if not entry:
continue
if not text_only:
docs.append(entry)
else:
if (not 'texts' in entry):
continue
if (not entry['texts']):
continue
if (type(entry['texts']) not in [list]):
continue
for one_text in entry['texts']:
source = None
if ('sources' in entry) and (type(entry['sources']) in (list, tuple)):
if len(entry['sources']) and (entry['sources'][0]):
source = entry['sources'][0]
if source:
if (type(one_text) is dict):
one_text['source'] = source
docs.append(one_text)
return (True, docs, {'total': total})
def do_patch_one(db, doc_id=None, data=None, force=None):
'''
starts/stops the stream
'''
try:
from citizendesk.feeds.twt.filter.storage import get_one as filter_get_one
except:
return (False, 'filter processor not available')
try:
from citizendesk.feeds.twt.oauth.storage import get_one as oauth_get_one
except:
return (False, 'oauth processor not available')
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
if data is None:
return (False, 'data not provided')
if type(data) is not dict:
return (False, 'invalid data provided')
if ('control' not in data) or (type(data['control']) is not dict):
return (False, '"control" part not provided')
if doc_id is None:
return (False, 'stream _id not provided')
doc_id = _get_id_value(doc_id)
try:
force_val = bool(_get_boolean(force))
except:
force_val = None
coll = db[collection]
entry = coll.find_one({'_id': doc_id})
if not entry:
return (False, '"stream" of the provided _id not found: ' + str(doc_id))
if ('spec' not in entry) or (type(entry['spec']) is not dict):
return (False, '"stream" of the provided _id does not contain "spec" part')
doc = {'control': {}}
for key in schema['control']:
doc['control'][key] = None
if key in data['control']:
doc['control'][key] = data['control'][key]
res = _check_schema(None, doc['control'])
if not res[0]:
return res
should_run = False
if doc['control']['switch_on']:
if not doc['control']['streamer_url']:
return (False, 'can not start the stream without the "streamer_url" being set')
else:
should_run = True
filter_id = None
oauth_id = None
if should_run:
if ('spec' in entry) and (type(entry['spec']) is dict):
if 'filter_id' in entry['spec']:
filter_id = entry['spec']['filter_id']
if 'oauth_id' in entry['spec']:
oauth_id = entry['spec']['oauth_id']
if (not filter_id) or (not oauth_id):
return (False, 'Can not start the stream without assigned filter_id and oauth_id')
new_stream_url = None
connector = None
if should_run:
new_stream_url = doc['control']['streamer_url']
connector = controller.NewstwisterConnector(new_stream_url)
# load the supplemental data
filter_info = None
oauth_info = None
if should_run:
res = filter_get_one(db, filter_id)
if not res[0]:
return (False, 'filter info with _id equal to "spec.filter_id" not found')
filter_info = res[1]
if ('spec' not in filter_info) or (type(filter_info['spec']) is not dict):
return (False, 'set filter without "spec" part')
res = oauth_get_one(db, oauth_id)
if not res[0]:
return (False, 'oauth info with _id equal to "spec.oauth_id" not found')
oauth_info = res[1]
if ('spec' not in oauth_info) or (type(oauth_info['spec']) is not dict):
return (False, 'set oauth without "spec" part')
filter_spec = {}
filter_spec_original = {}
if should_run and filter_info and filter_info['spec']:
try:
prep_res = _prepare_filter(db, connector, filter_info['spec'])
if not prep_res[0]:
return (False, prep_res[1])
filter_spec = prep_res[1]
filter_spec_original = prep_res[2]
except Exception as exc:
return (False, 'can not prepare filter params: ' + str(exc))
if not filter_spec:
return (False, 'no filter spec was prepared')
if not filter_spec_original:
return (False, 'no filter spec original was prepared')
# check if the oauth_id is not used by any other running feed
if should_run:
running_count = coll.find({'spec.oauth_id': oauth_id, 'control.switch_on': True, '_id': {'$ne': doc_id}}).count()
if running_count:
return (False, 'the "oauth_id" is already used at a running stream')
# stop first in any case, since filter/oauth specs might have changed
might_run = True
if not entry:
might_run = False
if might_run:
if ('control' not in entry) or (type(entry['control']) is not dict):
might_run = False
if might_run:
for key in schema['control']:
if (key not in entry['control']) or (not entry['control'][key]):
might_run = False
break
if might_run:
cur_stream_url = entry['control']['streamer_url']
connector = controller.NewstwisterConnector(cur_stream_url)
res = connector.request_stop(entry['control']['process_id'])
if (not res) and (not force_val):
return (False, 'can not stop the stream')
# update info in db when we know it has been stopped
timepoint = datetime.datetime.utcnow()
update_set = {'control.switch_on': False, 'control.process_id': None, 'logs.stopped': timepoint}
coll.update({'_id': doc_id}, {'$set': update_set}, upsert=False)
if should_run:
res = connector.request_start(str(doc_id), oauth_info['spec'], filter_spec, filter_spec_original)
if not res:
return (False, 'can not start the stream')
try:
proc_id = int(res)
except:
proc_id = res
# update info in db when we know it has been started
timepoint = datetime.datetime.utcnow()
update_set = {'control.switch_on': True, 'control.process_id': proc_id, 'control.streamer_url': new_stream_url, 'logs.started': timepoint}
coll.update({'_id': doc_id}, {'$set': update_set}, upsert=False)
return (True, {'_id': doc_id})
