def do_get_list(db, offset=None, limit=None, sort=None, other=None): ''' returns data of a set of twt citizen aliases ''' if not db: return (False, 'inner application error') list_spec = {'authority': AUTHORITY} sort_list = _get_sort(sort) if not sort_list: sort_list = [('produced', 1)] name_only = False if other and ('name_only' in other) and other['name_only']: try: name_only = bool(_get_boolean(other['name_only'])) except: name_only = False coll = db[collection] cursor = coll.find(list_spec).sort(sort_list) total = cursor.count() if limit is None: limit = DEFAULT_LIMIT if offset: cursor = cursor.skip(offset) if limit: cursor = cursor.limit(limit) docs = [] for entry in cursor: if not entry: continue if not name_only: docs.append(entry) else: if (not 'identifiers' in entry): continue if type(entry['identifiers']) not in (list, tuple): continue if not len(entry['identifiers']): continue if not entry['identifiers'][0]: continue one_name = entry['identifiers'][0] if type(one_name) is not dict: continue source = None if ('sources' in entry) and (type(entry['sources']) is list): if len(entry['sources']) and (entry['sources'][0]): source = entry['sources'][0] if source: one_name['source'] = source docs.append(one_name) return (True, docs, {'total': total})
def do_get_list(db, offset=None, limit=None, sort=None, other=None): ''' returns data of a set of twt citizen aliases ''' if not db: return (False, 'inner application error') list_spec = {'authority': AUTHORITY} sort_list = _get_sort(sort) if not sort_list: sort_list = [('produced', 1)] name_only = False if other and ('name_only' in other) and other['name_only']: try: name_only = bool(_get_boolean(other['name_only'])) except: name_only = False coll = db[collection] cursor = coll.find(list_spec).sort(sort_list) total = cursor.count() if limit is None: limit = DEFAULT_LIMIT if offset: cursor = cursor.skip(offset) if limit: cursor = cursor.limit(limit) docs = [] for entry in cursor: if not entry: continue if not name_only: docs.append(entry) else: if (not 'identifiers' in entry): continue if type(entry['identifiers']) is not dict: continue one_name = {} if 'user_id' in entry['identifiers']: one_name['user_id'] = entry['identifiers']['user_id'] if 'user_name' in entry['identifiers']: one_name['user_name'] = entry['identifiers']['user_name'] source = None if ('sources' in entry) and (type(entry['sources']) in (list, tuple)): if len(entry['sources']) and (entry['sources'][0]): source = entry['sources'][0] if source: one_name['source'] = source docs.append(one_name) return (True, docs, {'total': total})
def feed_any_coverage_insert_one(): from citizendesk.feeds.any.coverage import process logger = get_logger() client_ip = get_client_ip() coverage_data = {} try: data = request.get_json(True, False, False) if type(data) is not dict: data = None except: data = None if data is None: try: data = request.json if type(data) is not dict: data = None except: data = None if data is None: return (json.dumps('provided data are not valid json'), 404, { 'Content-Type': 'application/json' }) try: coverage_data['title'] = data['title'] coverage_data['description'] = data['description'] coverage_data['user_id'] = data['user_id'] except: return (json.dumps( 'provided data should contain "title", "description", "user_id" parts' ), 404, { 'Content-Type': 'application/json' }) if 'active' in data: coverage_data['active'] = _get_boolean(data['active']) try: if ('uuid' in data) and data['uuid']: coverage_data['uuid'] = str(data['uuid']) except: pass res = process.do_insert_one(mongo_dbs.get_db().db, coverage_data) if not res[0]: ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}} return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, { 'Content-Type': 'application/json' }) ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]} return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, { 'Content-Type': 'application/json' })
def extract_annotation_status(db, report): # taking annotation from status is_summary = False if (FIELD_SUMMARY_REPORT in report) and (report[FIELD_SUMMARY_REPORT] is not None): is_summary = _get_boolean(report[FIELD_SUMMARY_REPORT]) status_desc = None status_filled = False if (FIELD_STATUS_REPORT in report) and (report[FIELD_STATUS_REPORT]): status_filled = True status_ident = _get_id_value(report[FIELD_STATUS_REPORT]) if type(status_ident) is ObjectId: status_desc = take_status_desc_by_id(db, status_ident) else: status_desc = take_status_desc_by_key(db, status_ident) if not status_filled: if (FIELD_STATUS_VERIFIED_REPORT in report) and (report[FIELD_STATUS_VERIFIED_REPORT] is not None): verified_flag = _get_boolean(report[FIELD_STATUS_VERIFIED_REPORT]) if verified_flag: status_desc = take_status_desc_by_key(db, STATUS_VERIFIED_KEY) is_assigned = False if status_desc is None: if (FIELD_ASSIGNED_REPORT in report) and (type( report[FIELD_ASSIGNED_REPORT]) in (list, tuple)): for item in report[FIELD_ASSIGNED_REPORT]: if item is not None: is_assigned = True break if is_assigned: status_desc = take_status_desc_by_key(db, STATUS_ASSIGNED_KEY) if (status_desc is None) and (not is_assigned) and (not is_summary): status_desc = take_status_desc_by_key(db, STATUS_NEW_KEY) if not status_desc: status_desc = None return status_desc
def prepare_sms_send_report(targets, recipients, message, user_id=None, language=None, sensitive=None, client_ip=None): ''' preparing an sms-based report when the sms is sent not as a reply, i.e. starting a new session for it ''' channel_type = get_conf('channel_type') channel_value = get_conf('channel_value_send') feed_type = get_conf('feed_type') channel = { 'type': channel_type, 'value': channel_value, 'filter': None, 'reasons': None, 'request': None } current_timestamp = datetime.datetime.now() doc = { 'report_id': gen_id(feed_type, channel_type, channel_value, current_timestamp), # to generate the report_id 'channels': [channel], 'recipients': recipients, 'authors': [], # no citizen here 'endorsers': [], # no citizen here 'publisher': get_conf('publisher'), 'feed_type': feed_type, 'parent_id': None, # not a reply here 'session': None, 'user_id': user_id, # who requested the sending, if not automatic 'pinned_id': None, 'coverage_id': None, 'language': language, 'produced': current_timestamp, 'created': current_timestamp, 'assignments': [], 'original_id': None, 'original': {'message': message}, 'texts': [{'original': message, 'transcript': None}], 'tags': [], # we shall extract possible #tags, alike at sms receiving 'is_published': False, 'sensitive': None, 'summary': False, 'local': True, 'targets': targets, # alias_ids and/or group_ids for replies, on sent reports 'proto': False, 'client_ip': client_ip # for logging } if sensitive is not None: doc['sensitive'] = _get_boolean(sensitive) doc['session'] = doc['report_id'] doc['tags'] = _extract_tags(message) return doc
def do_get_list(db, offset, limit, sort, other): ''' returns list of coverages ''' if not db: return (False, 'inner application error') list_spec = {} sort_list = _get_sort(sort) if not sort_list: sort_list = [('produced', 1)] name_only = False if other and ('name_only' in other) and other['name_only']: try: name_only = bool(_get_boolean(other['name_only'])) except: name_only = False coll = db[collection] cursor = coll.find(list_spec).sort(sort_list) total = cursor.count() if limit is None: limit = DEFAULT_LIMIT if offset: cursor = cursor.skip(offset) if limit: cursor = cursor.limit(limit) docs = [] for entry in cursor: if not entry: continue if not name_only: docs.append(entry) else: if 'title' not in entry: continue one_name = { 'title': entry['title'], FIELD_ACTIVE: None, } if FIELD_ACTIVE in entry: one_name[FIELD_ACTIVE] = entry[FIELD_ACTIVE] docs.append(one_name) return (True, docs, {'total': total})
def feed_twt_citizen_alias_get_one(alias_type, alias_value): from citizendesk.feeds.twt.citizen_alias import process from citizendesk.feeds.config import get_config logger = get_logger() client_ip = get_client_ip() other = {'force': None} for key in other: if key in request.args: other[key] = request.args.get(key) force = False if other and ('force' in other): force = _get_boolean(other['force']) res = process.do_get_one(mongo_dbs.get_db().db, alias_type, alias_value) err_reason = None err_msg = None if not res[0]: err_msg = res[1] if (not res[0]) and force: err_msg = res[1] searcher_url = get_config('newstwister_url') res_aux = process.do_request_one(mongo_dbs.get_db().db, searcher_url, alias_type, alias_value) if not res_aux[0]: err_msg = res_aux[1] if 2 < len(res_aux): err_reason = res_aux[2] else: res = process.do_get_one(mongo_dbs.get_db().db, alias_type, alias_value) if not res[0]: err_msg = res_aux[1] err_reason = None if not res[0]: ret_data = {'_meta': {'schema': process.schema, 'message': err_msg}} if err_reason: ret_data['_meta']['reason'] = err_reason return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, { 'Content-Type': 'application/json' }) ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]} return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, { 'Content-Type': 'application/json' })
def extract_annotation_status(db, report): # taking annotation from status is_summary = False if (FIELD_SUMMARY_REPORT in report) and (report[FIELD_SUMMARY_REPORT] is not None): is_summary = _get_boolean(report[FIELD_SUMMARY_REPORT]) status_desc = None status_filled = False if (FIELD_STATUS_REPORT in report) and (report[FIELD_STATUS_REPORT]): status_filled = True status_ident = _get_id_value(report[FIELD_STATUS_REPORT]) if type(status_ident) is ObjectId: status_desc = take_status_desc_by_id(db, status_ident) else: status_desc = take_status_desc_by_key(db, status_ident) if not status_filled: if (FIELD_STATUS_VERIFIED_REPORT in report) and (report[FIELD_STATUS_VERIFIED_REPORT] is not None): verified_flag = _get_boolean(report[FIELD_STATUS_VERIFIED_REPORT]) if verified_flag: status_desc = take_status_desc_by_key(db, STATUS_VERIFIED_KEY) is_assigned = False if status_desc is None: if (FIELD_ASSIGNED_REPORT in report) and (type(report[FIELD_ASSIGNED_REPORT]) in (list, tuple)): for item in report[FIELD_ASSIGNED_REPORT]: if item is not None: is_assigned = True break if is_assigned: status_desc = take_status_desc_by_key(db, STATUS_ASSIGNED_KEY) if (status_desc is None) and (not is_assigned) and (not is_summary): status_desc = take_status_desc_by_key(db, STATUS_NEW_KEY) if not status_desc: status_desc = None return status_desc
def setup_config(workingon_config_data): if type(workingon_config_data) is not dict: return False if (CONFIG_USE_SETUP not in workingon_config_data) or (not workingon_config_data[CONFIG_USE_SETUP]): return False to_setup = _get_boolean(workingon_config_data[CONFIG_USE_SETUP]) if not to_setup: return False if CONFIG_USE_REPORTS in workingon_config_data: if (workingon_config_data[CONFIG_USE_REPORTS]) and (workingon_config_data[CONFIG_USE_REPORTS] != 'assignments'): set_conf('field_public_report', workingon_config_data[CONFIG_USE_REPORTS]) set_conf('condition_public_report', True) return True
def feed_any_coverage_insert_one(): from citizendesk.feeds.any.coverage import process logger = get_logger() client_ip = get_client_ip() coverage_data = {} try: data = request.get_json(True, False, False) if type(data) is not dict: data = None except: data = None if data is None: try: data = request.json if type(data) is not dict: data = None except: data = None if data is None: return (json.dumps('provided data are not valid json'), 404, {'Content-Type': 'application/json'}) try: coverage_data['title'] = data['title'] coverage_data['description'] = data['description'] coverage_data['user_id'] = data['user_id'] except: return (json.dumps('provided data should contain "title", "description", "user_id" parts'), 404, {'Content-Type': 'application/json'}) if 'active' in data: coverage_data['active'] = _get_boolean(data['active']) try: if ('uuid' in data) and data['uuid']: coverage_data['uuid'] = str(data['uuid']) except: pass res = process.do_insert_one(mongo_dbs.get_db().db, coverage_data) if not res[0]: ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}} return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'}) ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]} return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def feed_twt_citizen_alias_get_one(alias_type, alias_value): from citizendesk.feeds.twt.citizen_alias import process from citizendesk.feeds.config import get_config logger = get_logger() client_ip = get_client_ip() other = {'force': None} for key in other: if key in request.args: other[key] = request.args.get(key) force = False if other and ('force' in other): force = _get_boolean(other['force']) res = process.do_get_one(mongo_dbs.get_db().db, alias_type, alias_value) err_reason = None err_msg = None if not res[0]: err_msg = res[1] if (not res[0]) and force: err_msg = res[1] searcher_url = get_config('newstwister_url') res_aux = process.do_request_one(mongo_dbs.get_db().db, searcher_url, alias_type, alias_value) if not res_aux[0]: err_msg = res_aux[1] if 2 < len(res_aux): err_reason = res_aux[2] else: res = process.do_get_one(mongo_dbs.get_db().db, alias_type, alias_value) if not res[0]: err_msg = res_aux[1] err_reason = None if not res[0]: ret_data = {'_meta': {'schema': process.schema, 'message': err_msg}} if err_reason: ret_data['_meta']['reason'] = err_reason return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'}) ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]} return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def setup_config(workingon_config_data): if type(workingon_config_data) is not dict: return False if (CONFIG_USE_SETUP not in workingon_config_data) or ( not workingon_config_data[CONFIG_USE_SETUP]): return False to_setup = _get_boolean(workingon_config_data[CONFIG_USE_SETUP]) if not to_setup: return False if CONFIG_USE_REPORTS in workingon_config_data: if (workingon_config_data[CONFIG_USE_REPORTS]) and ( workingon_config_data[CONFIG_USE_REPORTS] != 'assignments'): set_conf('field_public_report', workingon_config_data[CONFIG_USE_REPORTS]) set_conf('condition_public_report', True) return True
def do_patch_one(db, doc_id=None, data=None): ''' sets the "proto" field value ''' if not db: return (False, 'inner application error') if data is None: return (False, 'data not provided') if doc_id is not None: doc_id = _get_id_value(doc_id) if type(data) is not dict: return (False, 'data should be a dict') if ('proto' not in data) or (data['proto'] is None): return (False, 'data should contain a "proto" field') try: proto = bool(_get_boolean(data['proto'])) except: return (False, 'the "proto" field has to be a boolean value') res = do_get_one(db, doc_id) if not res[0]: return res coll = db[collection] spec_field = 'report_id' if type(doc_id) is ObjectId: spec_field = '_id' update_set = {} update_set['proto'] = proto update_set[FIELD_UPDATED] = datetime.datetime.utcnow() #update_set['_etag'] = _get_etag() coll.update({'feed_type': FEED_TYPE, spec_field: doc_id}, {'$set': update_set}, upsert=False) return (True, {'report_id': doc_id})
def do_post_send(db, sender_url, authorized_id, user_id, endpoint_id, tweet_spec, report_id=None): ''' sends (a reply to) a tweet after it is sent and received, local=True, and user_id=user_id are set in the report ''' if not controller: return (False, 'external controller not available') if not db: return (False, 'inner application error') if not authorized_id: return (False, 'authorized_id not specified') if not user_id: return (False, 'user_id not specified') if not endpoint_id: return (False, 'endpoint_id not specified') if type(tweet_spec) is not dict: return (False, 'unknown form of tweet spec') if ('message' not in tweet_spec) or (not tweet_spec['message']): return (False, 'message text not provided') follow_part = [] tweet_data = { 'endpoint_id': endpoint_id, 'status': tweet_spec['message'], 'filter': {}, } if ('sensitive' in tweet_spec) and (tweet_spec['sensitive'] is not None): sensitive = _get_boolean(tweet_spec['sensitive']) if sensitive: tweet_data['possibly_sensitive'] = 'true' if ('display_coordinates' in tweet_spec) and (tweet_spec['display_coordinates'] is not None): display_coordinates = _get_boolean(tweet_spec['display_coordinates']) if display_coordinates: tweet_data['display_coordinates'] = 'true' else: tweet_data['display_coordinates'] = 'false' for key in ['lat', 'long', 'place_id']: if (key in tweet_spec) and (tweet_spec[key]): try: tweet_data[key] = str(tweet_spec[key]) except: return (False, 'wrong "' + str(key) + '" part in tweet spec') if report_id is not None: report_id = _get_id_value(report_id) search_spec = {'feed_type': FEED_TYPE} if type(report_id) is ObjectId: search_spec['_id'] = report_id else: search_spec['report_id'] = report_id coll = db[collection_reports] report = coll.find_one(search_spec) if not report: return (False, 'specified report not found') try: orig_user_screen_name = str( report['original']['user']['screen_name']).lower() if orig_user_screen_name not in follow_part: follow_part.append(orig_user_screen_name) except: return (False, 'can not find the original tweet sender') check_inclusion = '@' + orig_user_screen_name.lower() try: if check_inclusion not in tweet_spec['message'].lower(): return ( False, 'mentioning the original tweet sender not found in the tweet text' ) except: return (False, 'can not check inclusion of the original tweet sender') try: tweet_data['in_reply_to_status_id'] = str(report['original_id']) except: return (False, 'can not find id_str of the original tweet') coll = db[collection_authorized] authorized_id = _get_id_value(authorized_id) authorized_data = coll.find_one({'_id': authorized_id}) if not authorized_data: return (False, 'saved report on the send-tweet not found') try: authorized_spec = { 'consumer_key': authorized_data['spec']['app_consumer_key'], 'consumer_secret': authorized_data['spec']['app_consumer_secret'], 'access_token_key': authorized_data['spec']['authorized_access_token_key'], 'access_token_secret': authorized_data['spec']['authorized_access_token_secret'], } sender_screen_name = str(authorized_data['spec']['screen_name_search']) if sender_screen_name not in follow_part: follow_part.append(sender_screen_name) except Exception as exc: return (False, 'authorized info does not contain all the required data: ' + str(exc)) for key in authorized_spec: if not authorized_spec[key]: return (False, 'the "' + str(key) + '" part of authorized info is empty') tweet_data['filter']['follow'] = follow_part connector = controller.NewstwisterConnector(sender_url) res = connector.send_tweet(authorized_spec, tweet_data) if not res[0]: err_msg = 'error during send-tweet request dispatching: ' + res[1] return (False, err_msg) ret_envelope = res[1] if type(ret_envelope) is not dict: return (False, 'unknown form of returned send-tweet data: ' + str(type(ret_envelope))) if ('status' not in ret_envelope) or (not ret_envelope['status']): err_msg = '' if ('error' in ret_envelope) and (ret_envelope['error']): err_msg = ': ' + str(ret_envelope['error']) return (False, 'status not acknowledged in returned send-tweet data' + err_msg) if ('data' not in ret_envelope) or (not ret_envelope['data']): return (False, 'payload not provided in returned send-tweet data') ret_data = ret_envelope['data'] if type(ret_data) is not dict: return (False, 'unknown form of returned payload in send-tweet data: ' + str(type(ret_data))) if 'id_str' not in ret_data: return (False, 'returned send-tweet data without tweet identifier') coll = db[collection_reports] saved_tweet = coll.find_one({'original_id': ret_data['id_str']}) if not saved_tweet: return (False, 'saved report on the send-tweet not found') doc_id = saved_tweet['_id'] saved_update = { 'local': True, 'user_id': _get_id_value(user_id), 'proto': False, FIELD_UPDATED: datetime.datetime.utcnow(), #'_etag': _get_etag(), } coll.update({'_id': doc_id}, {'$set': saved_update}) return (True, {'_id': doc_id})
def prepare_sms_send_report(targets, recipients, message, user_id=None, language=None, sensitive=None, client_ip=None): ''' preparing an sms-based report when the sms is sent not as a reply, i.e. starting a new session for it ''' channel_type = get_conf('channel_type') channel_value = get_conf('channel_value_send') feed_type = get_conf('feed_type') if user_id: user_id = _get_id_value(user_id) channel = { 'type': channel_type, 'value': channel_value, 'filter': None, 'reasons': None, 'request': None } current_timestamp = datetime.datetime.utcnow() doc = { 'report_id': gen_id(feed_type, channel_type, channel_value, current_timestamp), # to generate the report_id 'channels': [channel], 'recipients': recipients, 'authors': [], # no citizen here 'endorsers': [], # no citizen here 'publisher': get_conf('publisher'), 'feed_type': feed_type, 'parent_id': None, # not a reply here 'session': None, 'user_id': user_id, # who requested the sending, if not automatic 'pinned_id': None, 'language': language, 'produced': current_timestamp, '_created': current_timestamp, '_updated': current_timestamp, 'assignments': [], 'original_id': None, 'original': { 'message': message }, 'texts': [{ 'original': message, 'transcript': None }], 'tags': [], # we shall extract possible #tags, alike at sms receiving 'sensitive': None, 'summary': False, 'local': True, 'targets': targets, # alias_ids and/or group_ids for replies, on sent reports 'proto': False, 'client_ip': client_ip # for logging } if sensitive is not None: doc['sensitive'] = _get_boolean(sensitive) doc['session'] = doc['report_id'] doc['tags'] = _extract_tags(message) return doc
def do_get_list(db, spec_type, spec_id, offset=None, limit=None, sort=None, other=None): ''' returns data of a set of sms reports ''' if not db: return (False, 'inner application error') list_spec = {'feed_type': FEED_TYPE} if 'session_id' == spec_type: list_spec['session'] = spec_id if 'sent_to' == spec_type: list_spec[ 'recipients.identifiers.user_id_search'] = normalize_phone_number( spec_id) if 'received_from' == spec_type: list_spec[ 'authors.identifiers.user_id_search'] = normalize_phone_number( spec_id) sort_list = _get_sort(sort) if not sort_list: sort_list = [('produced', 1)] text_only = False if other and ('text_only' in other) and other['text_only']: try: text_only = bool(_get_boolean(other['text_only'])) except: text_only = False coll = db[collection] cursor = coll.find(list_spec).sort(sort_list) total = cursor.count() if limit is None: limit = DEFAULT_LIMIT if offset: cursor = cursor.skip(offset) if limit: cursor = cursor.limit(limit) docs = [] for entry in cursor: if not entry: continue if not text_only: docs.append(entry) else: one_text = {'original': None, 'authors': None, 'recipients': None} authors = [] recipients = [] if 'original' in entry: one_text['original'] = entry['original'] if ('authors' in entry) and (type(entry['authors']) in (list, tuple)): for one_author in entry['authors']: if (type(one_author) is dict) and ( 'identifiers' in one_author) and (one_author['identifiers']): one_alias_set = one_author['identifiers'] if type(one_alias_set) is dict: if ('user_id' in one_alias_set ) and one_alias_set['user_id']: authors.append(one_alias_set['user_id']) if ('recipients' in entry) and (type(entry['recipients']) in (list, tuple)): for one_recipient in entry['recipients']: if (type(one_recipient) is dict) and ('identifiers' in one_recipient) and ( one_recipient['identifiers']): one_alias_set = one_recipient['identifiers'] if type(one_alias_set) is dict: if ('user_id' in one_alias_set ) and one_alias_set['user_id']: recipients.append(one_alias_set['user_id']) if authors: try: one_text['authors'] = ', '.join(authors) except: pass if recipients: try: one_text['recipients'] = ', '.join(recipients) except: pass if (not one_text['original']) and (not one_text['authors']) and ( not one_text['recipients']): continue docs.append(one_text) return (True, docs, {'total': total})
def do_post_send(db, sender_url, authorized_id, user_id, endpoint_id, tweet_spec, report_id=None): ''' sends (a reply to) a tweet after it is sent and received, local=True, and user_id=user_id are set in the report ''' if not controller: return (False, 'external controller not available') if not db: return (False, 'inner application error') if not authorized_id: return (False, 'authorized_id not specified') if not user_id: return (False, 'user_id not specified') if not endpoint_id: return (False, 'endpoint_id not specified') if type(tweet_spec) is not dict: return (False, 'unknown form of tweet spec') if ('message' not in tweet_spec) or (not tweet_spec['message']): return (False, 'message text not provided') follow_part = [] tweet_data = { 'endpoint_id': endpoint_id, 'status': tweet_spec['message'], 'filter': {}, } if ('sensitive' in tweet_spec) and (tweet_spec['sensitive'] is not None): sensitive = _get_boolean(tweet_spec['sensitive']) if sensitive: tweet_data['possibly_sensitive'] = 'true' if ('display_coordinates' in tweet_spec) and (tweet_spec['display_coordinates'] is not None): display_coordinates = _get_boolean(tweet_spec['display_coordinates']) if display_coordinates: tweet_data['display_coordinates'] = 'true' else: tweet_data['display_coordinates'] = 'false' for key in ['lat', 'long', 'place_id']: if (key in tweet_spec) and (tweet_spec[key]): try: tweet_data[key] = str(tweet_spec[key]) except: return (False, 'wrong "' + str(key) + '" part in tweet spec') if report_id is not None: report_id = _get_id_value(report_id) search_spec = {'feed_type': FEED_TYPE} if type(report_id) is ObjectId: search_spec['_id'] = report_id else: search_spec['report_id'] = report_id coll = db[collection_reports] report = coll.find_one(search_spec) if not report: return (False, 'specified report not found') try: orig_user_screen_name = str(report['original']['user']['screen_name']).lower() if orig_user_screen_name not in follow_part: follow_part.append(orig_user_screen_name) except: return (False, 'can not find the original tweet sender') check_inclusion = '@' + orig_user_screen_name.lower() try: if check_inclusion not in tweet_spec['message'].lower(): return (False, 'mentioning the original tweet sender not found in the tweet text') except: return (False, 'can not check inclusion of the original tweet sender') try: tweet_data['in_reply_to_status_id'] = str(report['original_id']) except: return (False, 'can not find id_str of the original tweet') coll = db[collection_authorized] authorized_id = _get_id_value(authorized_id) authorized_data = coll.find_one({'_id': authorized_id}) if not authorized_data: return (False, 'saved report on the send-tweet not found') try: authorized_spec = { 'consumer_key': authorized_data['spec']['app_consumer_key'], 'consumer_secret': authorized_data['spec']['app_consumer_secret'], 'access_token_key': authorized_data['spec']['authorized_access_token_key'], 'access_token_secret': authorized_data['spec']['authorized_access_token_secret'], } sender_screen_name = str(authorized_data['spec']['screen_name_search']) if sender_screen_name not in follow_part: follow_part.append(sender_screen_name) except Exception as exc: return (False, 'authorized info does not contain all the required data: ' + str(exc)) for key in authorized_spec: if not authorized_spec[key]: return (False, 'the "' + str(key) + '" part of authorized info is empty') tweet_data['filter']['follow'] = follow_part connector = controller.NewstwisterConnector(sender_url) res = connector.send_tweet(authorized_spec, tweet_data) if not res[0]: err_msg = 'error during send-tweet request dispatching: ' + res[1] return (False, err_msg) ret_envelope = res[1] if type(ret_envelope) is not dict: return (False, 'unknown form of returned send-tweet data: ' + str(type(ret_envelope))) if ('status' not in ret_envelope) or (not ret_envelope['status']): err_msg = '' if ('error' in ret_envelope) and (ret_envelope['error']): err_msg = ': ' + str(ret_envelope['error']) return (False, 'status not acknowledged in returned send-tweet data' + err_msg) if ('data' not in ret_envelope) or (not ret_envelope['data']): return (False, 'payload not provided in returned send-tweet data') ret_data = ret_envelope['data'] if type(ret_data) is not dict: return (False, 'unknown form of returned payload in send-tweet data: ' + str(type(ret_data))) if 'id_str' not in ret_data: return (False, 'returned send-tweet data without tweet identifier') coll = db[collection_reports] saved_tweet = coll.find_one({'original_id': ret_data['id_str']}) if not saved_tweet: return (False, 'saved report on the send-tweet not found') doc_id = saved_tweet['_id'] saved_update = { 'local': True, 'user_id': _get_id_value(user_id), 'proto': False, FIELD_UPDATED: datetime.datetime.utcnow(), #'_etag': _get_etag(), } coll.update({'_id': doc_id}, {'$set': saved_update}) return (True, {'_id': doc_id})
def do_get_list(db, offset=None, limit=None, sort=None, other=None): ''' returns data of a set of sms citizen aliases ''' if not db: return (False, 'inner application error') list_spec = {'authority': AUTHORITY} sort_list = _get_sort(sort) if not sort_list: sort_list = [('produced', 1)] name_only = False if other and ('name_only' in other) and other['name_only']: try: name_only = bool(_get_boolean(other['name_only'])) except: name_only = False coll = db[collection] cursor = coll.find(list_spec).sort(sort_list) total = cursor.count() if limit is None: limit = DEFAULT_LIMIT if offset: cursor = cursor.skip(offset) if limit: cursor = cursor.limit(limit) docs = [] for entry in cursor: if not entry: continue if not name_only: docs.append(entry) else: if (not 'identifiers' in entry): continue if type(entry['identifiers']) is not dict: continue one_name = {} if 'user_id' not in entry['identifiers']: continue one_name['phone_number'] = entry['identifiers']['user_id'] if not one_name['phone_number']: continue description = None if ('description' in entry) and entry['description']: description = entry['description'] if description: one_name['description'] = description name_list = [] for name_key in ['name_first', 'name_last', 'name_full']: if (name_key in entry) and entry[name_key]: name_list.append(entry[name_key]) if name_list: one_name['names'] = ' - '.join(name_list) docs.append(one_name) return (True, docs, {'total': total})
def do_get_list(db, endpoint_type, endpoint_id, proto=None, offset=None, limit=None, sort=None, other=None): ''' returns data of a set of reports saved by a stream or a search ''' if not db: return (False, 'inner application error') endpoint_id = _get_id_value(endpoint_id) list_spec = {'feed_type': FEED_TYPE, 'channels': {'$elemMatch': {'type': endpoint_type, 'value': endpoint_id}}} if proto is not None: try: proto = bool(_get_boolean(proto)) except: return (False, 'the "proto" specifier has to be a boolean value') list_spec['proto'] = proto sort_list = _get_sort(sort) if not sort_list: sort_list = [('produced', 1)] text_only = False if other and ('text_only' in other) and other['text_only']: try: text_only = bool(_get_boolean(other['text_only'])) except: text_only = False coll = db[collection] cursor = coll.find(list_spec).sort(sort_list) total = cursor.count() if limit is None: limit = DEFAULT_LIMIT if offset: cursor = cursor.skip(offset) if limit: cursor = cursor.limit(limit) docs = [] for entry in cursor: if not entry: continue if not text_only: docs.append(entry) else: if (not 'texts' in entry): continue if (not entry['texts']): continue if (type(entry['texts']) not in [list]): continue for one_text in entry['texts']: source = None if ('sources' in entry) and (type(entry['sources']) is list): if len(entry['sources']) and (entry['sources'][0]): source = entry['sources'][0] if source: if (type(one_text) is dict): one_text['source'] = source docs.append(one_text) return (True, docs, {'total': total})
def do_get_list(db, spec_type, spec_id, offset=None, limit=None, sort=None, other=None): ''' returns data of a set of sms reports ''' if not db: return (False, 'inner application error') list_spec = {'feed_type': FEED_TYPE} if 'session_id' == spec_type: list_spec['session'] = spec_id if 'sent_to' == spec_type: list_spec['recipients.identifiers.user_id_search'] = normalize_phone_number(spec_id) if 'received_from' == spec_type: list_spec['authors.identifiers.user_id_search'] = normalize_phone_number(spec_id) sort_list = _get_sort(sort) if not sort_list: sort_list = [('produced', 1)] text_only = False if other and ('text_only' in other) and other['text_only']: try: text_only = bool(_get_boolean(other['text_only'])) except: text_only = False coll = db[collection] cursor = coll.find(list_spec).sort(sort_list) total = cursor.count() if limit is None: limit = DEFAULT_LIMIT if offset: cursor = cursor.skip(offset) if limit: cursor = cursor.limit(limit) docs = [] for entry in cursor: if not entry: continue if not text_only: docs.append(entry) else: one_text = { 'original': None, 'authors': None, 'recipients': None } authors = [] recipients = [] if 'original' in entry: one_text['original'] = entry['original'] if ('authors' in entry) and (type(entry['authors']) in (list, tuple)): for one_author in entry['authors']: if (type(one_author) is dict) and ('identifiers' in one_author) and (one_author['identifiers']): one_alias_set = one_author['identifiers'] if type(one_alias_set) is dict: if ('user_id' in one_alias_set) and one_alias_set['user_id']: authors.append(one_alias_set['user_id']) if ('recipients' in entry) and (type(entry['recipients']) in (list, tuple)): for one_recipient in entry['recipients']: if (type(one_recipient) is dict) and ('identifiers' in one_recipient) and (one_recipient['identifiers']): one_alias_set = one_recipient['identifiers'] if type(one_alias_set) is dict: if ('user_id' in one_alias_set) and one_alias_set['user_id']: recipients.append(one_alias_set['user_id']) if authors: try: one_text['authors'] = ', '.join(authors) except: pass if recipients: try: one_text['recipients'] = ', '.join(recipients) except: pass if (not one_text['original']) and (not one_text['authors']) and (not one_text['recipients']): continue docs.append(one_text) return (True, docs, {'total': total})
def do_get_list(db, endpoint_type, endpoint_id, proto=None, offset=None, limit=None, sort=None, other=None): ''' returns data of a set of reports saved by a stream or a search ''' if not db: return (False, 'inner application error') endpoint_id = _get_id_value(endpoint_id) list_spec = {'feed_type': FEED_TYPE, 'channels': {'$elemMatch': {'type': endpoint_type, 'value': endpoint_id}}} if proto is not None: try: proto = bool(_get_boolean(proto)) except: return (False, 'the "proto" specifier has to be a boolean value') list_spec['proto'] = proto sort_list = _get_sort(sort) if not sort_list: sort_list = [('produced', 1)] text_only = False if other and ('text_only' in other) and other['text_only']: try: text_only = bool(_get_boolean(other['text_only'])) except: text_only = False coll = db[collection] cursor = coll.find(list_spec).sort(sort_list) total = cursor.count() if limit is None: limit = DEFAULT_LIMIT if offset: cursor = cursor.skip(offset) if limit: cursor = cursor.limit(limit) docs = [] for entry in cursor: if not entry: continue if not text_only: docs.append(entry) else: if (not 'texts' in entry): continue if (not entry['texts']): continue if (type(entry['texts']) not in [list]): continue for one_text in entry['texts']: source = None if ('sources' in entry) and (type(entry['sources']) in (list, tuple)): if len(entry['sources']) and (entry['sources'][0]): source = entry['sources'][0] if source: if (type(one_text) is dict): one_text['source'] = source docs.append(one_text) return (True, docs, {'total': total})
def do_patch_one(db, doc_id=None, data=None, force=None): ''' starts/stops the stream ''' try: from citizendesk.feeds.twt.filter.storage import get_one as filter_get_one except: return (False, 'filter processor not available') try: from citizendesk.feeds.twt.oauth.storage import get_one as oauth_get_one except: return (False, 'oauth processor not available') if not controller: return (False, 'external controller not available') if not db: return (False, 'inner application error') if data is None: return (False, 'data not provided') if type(data) is not dict: return (False, 'invalid data provided') if ('control' not in data) or (type(data['control']) is not dict): return (False, '"control" part not provided') if doc_id is None: return (False, 'stream _id not provided') doc_id = _get_id_value(doc_id) try: force_val = bool(_get_boolean(force)) except: force_val = None coll = db[collection] entry = coll.find_one({'_id': doc_id}) if not entry: return (False, '"stream" of the provided _id not found: ' + str(doc_id)) if ('spec' not in entry) or (type(entry['spec']) is not dict): return (False, '"stream" of the provided _id does not contain "spec" part') doc = {'control': {}} for key in schema['control']: doc['control'][key] = None if key in data['control']: doc['control'][key] = data['control'][key] res = _check_schema(None, doc['control']) if not res[0]: return res should_run = False if doc['control']['switch_on']: if not doc['control']['streamer_url']: return (False, 'can not start the stream without the "streamer_url" being set') else: should_run = True filter_id = None oauth_id = None if should_run: if ('spec' in entry) and (type(entry['spec']) is dict): if 'filter_id' in entry['spec']: filter_id = entry['spec']['filter_id'] if 'oauth_id' in entry['spec']: oauth_id = entry['spec']['oauth_id'] if (not filter_id) or (not oauth_id): return (False, 'Can not start the stream without assigned filter_id and oauth_id') new_stream_url = None connector = None if should_run: new_stream_url = doc['control']['streamer_url'] connector = controller.NewstwisterConnector(new_stream_url) # load the supplemental data filter_info = None oauth_info = None if should_run: res = filter_get_one(db, filter_id) if not res[0]: return (False, 'filter info with _id equal to "spec.filter_id" not found') filter_info = res[1] if ('spec' not in filter_info) or (type(filter_info['spec']) is not dict): return (False, 'set filter without "spec" part') res = oauth_get_one(db, oauth_id) if not res[0]: return (False, 'oauth info with _id equal to "spec.oauth_id" not found') oauth_info = res[1] if ('spec' not in oauth_info) or (type(oauth_info['spec']) is not dict): return (False, 'set oauth without "spec" part') filter_spec = {} filter_spec_original = {} if should_run and filter_info and filter_info['spec']: try: prep_res = _prepare_filter(db, connector, filter_info['spec']) if not prep_res[0]: return (False, prep_res[1]) filter_spec = prep_res[1] filter_spec_original = prep_res[2] except Exception as exc: return (False, 'can not prepare filter params: ' + str(exc)) if not filter_spec: return (False, 'no filter spec was prepared') if not filter_spec_original: return (False, 'no filter spec original was prepared') # check if the oauth_id is not used by any other running feed if should_run: running_count = coll.find({'spec.oauth_id': oauth_id, 'control.switch_on': True, '_id': {'$ne': doc_id}}).count() if running_count: return (False, 'the "oauth_id" is already used at a running stream') # stop first in any case, since filter/oauth specs might have changed might_run = True if not entry: might_run = False if might_run: if ('control' not in entry) or (type(entry['control']) is not dict): might_run = False if might_run: for key in schema['control']: if (key not in entry['control']) or (not entry['control'][key]): might_run = False break if might_run: cur_stream_url = entry['control']['streamer_url'] connector = controller.NewstwisterConnector(cur_stream_url) res = connector.request_stop(entry['control']['process_id']) if (not res) and (not force_val): return (False, 'can not stop the stream') # update info in db when we know it has been stopped timepoint = datetime.datetime.utcnow() update_set = {'control.switch_on': False, 'control.process_id': None, 'logs.stopped': timepoint} coll.update({'_id': doc_id}, {'$set': update_set}, upsert=False) if should_run: res = connector.request_start(str(doc_id), oauth_info['spec'], filter_spec, filter_spec_original) if not res: return (False, 'can not start the stream') try: proc_id = int(res) except: proc_id = res # update info in db when we know it has been started timepoint = datetime.datetime.utcnow() update_set = {'control.switch_on': True, 'control.process_id': proc_id, 'control.streamer_url': new_stream_url, 'logs.started': timepoint} coll.update({'_id': doc_id}, {'$set': update_set}, upsert=False) return (True, {'_id': doc_id})