def do_on_behalf_of(db, report_id, user_id=None): ''' un/sets report as on behalf of user ''' if not db: return (False, 'inner application error') report_id = _get_id_value(report_id) report_get = get_report_by_id(db, report_id) if not report_get[0]: return (False, 'report not found') report = report_get[1] if user_id: user_id = _get_id_value(user_id) timepoint = datetime.datetime.utcnow() properties_set = { FIELD_UPDATED: timepoint, 'proto': False, } removal_set = None if user_id: properties_set['on_behalf_id'] = user_id else: removal_set = {'on_behalf_id': ''} update_report_set(db, report_id, properties_set, removal_set) return (True, {'_id': report_id})
def get_coverage_info(db, coverage_id): coverage_id = _get_id_value(coverage_id) coll = db[COLL_COVERAGES] if type(coverage_id) is ObjectId: return (False, 'old form of coverage info retrieval') else: coverage = coll.find_one({'uuid': coverage_id}) if not coverage: return (False, 'coverage not found') base_url = get_conf('reports_url') reports_url = base_url.replace(PUBLISHED_REPORTS_PLACEHOLDER, urllib.quote_plus(str(coverage_id))) title = '' if ('title' in coverage) and coverage['title']: title = coverage['title'] description = '' if ('description' in coverage) and coverage['description']: description = coverage['description'] active = False if (FIELD_ACTIVE_COVERAGE in coverage) and coverage[FIELD_ACTIVE_COVERAGE]: active = coverage[FIELD_ACTIVE_COVERAGE] info = { 'IsLive': active, 'Title': title, 'Description': description, 'PostPublished': {'href': reports_url,}, } return (True, info)
def do_set_active_one(db, coverage_id, set_active): ''' de/activate a coverage ''' if not db: return (False, 'inner application error') coverage_id = _get_id_value(coverage_id) coverage_get = get_coverage_by_id(db, coverage_id) if not coverage_get[0]: return (False, 'coverage not found') coverage = coverage_get[1] if set_active: if (FIELD_DECAYED in coverage) and coverage[FIELD_DECAYED]: return (False, 'can not activate decayed coverage') timepoint = datetime.datetime.utcnow() if type(coverage_id) is ObjectId: sel_part = {'_id': coverage_id} else: sel_part = {'uuid': coverage_id} update_coverage_set(db, sel_part, {FIELD_ACTIVE: set_active, '_updated': timepoint}) return (True, {'_id': coverage_id})
def do_get_session(db, session, offset=None, limit=None): ''' returns data of a set of reports saved by the stream ''' if not db: return (False, 'inner application error') session = _get_id_value(session) list_spec = {'feed_type': FEED_TYPE, 'session': session} coll = db[collection] cursor = coll.find(list_spec).sort([('produced', 1)]) total = cursor.count() if limit is None: limit = DEFAULT_LIMIT if offset: cursor = cursor.skip(offset) if limit: cursor = cursor.limit(limit) docs = [] for entry in cursor: if not entry: continue docs.append(entry) return (True, docs, {'total': total})
def get_report_author(db, report_id, author_form): coll = db[COLL_REPORTS] if author_form not in ['author', 'creator', 'icon']: return (False, 'unknown author form') report_id = _get_id_value(report_id) report = coll.find_one({'_id': report_id}) if not report: return (False, 'respective report not found') if 'feed_type' not in report: report['feed_type'] = DEFAULT_FIELD_TYPE feed_type = report['feed_type'] if feed_type not in OUTPUT_FEED_TYPES: return (False, 'unsupported report type') author = None user = None if 'on_behalf_id' in report: user_id = report['on_behalf_id'] user = load_local_user(db, user_id) if not user: user = None if 'sms' == feed_type: if 'author' == author_form: author = get_sms_report_author(report_id, report, user) if 'creator' == author_form: author = get_sms_report_creator(report_id, report, user) if 'icon' == author_form: author = get_sms_report_icon(report_id, report, user) if 'tweet' == feed_type: if 'author' == author_form: author = get_tweet_report_author(report_id, report, user) if 'creator' == author_form: author = get_tweet_report_creator(report_id, report, user) if 'icon' == author_form: author = get_tweet_report_icon(report_id, report, user) if feed_type in ('plain', 'web_link'): if 'author' == author_form: author = get_plain_report_author(report_id, report, user) if 'creator' == author_form: author = get_plain_report_creator(report_id, report, user) if 'icon' == author_form: author = get_plain_report_icon(report_id, report, user) return (True, author)
def do_get_one(db, coverage_id): ''' returns data of a single coverage ''' if not db: return (False, 'inner application error') coverage_id = _get_id_value(coverage_id) res = get_coverage_by_id(db, coverage_id) return res
def do_get_one(db, doc_id): ''' returns data of a single sms report ''' if not db: return (False, 'inner application error') doc_id = _get_id_value(doc_id) res = do_get_one_by_id(db, doc_id) return res
def do_get_one(db, service_id): ''' returns data of a single service ''' if not db: return (False, 'inner application error') service_id = _get_id_value(service_id) res = get_service_by_id(db, service_id) return res
def load_local_user(db, user_id): user_id = _get_id_value(user_id) coll = db[COLL_USERS] user = coll.find_one({'_id': user_id}) if not user: return None return user
def do_insert_one(db, coverage_data): ''' creates a coverage ''' if not db: return (False, 'inner application error') timepoint = datetime.datetime.utcnow() uuid_value = None if ('uuid' in coverage_data) and coverage_data['uuid']: try: uuid_value = str(uuid.UUID(str(coverage_data['uuid'])).hex) except: uuid_value = None coverage_set = None if uuid_value: coverage_get = get_coverage_by_uuid(db, uuid_value) if coverage_get[0]: coverage_set = coverage_get[1] if not coverage_set: coverage_set = {} if uuid_value: coverage_set['uuid'] = uuid_value else: coverage_set['uuid'] = str(uuid.uuid4().hex) try: coverage_set['title'] = str(coverage_data['title']) coverage_set['description'] = str(coverage_data['description']) coverage_set['user_id'] = _get_id_value(coverage_data['user_id']) coverage_set[FIELD_ACTIVE] = False coverage_set[FIELD_DECAYED] = False coverage_set['_created'] = timepoint coverage_set['_updated'] = timepoint #coverage_set['_etag'] = _get_etag() except: return (False, 'can not setup the coverage') if ('active' in coverage_data) and (coverage_data['active']): coverage_set[FIELD_ACTIVE] = True coll = db[collection] try: coverage_id = coll.save(coverage_set) except: return (False, 'can not save the coverage data') return (True, {'_id': coverage_id})
def do_delete_one(db, doc_id): ''' deletes data of a single oauth info ''' if not db: return (False, 'inner application error') doc_id = _get_id_value(doc_id) coll = db[collection] coll.remove({'_id': doc_id}) return (True, {'_id': doc_id})
def get_one(db, doc_id): ''' returns data of a single oauth info ''' if not db: return (False, 'inner application error') doc_id = _get_id_value(doc_id) coll = db[collection] doc = coll.find_one({'_id': doc_id}) if not doc: return (False, 'oauth info not found') return (True, doc)
def get_one(db, doc_id): ''' returns data of a single filter info ''' if not db: return (False, 'inner application error') doc_id = _get_id_value(doc_id) coll = db[collection] doc = coll.find_one({'_id': doc_id}) if not doc: return (False, 'filter info not found') return (True, doc)
def do_get_one(db, doc_id): ''' returns data of a single stream info ''' if not controller: return (False, 'external controller not available') if not db: return (False, 'inner application error') doc_id = _get_id_value(doc_id) coll = db[collection] doc = coll.find_one({'_id': doc_id}) if not doc: return (False, 'stream info not found') might_run = True if ('control' not in doc) or (type(doc['control']) is not dict): might_run = False if might_run: for key in schema['control']: if (key not in doc['control']) or (not doc['control'][key]): might_run = False break if might_run: connector = controller.NewstwisterConnector(doc['control']['streamer_url']) res = connector.request_status(doc['control']['process_id']) # let None here when can not check that it runs if res is not None: doc['control']['switch_on'] = True if res else False else: doc['control']['switch_on'] = None if res is False: # update info in db when we know it has been stopped timepoint = datetime.datetime.utcnow() update_set = {'control.switch_on': False, 'control.process_id': None, 'logs.stopped': timepoint} coll.update({'_id': doc_id}, {'$set': update_set}, upsert=False) if ('spec' not in doc) or (type(doc['spec']) is not dict): return (False, 'wrong stream info (spec) in db') if ('control' not in doc) or (type(doc['control']) is not dict): return (False, 'wrong stream info (control) in db') return (True, doc)
def do_delete_one(db, service_id): ''' remove one service ''' if not db: return (False, 'inner application error') service_id = _get_id_value(service_id) service_get = get_service_by_id(db, service_id) if not service_get[0]: return (False, 'service not found') service = service_get[1] coll = db[collection] cursor = coll.remove({'_id': service_id}) return (True, {'_id': service_id})
def do_set_active_one(db, service_id, set_active): ''' de/activate a service ''' if not db: return (False, 'inner application error') service_id = _get_id_value(service_id) service_get = get_service_by_id(db, service_id) if not service_get[0]: return (False, 'service not found') service = service_get[1] timepoint = datetime.datetime.utcnow() update_service_set(db, service_id, {FIELD_ACTIVE: set_active, '_updated': timepoint}) return (True, {'_id': service_id})
def do_get_one(db, doc_id): ''' returns data of a single stream control ''' if not db: return (False, 'inner application error') doc_id = _get_id_value(doc_id) spec_field = 'report_id' if type(doc_id) is ObjectId: spec_field = '_id' coll = db[collection] doc = coll.find_one({'feed_type': FEED_TYPE, spec_field: doc_id}) if not doc: return (False, 'report not found') return (True, doc)
def extract_annotation_status(db, report): # taking annotation from status is_summary = False if (FIELD_SUMMARY_REPORT in report) and (report[FIELD_SUMMARY_REPORT] is not None): is_summary = _get_boolean(report[FIELD_SUMMARY_REPORT]) status_desc = None status_filled = False if (FIELD_STATUS_REPORT in report) and (report[FIELD_STATUS_REPORT]): status_filled = True status_ident = _get_id_value(report[FIELD_STATUS_REPORT]) if type(status_ident) is ObjectId: status_desc = take_status_desc_by_id(db, status_ident) else: status_desc = take_status_desc_by_key(db, status_ident) if not status_filled: if (FIELD_STATUS_VERIFIED_REPORT in report) and (report[FIELD_STATUS_VERIFIED_REPORT] is not None): verified_flag = _get_boolean(report[FIELD_STATUS_VERIFIED_REPORT]) if verified_flag: status_desc = take_status_desc_by_key(db, STATUS_VERIFIED_KEY) is_assigned = False if status_desc is None: if (FIELD_ASSIGNED_REPORT in report) and (type( report[FIELD_ASSIGNED_REPORT]) in (list, tuple)): for item in report[FIELD_ASSIGNED_REPORT]: if item is not None: is_assigned = True break if is_assigned: status_desc = take_status_desc_by_key(db, STATUS_ASSIGNED_KEY) if (status_desc is None) and (not is_assigned) and (not is_summary): status_desc = take_status_desc_by_key(db, STATUS_NEW_KEY) if not status_desc: status_desc = None return status_desc
def do_unpublish_one(db, coverage_id): ''' unpublish all reports from a coverage ''' if not db: return (False, 'inner application error') coverage_id = _get_id_value(coverage_id) timepoint = datetime.datetime.utcnow() update_set = { FIELD_UPDATED_REPORT: timepoint, #'_etag': _get_etag(), } excise_set = {FIELD_COVERAGES_PUBLISHED: [coverage_id]} coll = db[collection_reports] coll.update({FIELD_COVERAGES_PUBLISHED: coverage_id}, {'$pullAll': excise_set, '$set': update_set}, multi=True, upsert=False) return (True, {'_id': coverage_id})
def do_patch_one(db, doc_id=None, data=None): ''' sets the "proto" field value ''' if not db: return (False, 'inner application error') if data is None: return (False, 'data not provided') if doc_id is not None: doc_id = _get_id_value(doc_id) if type(data) is not dict: return (False, 'data should be a dict') if ('proto' not in data) or (data['proto'] is None): return (False, 'data should contain a "proto" field') try: proto = bool(_get_boolean(data['proto'])) except: return (False, 'the "proto" field has to be a boolean value') res = do_get_one(db, doc_id) if not res[0]: return res coll = db[collection] spec_field = 'report_id' if type(doc_id) is ObjectId: spec_field = '_id' update_set = {} update_set['proto'] = proto update_set[FIELD_UPDATED] = datetime.datetime.utcnow() #update_set['_etag'] = _get_etag() coll.update({'feed_type': FEED_TYPE, spec_field: doc_id}, {'$set': update_set}, upsert=False) return (True, {'report_id': doc_id})
def do_set_active_one(db, service_id, set_active): ''' de/activate a service ''' if not db: return (False, 'inner application error') service_id = _get_id_value(service_id) service_get = get_service_by_id(db, service_id) if not service_get[0]: return (False, 'service not found') service = service_get[1] timepoint = datetime.datetime.utcnow() update_service_set(db, service_id, { FIELD_ACTIVE: set_active, '_updated': timepoint }) return (True, {'_id': service_id})
def extract_annotation_status(db, report): # taking annotation from status is_summary = False if (FIELD_SUMMARY_REPORT in report) and (report[FIELD_SUMMARY_REPORT] is not None): is_summary = _get_boolean(report[FIELD_SUMMARY_REPORT]) status_desc = None status_filled = False if (FIELD_STATUS_REPORT in report) and (report[FIELD_STATUS_REPORT]): status_filled = True status_ident = _get_id_value(report[FIELD_STATUS_REPORT]) if type(status_ident) is ObjectId: status_desc = take_status_desc_by_id(db, status_ident) else: status_desc = take_status_desc_by_key(db, status_ident) if not status_filled: if (FIELD_STATUS_VERIFIED_REPORT in report) and (report[FIELD_STATUS_VERIFIED_REPORT] is not None): verified_flag = _get_boolean(report[FIELD_STATUS_VERIFIED_REPORT]) if verified_flag: status_desc = take_status_desc_by_key(db, STATUS_VERIFIED_KEY) is_assigned = False if status_desc is None: if (FIELD_ASSIGNED_REPORT in report) and (type(report[FIELD_ASSIGNED_REPORT]) in (list, tuple)): for item in report[FIELD_ASSIGNED_REPORT]: if item is not None: is_assigned = True break if is_assigned: status_desc = take_status_desc_by_key(db, STATUS_ASSIGNED_KEY) if (status_desc is None) and (not is_assigned) and (not is_summary): status_desc = take_status_desc_by_key(db, STATUS_NEW_KEY) if not status_desc: status_desc = None return status_desc
def get_coverage_info(db, coverage_id): coverage_id = _get_id_value(coverage_id) coll = db[COLL_COVERAGES] if type(coverage_id) is ObjectId: return (False, 'old form of coverage info retrieval') else: coverage = coll.find_one({'uuid': coverage_id}) if not coverage: return (False, 'coverage not found') base_url = get_conf('reports_url') reports_url = base_url.replace(PUBLISHED_REPORTS_PLACEHOLDER, urllib.quote_plus(str(coverage_id))) title = '' if ('title' in coverage) and coverage['title']: title = coverage['title'] description = '' if ('description' in coverage) and coverage['description']: description = coverage['description'] active = False if (FIELD_ACTIVE_COVERAGE in coverage) and coverage[FIELD_ACTIVE_COVERAGE]: active = coverage[FIELD_ACTIVE_COVERAGE] info = { 'IsLive': active, 'Title': title, 'Description': description, 'PostPublished': { 'href': reports_url, }, } return (True, info)
def do_get_one(db, alias_type, alias_value): ''' returns data of a single citizen alias ''' if not db: return (False, 'inner application error') res = None if 'alias_id' == alias_type: alias_value = _get_id_value(alias_value) res = get_one_by_id(db, alias_value) if 'phone_number' == alias_type: try: phone_number = str(alias_value) except: return (False, 'wrong phone number form') res = get_one_by_phone_number(db, phone_number) if not res: return (False, 'unknown form of citizen alias') return res
def do_delete_one(db, doc_id): ''' deletes data of a single stream info ''' if not controller: return (False, 'external controller not available') if not db: return (False, 'inner application error') doc_id = _get_id_value(doc_id) coll = db[collection] doc = coll.find_one({'_id': doc_id}) if not doc: return (False, 'requested stream not found') might_run = True if ('control' not in doc) or (type(doc['control']) is not dict): might_run = False if might_run: for key in schema['control']: if (key not in doc['control']) or (not doc['control'][key]): might_run = False break if might_run: connector = controller.NewstwisterConnector(doc['control']['streamer_url']) res = connector.request_stop(doc['control']['process_id']) if not res: return (False, 'can not stop the stream') coll.remove({'_id': doc_id}) return (True, {'_id': doc_id})
def do_post(holder, tweet_id, tweet, channel_type, endpoint, request_id, feed_filter, client_ip): try: endpoint_id = endpoint['endpoint_id'] except: return (False, 'endpoint[endpoint_id] not provided',) endpoint_id = _get_id_value(endpoint_id) request_id = _get_id_value(request_id) feed_type = get_conf('feed_type') # check if the tweet is a new tweet, already saved tweet, a retweet, or a retweet on an already saved tweet # for a retweet, set endorsers (and may be comments if a new one) retweeted_report = None retweeted_id = None retweeted_tweet = None try: current_sub = tweet while ('retweeted_status' in current_sub) and current_sub['retweeted_status']: current_sub = current_sub['retweeted_status'] retweeted_tweet = current_sub retweeted_id = retweeted_tweet['id_str'] except: retweeted_id = None retweeted_tweet = None # we need to have endorsers extracted early for extracting the reasons report_endorsers = [] if retweeted_id: try: endorser_ids = [{'type':'user_id', 'value':tweet['user']['id_str']}, {'type':'user_name', 'value':tweet['user']['screen_name']}] report_endorsers = [{'authority': 'twitter', 'identifiers': endorser_ids}] except: return (False, 'can not create endorsers',) # if tweet already saved, add channels part main_report_id = gen_id(feed_type, (retweeted_id if retweeted_id else tweet_id)) tweet_report = get_tweet(main_report_id) if tweet_report: main_report_id = tweet_report['report_id'] one_channel = {'type': channel_type, 'value': endpoint_id, 'request': request_id, 'filter': feed_filter, 'reasons': {}} try: expanded_text = _get_expanded_text(tweet_report['original']) reasons = {} if 'stream' == channel_type: reasons = find_stream_reason(feed_filter, expanded_text, tweet_report['authors'], report_endorsers, tweet_report['recipients']) if 'search' == channel_type: reasons = find_search_reason(feed_filter, expanded_text, tweet_report['authors'], report_endorsers, tweet_report['recipients']) one_channel['reasons'] = reasons except Exception as exc: sys.stderr.write(str(exc) + '\n') return (False, 'can extract the reasons') holder.add_channels(feed_type, main_report_id, [one_channel]) # for a retweet, set endorsers for the original tweet if retweeted_id: try: holder.add_endorsers(feed_type, main_report_id, report_endorsers) except: return (False, 'can not add endorsers',) else: main_tweet_id = retweeted_id if retweeted_id else tweet_id main_tweet = retweeted_tweet if retweeted_tweet else tweet res = process_new_tweet(holder, main_tweet_id, main_tweet, channel_type, endpoint_id, request_id, feed_filter, report_endorsers, client_ip) if not res[0]: return (False, res[1]) return (True,)
def do_post_reply(db, sms_gateway_api, sms_gateway_url, sms_gateway_key, message, report_id, user_id, language, sensitive, client_ip): try: controller = importlib.import_module('citizendesk.feeds.sms.external.' + sms_gateway_api) except: controller = None if not controller: return (False, 'no sms gateway controller available') if not db: return (False, 'no database available') if not sms_gateway_url: return (False, 'no sms gateway url configured') if not sms_gateway_key: return (False, 'no sms gateway key configured') if not message: return (False, 'no message provided') if not report_id: return (False, 'report_id not provided') from citizendesk.feeds.sms.report.storage import do_get_one_by_id as get_one_report_by_id from citizendesk.feeds.sms.citizen_alias.storage import get_one_by_id as get_one_citizen_alias_by_id from citizendesk.feeds.sms.citizen_alias.storage import get_one_by_phone_number as get_one_citizen_alias_by_phone_number report_id = _get_id_value(report_id) report_res = get_one_report_by_id(db, report_id) if not report_res[0]: return (False, 'report of given id not found') report_source = report_res[1] was_sent = False was_received = False channel_type = get_conf('channel_type') channel_value_send = get_conf('channel_value_send') channel_value_receive = get_conf('channel_value_receive') alias_doctype = get_conf('alias_doctype') authority = get_conf('authority') if type(report_source) is not dict: return (False, 'wrong structure of report') if ('channels' in report_source) and (type(report_source['channels']) in (list, tuple)): for one_channel in report_source['channels']: if (type(one_channel) is dict) and ('type' in one_channel) and ('value' in one_channel): if one_channel['type'] != channel_type: continue if one_channel['value'] == channel_value_send: was_sent = True if one_channel['value'] == channel_value_receive: was_received = True if (not was_sent) and (not was_received): return (False, 'unknown form of report') citizen_aliases = [] if was_sent and ('targets' in report_source) and (type(report_source['targets']) in (list, tuple)): for one_target in report_source['targets']: if (type(one_target) is dict) and ('type' in one_target) and ('value' in one_target): if one_target['type'] == alias_doctype: one_target_value = _get_id_value(one_target['value']) one_citizen_alias_res = get_one_citizen_alias_by_id(db, one_target_value) if one_citizen_alias_res[0]: citizen_aliases.append(one_citizen_alias_res[1]) if was_received and ('authors' in report_source) and (type(report_source['authors']) in (list, tuple)): for one_author in report_source['authors']: one_phone_number = _get_phone_number(one_author) if not one_phone_number: continue one_citizen_alias_res = get_one_citizen_alias_by_phone_number(db, one_phone_number) if one_citizen_alias_res[0]: citizen_aliases.append(one_citizen_alias_res[1]) if not citizen_aliases: return (False, 'not target suitable citizen alias found') use_targets = [] use_recipients = [] use_phone_numbers = [] for one_citizen_alias in citizen_aliases: if (type(one_citizen_alias) is not dict): continue if ('_id' not in one_citizen_alias) or (not one_citizen_alias['_id']): continue one_target = {'type': alias_doctype, 'value': one_citizen_alias['_id']} if ('identifiers' not in one_citizen_alias) or (not one_citizen_alias['identifiers']): continue one_recipient = {'authority': authority, 'identifiers': one_citizen_alias['identifiers']} one_phone_number = _get_phone_number(one_citizen_alias) if not one_phone_number: continue use_phone_numbers.append(one_phone_number) use_recipients.append(one_recipient) use_targets.append(one_target) if not use_targets: return (False, 'no valid recipient found') # putting all the recipients into a single report; and thus using its session for next communication with all the recipients try: report = _prepare_sms_reply_report(report_source, use_targets, use_recipients, message, user_id, language, sensitive, client_ip) except: report = None if not report: return (False, 'report could not be prepared') # we either save the report before sms sending, and deleting it if sending fails, # or we first send sms, and if success on it, we save the report then; # thus either transiently having a false report, or a possibility of not having the report # on sent sms if the report saving fails at the end (should not hopefully though) doc_id = report_holder.save_report(report) if not doc_id: return (False, 'report could not be saved') connector = controller.SMSConnector(sms_gateway_url, sms_gateway_key) res = connector.send_sms(message, {'phone_numbers': use_phone_numbers}) if not res[0]: report_holder.delete_report(doc_id) return (False, 'message could not be sent', res[1]) return (True, {'_id': doc_id})
def prepare_sms_send_report(targets, recipients, message, user_id=None, language=None, sensitive=None, client_ip=None): ''' preparing an sms-based report when the sms is sent not as a reply, i.e. starting a new session for it ''' channel_type = get_conf('channel_type') channel_value = get_conf('channel_value_send') feed_type = get_conf('feed_type') if user_id: user_id = _get_id_value(user_id) channel = { 'type': channel_type, 'value': channel_value, 'filter': None, 'reasons': None, 'request': None } current_timestamp = datetime.datetime.utcnow() doc = { 'report_id': gen_id(feed_type, channel_type, channel_value, current_timestamp), # to generate the report_id 'channels': [channel], 'recipients': recipients, 'authors': [], # no citizen here 'endorsers': [], # no citizen here 'publisher': get_conf('publisher'), 'feed_type': feed_type, 'parent_id': None, # not a reply here 'session': None, 'user_id': user_id, # who requested the sending, if not automatic 'pinned_id': None, 'language': language, 'produced': current_timestamp, '_created': current_timestamp, '_updated': current_timestamp, 'assignments': [], 'original_id': None, 'original': { 'message': message }, 'texts': [{ 'original': message, 'transcript': None }], 'tags': [], # we shall extract possible #tags, alike at sms receiving 'sensitive': None, 'summary': False, 'local': True, 'targets': targets, # alias_ids and/or group_ids for replies, on sent reports 'proto': False, 'client_ip': client_ip # for logging } if sensitive is not None: doc['sensitive'] = _get_boolean(sensitive) doc['session'] = doc['report_id'] doc['tags'] = _extract_tags(message) return doc
def do_publish_one(db, report_id, coverage_id=None): ''' sets report as published in a coverage; the coverage_id can be either _id or uuid ''' if not db: return (False, 'inner application error') report_id = _get_id_value(report_id) report_get = get_report_by_id(db, report_id) if not report_get[0]: return (False, 'report not found') report = report_get[1] if 'coverages' not in report: coverages = {} for cov_set in COVERAGE_SETS: coverages[cov_set] = [] update_report_set(db, report_id, {'coverages': coverages}) report['coverages'] = coverages coverages = report['coverages'] if type(coverages) is not dict: return (False, 'report coverages not a dict') for cov_set in COVERAGE_SETS: if (cov_set not in coverages) or (type(coverages[cov_set]) is not list): return (False, 'report coverage parts missing or wrong') to_publish = coverages['targeting'] if coverage_id: coverage_uuid = None coverage_id = _get_id_value(coverage_id) if type(coverage_id) is ObjectId: coverage_get = get_coverage_by_id(db, coverage_id) else: coverage_uuid = coverage_id coverage_get = get_coverage_by_uuid(db, coverage_uuid) if not coverage_get[0]: return (False, 'coverage not found') if not coverage_uuid: # here we know the coverage_id was _id of an existent coverage coverage_got = coverage_get[1] if ('uuid' in coverage_got) and coverage_got['uuid']: coverage_uuid = coverage_got['uuid'] else: coverage_uuid = str(uuid.uuid4().hex) update_coverage_set({'_id': coverage_id}, {'uuid': coverage_uuid}) to_publish = [coverage_uuid] cov_published = coverages['published'] cov_outgested = coverages['outgested'] set_published = False set_outgested = False if not to_publish: return (False, 'no coverage to be published in') for one_cov in to_publish: if one_cov not in cov_published: set_published = True cov_published.append(one_cov) if one_cov not in cov_outgested: set_outgested = True cov_outgested.append(one_cov) if set_published: update_report_set(db, report_id, {'coverages.published': cov_published}) if set_outgested: update_report_set(db, report_id, {'coverages.outgested': cov_outgested}) timepoint = datetime.datetime.utcnow() adjective_set = { FIELD_UPDATED: timepoint, 'proto': False, } if FIELD_UUID not in report: adjective_set[FIELD_UUID] = str(uuid.uuid4().hex) update_report_set(db, report_id, adjective_set) return (True, {'_id': report_id})
def do_unpublish_one(db, report_id, coverage_id=None): ''' sets report as unpublished in a coverage; the coverage_id can be either _id or uuid ''' if not db: return (False, 'inner application error') report_id = _get_id_value(report_id) report_get = get_report_by_id(db, report_id) if not report_get[0]: return (False, 'report not found') report = report_get[1] if 'coverages' not in report: coverages = {} for cov_set in COVERAGE_SETS: coverages[cov_set] = [] update_report_set(db, report_id, {'coverages': coverages}) report['coverages'] = coverages coverages = report['coverages'] if type(coverages) is not dict: return (False, 'report coverages not a dict') if ('published' not in coverages) or (type(coverages['published']) is not list): return (False, 'published coverages missing or wrong in report') cov_published = [] if coverage_id: coverage_uuid = None coverage_id = _get_id_value(coverage_id) if type(coverage_id) is ObjectId: coverage_get = get_coverage_by_id(db, coverage_id) if coverage_get[0] and ( 'uuid' in coverage_get[1]) and coverage_get[1]['uuid']: coverage_uuid = coverage_get[1]['uuid'] else: coverage_uuid = coverage_id curr_specifiers = [coverage_id] if coverage_uuid: curr_specifiers.append(coverage_uuid) was_published = False for one_spec in curr_specifiers: if (coverage_id in coverages['published']): was_published = True break if not was_published: return (False, 'not published into the coverage') for one_cov in coverages['published']: if one_cov and (one_cov != coverage_id) and (one_cov != coverage_uuid): cov_published.append(one_cov) update_report_set(db, report_id, {'coverages.published': cov_published}) timepoint = datetime.datetime.utcnow() adjective_set = { FIELD_UPDATED: timepoint, 'proto': False, } update_report_set(db, report_id, adjective_set) return (True, {'_id': report_id})
def do_post_one(db, doc_id=None, data=None): ''' sets data of a single oauth info ''' if not db: return (False, 'inner application error') if data is None: return (False, 'data not provided') if ('spec' not in data) or (type(data['spec']) is not dict): return (False, '"spec" part not provided') spec = data['spec'] doc_id = _get_id_value(doc_id) coll = db[collection] timepoint = datetime.datetime.utcnow() created = timepoint updated = timepoint if doc_id is not None: entry = coll.find_one({'_id': doc_id}) if not entry: return (False, '"oauth" of the provided _id not found') try: if ('_created' in entry) and entry['_created']: created = entry['_created'] except: created = timepoint doc = { '_created': created, '_updated': updated, 'spec': {} } for key in schema['spec']: doc['spec'][key] = None if key in spec: doc['spec'][key] = spec[key] res = _check_schema(doc['spec']) if not res[0]: return res if not doc_id: try: if USE_SEQUENCES: entry = db['counters'].find_and_modify(query={'_id': collection}, update={'$inc': {'next':1}}, new=True, upsert=True, full_response=False) doc_id = entry['next'] else: doc_id = ObjectId() except: return (False, 'can not create document id') doc['_id'] = doc_id doc_id = coll.save(doc) return (True, {'_id': doc_id})
def do_get_list(db, endpoint_type, endpoint_id, proto=None, offset=None, limit=None, sort=None, other=None): ''' returns data of a set of reports saved by a stream or a search ''' if not db: return (False, 'inner application error') endpoint_id = _get_id_value(endpoint_id) list_spec = {'feed_type': FEED_TYPE, 'channels': {'$elemMatch': {'type': endpoint_type, 'value': endpoint_id}}} if proto is not None: try: proto = bool(_get_boolean(proto)) except: return (False, 'the "proto" specifier has to be a boolean value') list_spec['proto'] = proto sort_list = _get_sort(sort) if not sort_list: sort_list = [('produced', 1)] text_only = False if other and ('text_only' in other) and other['text_only']: try: text_only = bool(_get_boolean(other['text_only'])) except: text_only = False coll = db[collection] cursor = coll.find(list_spec).sort(sort_list) total = cursor.count() if limit is None: limit = DEFAULT_LIMIT if offset: cursor = cursor.skip(offset) if limit: cursor = cursor.limit(limit) docs = [] for entry in cursor: if not entry: continue if not text_only: docs.append(entry) else: if (not 'texts' in entry): continue if (not entry['texts']): continue if (type(entry['texts']) not in [list]): continue for one_text in entry['texts']: source = None if ('sources' in entry) and (type(entry['sources']) is list): if len(entry['sources']) and (entry['sources'][0]): source = entry['sources'][0] if source: if (type(one_text) is dict): one_text['source'] = source docs.append(one_text) return (True, docs, {'total': total})
def do_patch_one(db, doc_id=None, data=None, force=None): ''' starts/stops the stream ''' try: from citizendesk.feeds.twt.filter.storage import get_one as filter_get_one except: return (False, 'filter processor not available') try: from citizendesk.feeds.twt.oauth.storage import get_one as oauth_get_one except: return (False, 'oauth processor not available') if not controller: return (False, 'external controller not available') if not db: return (False, 'inner application error') if data is None: return (False, 'data not provided') if type(data) is not dict: return (False, 'invalid data provided') if ('control' not in data) or (type(data['control']) is not dict): return (False, '"control" part not provided') if doc_id is None: return (False, 'stream _id not provided') doc_id = _get_id_value(doc_id) try: force_val = bool(_get_boolean(force)) except: force_val = None coll = db[collection] entry = coll.find_one({'_id': doc_id}) if not entry: return (False, '"stream" of the provided _id not found: ' + str(doc_id)) if ('spec' not in entry) or (type(entry['spec']) is not dict): return (False, '"stream" of the provided _id does not contain "spec" part') doc = {'control': {}} for key in schema['control']: doc['control'][key] = None if key in data['control']: doc['control'][key] = data['control'][key] res = _check_schema(None, doc['control']) if not res[0]: return res should_run = False if doc['control']['switch_on']: if not doc['control']['streamer_url']: return (False, 'can not start the stream without the "streamer_url" being set') else: should_run = True filter_id = None oauth_id = None if should_run: if ('spec' in entry) and (type(entry['spec']) is dict): if 'filter_id' in entry['spec']: filter_id = entry['spec']['filter_id'] if 'oauth_id' in entry['spec']: oauth_id = entry['spec']['oauth_id'] if (not filter_id) or (not oauth_id): return (False, 'Can not start the stream without assigned filter_id and oauth_id') new_stream_url = None connector = None if should_run: new_stream_url = doc['control']['streamer_url'] connector = controller.NewstwisterConnector(new_stream_url) # load the supplemental data filter_info = None oauth_info = None if should_run: res = filter_get_one(db, filter_id) if not res[0]: return (False, 'filter info with _id equal to "spec.filter_id" not found') filter_info = res[1] if ('spec' not in filter_info) or (type(filter_info['spec']) is not dict): return (False, 'set filter without "spec" part') res = oauth_get_one(db, oauth_id) if not res[0]: return (False, 'oauth info with _id equal to "spec.oauth_id" not found') oauth_info = res[1] if ('spec' not in oauth_info) or (type(oauth_info['spec']) is not dict): return (False, 'set oauth without "spec" part') filter_spec = {} filter_spec_original = {} if should_run and filter_info and filter_info['spec']: try: prep_res = _prepare_filter(db, connector, filter_info['spec']) if not prep_res[0]: return (False, prep_res[1]) filter_spec = prep_res[1] filter_spec_original = prep_res[2] except Exception as exc: return (False, 'can not prepare filter params: ' + str(exc)) if not filter_spec: return (False, 'no filter spec was prepared') if not filter_spec_original: return (False, 'no filter spec original was prepared') # check if the oauth_id is not used by any other running feed if should_run: running_count = coll.find({'spec.oauth_id': oauth_id, 'control.switch_on': True, '_id': {'$ne': doc_id}}).count() if running_count: return (False, 'the "oauth_id" is already used at a running stream') # stop first in any case, since filter/oauth specs might have changed might_run = True if not entry: might_run = False if might_run: if ('control' not in entry) or (type(entry['control']) is not dict): might_run = False if might_run: for key in schema['control']: if (key not in entry['control']) or (not entry['control'][key]): might_run = False break if might_run: cur_stream_url = entry['control']['streamer_url'] connector = controller.NewstwisterConnector(cur_stream_url) res = connector.request_stop(entry['control']['process_id']) if (not res) and (not force_val): return (False, 'can not stop the stream') # update info in db when we know it has been stopped timepoint = datetime.datetime.utcnow() update_set = {'control.switch_on': False, 'control.process_id': None, 'logs.stopped': timepoint} coll.update({'_id': doc_id}, {'$set': update_set}, upsert=False) if should_run: res = connector.request_start(str(doc_id), oauth_info['spec'], filter_spec, filter_spec_original) if not res: return (False, 'can not start the stream') try: proc_id = int(res) except: proc_id = res # update info in db when we know it has been started timepoint = datetime.datetime.utcnow() update_set = {'control.switch_on': True, 'control.process_id': proc_id, 'control.streamer_url': new_stream_url, 'logs.started': timepoint} coll.update({'_id': doc_id}, {'$set': update_set}, upsert=False) return (True, {'_id': doc_id})
def do_post_one(db, alias_id, alias_spec, user_id): ''' sets a basic info on phone_number-based citizen alias ''' if not db: return (False, 'inner application error') if type(alias_spec) is not dict: return (False, 'citizen specification should be a dict') if 'phone_number' not in alias_spec: return (False, 'phone number specification not provided') use_phone_number = alias_spec['phone_number'] try: use_phone_number = str(use_phone_number) except: return (False, 'wrong specification of phone number') if not use_phone_number: return (False, 'empty phone number specification') user_id = _get_id_value(user_id) if not alias_id: search_res = get_one_by_phone_number(db, use_phone_number) if search_res[0]: return (False, 'active citizen alias with given phone number already exists', search_res[1]['_id']) alias_doc = None if alias_id: alias_id = _get_id_value(alias_id) alias_doc = get_one_by_id(db, alias_id) if not alias_doc: return (False, 'citizen alias of provided id not found') spec_keys = { 'description': [str, unicode], 'name_first': [str, unicode], 'name_last': [str, unicode], 'name_full': [str, unicode], 'languages': [list, tuple], 'places': [list, tuple], 'home_pages': [list, tuple], 'verified': [bool] } use_identifiers = _create_phone_number_identities(use_phone_number) if not alias_doc: alias_use = { 'authority': AUTHORITY, 'identifiers': use_identifiers, 'verified': False, 'local': True, 'created_by': user_id } for key in spec_keys: if (key in alias_spec) and (type(alias_spec[key]) in spec_keys[key]): alias_use[key] = alias_spec[key] alias_id = citizen_holder.save_alias(alias_use) if not alias_id: return (False, 'can not save the citizen alias') else: alias_use = {} alias_use['identifiers'] = use_identifiers alias_use['updated_by'] = user_id alias_use['local'] = True for key in spec_keys: if (key in alias_spec) and (type(alias_spec[key]) in spec_keys[key]): alias_use[key] = alias_spec[key] #alias_use['_etag'] = _get_etag() coll = db[collection] alias_id = coll.update({'_id': alias_doc['_id']}, {'$set': alias_use}, upsert=False) if not alias_id: return (False, 'can not update the citizen alias') return (True, {'_id': alias_id})
def get_coverage_published_report_list(db, coverage_id, cid_last): ''' SMS: original/transcribed text, can have two comments both creator and author are sms-based user/citizen, of smsblog source type (source name is name of sms feed) http://sourcefabric.superdesk.pro/resources/LiveDesk/Blog/1/Post/854 tweet: the original tweet (with some predefined adjusting), without any (other) changes, can have two comments creator is a local user, author is just the twitter source http://sourcefabric.superdesk.pro/resources/LiveDesk/Blog/1/Post/855 plain: a text http://sourcefabric.superdesk.pro/resources/LiveDesk/Blog/1/Post/856 link: a web link http://sourcefabric.superdesk.pro/resources/LiveDesk/Blog/38/Post/1070 other: ignoring by now ''' try: coverage_id = _get_id_value(coverage_id) except: pass res_none = { 'total': 0, 'limit': 0, 'offset': 0, 'PostList': [] } coll = db[COLL_COVERAGES] if type(coverage_id) is ObjectId: return (False, 'old form of coverage reports retrieval') else: coverage = coll.find_one({'uuid': coverage_id}) if not coverage: return (False, 'coverage not found') if (FIELD_ACTIVE_COVERAGE not in coverage) or (not coverage[FIELD_ACTIVE_COVERAGE]): return (True, res_none) author_url_template = get_conf('author_url') creator_url_template = get_conf('creator_url') coll = db[COLL_REPORTS] reports = [] put_up_border = datetime.datetime.utcfromtimestamp(0) search_spec = { 'coverages.outgested': {'$in': [coverage_id, coverage['_id']]}, FIELD_DECAYED_REPORT: {'$ne': True}, } if cid_last: update_last = update_from_cid(cid_last) search_spec[FIELD_UPDATED_REPORT] = {'$gt': update_last} put_up_border = update_last else: search_spec[FIELD_UPDATED_REPORT] = {'$exists': True} # probably some adjusted dealing with: local, summary, unpublished/deleted reports cursor = coll.find(search_spec).sort([(FIELD_UPDATED_REPORT, 1)]) for entry in cursor: if 'feed_type' not in entry: entry['feed_type'] = DEFAULT_FIELD_TYPE with_fields = True for key in [FIELD_UPDATED_REPORT, FIELD_UUID_REPORT, 'feed_type', 'texts', 'coverages']: if (key not in entry) or (not entry[key]): with_fields = False break if not with_fields: continue if type(entry['coverages']) is not dict: continue if ('published' not in entry['coverages']): continue if type(entry['coverages']['published']) not in (list, tuple): continue feed_type = entry['feed_type'] if feed_type not in OUTPUT_FEED_TYPES: continue content = None meta = None post_type_key = 'normal' if 'sms' == feed_type: adapted = adapt_sms_report(db, entry) content = adapted['content'] meta = adapted['meta'] if 'tweet' == feed_type: adapted = adapt_tweet_report(db, entry) content = adapted['content'] meta = adapted['meta'] if 'plain' == feed_type: adapted = adapt_plain_report(db, entry) content = adapted['content'] meta = adapted['meta'] if 'web_link' == feed_type: adapted = adapt_link_report(db, entry) content = adapted['content'] meta = adapted['meta'] post_type_key = 'link' use_cid = 0 if FIELD_UPDATED_REPORT in entry: use_cid = cid_from_update(entry[FIELD_UPDATED_REPORT]) one_report = { 'CId': use_cid, 'IsPublished': 'True', 'Type': {'Key': post_type_key}, 'Content': content, 'Meta': meta, } if FIELD_UUID_REPORT in entry: one_report['Uuid'] = entry[FIELD_UUID_REPORT] if (FIELD_PUTUP_REPORT in entry) and entry[FIELD_PUTUP_REPORT]: try: if entry[FIELD_PUTUP_REPORT] > put_up_border: one_report['PutUp'] = 'True' except: pass if coverage_id not in entry['coverages']['published']: if coverage['_id'] not in entry['coverages']['published']: one_report['DeletedOn'] = '01/01/70 12:01 AM' link_id = urllib.quote_plus(str(entry['_id'])) author_url = author_url_template.replace(REPORT_LINK_ID_PLACEHOLDER, link_id) creator_url = creator_url_template.replace(REPORT_LINK_ID_PLACEHOLDER, link_id) one_report['Author'] = {'href': author_url} one_report['Creator'] = {'href': creator_url} reports.append(one_report) res = { 'total': len(reports), 'limit': len(reports), 'offset': 0, 'PostList': reports } return (True, res)
def do_finalize_one(db, auther_url, doc_id, data): ''' finalizes authorization initialization on info of one app data ''' if not controller: return (False, 'external controller not available') if not db: return (False, 'inner application error') if data is None: return (False, 'data not provided') if ('spec' not in data) or (type(data['spec']) is not dict): return (False, '"spec" part not provided') spec = data['spec'] coll = db[collection] if doc_id is None: return (False, 'document id not provided') doc_id = _get_id_value(doc_id) entry = coll.find_one({'_id': doc_id}) if not entry: return (False, 'document of the provided _id not found') if ('spec' not in entry) or (type(entry['spec']) is not dict): return (False, 'damaged document structure') spec = entry['spec'] needed_keys = ['app_consumer_key', 'app_consumer_secret', 'temporary_access_token_key', 'temporary_access_token_secret'] for one_key in needed_keys: if (one_key not in spec) or (not spec[one_key]): return (False, 'document missing the "' + str(one_key) + '" part') if ('spec' not in data) or (type(data['spec']) is not dict): return (False, '"spec" part not provided') spec_data = data['spec'] for one_key in ['verifier_pin']: if (one_key not in spec_data) or (not spec_data[one_key]): return (False, str(one_key) + ' not provided') authfin_data = { 'oauth_info': { 'consumer_key': spec['app_consumer_key'], 'consumer_secret': spec['app_consumer_secret'], 'access_token_key': spec['temporary_access_token_key'], 'access_token_secret': spec['temporary_access_token_secret'], }, 'payload': { 'verifier': spec_data['verifier_pin'], } } connector = controller.NewstwisterConnector(auther_url) res = connector.request_authfin(authfin_data['oauth_info'], authfin_data['payload']) if not res[0]: err_msg = 'error during authfin request dispatching: ' + res[1] return (False, err_msg) ret_envelope = res[1] if type(ret_envelope) is not dict: return (False, 'unknown form of returned authfin data: ' + str(type(ret_envelope))) if ('status' not in ret_envelope) or (not ret_envelope['status']): err_msg = '' if ('error' in ret_envelope) and (ret_envelope['error']): err_msg = ': ' + str(ret_envelope['error']) return (False, 'status not acknowledged in returned authfin data' + err_msg) if ('data' not in ret_envelope) or (not ret_envelope['data']): return (False, 'payload not provided in returned authfin data') ret_data = ret_envelope['data'] if type(ret_data) is not dict: return (False, 'unknown form of returned payload in authfin data: ' + str(type(ret_data))) for part in ['oauth_token_key', 'oauth_token_secret', 'user_id', 'screen_name']: if (part not in ret_data) or (not ret_data[part]): return (False, 'returned authfin data missing the "' + str(part) + '" part') try: screen_name_search = ret_data['screen_name'].lower() except: screen_name_search = ret_data['screen_name'] timepoint = datetime.datetime.utcnow() created = timepoint updated = timepoint try: if ('_created' in entry) and entry['_created']: created = entry['_created'] except: created = timepoint save_data = { '_id': doc_id, 'spec': { 'app_consumer_key': spec['app_consumer_key'], 'app_consumer_secret': spec['app_consumer_secret'], 'temporary_access_token_key': None, 'temporary_access_token_secret': None, 'authorized_access_token_key': ret_data['oauth_token_key'], 'authorized_access_token_secret': ret_data['oauth_token_secret'], 'verifier_url': None, 'user_id': ret_data['user_id'], 'screen_name': ret_data['screen_name'], 'screen_name_search': screen_name_search, }, '_created': created, '_updated': updated, } coll = db[collection] doc_id = coll.save(save_data) if not doc_id: return (False, 'can not save the authfin data') return (True, {'_id': doc_id})
def do_post_send(db, sender_url, authorized_id, user_id, endpoint_id, tweet_spec, report_id=None): ''' sends (a reply to) a tweet after it is sent and received, local=True, and user_id=user_id are set in the report ''' if not controller: return (False, 'external controller not available') if not db: return (False, 'inner application error') if not authorized_id: return (False, 'authorized_id not specified') if not user_id: return (False, 'user_id not specified') if not endpoint_id: return (False, 'endpoint_id not specified') if type(tweet_spec) is not dict: return (False, 'unknown form of tweet spec') if ('message' not in tweet_spec) or (not tweet_spec['message']): return (False, 'message text not provided') follow_part = [] tweet_data = { 'endpoint_id': endpoint_id, 'status': tweet_spec['message'], 'filter': {}, } if ('sensitive' in tweet_spec) and (tweet_spec['sensitive'] is not None): sensitive = _get_boolean(tweet_spec['sensitive']) if sensitive: tweet_data['possibly_sensitive'] = 'true' if ('display_coordinates' in tweet_spec) and (tweet_spec['display_coordinates'] is not None): display_coordinates = _get_boolean(tweet_spec['display_coordinates']) if display_coordinates: tweet_data['display_coordinates'] = 'true' else: tweet_data['display_coordinates'] = 'false' for key in ['lat', 'long', 'place_id']: if (key in tweet_spec) and (tweet_spec[key]): try: tweet_data[key] = str(tweet_spec[key]) except: return (False, 'wrong "' + str(key) + '" part in tweet spec') if report_id is not None: report_id = _get_id_value(report_id) search_spec = {'feed_type': FEED_TYPE} if type(report_id) is ObjectId: search_spec['_id'] = report_id else: search_spec['report_id'] = report_id coll = db[collection_reports] report = coll.find_one(search_spec) if not report: return (False, 'specified report not found') try: orig_user_screen_name = str( report['original']['user']['screen_name']).lower() if orig_user_screen_name not in follow_part: follow_part.append(orig_user_screen_name) except: return (False, 'can not find the original tweet sender') check_inclusion = '@' + orig_user_screen_name.lower() try: if check_inclusion not in tweet_spec['message'].lower(): return ( False, 'mentioning the original tweet sender not found in the tweet text' ) except: return (False, 'can not check inclusion of the original tweet sender') try: tweet_data['in_reply_to_status_id'] = str(report['original_id']) except: return (False, 'can not find id_str of the original tweet') coll = db[collection_authorized] authorized_id = _get_id_value(authorized_id) authorized_data = coll.find_one({'_id': authorized_id}) if not authorized_data: return (False, 'saved report on the send-tweet not found') try: authorized_spec = { 'consumer_key': authorized_data['spec']['app_consumer_key'], 'consumer_secret': authorized_data['spec']['app_consumer_secret'], 'access_token_key': authorized_data['spec']['authorized_access_token_key'], 'access_token_secret': authorized_data['spec']['authorized_access_token_secret'], } sender_screen_name = str(authorized_data['spec']['screen_name_search']) if sender_screen_name not in follow_part: follow_part.append(sender_screen_name) except Exception as exc: return (False, 'authorized info does not contain all the required data: ' + str(exc)) for key in authorized_spec: if not authorized_spec[key]: return (False, 'the "' + str(key) + '" part of authorized info is empty') tweet_data['filter']['follow'] = follow_part connector = controller.NewstwisterConnector(sender_url) res = connector.send_tweet(authorized_spec, tweet_data) if not res[0]: err_msg = 'error during send-tweet request dispatching: ' + res[1] return (False, err_msg) ret_envelope = res[1] if type(ret_envelope) is not dict: return (False, 'unknown form of returned send-tweet data: ' + str(type(ret_envelope))) if ('status' not in ret_envelope) or (not ret_envelope['status']): err_msg = '' if ('error' in ret_envelope) and (ret_envelope['error']): err_msg = ': ' + str(ret_envelope['error']) return (False, 'status not acknowledged in returned send-tweet data' + err_msg) if ('data' not in ret_envelope) or (not ret_envelope['data']): return (False, 'payload not provided in returned send-tweet data') ret_data = ret_envelope['data'] if type(ret_data) is not dict: return (False, 'unknown form of returned payload in send-tweet data: ' + str(type(ret_data))) if 'id_str' not in ret_data: return (False, 'returned send-tweet data without tweet identifier') coll = db[collection_reports] saved_tweet = coll.find_one({'original_id': ret_data['id_str']}) if not saved_tweet: return (False, 'saved report on the send-tweet not found') doc_id = saved_tweet['_id'] saved_update = { 'local': True, 'user_id': _get_id_value(user_id), 'proto': False, FIELD_UPDATED: datetime.datetime.utcnow(), #'_etag': _get_etag(), } coll.update({'_id': doc_id}, {'$set': saved_update}) return (True, {'_id': doc_id})
def get_coverage_published_report_list(db, coverage_id, cid_last): ''' SMS: original/transcribed text, can have two comments both creator and author are sms-based user/citizen, of smsblog source type (source name is name of sms feed) http://sourcefabric.superdesk.pro/resources/LiveDesk/Blog/1/Post/854 tweet: the original tweet (with some predefined adjusting), without any (other) changes, can have two comments creator is a local user, author is just the twitter source http://sourcefabric.superdesk.pro/resources/LiveDesk/Blog/1/Post/855 plain: a text http://sourcefabric.superdesk.pro/resources/LiveDesk/Blog/1/Post/856 link: a web link http://sourcefabric.superdesk.pro/resources/LiveDesk/Blog/38/Post/1070 other: ignoring by now ''' try: coverage_id = _get_id_value(coverage_id) except: pass res_none = {'total': 0, 'limit': 0, 'offset': 0, 'PostList': []} coll = db[COLL_COVERAGES] if type(coverage_id) is ObjectId: return (False, 'old form of coverage reports retrieval') else: coverage = coll.find_one({'uuid': coverage_id}) if not coverage: return (False, 'coverage not found') if (FIELD_ACTIVE_COVERAGE not in coverage) or (not coverage[FIELD_ACTIVE_COVERAGE]): return (True, res_none) author_url_template = get_conf('author_url') creator_url_template = get_conf('creator_url') coll = db[COLL_REPORTS] reports = [] put_up_border = datetime.datetime.utcfromtimestamp(0) search_spec = { 'coverages.outgested': { '$in': [coverage_id, coverage['_id']] }, FIELD_DECAYED_REPORT: { '$ne': True }, } if cid_last: update_last = update_from_cid(cid_last) search_spec[FIELD_UPDATED_REPORT] = {'$gt': update_last} put_up_border = update_last else: search_spec[FIELD_UPDATED_REPORT] = {'$exists': True} # probably some adjusted dealing with: local, summary, unpublished/deleted reports cursor = coll.find(search_spec).sort([(FIELD_UPDATED_REPORT, 1)]) for entry in cursor: if 'feed_type' not in entry: entry['feed_type'] = DEFAULT_FIELD_TYPE with_fields = True for key in [ FIELD_UPDATED_REPORT, FIELD_UUID_REPORT, 'feed_type', 'texts', 'coverages' ]: if (key not in entry) or (not entry[key]): with_fields = False break if not with_fields: continue if type(entry['coverages']) is not dict: continue if ('published' not in entry['coverages']): continue if type(entry['coverages']['published']) not in (list, tuple): continue feed_type = entry['feed_type'] if feed_type not in OUTPUT_FEED_TYPES: continue content = None meta = None post_type_key = 'normal' if 'sms' == feed_type: adapted = adapt_sms_report(db, entry) content = adapted['content'] meta = adapted['meta'] if 'tweet' == feed_type: adapted = adapt_tweet_report(db, entry) content = adapted['content'] meta = adapted['meta'] if 'plain' == feed_type: adapted = adapt_plain_report(db, entry) content = adapted['content'] meta = adapted['meta'] if 'web_link' == feed_type: adapted = adapt_link_report(db, entry) content = adapted['content'] meta = adapted['meta'] post_type_key = 'link' use_cid = 0 if FIELD_UPDATED_REPORT in entry: use_cid = cid_from_update(entry[FIELD_UPDATED_REPORT]) one_report = { 'CId': use_cid, 'IsPublished': 'True', 'Type': { 'Key': post_type_key }, 'Content': content, 'Meta': meta, } if FIELD_UUID_REPORT in entry: one_report['Uuid'] = entry[FIELD_UUID_REPORT] if (FIELD_PUTUP_REPORT in entry) and entry[FIELD_PUTUP_REPORT]: try: if entry[FIELD_PUTUP_REPORT] > put_up_border: one_report['PutUp'] = 'True' except: pass if coverage_id not in entry['coverages']['published']: if coverage['_id'] not in entry['coverages']['published']: one_report['DeletedOn'] = '01/01/70 12:01 AM' link_id = urllib.quote_plus(str(entry['_id'])) author_url = author_url_template.replace(REPORT_LINK_ID_PLACEHOLDER, link_id) creator_url = creator_url_template.replace(REPORT_LINK_ID_PLACEHOLDER, link_id) one_report['Author'] = {'href': author_url} one_report['Creator'] = {'href': creator_url} reports.append(one_report) res = { 'total': len(reports), 'limit': len(reports), 'offset': 0, 'PostList': reports } return (True, res)
def do_post(db, url, channel_type, request_id, client_ip): ''' * assure citizen_alias exists (i.e. create/save if does not yet) * create and save the report * channel_type: frontend, bookmarklet, etc. * request_id: commonly _id or uuid for filtering ''' logger = get_logger() timestamp = datetime.datetime.utcnow() channel_type = _get_id_value(channel_type) request_id = _get_id_value(request_id) if not url: return (False, 'emtpy url') try: url = url.strip() url_parsed = urlparse(url, scheme='http') except: url_parsed = None if not url_parsed: return (False, 'can not parse the url') if not url_parsed.netloc: return (False, 'url without domain part') # taking params feed_type = get_conf('feed_type') eff_tlds = get_conf('eff_tlds') # taking info page_info_got = get_page_info(url) if not page_info_got[0]: return (False, 'can not get page info: ' + page_info_got[1]) page_info = page_info_got[1] report_id = report_holder.gen_id(feed_type) session_url = '' if ('url' in page_info) and page_info['url']: try: session_url = page_info['url'].split('#')[0].strip().lower() except: pass if not session_url: session_url = url.split('#')[0].strip().lower() session = feed_type + '||' + session_url source_url = '' if ('url' in page_info) and page_info['url']: try: source_url = page_info['url'].strip() except: pass if not source_url: source_url = url.strip() domain_effective = '' if ('domain_name' in page_info) and page_info['domain_name']: try: domain_effective = str(page_info['domain_name']).split(':')[0].strip() except: pass if not domain_effective: domain_effective = url_parsed.netloc.split(':')[0].strip() if eff_tlds: citizen_id = take_specific_domain(eff_tlds, domain_effective) else: citizen_id = domain_effective if not citizen_id: citizen_id = source_url parent_id = None publisher = None if ('feed_name' in page_info) and page_info['feed_name']: feed_name = page_info['feed_name'] else: feed_name = None use_language = None if ('language' in page_info) and page_info['language']: use_language = page_info['language'] authority = get_conf('authority') use_identifiers = { 'user_id': citizen_id, 'user_id_search': citizen_id.lower(), 'user_name': citizen_id, 'user_name_search': citizen_id.lower(), } channel_value = None if ('type' in page_info) and page_info['type']: try: channel_value = page_info['type'].strip().lower() if not channel_value: channel_value = None except: channel_value = None channels = [{'type': channel_type, 'value': channel_value, 'filter': None, 'request': request_id, 'reasons': None}] authors = [{'authority': authority, 'identifiers': use_identifiers}] author_name = '' if 'author' in page_info: author_name = page_info['author'] alias_res_got = assure_citizen_alias(db, authority, use_identifiers, author_name) endorsers = [] original = page_info original_id = url # notice that we group the (effective) URLs via session tags = [] texts = [] text_parts = ['title', 'description'] for one_text_part in text_parts: if (one_text_part in page_info) and page_info[one_text_part]: texts.append({'original': page_info[one_text_part], 'transcript': None}) estimated = timestamp if ('date' in page_info) and page_info['date'] and (type(page_info['date']) is datetime.datetime): estimated = page_info['date'] # site_icon added at the end of the media list media = [] if ('image' in page_info) and page_info['image'] and (type(page_info['image']) in (list, tuple)): for one_image_link in page_info['image']: try: link_ssl = None if one_image_link.startswith('https'): link_ssl = one_image_link media.append({ 'link': one_image_link, 'link_ssl': link_ssl, 'data': None, 'name': None, 'width': None, 'height': None, }) except: pass if ('site_icon' in page_info) and page_info['site_icon']: try: link_ssl = None if page_info['site_icon'].startswith('https'): link_ssl = page_info['site_icon'] media.append({ 'link': page_info['site_icon'], 'link_ssl': link_ssl, 'data': None, 'name': None, 'width': None, 'height': None, }) except: pass report = {} report['report_id'] = report_id report['parent_id'] = None report['client_ip'] = client_ip report['feed_type'] = feed_type report['produced'] = estimated report['session'] = session report['publisher'] = publisher report['channels'] = channels report['authors'] = authors report['original'] = original report['texts'] = texts report['tags'] = tags report['language'] = use_language report['sources'] = [source_url] report['media'] = media report['editable'] = False report['proto'] = False report_doc_id = report_holder.save_report(report) if not report_doc_id: return (False, 'can not save URL report') return (True, {'_id': report_doc_id})
def do_post_send(db, sms_gateway_api, sms_gateway_url, sms_gateway_key, message, targets, user_id, language, sensitive, client_ip): try: controller = importlib.import_module('citizendesk.feeds.sms.external.' + sms_gateway_api) except: controller = None if not controller: return (False, 'no sms gateway controller available') if not db: return (False, 'no database available') if not sms_gateway_url: return (False, 'no sms gateway url configured') if not sms_gateway_key: return (False, 'no sms gateway key configured') if not message: return (False, 'no message provided') if not targets: return (False, 'no targets provided') from citizendesk.feeds.sms.citizen_alias.storage import get_one_by_id as get_one_citizen_alias_by_id use_targets = [] use_recipients = [] use_phone_numbers = [] authority = get_conf('authority') alias_doctype = get_conf('alias_doctype') if type(targets) not in [list, tuple]: return (False, '') for one_target in targets: if type(one_target) is not dict: continue if 'type' not in one_target: continue if one_target['type'] != alias_doctype: continue if 'value' not in one_target: continue one_target_id = _get_id_value(one_target['value']) alias_res = get_one_citizen_alias_by_id(db, one_target_id) if (not alias_res) or (not alias_res[0]): continue alias = alias_res[1] if (type(alias) is not dict) or (not alias): continue if ('identifiers' not in alias) or (not alias['identifiers']): continue one_phone_number = _get_phone_number(alias) if not one_phone_number: continue one_recipient = {'authority': authority, 'identifiers': alias['identifiers']} use_phone_numbers.append(one_phone_number) use_recipients.append(one_recipient) use_targets.append(one_target) if not use_targets: return (False, 'no valid recipient found') # putting all the recipients into a single report; and thus using its session for next communication with all the recipients try: report = _prepare_sms_send_report(use_targets, use_recipients, message, user_id, language, sensitive, client_ip) except: report = None if not report: return (False, 'report could not be prepared') # we either save the report before sms sending, and deleting it if sending fails, # or we first send sms, and if success on it, we save the report then; # thus either transiently having a false report, or a possibility of not having the report # on sent sms if the report saving fails at the end (should not hopefully though) doc_id = report_holder.save_report(report) if not doc_id: return (False, 'report could not be saved') connector = controller.SMSConnector(sms_gateway_url, sms_gateway_key) res = connector.send_sms(message, {'phone_numbers': use_phone_numbers}) if not res[0]: report_holder.delete_report(doc_id) return (False, 'message could not be sent', res[1]) return (True, {'_id': doc_id})