def do_request_one(db, searcher_url, alias_type, alias_value):
'''
asks Newstwister for info on a given twt user
'''
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
if not alias_value:
return (False, 'empty provided citizen alias')
request_type = None
if 'alias_id' == alias_type:
request_type = 'user_id'
if 'alias_name' == alias_type:
request_type = 'user_name'
if not request_type:
return (False, 'unknown form of requested citizen alias')
search_spec = {}
search_spec[request_type] = alias_value
connector = controller.NewstwisterConnector(searcher_url)
res = connector.request_user(search_spec)
if not res[0]:
return (False, 'error during user-search request dispatching',
str(res[1]))
return (True, res[1])
def do_get_list(db, offset=None, limit=None):
'''
returns data of a set of stream control
'''
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
coll = db[collection]
cursor = coll.find().sort([('_id', 1)])
total = cursor.count()
if limit is None:
limit = DEFAULT_LIMIT
if offset:
cursor = cursor.skip(offset)
if limit:
cursor = cursor.limit(limit)
docs = []
for entry in cursor:
if not entry:
continue
if ('spec' not in entry) or (type(entry['spec']) is not dict):
continue
if ('control' not in entry) or (type(entry['control']) is not dict):
continue
might_run = True
for key in schema['control']:
if (key not in entry['control']) or (not entry['control'][key]):
might_run = False
break
if might_run:
connector = controller.NewstwisterConnector(entry['control']['streamer_url'])
res = connector.request_status(entry['control']['process_id'])
# let None here when can not check that it runs
if res is not None:
entry['control']['switch_on'] = True if res else False
else:
entry['control']['switch_on'] = None
if res is False:
# update info in db when we know it has been stopped
timepoint = datetime.datetime.utcnow()
update_set = {'control.switch_on': False, 'control.process_id': None, 'logs.stopped': timepoint}
coll.update({'_id': entry['_id']}, {'$set': update_set}, upsert=False)
docs.append(entry)
return (True, docs, {'total': total})
def do_post_search(db, searcher_url, user_id, request_id, search_spec):
'''
executes twitter search
'''
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
for (part_name, part_value) in [['user_id', user_id],
['request_id', request_id],
['search_spec', search_spec]]:
if not part_value:
return (False, str(part_name) + ' not provided')
if type(user_id) in [str, unicode]:
if user_id.isdigit():
try:
user_id = int(user_id)
except:
pass
if type(request_id) in [str, unicode]:
if request_id.isdigit():
try:
request_id = int(request_id)
except:
pass
try:
parsed_res = _parse_data(search_spec)
if not parsed_res[0]:
return parsed_res
except:
return (False, 'could not parse the search specification data')
'''
coll = db[collection]
timepoint = datetime.datetime.utcnow()
created = timepoint
updated = timepoint
searched = None
'''
parsed_spec = parsed_res[1]
connector = controller.NewstwisterConnector(searcher_url)
res = connector.request_search(user_id, request_id, parsed_spec,
search_spec)
if not res:
return (False, 'error during search request dispatching')
return (True, res)
def do_get_one(db, doc_id):
'''
returns data of a single stream info
'''
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
doc_id = _get_id_value(doc_id)
coll = db[collection]
doc = coll.find_one({'_id': doc_id})
if not doc:
return (False, 'stream info not found')
might_run = True
if ('control' not in doc) or (type(doc['control']) is not dict):
might_run = False
if might_run:
for key in schema['control']:
if (key not in doc['control']) or (not doc['control'][key]):
might_run = False
break
if might_run:
connector = controller.NewstwisterConnector(doc['control']['streamer_url'])
res = connector.request_status(doc['control']['process_id'])
# let None here when can not check that it runs
if res is not None:
doc['control']['switch_on'] = True if res else False
else:
doc['control']['switch_on'] = None
if res is False:
# update info in db when we know it has been stopped
timepoint = datetime.datetime.utcnow()
update_set = {'control.switch_on': False, 'control.process_id': None, 'logs.stopped': timepoint}
coll.update({'_id': doc_id}, {'$set': update_set}, upsert=False)
if ('spec' not in doc) or (type(doc['spec']) is not dict):
return (False, 'wrong stream info (spec) in db')
if ('control' not in doc) or (type(doc['control']) is not dict):
return (False, 'wrong stream info (control) in db')
return (True, doc)
def do_delete_one(db, doc_id):
'''
deletes data of a single stream info
'''
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
doc_id = _get_id_value(doc_id)
coll = db[collection]
doc = coll.find_one({'_id': doc_id})
if not doc:
return (False, 'requested stream not found')
might_run = True
if ('control' not in doc) or (type(doc['control']) is not dict):
might_run = False
if might_run:
for key in schema['control']:
if (key not in doc['control']) or (not doc['control'][key]):
might_run = False
break
if might_run:
connector = controller.NewstwisterConnector(doc['control']['streamer_url'])
res = connector.request_stop(doc['control']['process_id'])
if not res:
return (False, 'can not stop the stream')
coll.remove({'_id': doc_id})
return (True, {'_id': doc_id})
def do_post_send(db,
sender_url,
authorized_id,
user_id,
endpoint_id,
tweet_spec,
report_id=None):
'''
sends (a reply to) a tweet
after it is sent and received, local=True, and user_id=user_id are set in the report
'''
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
if not authorized_id:
return (False, 'authorized_id not specified')
if not user_id:
return (False, 'user_id not specified')
if not endpoint_id:
return (False, 'endpoint_id not specified')
if type(tweet_spec) is not dict:
return (False, 'unknown form of tweet spec')
if ('message' not in tweet_spec) or (not tweet_spec['message']):
return (False, 'message text not provided')
follow_part = []
tweet_data = {
'endpoint_id': endpoint_id,
'status': tweet_spec['message'],
'filter': {},
}
if ('sensitive' in tweet_spec) and (tweet_spec['sensitive'] is not None):
sensitive = _get_boolean(tweet_spec['sensitive'])
if sensitive:
tweet_data['possibly_sensitive'] = 'true'
if ('display_coordinates'
in tweet_spec) and (tweet_spec['display_coordinates'] is not None):
display_coordinates = _get_boolean(tweet_spec['display_coordinates'])
if display_coordinates:
tweet_data['display_coordinates'] = 'true'
else:
tweet_data['display_coordinates'] = 'false'
for key in ['lat', 'long', 'place_id']:
if (key in tweet_spec) and (tweet_spec[key]):
try:
tweet_data[key] = str(tweet_spec[key])
except:
return (False, 'wrong "' + str(key) + '" part in tweet spec')
if report_id is not None:
report_id = _get_id_value(report_id)
search_spec = {'feed_type': FEED_TYPE}
if type(report_id) is ObjectId:
search_spec['_id'] = report_id
else:
search_spec['report_id'] = report_id
coll = db[collection_reports]
report = coll.find_one(search_spec)
if not report:
return (False, 'specified report not found')
try:
orig_user_screen_name = str(
report['original']['user']['screen_name']).lower()
if orig_user_screen_name not in follow_part:
follow_part.append(orig_user_screen_name)
except:
return (False, 'can not find the original tweet sender')
check_inclusion = '@' + orig_user_screen_name.lower()
try:
if check_inclusion not in tweet_spec['message'].lower():
return (
False,
'mentioning the original tweet sender not found in the tweet text'
)
except:
return (False,
'can not check inclusion of the original tweet sender')
try:
tweet_data['in_reply_to_status_id'] = str(report['original_id'])
except:
return (False, 'can not find id_str of the original tweet')
coll = db[collection_authorized]
authorized_id = _get_id_value(authorized_id)
authorized_data = coll.find_one({'_id': authorized_id})
if not authorized_data:
return (False, 'saved report on the send-tweet not found')
try:
authorized_spec = {
'consumer_key':
authorized_data['spec']['app_consumer_key'],
'consumer_secret':
authorized_data['spec']['app_consumer_secret'],
'access_token_key':
authorized_data['spec']['authorized_access_token_key'],
'access_token_secret':
authorized_data['spec']['authorized_access_token_secret'],
}
sender_screen_name = str(authorized_data['spec']['screen_name_search'])
if sender_screen_name not in follow_part:
follow_part.append(sender_screen_name)
except Exception as exc:
return (False,
'authorized info does not contain all the required data: ' +
str(exc))
for key in authorized_spec:
if not authorized_spec[key]:
return (False,
'the "' + str(key) + '" part of authorized info is empty')
tweet_data['filter']['follow'] = follow_part
connector = controller.NewstwisterConnector(sender_url)
res = connector.send_tweet(authorized_spec, tweet_data)
if not res[0]:
err_msg = 'error during send-tweet request dispatching: ' + res[1]
return (False, err_msg)
ret_envelope = res[1]
if type(ret_envelope) is not dict:
return (False, 'unknown form of returned send-tweet data: ' +
str(type(ret_envelope)))
if ('status' not in ret_envelope) or (not ret_envelope['status']):
err_msg = ''
if ('error' in ret_envelope) and (ret_envelope['error']):
err_msg = ': ' + str(ret_envelope['error'])
return (False, 'status not acknowledged in returned send-tweet data' +
err_msg)
if ('data' not in ret_envelope) or (not ret_envelope['data']):
return (False, 'payload not provided in returned send-tweet data')
ret_data = ret_envelope['data']
if type(ret_data) is not dict:
return (False,
'unknown form of returned payload in send-tweet data: ' +
str(type(ret_data)))
if 'id_str' not in ret_data:
return (False, 'returned send-tweet data without tweet identifier')
coll = db[collection_reports]
saved_tweet = coll.find_one({'original_id': ret_data['id_str']})
if not saved_tweet:
return (False, 'saved report on the send-tweet not found')
doc_id = saved_tweet['_id']
saved_update = {
'local': True,
'user_id': _get_id_value(user_id),
'proto': False,
FIELD_UPDATED: datetime.datetime.utcnow(),
#'_etag': _get_etag(),
}
coll.update({'_id': doc_id}, {'$set': saved_update})
return (True, {'_id': doc_id})
def do_post_one(db, auther_url, data):
'''
sets info of one app data and requests authorization initialization on it
'''
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
if data is None:
return (False, 'data not provided')
if ('spec' not in data) or (type(data['spec']) is not dict):
return (False, '"spec" part not provided')
spec_data = data['spec']
for one_key in ['app_consumer_key', 'app_consumer_secret']:
if (one_key not in spec_data) or (not spec_data[one_key]):
return (False, str(one_key) + ' not provided')
authini_data = {
'oauth_info': {
'consumer_key': spec_data['app_consumer_key'],
'consumer_secret': spec_data['app_consumer_secret'],
},
'payload': {}
}
connector = controller.NewstwisterConnector(auther_url)
res = connector.request_authini(authini_data['oauth_info'], authini_data['payload'])
if not res[0]:
err_msg = 'error during authini request dispatching: ' + res[1]
return (False, err_msg)
ret_envelope = res[1]
if type(ret_envelope) is not dict:
return (False, 'unknown form of returned authini data: ' + str(type(ret_envelope)))
if ('status' not in ret_envelope) or (not ret_envelope['status']):
err_msg = ''
if ('error' in ret_envelope) and (ret_envelope['error']):
err_msg = ': ' + str(ret_envelope['error'])
return (False, 'status not acknowledged in returned authini data' + err_msg)
if ('data' not in ret_envelope) or (not ret_envelope['data']):
return (False, 'payload not provided in returned authini data')
ret_data = ret_envelope['data']
if type(ret_data) is not dict:
return (False, 'unknown form of returned payload in authini data: ' + str(type(ret_data)))
for part in ['oauth_token_key', 'oauth_token_secret', 'pin_url']:
if (part not in ret_data) or (not ret_data[part]):
return (False, 'returned authini data missing the "' + str(part) + '" part')
try:
if USE_SEQUENCES:
entry = db['counters'].find_and_modify(query={'_id': collection}, update={'$inc': {'next':1}}, new=True, upsert=True, full_response=False)
doc_id = entry['next']
else:
doc_id = ObjectId()
except:
return (False, 'can not create document id')
timepoint = datetime.datetime.utcnow()
created = timepoint
updated = timepoint
save_data = {
'_id': doc_id,
'spec': {
'app_consumer_key': spec_data['app_consumer_key'],
'app_consumer_secret': spec_data['app_consumer_secret'],
'temporary_access_token_key': ret_data['oauth_token_key'],
'temporary_access_token_secret': ret_data['oauth_token_secret'],
'verifier_url': ret_data['pin_url'],
},
'_created': created,
'_updated': updated,
}
coll = db[collection]
doc_id = coll.save(save_data)
if not doc_id:
return (False, 'can not save the authini data')
return (True, {'_id': doc_id})
def do_finalize_one(db, auther_url, doc_id, data):
'''
finalizes authorization initialization on info of one app data
'''
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
if data is None:
return (False, 'data not provided')
if ('spec' not in data) or (type(data['spec']) is not dict):
return (False, '"spec" part not provided')
spec = data['spec']
coll = db[collection]
if doc_id is None:
return (False, 'document id not provided')
doc_id = _get_id_value(doc_id)
entry = coll.find_one({'_id': doc_id})
if not entry:
return (False, 'document of the provided _id not found')
if ('spec' not in entry) or (type(entry['spec']) is not dict):
return (False, 'damaged document structure')
spec = entry['spec']
needed_keys = ['app_consumer_key', 'app_consumer_secret', 'temporary_access_token_key', 'temporary_access_token_secret']
for one_key in needed_keys:
if (one_key not in spec) or (not spec[one_key]):
return (False, 'document missing the "' + str(one_key) + '" part')
if ('spec' not in data) or (type(data['spec']) is not dict):
return (False, '"spec" part not provided')
spec_data = data['spec']
for one_key in ['verifier_pin']:
if (one_key not in spec_data) or (not spec_data[one_key]):
return (False, str(one_key) + ' not provided')
authfin_data = {
'oauth_info': {
'consumer_key': spec['app_consumer_key'],
'consumer_secret': spec['app_consumer_secret'],
'access_token_key': spec['temporary_access_token_key'],
'access_token_secret': spec['temporary_access_token_secret'],
},
'payload': {
'verifier': spec_data['verifier_pin'],
}
}
connector = controller.NewstwisterConnector(auther_url)
res = connector.request_authfin(authfin_data['oauth_info'], authfin_data['payload'])
if not res[0]:
err_msg = 'error during authfin request dispatching: ' + res[1]
return (False, err_msg)
ret_envelope = res[1]
if type(ret_envelope) is not dict:
return (False, 'unknown form of returned authfin data: ' + str(type(ret_envelope)))
if ('status' not in ret_envelope) or (not ret_envelope['status']):
err_msg = ''
if ('error' in ret_envelope) and (ret_envelope['error']):
err_msg = ': ' + str(ret_envelope['error'])
return (False, 'status not acknowledged in returned authfin data' + err_msg)
if ('data' not in ret_envelope) or (not ret_envelope['data']):
return (False, 'payload not provided in returned authfin data')
ret_data = ret_envelope['data']
if type(ret_data) is not dict:
return (False, 'unknown form of returned payload in authfin data: ' + str(type(ret_data)))
for part in ['oauth_token_key', 'oauth_token_secret', 'user_id', 'screen_name']:
if (part not in ret_data) or (not ret_data[part]):
return (False, 'returned authfin data missing the "' + str(part) + '" part')
try:
screen_name_search = ret_data['screen_name'].lower()
except:
screen_name_search = ret_data['screen_name']
timepoint = datetime.datetime.utcnow()
created = timepoint
updated = timepoint
try:
if ('_created' in entry) and entry['_created']:
created = entry['_created']
except:
created = timepoint
save_data = {
'_id': doc_id,
'spec': {
'app_consumer_key': spec['app_consumer_key'],
'app_consumer_secret': spec['app_consumer_secret'],
'temporary_access_token_key': None,
'temporary_access_token_secret': None,
'authorized_access_token_key': ret_data['oauth_token_key'],
'authorized_access_token_secret': ret_data['oauth_token_secret'],
'verifier_url': None,
'user_id': ret_data['user_id'],
'screen_name': ret_data['screen_name'],
'screen_name_search': screen_name_search,
},
'_created': created,
'_updated': updated,
}
coll = db[collection]
doc_id = coll.save(save_data)
if not doc_id:
return (False, 'can not save the authfin data')
return (True, {'_id': doc_id})
def do_patch_one(db, doc_id=None, data=None, force=None):
'''
starts/stops the stream
'''
try:
from citizendesk.feeds.twt.filter.storage import get_one as filter_get_one
except:
return (False, 'filter processor not available')
try:
from citizendesk.feeds.twt.oauth.storage import get_one as oauth_get_one
except:
return (False, 'oauth processor not available')
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
if data is None:
return (False, 'data not provided')
if type(data) is not dict:
return (False, 'invalid data provided')
if ('control' not in data) or (type(data['control']) is not dict):
return (False, '"control" part not provided')
if doc_id is None:
return (False, 'stream _id not provided')
doc_id = _get_id_value(doc_id)
try:
force_val = bool(_get_boolean(force))
except:
force_val = None
coll = db[collection]
entry = coll.find_one({'_id': doc_id})
if not entry:
return (False, '"stream" of the provided _id not found: ' + str(doc_id))
if ('spec' not in entry) or (type(entry['spec']) is not dict):
return (False, '"stream" of the provided _id does not contain "spec" part')
doc = {'control': {}}
for key in schema['control']:
doc['control'][key] = None
if key in data['control']:
doc['control'][key] = data['control'][key]
res = _check_schema(None, doc['control'])
if not res[0]:
return res
should_run = False
if doc['control']['switch_on']:
if not doc['control']['streamer_url']:
return (False, 'can not start the stream without the "streamer_url" being set')
else:
should_run = True
filter_id = None
oauth_id = None
if should_run:
if ('spec' in entry) and (type(entry['spec']) is dict):
if 'filter_id' in entry['spec']:
filter_id = entry['spec']['filter_id']
if 'oauth_id' in entry['spec']:
oauth_id = entry['spec']['oauth_id']
if (not filter_id) or (not oauth_id):
return (False, 'Can not start the stream without assigned filter_id and oauth_id')
new_stream_url = None
connector = None
if should_run:
new_stream_url = doc['control']['streamer_url']
connector = controller.NewstwisterConnector(new_stream_url)
# load the supplemental data
filter_info = None
oauth_info = None
if should_run:
res = filter_get_one(db, filter_id)
if not res[0]:
return (False, 'filter info with _id equal to "spec.filter_id" not found')
filter_info = res[1]
if ('spec' not in filter_info) or (type(filter_info['spec']) is not dict):
return (False, 'set filter without "spec" part')
res = oauth_get_one(db, oauth_id)
if not res[0]:
return (False, 'oauth info with _id equal to "spec.oauth_id" not found')
oauth_info = res[1]
if ('spec' not in oauth_info) or (type(oauth_info['spec']) is not dict):
return (False, 'set oauth without "spec" part')
filter_spec = {}
filter_spec_original = {}
if should_run and filter_info and filter_info['spec']:
try:
prep_res = _prepare_filter(db, connector, filter_info['spec'])
if not prep_res[0]:
return (False, prep_res[1])
filter_spec = prep_res[1]
filter_spec_original = prep_res[2]
except Exception as exc:
return (False, 'can not prepare filter params: ' + str(exc))
if not filter_spec:
return (False, 'no filter spec was prepared')
if not filter_spec_original:
return (False, 'no filter spec original was prepared')
# check if the oauth_id is not used by any other running feed
if should_run:
running_count = coll.find({'spec.oauth_id': oauth_id, 'control.switch_on': True, '_id': {'$ne': doc_id}}).count()
if running_count:
return (False, 'the "oauth_id" is already used at a running stream')
# stop first in any case, since filter/oauth specs might have changed
might_run = True
if not entry:
might_run = False
if might_run:
if ('control' not in entry) or (type(entry['control']) is not dict):
might_run = False
if might_run:
for key in schema['control']:
if (key not in entry['control']) or (not entry['control'][key]):
might_run = False
break
if might_run:
cur_stream_url = entry['control']['streamer_url']
connector = controller.NewstwisterConnector(cur_stream_url)
res = connector.request_stop(entry['control']['process_id'])
if (not res) and (not force_val):
return (False, 'can not stop the stream')
# update info in db when we know it has been stopped
timepoint = datetime.datetime.utcnow()
update_set = {'control.switch_on': False, 'control.process_id': None, 'logs.stopped': timepoint}
coll.update({'_id': doc_id}, {'$set': update_set}, upsert=False)
if should_run:
res = connector.request_start(str(doc_id), oauth_info['spec'], filter_spec, filter_spec_original)
if not res:
return (False, 'can not start the stream')
try:
proc_id = int(res)
except:
proc_id = res
# update info in db when we know it has been started
timepoint = datetime.datetime.utcnow()
update_set = {'control.switch_on': True, 'control.process_id': proc_id, 'control.streamer_url': new_stream_url, 'logs.started': timepoint}
coll.update({'_id': doc_id}, {'$set': update_set}, upsert=False)
return (True, {'_id': doc_id})
def do_post_one(db, doc_id=None, data=None):
'''
sets data of a single stream info
'''
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
if data is None:
return (False, 'data not provided')
if ('spec' not in data) or (type(data['spec']) is not dict):
return (False, '"spec" part not provided')
doc_id = _get_id_value(doc_id)
coll = db[collection]
timepoint = datetime.datetime.utcnow()
created = timepoint
updated = timepoint
started = None
stopped = None
entry = None
if doc_id is not None:
entry = coll.find_one({'_id': doc_id})
if not entry:
return (False, '"stream" of the provided _id not found')
try:
if ('logs' in entry) and (entry['logs']):
if ('started' in entry['logs']) and entry['logs']['started']:
started = entry['logs']['started']
if ('stopped' in entry['logs']) and entry['logs']['stopped']:
started = entry['logs']['stopped']
except:
started = None
stopped = None
try:
if ('_created' in entry) and entry['_created']:
created = entry['_created']
except:
created = timepoint
doc = {
'logs': {
'started': started,
'stopped': stopped,
},
'spec': {},
'control': {
'streamer_url': None,
'process_id': None,
'switch_on': False,
},
'_created': created,
'_updated': updated,
}
for key in schema['spec']:
doc['spec'][key] = None
if key in data['spec']:
doc['spec'][key] = _get_id_value(data['spec'][key])
res = _check_schema(doc['spec'], None)
if not res[0]:
return res
# stop in any case, since filter/oauth specs might have changed
might_run = True
if not entry:
might_run = False
if might_run:
if ('control' not in entry) or (type(entry['control']) is not dict):
might_run = False
if might_run:
for key in schema['control']:
if (key not in entry['control']) or (not entry['control'][key]):
might_run = False
break
if might_run:
connector = controller.NewstwisterConnector(entry['control']['streamer_url'])
res = connector.request_stop(entry['control']['process_id'])
if not res:
return (False, 'can not stop the stream')
# update info in db when we know it has been stopped
timepoint = datetime.datetime.utcnow()
doc['logs']['stopped'] = timepoint
update_set = {'control.switch_on': False, 'control.process_id': None, 'logs.stopped': timepoint}
coll.update({'_id': entry['_id']}, {'$set': update_set}, upsert=False)
if not doc_id:
try:
if USE_SEQUENCES:
entry = db['counters'].find_and_modify(query={'_id': collection}, update={'$inc': {'next':1}}, new=True, upsert=True, full_response=False);
doc_id = entry['next']
else:
doc_id = ObjectId()
except:
return (False, 'can not create document id')
doc['_id'] = doc_id
doc_id = coll.save(doc)
return (True, {'_id': doc_id})
def do_post_pick(db, picker_url, endpoint_id, tweet_spec):
'''
picks a tweet
'''
if not controller:
return (False, 'external controller not available')
if not db:
return (False, 'inner application error')
if not endpoint_id:
return (False, 'endpoint_id not specified')
if type(tweet_spec) is not dict:
return (False, 'unknown form of tweet spec')
tweet_id = None
if ('tweet_id' in tweet_spec) and tweet_spec['tweet_id']:
tweet_id = str(tweet_spec['tweet_id'])
if (not tweet_id) and ('tweet_url' in tweet_spec) and tweet_spec['tweet_url']:
tweet_url = str(tweet_spec['tweet_url']).lower()
url_scheme, url_netloc, url_path, url_params, url_query, url_index = urllib2.urlparse.urlparse(tweet_url)
if TWEET_DOMAIN != url_netloc:
if url_path.startswith(TWEET_DOMAIN):
url_netloc = TWEET_DOMAIN
url_path = url_path[len(TWEET_DOMAIN):]
if TWEET_DOMAIN != url_netloc:
return (False, 'unknown tweet URL domain')
path_parts = []
for one_part in url_path.split('/'):
if not one_part:
continue
path_parts.append(one_part)
if 3 != len(path_parts):
return (False, 'unrecognized tweet URL path')
if path_parts[1] not in ('status', 'statuses'):
return (False, 'unrecognized tweet URL path segments')
tweet_id = path_parts[2].strip()
if not tweet_id:
return (False, 'tweet id part of provided URL is empty')
if not tweet_id:
return (False, 'no tweet specifier found')
if not tweet_id.isdigit():
return (False, 'tweet specifier is not a numerical string')
pick_data = {
'endpoint_id': endpoint_id,
'tweet_id': tweet_id,
}
connector = controller.NewstwisterConnector(picker_url)
res = connector.pick_tweet(pick_data)
if not res[0]:
err_msg = 'error during pick-tweet request dispatching: ' + res[1]
return (False, err_msg)
coll = db[collection_reports]
saved_tweet = coll.find_one({'original_id': tweet_id})
if not saved_tweet:
return (False, 'saved report on the pick-tweet not found')
doc_id = saved_tweet['_id']
saved_update = {'proto': False}
saved_update[FIELD_UPDATED] = datetime.datetime.utcnow()
#saved_update['_etag'] = _get_etag()
coll.update({'_id': doc_id}, {'$set': saved_update})
return (True, {'_id': doc_id})