def feed_twt_authorized_get_list():
from citizendesk.feeds.twt.authorized import process
logger = get_logger()
client_ip = get_client_ip()
is_local = not is_remote_ip(client_ip)
params = {'offset': None, 'limit': None}
for key in params:
if key in request.args:
try:
params[key] = int(request.args.get(key))
except:
params[key] = None
res = process.do_get_list(mongo_dbs.get_db().db, is_local, params['offset'], params['limit'])
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
if 2 < len(res):
ret_data['_meta']['list'] = res[2]
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def feed_twt_authorized_get_list():
from citizendesk.feeds.twt.authorized import process
logger = get_logger()
client_ip = get_client_ip()
is_local = not is_remote_ip(client_ip)
params = {'offset': None, 'limit': None}
for key in params:
if key in request.args:
try:
params[key] = int(request.args.get(key))
except:
params[key] = None
res = process.do_get_list(mongo_dbs.get_db().db, is_local,
params['offset'], params['limit'])
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 404, {
'Content-Type': 'application/json'
})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
if 2 < len(res):
ret_data['_meta']['list'] = res[2]
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 200, {
'Content-Type': 'application/json'
})
def take_workingon_reports():
from citizendesk.outgest.workingon import process
logger = get_logger()
client_ip = get_client_ip()
params = {'offset': None, 'limit': None}
for key in params:
if key in request.args:
try:
params[key] = int(request.args.get(key))
except:
params[key] = None
for key in ['sort']:
params[key] = None
if key in request.args:
params[key] = request.args.get(key)
try:
res = process.get_workingon_reports_list(mongo_dbs.get_db().db, params['offset'], params['limit'], params['sort'])
if not res[0]:
ret_data = {'_meta': {'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {}, '_data': res[1]}
if 2 < len(res):
ret_data['_meta']['list'] = res[2]
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
except Exception as exc:
logger.warning('problem on workingon-oriented reports listing')
return ('problem on workingon-oriented reports listing', 404,)
def prepare_reporting(mongo_addr, dbname, config_path):
from citizendesk.common.utils import get_logger
from citizendesk.common.dbc import mongo_dbs
from citizendesk.feeds.config import set_config
import citizendesk.outgest.dispatch as dispatch
logger = get_logger()
mongo_dbs.set_dbname(dbname)
DbHolder = namedtuple('DbHolder', 'db')
mongo_dbs.set_db(DbHolder(db=MongoClient(mongo_addr[0], mongo_addr[1])[mongo_dbs.get_dbname()]))
config_data = None
if config_path:
try:
cfghf = open(config_path)
config_data = cfghf.read()
cfghf.close()
config_yaml = yaml.load_all(config_data)
config_data = config_yaml.next()
config_yaml.close()
except:
config_data = None
logger.error('can not read config file: ' + str(config_path))
return False
dispatch.setup_blueprints(app, config_data)
return True
def feed_sms_citizen_alias_get_list():
from citizendesk.feeds.sms.citizen_alias import process
logger = get_logger()
client_ip = get_client_ip()
params = {'offset': None, 'limit': None}
for key in params:
if key in request.args:
try:
params[key] = int(request.args.get(key))
except:
params[key] = None
for key in ['sort']:
params[key] = None
if key in request.args:
params[key] = request.args.get(key)
other = {'name_only': None}
for key in other:
if key in request.args:
other[key] = request.args.get(key)
res = process.do_get_list(mongo_dbs.get_db().db, params['offset'], params['limit'], params['sort'], other)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
if 2 < len(res):
ret_data['_meta']['list'] = res[2]
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def feed_twt_authorized_post_one():
from citizendesk.feeds.twt.authorized import process
from citizendesk.feeds.config import get_config
logger = get_logger()
client_ip = get_client_ip()
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {'Content-Type': 'application/json'})
auther_url = get_config('newstwister_url')
res = process.do_post_one(mongo_dbs.get_db().db, auther_url, data)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def feed_sms_report_get_list(value_type, value_id):
from citizendesk.feeds.sms.report import process
logger = get_logger()
client_ip = get_client_ip()
params = {'offset': None, 'limit': None}
for key in params:
if key in request.args:
try:
params[key] = int(request.args.get(key))
except:
params[key] = None
for key in ['sort']:
params[key] = None
if key in request.args:
params[key] = request.args.get(key)
other = {'text_only': None}
for key in other:
if key in request.args:
other[key] = request.args.get(key)
res = process.do_get_list(mongo_dbs.get_db().db, value_type, value_id, params['offset'], params['limit'], params['sort'], other)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
if 3 >= len(res):
ret_data['_meta']['list'] = res[2]
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def feed_twt_citizen_alias_request_one(alias_type, alias_value):
from citizendesk.feeds.twt.citizen_alias import process
from citizendesk.feeds.config import get_config
logger = get_logger()
client_ip = get_client_ip()
searcher_url = get_config('newstwister_url')
res = process.do_request_one(mongo_dbs.get_db().db, searcher_url,
alias_type, alias_value)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
if 2 < len(res):
ret_data['_meta']['reason'] = res[2]
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 404, {
'Content-Type': 'application/json'
})
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 200, {
'Content-Type': 'application/json'
})
def feed_twt_report_patch_one(report_id):
from citizendesk.feeds.twt.report import process
logger = get_logger()
client_ip = get_client_ip()
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {'Content-Type': 'application/json'})
res = process.do_patch_one(mongo_dbs.get_db().db, report_id, data)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def feed_twt_stream_patch_one_emulate(stream_id, switch_on):
from citizendesk.feeds.twt.stream import process
from citizendesk.feeds.config import get_config
logger = get_logger()
client_ip = get_client_ip()
data = {
'control': {
'streamer_url': get_config('newstwister_url'),
'switch_on': switch_on
},
}
params = {'force': True}
res = process.do_patch_one(mongo_dbs.get_db().db, stream_id, data,
params['force'])
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 404, {
'Content-Type': 'application/json'
})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 200, {
'Content-Type': 'application/json'
})
def take_coverages():
setup_urls()
use_liveblog_configuration(mongo_dbs.get_db().db)
from citizendesk.outgest.liveblog.process import get_coverage_list
logger = get_logger()
client_ip = get_client_ip()
try:
res = get_coverage_list(mongo_dbs.get_db().db)
if (not res) or (not res[0]):
logger.info(str(res[1]))
return (
res[1],
404,
)
return (json.dumps(res[1]), 200, {
'Content-Type': 'application/json',
'Access-Control-Allow-Origin': '*'
})
except Exception as exc:
logger.warning('problem on liveblog-oriented coverages listing')
return (
'problem on liveblog-oriented coverages listing',
404,
)
def feed_any_coverage_insert_one():
from citizendesk.feeds.any.coverage import process
logger = get_logger()
client_ip = get_client_ip()
coverage_data = {}
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {
'Content-Type': 'application/json'
})
try:
coverage_data['title'] = data['title']
coverage_data['description'] = data['description']
coverage_data['user_id'] = data['user_id']
except:
return (json.dumps(
'provided data should contain "title", "description", "user_id" parts'
), 404, {
'Content-Type': 'application/json'
})
if 'active' in data:
coverage_data['active'] = _get_boolean(data['active'])
try:
if ('uuid' in data) and data['uuid']:
coverage_data['uuid'] = str(data['uuid'])
except:
pass
res = process.do_insert_one(mongo_dbs.get_db().db, coverage_data)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 404, {
'Content-Type': 'application/json'
})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 200, {
'Content-Type': 'application/json'
})
def page_not_found(error):
from citizendesk.common.utils import get_client_ip
from citizendesk.common.utils import get_logger
logger = get_logger()
logger.info('page not found: ' + str(request.method) + ' on ' + str(request.url) + ', by ' + str(get_client_ip()))
return 'page not found', 404
def run_flask(dbname, server, mongo, config_path, debug=False):
from citizendesk.common.utils import get_logger
logger = get_logger()
state = prepare_reporting(mongo, dbname, config_path)
if not state:
logger.warning('quiting the feeds daemon for not successful startup')
return
app.run(host=server[0], port=server[1], debug=debug, threaded=True)
def feed_twt_send_one_post(report_id):
from citizendesk.feeds.twt.send import process
from citizendesk.feeds.config import get_config
logger = get_logger()
client_ip = get_client_ip()
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {
'Content-Type': 'application/json'
})
try:
authorized_id = data['authorized_id']
user_id = data['user_id']
endpoint_id = data['endpoint_id']
tweet_spec = data['tweet_spec']
except:
return (json.dumps(
'provided data should contain "authorized_id", "user_id", "endpoint_id", "tweet_spec" parts'
), 404, {
'Content-Type': 'application/json'
})
sender_url = get_config('newstwister_url')
res = process.do_post_send(mongo_dbs.get_db().db, sender_url,
authorized_id, user_id, endpoint_id, tweet_spec,
report_id)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 404, {
'Content-Type': 'application/json'
})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 200, {
'Content-Type': 'application/json'
})
def feed_twt_citizen_alias_get_one(alias_type, alias_value):
from citizendesk.feeds.twt.citizen_alias import process
from citizendesk.feeds.config import get_config
logger = get_logger()
client_ip = get_client_ip()
other = {'force': None}
for key in other:
if key in request.args:
other[key] = request.args.get(key)
force = False
if other and ('force' in other):
force = _get_boolean(other['force'])
res = process.do_get_one(mongo_dbs.get_db().db, alias_type, alias_value)
err_reason = None
err_msg = None
if not res[0]:
err_msg = res[1]
if (not res[0]) and force:
err_msg = res[1]
searcher_url = get_config('newstwister_url')
res_aux = process.do_request_one(mongo_dbs.get_db().db, searcher_url,
alias_type, alias_value)
if not res_aux[0]:
err_msg = res_aux[1]
if 2 < len(res_aux):
err_reason = res_aux[2]
else:
res = process.do_get_one(mongo_dbs.get_db().db, alias_type,
alias_value)
if not res[0]:
err_msg = res_aux[1]
err_reason = None
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': err_msg}}
if err_reason:
ret_data['_meta']['reason'] = err_reason
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 404, {
'Content-Type': 'application/json'
})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 200, {
'Content-Type': 'application/json'
})
def feed_err_ingest_insert_one(notice_timestamp):
from citizendesk.feeds.err.ingest import process
logger = get_logger()
client_ip = get_client_ip()
notice_data = {}
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {
'Content-Type': 'application/json'
})
try:
notice_data['feed_type'] = data['feed_type']
notice_data['channel'] = data['channel']
notice_data['message'] = data['message']
except:
return (json.dumps(
'provided data should contain "feed_type", "channel", "message" parts'
), 404, {
'Content-Type': 'application/json'
})
res = process.do_insert_one(mongo_dbs.get_db().db, notice_timestamp,
notice_data)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 404, {
'Content-Type': 'application/json'
})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 200, {
'Content-Type': 'application/json'
})
def take_twt(tweet_id):
from citizendesk.ingest.twt.process import do_post
logger = get_logger()
client_ip = get_client_ip()
allowed_ips = get_allowed_ips()
if allowed_ips and ('*' not in allowed_ips):
if not client_ip in allowed_ips:
logger.info('unallowed client from: '+ str(client_ip))
return ('Client not allowed\n\n', 403,)
logger.info('allowed client from: '+ str(client_ip))
try:
json_data = request.get_json(True, False, False)
if type(json_data) is not dict:
json_data = None
except:
json_data = None
if json_data is None:
try:
json_data = request.json
if type(json_data) is not dict:
json_data = None
except:
json_data = None
if not json_data:
logger.info('data not provided in the request')
return ('Data not provided', 404,)
for part in ['filter', 'tweet', 'endpoint', 'type']:
if (part not in json_data) or (not json_data[part]):
logger.info('No ' + str(part) + ' provided')
return ('No ' + str(part) + ' provided', 404,)
tweet = json_data['tweet']
feed_filter = json_data['filter']
endpoint = json_data['endpoint']
channel_type = json_data['type']
request_id = None
if ('request' in json_data) and json_data['request']:
request_id = json_data['request']
try:
res = do_post(holder, tweet_id, tweet, channel_type, endpoint, request_id, feed_filter, client_ip)
if (not res) or (not res[0]):
logger.info(str(res[1]))
return (res[1], 404,)
return ('tweet received\n\n', 200,)
except Exception as exc:
logger.warning('problem on tweet processing or saving')
return ('problem on tweet processing or saving', 404,)
def feed_sms_report_get_one(doc_id):
from citizendesk.feeds.sms.report import process
logger = get_logger()
client_ip = get_client_ip()
res = process.do_get_one(mongo_dbs.get_db().db, doc_id)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def feed_sms_citizen_alias_get_one(alias_type, alias_value):
from citizendesk.feeds.sms.citizen_alias import process
logger = get_logger()
client_ip = get_client_ip()
res = process.do_get_one(mongo_dbs.get_db().db, alias_type, alias_value)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def feed_any_coverage_activate_one(coverage_id, set_active):
from citizendesk.feeds.any.coverage import process
logger = get_logger()
client_ip = get_client_ip()
res = process.do_set_active_one(mongo_dbs.get_db().db, coverage_id, set_active)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def prepare_reporting(mongo_addr, dbname, newstwister_url, sms_config_path):
from citizendesk.common.utils import get_logger
from citizendesk.common.dbc import mongo_dbs
from citizendesk.feeds.config import set_config
import citizendesk.feeds.err.dispatch as err_dispatch
import citizendesk.feeds.any.dispatch as any_dispatch
import citizendesk.feeds.img.dispatch as img_dispatch
import citizendesk.feeds.twt.dispatch as twt_dispatch
import citizendesk.feeds.sms.dispatch as sms_dispatch
logger = get_logger()
mongo_dbs.set_dbname(dbname)
DbHolder = namedtuple('DbHolder', 'db')
mongo_dbs.set_db(
DbHolder(db=MongoClient(mongo_addr[0], mongo_addr[1])[
mongo_dbs.get_dbname()]))
set_config('newstwister_url', newstwister_url)
if sms_config_path:
try:
shf = open(sms_config_path)
sms_config_data = shf.read()
shf.close()
sms_yaml = yaml.load_all(sms_config_data)
sms_config = sms_yaml.next()
sms_yaml.close()
except:
logger.error('can not read sms config file: ' +
str(sms_config_path))
return False
if ('gateway_api' in sms_config) and sms_config['gateway_api']:
set_config('sms_gateway_api', sms_config['gateway_api'])
if ('gateway_url' in sms_config) and sms_config['gateway_url']:
set_config('sms_gateway_url', sms_config['gateway_url'])
if ('gateway_key' in sms_config) and sms_config['gateway_key']:
set_config('sms_gateway_key', sms_config['gateway_key'])
if ('allowed_ips' in sms_config) and sms_config['allowed_ips']:
set_config('sms_allowed_ips', sms_config['allowed_ips'])
err_dispatch.setup_blueprints(app)
any_dispatch.setup_blueprints(app)
img_dispatch.setup_blueprints(app)
twt_dispatch.setup_blueprints(app)
sms_dispatch.setup_blueprints(app)
return True
def feed_twt_oauth_get_one(oauth_id):
from citizendesk.feeds.twt.oauth import process
logger = get_logger()
client_ip = get_client_ip()
is_local = not is_remote_ip(client_ip)
res = process.do_get_one(mongo_dbs.get_db().db, oauth_id, is_local)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def check_client():
from citizendesk.common.utils import get_logger, get_client_ip, get_allowed_ips
logger = get_logger()
client_ip = get_client_ip()
message = '' + str(request.method) + ' request on ' + str(request.url) + ', by ' + str(get_client_ip())
allowed_ips = get_allowed_ips()
if (allowed_ips is not None) and ('*' not in allowed_ips):
if not client_ip in allowed_ips:
logger.info('unallowed ' + message)
return make_response('Client not allowed\n\n', 403,)
logger.info('allowed ' + message)
def take_twt_user(user_id):
from citizendesk.ingest.twt.process_user import do_post as do_post_user
logger = get_logger()
client_ip = get_client_ip()
allowed_ips = get_allowed_ips()
if allowed_ips and ('*' not in allowed_ips):
if not client_ip in allowed_ips:
logger.info('unallowed client from: '+ str(client_ip))
return ('Client not allowed\n\n', 403,)
logger.info('allowed client from: '+ str(client_ip))
try:
json_data = request.get_json(True, False, False)
if type(json_data) is not dict:
json_data = None
except:
json_data = None
if json_data is None:
try:
json_data = request.json
if type(json_data) is not dict:
json_data = None
except:
json_data = None
if not json_data:
logger.info('data not provided in the request')
return ('Data not provided', 404,)
for part in ['user']:
if (part not in json_data) or (not json_data[part]):
logger.info('No ' + str(part) + ' provided')
return ('No ' + str(part) + ' provided', 404,)
user_info = json_data['user']
try:
res = do_post_user(citizen_holder, user_id, user_info, client_ip)
if (not res) or (not res[0]):
logger.info(str(res[1]))
return (res[1], 404,)
message = 'user info received'
if len(res) > 1:
message = res[1]
return (json.dumps(message) + '\r\n', 200,)
except Exception as exc:
logger.warning('problem on user info processing or saving')
return ('problem on user info processing or saving', 404,)
def feed_twt_stream_patch_one(stream_id):
from citizendesk.feeds.twt.stream import process
from citizendesk.feeds.config import get_config
logger = get_logger()
client_ip = get_client_ip()
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {
'Content-Type': 'application/json'
})
if ('streamer_url' not in data) or (not data['streamer_url']):
data['streamer_url'] = get_config('newstwister_url')
params = {'force': None}
for key in params:
if key in request.args:
params[key] = request.args.get(key)
res = process.do_patch_one(mongo_dbs.get_db().db, stream_id, data,
params['force'])
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 404, {
'Content-Type': 'application/json'
})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 200, {
'Content-Type': 'application/json'
})
def feed_any_coverage_insert_one():
from citizendesk.feeds.any.coverage import process
logger = get_logger()
client_ip = get_client_ip()
coverage_data = {}
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {'Content-Type': 'application/json'})
try:
coverage_data['title'] = data['title']
coverage_data['description'] = data['description']
coverage_data['user_id'] = data['user_id']
except:
return (json.dumps('provided data should contain "title", "description", "user_id" parts'), 404, {'Content-Type': 'application/json'})
if 'active' in data:
coverage_data['active'] = _get_boolean(data['active'])
try:
if ('uuid' in data) and data['uuid']:
coverage_data['uuid'] = str(data['uuid'])
except:
pass
res = process.do_insert_one(mongo_dbs.get_db().db, coverage_data)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def prepare_reporting(mongo_addr, dbname, newstwister_url, sms_config_path):
from citizendesk.common.utils import get_logger
from citizendesk.common.dbc import mongo_dbs
from citizendesk.feeds.config import set_config
import citizendesk.feeds.err.dispatch as err_dispatch
import citizendesk.feeds.any.dispatch as any_dispatch
import citizendesk.feeds.img.dispatch as img_dispatch
import citizendesk.feeds.twt.dispatch as twt_dispatch
import citizendesk.feeds.sms.dispatch as sms_dispatch
logger = get_logger()
mongo_dbs.set_dbname(dbname)
DbHolder = namedtuple('DbHolder', 'db')
mongo_dbs.set_db(DbHolder(db=MongoClient(mongo_addr[0], mongo_addr[1])[mongo_dbs.get_dbname()]))
set_config('newstwister_url', newstwister_url)
if sms_config_path:
try:
shf = open(sms_config_path)
sms_config_data = shf.read()
shf.close()
sms_yaml = yaml.load_all(sms_config_data)
sms_config = sms_yaml.next()
sms_yaml.close()
except:
logger.error('can not read sms config file: ' + str(sms_config_path))
return False
if ('gateway_api' in sms_config) and sms_config['gateway_api']:
set_config('sms_gateway_api', sms_config['gateway_api'])
if ('gateway_url' in sms_config) and sms_config['gateway_url']:
set_config('sms_gateway_url', sms_config['gateway_url'])
if ('gateway_key' in sms_config) and sms_config['gateway_key']:
set_config('sms_gateway_key', sms_config['gateway_key'])
if ('allowed_ips' in sms_config) and sms_config['allowed_ips']:
set_config('sms_allowed_ips', sms_config['allowed_ips'])
err_dispatch.setup_blueprints(app)
any_dispatch.setup_blueprints(app)
img_dispatch.setup_blueprints(app)
twt_dispatch.setup_blueprints(app)
sms_dispatch.setup_blueprints(app)
return True
def feed_twt_citizen_alias_get_one(alias_type, alias_value):
from citizendesk.feeds.twt.citizen_alias import process
from citizendesk.feeds.config import get_config
logger = get_logger()
client_ip = get_client_ip()
other = {'force': None}
for key in other:
if key in request.args:
other[key] = request.args.get(key)
force = False
if other and ('force' in other):
force = _get_boolean(other['force'])
res = process.do_get_one(mongo_dbs.get_db().db, alias_type, alias_value)
err_reason = None
err_msg = None
if not res[0]:
err_msg = res[1]
if (not res[0]) and force:
err_msg = res[1]
searcher_url = get_config('newstwister_url')
res_aux = process.do_request_one(mongo_dbs.get_db().db, searcher_url, alias_type, alias_value)
if not res_aux[0]:
err_msg = res_aux[1]
if 2 < len(res_aux):
err_reason = res_aux[2]
else:
res = process.do_get_one(mongo_dbs.get_db().db, alias_type, alias_value)
if not res[0]:
err_msg = res_aux[1]
err_reason = None
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': err_msg}}
if err_reason:
ret_data['_meta']['reason'] = err_reason
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def feed_twt_citizen_alias_request_one(alias_type, alias_value):
from citizendesk.feeds.twt.citizen_alias import process
from citizendesk.feeds.config import get_config
logger = get_logger()
client_ip = get_client_ip()
searcher_url = get_config('newstwister_url')
res = process.do_request_one(mongo_dbs.get_db().db, searcher_url, alias_type, alias_value)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
if 2 < len(res):
ret_data['_meta']['reason'] = res[2]
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def take_sms(feed_name):
from citizendesk.common.dbc import mongo_dbs
db = mongo_dbs.get_db().db
from citizendesk.ingest.sms.process import do_post
logger = get_logger()
client_ip = get_client_ip()
allowed_ips = get_allowed_ips()
if allowed_ips and ('*' not in allowed_ips):
if not client_ip in allowed_ips:
logger.info('unallowed client from: '+ str(client_ip))
return ('Client not allowed\n\n', 403,)
logger.info('allowed client from: '+ str(client_ip))
params = {'feed': feed_name}
for part in ['feed', 'phone', 'time', 'text']:
if part not in params:
params[part] = None
if part in request.form:
try:
params[part] = str(request.form[part].encode('utf8'))
except:
pass
timepoint = datetime.datetime.now()
if not params['time']:
params['time'] = timepoint
for part in ['phone', 'text']:
if not params[part]:
return ('No ' + str(part) + ' provided', 404)
try:
res = do_post(db, holder, params, client_ip)
if (not res) or (not res[0]):
logger.info(str(res[1]))
return (res[1], 404,)
return ('SMS received\n\n', 200,)
except Exception as exc:
logger.warning('problem on tweet processing or saving')
return ('problem on tweet processing or saving', 404,)
def feed_sms_citizen_alias_post_one(alias_id):
from citizendesk.feeds.sms.citizen_alias import process
logger = get_logger()
client_ip = get_client_ip()
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {'Content-Type': 'application/json'})
try:
alias_spec = data['spec']
user_id = data['user_id']
except:
return (json.dumps('provided data should contain "spec", "user_id" parts'), 404, {'Content-Type': 'application/json'})
if not alias_id:
if ('_id' in data) and (data['_id']):
alias_id = data['_id']
res = process.do_post_one(mongo_dbs.get_db().db, alias_id, alias_spec, user_id)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
if 2 < len(res):
ret_data['_meta']['reason'] = res[2]
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def feed_twt_search_one_post():
from citizendesk.feeds.twt.search import process
from citizendesk.feeds.config import get_config
logger = get_logger()
client_ip = get_client_ip()
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {'Content-Type': 'application/json'})
try:
user_id = data['user_id']
request_id = data['request_id']
search_spec = data['search_spec']
except:
return (json.dumps('provided data should contain "user_id", "request_id", "search_spec" parts'), 404, {'Content-Type': 'application/json'})
searcher_url = get_config('newstwister_url')
if ('searcher_url' in data) and data['searcher_url']:
searcher_url = data['searcher_url']
res = process.do_post_search(mongo_dbs.get_db().db, searcher_url, user_id, request_id, search_spec)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def feed_twt_authorized_finalize_one(doc_id):
from citizendesk.feeds.twt.authorized import process
from citizendesk.feeds.config import get_config
logger = get_logger()
client_ip = get_client_ip()
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {
'Content-Type': 'application/json'
})
auther_url = get_config('newstwister_url')
res = process.do_finalize_one(mongo_dbs.get_db().db, auther_url, doc_id,
data)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 404, {
'Content-Type': 'application/json'
})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 200, {
'Content-Type': 'application/json'
})
def feed_sms_citizen_alias_post_one(alias_id):
from citizendesk.feeds.sms.citizen_alias import process
logger = get_logger()
client_ip = get_client_ip()
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {'Content-Type': 'application/json'})
try:
alias_spec = data['spec']
user_id = data['user_id']
except:
return (json.dumps('provided data should contain "spec", "user_id" parts'), 404, {'Content-Type': 'application/json'})
if not alias_id:
if ('_id' in data) and (data['_id']):
alias_id = data['_id']
res = process.do_post_one(mongo_dbs.get_db().db, alias_id, alias_spec, user_id)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
if 2 < len(res):
ret_data['_meta']['reason'] = res[2]
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def feed_twt_authorized_delete_one(doc_id):
from citizendesk.feeds.twt.authorized import process
logger = get_logger()
client_ip = get_client_ip()
res = process.do_delete_one(mongo_dbs.get_db().db, doc_id)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 404, {
'Content-Type': 'application/json'
})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 200, {
'Content-Type': 'application/json'
})
def feed_any_coverage_unpublish_one(coverage_id):
from citizendesk.feeds.any.coverage import process
logger = get_logger()
client_ip = get_client_ip()
res = process.do_unpublish_one(mongo_dbs.get_db().db, coverage_id)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 404, {
'Content-Type': 'application/json'
})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 200, {
'Content-Type': 'application/json'
})
def feed_twt_oauth_get_one(oauth_id):
from citizendesk.feeds.twt.oauth import process
logger = get_logger()
client_ip = get_client_ip()
is_local = not is_remote_ip(client_ip)
res = process.do_get_one(mongo_dbs.get_db().db, oauth_id, is_local)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 404, {
'Content-Type': 'application/json'
})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 200, {
'Content-Type': 'application/json'
})
def feed_err_ingest_insert_one(notice_timestamp):
from citizendesk.feeds.err.ingest import process
logger = get_logger()
client_ip = get_client_ip()
notice_data = {}
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {'Content-Type': 'application/json'})
try:
notice_data['feed_type'] = data['feed_type']
notice_data['channel'] = data['channel']
notice_data['message'] = data['message']
except:
return (json.dumps('provided data should contain "feed_type", "channel", "message" parts'), 404, {'Content-Type': 'application/json'})
res = process.do_insert_one(mongo_dbs.get_db().db, notice_timestamp, notice_data)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def feed_twt_stream_patch_one(stream_id):
from citizendesk.feeds.twt.stream import process
from citizendesk.feeds.config import get_config
logger = get_logger()
client_ip = get_client_ip()
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {'Content-Type': 'application/json'})
if ('streamer_url' not in data) or (not data['streamer_url']):
data['streamer_url'] = get_config('newstwister_url')
params = {'force': None}
for key in params:
if key in request.args:
params[key] = request.args.get(key)
res = process.do_patch_one(mongo_dbs.get_db().db, stream_id, data, params['force'])
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def feed_twt_stream_post_one(stream_id):
from citizendesk.feeds.twt.stream import process
logger = get_logger()
client_ip = get_client_ip()
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {
'Content-Type': 'application/json'
})
res = process.do_post_one(mongo_dbs.get_db().db, stream_id, data)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 404, {
'Content-Type': 'application/json'
})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 200, {
'Content-Type': 'application/json'
})
def feed_twt_stream_patch_one_emulate(stream_id, switch_on):
from citizendesk.feeds.twt.stream import process
from citizendesk.feeds.config import get_config
logger = get_logger()
client_ip = get_client_ip()
data = {
'control': {
'streamer_url': get_config('newstwister_url'),
'switch_on': switch_on
},
}
params = {'force': True}
res = process.do_patch_one(mongo_dbs.get_db().db, stream_id, data, params['force'])
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
from citizendesk.common.utils import setup_logger, get_logger, set_allowed_ips
from citizendesk.common.utils import set_pid_path, set_daemon_name
from citizendesk.common.utils import daemonize, set_user, cleanup, exit_handler
from citizendesk.outgest.run import run_flask
except:
sys.stderr.write('citizen modules not installed\n')
os._exit(1)
params.use_specs()
setup_logger(params.get_log_path())
set_pid_path(params.get_pid_path())
set_allowed_ips(params.get_allowed_ips())
set_daemon_name(DAEMON_NAME)
atexit.register(cleanup)
signal.signal(signal.SIGTERM, exit_handler)
signal.signal(signal.SIGINT, exit_handler)
if params.to_daemonize():
daemonize(params.get_home_dir(), params.get_pid_path())
set_user(params.get_user_id(), params.get_group_id(),
params.get_pid_path())
logger = get_logger()
try:
logger.info('starting the ' + str(DAEMON_NAME))
run_server()
except Exception as exc:
logger.error('can not start the ' + str(DAEMON_NAME) + ': ' + str(exc))
cleanup(1)
def ingest_sms_feed_take_one(feed_name):
from citizendesk.common.dbc import mongo_dbs
db = mongo_dbs.get_db().db
from citizendesk.ingest.sms.process import do_post
logger = get_logger()
client_ip = get_client_ip()
allowed_ips = get_allowed_ips()
if (allowed_ips is not None) and ('*' not in allowed_ips):
if not client_ip in allowed_ips:
logger.info('unallowed client from: ' + str(client_ip))
return (
'Client not allowed\n\n',
403,
)
logger.info('allowed client from: ' + str(client_ip))
store_keys = {
'phone': 'phone_param',
'text': 'text_param',
'feed': 'feed_param',
'time': 'time_param',
'pass': '******',
}
param_keys = {}
for key in store_keys:
param_keys[store_keys[key]] = key
main_config = get_sms_configuration(db)
sms_password_key = main_config['sms_password_key']
for key in param_keys:
take_key = 'sms_' + key
if take_key in main_config:
param_keys[key] = main_config[take_key]
params = {'feed': feed_name}
for store_key in store_keys:
part_key = store_keys[store_key]
part_param = param_keys[part_key]
if store_key not in params:
params[store_key] = None
if part_param in request.args:
try:
params[store_key] = str(
request.args[part_param].encode('utf8'))
except:
pass
if part_param in request.form:
try:
params[store_key] = str(
request.form[part_param].encode('utf8'))
except:
pass
timepoint = datetime.datetime.utcnow()
if params['time']:
try:
dt_format = '%Y-%m-%dT%H:%M:%S'
if '.' in params['time']:
dt_format = '%Y-%m-%dT%H:%M:%S.%f'
params['time'] = datetime.datetime.strptime(
params['time'], dt_format)
except:
params['time'] = None
if not params['time']:
params['time'] = timepoint
for part in ['phone', 'text']:
if not params[part]:
return ('No ' + str(part) + ' provided', 404)
if sms_password_key:
if sms_password_key != params['pass']:
logger.info('request with wrong pass-phrase from: ' +
str(client_ip))
return (
'wrong pass-phrase\n\n',
403,
)
try:
res = do_post(db, params, main_config, client_ip)
if not res[0]:
logger.info(str(res[1]))
return (
res[1],
404,
)
return (
'SMS received\n\n',
200,
)
except Exception as exc:
logger.warning('problem on sms processing or saving')
return (
'problem on sms processing or saving',
404,
)
def take_twt_user(user_id):
from citizendesk.ingest.twt.process_user import do_post as do_post_user
logger = get_logger()
client_ip = get_client_ip()
allowed_ips = get_allowed_ips()
if allowed_ips and ('*' not in allowed_ips):
if not client_ip in allowed_ips:
logger.info('unallowed client from: ' + str(client_ip))
return (
'Client not allowed\n\n',
403,
)
logger.info('allowed client from: ' + str(client_ip))
try:
json_data = request.get_json(True, False, False)
if type(json_data) is not dict:
json_data = None
except:
json_data = None
if json_data is None:
try:
json_data = request.json
if type(json_data) is not dict:
json_data = None
except:
json_data = None
if not json_data:
logger.info('data not provided in the request')
return (
'Data not provided',
404,
)
for part in ['user']:
if (part not in json_data) or (not json_data[part]):
logger.info('No ' + str(part) + ' provided')
return (
'No ' + str(part) + ' provided',
404,
)
user_info = json_data['user']
try:
res = do_post_user(citizen_holder, user_id, user_info, client_ip)
if (not res) or (not res[0]):
logger.info(str(res[1]))
return (
res[1],
404,
)
message = 'user info received'
if len(res) > 1:
message = res[1]
return (
json.dumps(message) + '\r\n',
200,
)
except Exception as exc:
logger.warning('problem on user info processing or saving')
return (
'problem on user info processing or saving',
404,
)
def take_twt(tweet_id):
from citizendesk.ingest.twt.process import do_post
logger = get_logger()
client_ip = get_client_ip()
allowed_ips = get_allowed_ips()
if allowed_ips and ('*' not in allowed_ips):
if not client_ip in allowed_ips:
logger.info('unallowed client from: ' + str(client_ip))
return (
'Client not allowed\n\n',
403,
)
logger.info('allowed client from: ' + str(client_ip))
try:
json_data = request.get_json(True, False, False)
if type(json_data) is not dict:
json_data = None
except:
json_data = None
if json_data is None:
try:
json_data = request.json
if type(json_data) is not dict:
json_data = None
except:
json_data = None
if not json_data:
logger.info('data not provided in the request')
return (
'Data not provided',
404,
)
for part in ['filter', 'tweet', 'endpoint', 'type']:
if (part not in json_data) or (not json_data[part]):
logger.info('No ' + str(part) + ' provided')
return (
'No ' + str(part) + ' provided',
404,
)
tweet = json_data['tweet']
feed_filter = json_data['filter']
endpoint = json_data['endpoint']
channel_type = json_data['type']
request_id = None
if ('request' in json_data) and json_data['request']:
request_id = json_data['request']
try:
res = do_post(holder, tweet_id, tweet, channel_type, endpoint,
request_id, feed_filter, client_ip)
if (not res) or (not res[0]):
logger.info(str(res[1]))
return (
res[1],
404,
)
return (
'tweet received\n\n',
200,
)
except Exception as exc:
logger.warning('problem on tweet processing or saving')
return (
'problem on tweet processing or saving',
404,
)
def ingest_url_feed_take_one(feed_name):
from citizendesk.common.dbc import mongo_dbs
db = mongo_dbs.get_db().db
from citizendesk.ingest.url.process import do_post
logger = get_logger()
client_ip = get_client_ip()
allowed_ips = get_allowed_ips()
if (allowed_ips is not None) and ('*' not in allowed_ips):
if not client_ip in allowed_ips:
logger.info('unallowed client from: ' + str(client_ip))
return ('Client not allowed\n\n', 403,)
logger.info('allowed client from: '+ str(client_ip))
param_keys = [
'feed_name',
'url_link',
'request_id',
]
params = {'feed_name': feed_name}
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {'Content-Type': 'application/json'})
for one_param in param_keys:
if one_param in data:
try:
params[one_param] = str(data[one_param].encode('utf8'))
except:
pass
for one_param in param_keys:
if (not one_param in params) or (not params[one_param]):
return ('No ' + str(one_param) + ' provided', 404)
try:
res = do_post(db, params['url_link'], params['feed_name'], params['request_id'], client_ip)
if not res[0]:
logger.info(str(res[1]))
return (res[1], 404,)
ret_data = res[1]
return json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'}
except Exception as exc:
logger.warning('problem on url processing or saving: ' + str(exc))
return ('problem on url processing or saving: ' + str(exc), 404,)
def ingest_sms_feed_take_one(feed_name):
from citizendesk.common.dbc import mongo_dbs
db = mongo_dbs.get_db().db
from citizendesk.ingest.sms.process import do_post
logger = get_logger()
client_ip = get_client_ip()
allowed_ips = get_allowed_ips()
if (allowed_ips is not None) and ('*' not in allowed_ips):
if not client_ip in allowed_ips:
logger.info('unallowed client from: ' + str(client_ip))
return ('Client not allowed\n\n', 403,)
logger.info('allowed client from: '+ str(client_ip))
store_keys = {
'phone': 'phone_param',
'text': 'text_param',
'feed': 'feed_param',
'time': 'time_param',
'pass': '******',
}
param_keys = {}
for key in store_keys:
param_keys[store_keys[key]] = key
main_config = get_sms_configuration(db)
sms_password_key = main_config['sms_password_key']
for key in param_keys:
take_key = 'sms_' + key
if take_key in main_config:
param_keys[key] = main_config[take_key]
params = {'feed': feed_name}
for store_key in store_keys:
part_key = store_keys[store_key]
part_param = param_keys[part_key]
if store_key not in params:
params[store_key] = None
if part_param in request.args:
try:
params[store_key] = str(request.args[part_param].encode('utf8'))
except:
pass
if part_param in request.form:
try:
params[store_key] = str(request.form[part_param].encode('utf8'))
except:
pass
timepoint = datetime.datetime.now()
if params['time']:
try:
dt_format = '%Y-%m-%dT%H:%M:%S'
if '.' in params['time']:
dt_format = '%Y-%m-%dT%H:%M:%S.%f'
params['time'] = datetime.datetime.strptime(params['time'], dt_format)
except:
params['time'] = None
if not params['time']:
params['time'] = timepoint
for part in ['phone', 'text']:
if not params[part]:
return ('No ' + str(part) + ' provided', 404)
if sms_password_key:
if sms_password_key != params['pass']:
logger.info('request with wrong pass-phrase from: ' + str(client_ip))
return ('wrong pass-phrase\n\n', 403,)
try:
res = do_post(db, params, main_config, client_ip)
if not res[0]:
logger.info(str(res[1]))
return (res[1], 404,)
return ('SMS received\n\n', 200,)
except Exception as exc:
logger.warning('problem on sms processing or saving')
return ('problem on sms processing or saving', 404,)
from citizendesk.common.utils import setup_logger, get_logger, set_allowed_ips
from citizendesk.common.utils import set_pid_path, set_daemon_name
from citizendesk.common.utils import daemonize, set_user, cleanup, exit_handler
from citizendesk.ingest.url.run import run_flask
except:
sys.stderr.write('citizen modules not installed\n')
os._exit(1)
params.use_specs()
setup_logger(params.get_log_path())
set_pid_path(params.get_pid_path())
set_allowed_ips(params.get_allowed_ips())
set_daemon_name(DAEMON_NAME)
atexit.register(cleanup)
signal.signal(signal.SIGTERM, exit_handler)
signal.signal(signal.SIGINT, exit_handler)
if params.to_daemonize():
daemonize(params.get_home_dir(), params.get_pid_path())
set_user(params.get_user_id(), params.get_group_id(), params.get_pid_path())
logger = get_logger()
try:
logger.info('starting the ' + str(DAEMON_NAME))
run_server()
except Exception as exc:
logger.error('can not start the ' + str(DAEMON_NAME) + ': ' + str(exc))
cleanup(1)
def do_post(db, url, channel_type, request_id, client_ip):
'''
* assure citizen_alias exists (i.e. create/save if does not yet)
* create and save the report
* channel_type: frontend, bookmarklet, etc.
* request_id: commonly _id or uuid for filtering
'''
logger = get_logger()
timestamp = datetime.datetime.utcnow()
channel_type = _get_id_value(channel_type)
request_id = _get_id_value(request_id)
if not url:
return (False, 'emtpy url')
try:
url = url.strip()
url_parsed = urlparse(url, scheme='http')
except:
url_parsed = None
if not url_parsed:
return (False, 'can not parse the url')
if not url_parsed.netloc:
return (False, 'url without domain part')
# taking params
feed_type = get_conf('feed_type')
eff_tlds = get_conf('eff_tlds')
# taking info
page_info_got = get_page_info(url)
if not page_info_got[0]:
return (False, 'can not get page info: ' + page_info_got[1])
page_info = page_info_got[1]
report_id = report_holder.gen_id(feed_type)
session_url = ''
if ('url' in page_info) and page_info['url']:
try:
session_url = page_info['url'].split('#')[0].strip().lower()
except:
pass
if not session_url:
session_url = url.split('#')[0].strip().lower()
session = feed_type + '||' + session_url
source_url = ''
if ('url' in page_info) and page_info['url']:
try:
source_url = page_info['url'].strip()
except:
pass
if not source_url:
source_url = url.strip()
domain_effective = ''
if ('domain_name' in page_info) and page_info['domain_name']:
try:
domain_effective = str(page_info['domain_name']).split(':')[0].strip()
except:
pass
if not domain_effective:
domain_effective = url_parsed.netloc.split(':')[0].strip()
if eff_tlds:
citizen_id = take_specific_domain(eff_tlds, domain_effective)
else:
citizen_id = domain_effective
if not citizen_id:
citizen_id = source_url
parent_id = None
publisher = None
if ('feed_name' in page_info) and page_info['feed_name']:
feed_name = page_info['feed_name']
else:
feed_name = None
use_language = None
if ('language' in page_info) and page_info['language']:
use_language = page_info['language']
authority = get_conf('authority')
use_identifiers = {
'user_id': citizen_id,
'user_id_search': citizen_id.lower(),
'user_name': citizen_id,
'user_name_search': citizen_id.lower(),
}
channel_value = None
if ('type' in page_info) and page_info['type']:
try:
channel_value = page_info['type'].strip().lower()
if not channel_value:
channel_value = None
except:
channel_value = None
channels = [{'type': channel_type, 'value': channel_value, 'filter': None, 'request': request_id, 'reasons': None}]
authors = [{'authority': authority, 'identifiers': use_identifiers}]
author_name = ''
if 'author' in page_info:
author_name = page_info['author']
alias_res_got = assure_citizen_alias(db, authority, use_identifiers, author_name)
endorsers = []
original = page_info
original_id = url # notice that we group the (effective) URLs via session
tags = []
texts = []
text_parts = ['title', 'description']
for one_text_part in text_parts:
if (one_text_part in page_info) and page_info[one_text_part]:
texts.append({'original': page_info[one_text_part], 'transcript': None})
estimated = timestamp
if ('date' in page_info) and page_info['date'] and (type(page_info['date']) is datetime.datetime):
estimated = page_info['date']
# site_icon added at the end of the media list
media = []
if ('image' in page_info) and page_info['image'] and (type(page_info['image']) in (list, tuple)):
for one_image_link in page_info['image']:
try:
link_ssl = None
if one_image_link.startswith('https'):
link_ssl = one_image_link
media.append({
'link': one_image_link,
'link_ssl': link_ssl,
'data': None,
'name': None,
'width': None,
'height': None,
})
except:
pass
if ('site_icon' in page_info) and page_info['site_icon']:
try:
link_ssl = None
if page_info['site_icon'].startswith('https'):
link_ssl = page_info['site_icon']
media.append({
'link': page_info['site_icon'],
'link_ssl': link_ssl,
'data': None,
'name': None,
'width': None,
'height': None,
})
except:
pass
report = {}
report['report_id'] = report_id
report['parent_id'] = None
report['client_ip'] = client_ip
report['feed_type'] = feed_type
report['produced'] = estimated
report['session'] = session
report['publisher'] = publisher
report['channels'] = channels
report['authors'] = authors
report['original'] = original
report['texts'] = texts
report['tags'] = tags
report['language'] = use_language
report['sources'] = [source_url]
report['media'] = media
report['editable'] = False
report['proto'] = False
report_doc_id = report_holder.save_report(report)
if not report_doc_id:
return (False, 'can not save URL report')
return (True, {'_id': report_doc_id})
def do_post(db, params, main_config, client_ip):
'''
* assure citizen_alias exists (i.e. create/save if does not yet)
* find if following a previous report (incl. taking its session)
* create and save the report
* if starting a new session and with auto-replies:
* create a reply-report
* send a reply message (via gateway)
'''
logger = get_logger()
# taking given params
feed_type = get_conf('feed_type')
publisher = get_conf('publisher')
feed_name = params['feed']
phone_number = params['phone']
message = params['text']
received = params['time']
timestamp = datetime.datetime.now()
# assuring the citizen
alias_id = None
alias_info = None
alias_res = assure_citizen_alias(db, phone_number)
if alias_res[0]:
alias_info = alias_res[1]
alias_id = alias_info['_id']
# finding the followed report
last_report = get_sms(phone_number)
if last_report:
for key in ['produced', 'session', 'report_id']:
if key not in last_report:
last_report = None
# creating the report
authority = get_conf('authority')
phone_identifier_type = get_conf('phone_identifier_type')
channel_type = get_conf('channel_type')
channel_value_receive = get_conf('channel_value_receive')
sms_filter = None
if feed_name:
sms_filter = {'feed_name': feed_name}
channels = [{'type': channel_type, 'value': channel_value_receive, 'filter': sms_filter, 'request': None, 'reasons': None}]
authors = [{'authority': authority, 'identifiers': [{'type': phone_identifier_type, 'value': phone_number}]}]
endorsers = []
original = {'message': message}
texts = [{'original': message, 'transcript': None}]
tags = []
if message:
tags = _extract_tags(message)
report_id = gen_id(feed_type, channel_type, channel_value_receive, timestamp)
session = report_id
parent_id = None
new_session = True
pinned_id = None
assignments = []
if last_report:
if is_within_session(last_report['produced'], received, main_config):
session = last_report['session']
parent_id = last_report['report_id']
if 'pinned_id' in last_report:
pinned_id = last_report['pinned_id']
if 'assignments' in last_report:
assignments = last_report['assignments']
new_session = False
report = {}
report['report_id'] = report_id
report['parent_id'] = parent_id
report['client_ip'] = client_ip
report['feed_type'] = feed_type
report['produced'] = received
report['session'] = session
report['publisher'] = publisher
report['channels'] = channels
report['authors'] = authors
report['original'] = original
report['texts'] = texts
report['tags'] = tags
report['pinned_id'] = pinned_id
report['assignments'] = assignments
report['proto'] = False
report_doc_id = report_holder.save_report(report)
if not report_doc_id:
return (False, 'can not save SMS report')
report = report_holder.provide_report(feed_type, report_id)
reply_res = ask_sender(db, new_session, report, alias_info, phone_number, main_config)
if not reply_res[0]:
reason = str(reply_res[1])
if 2 < len(reply_res):
reason += ', ' + str(reply_res[2])
logger.warning('Issue during auto-reply SMS: ' + reason)
return (True, str(report_doc_id))
def feed_sms_reply_one_post():
from citizendesk.feeds.sms.reply import process
from citizendesk.feeds.config import get_config as get_main_config
sms_allowed_ips = get_main_config('sms_allowed_ips')
sms_gateway_url = get_main_config('sms_gateway_url')
sms_gateway_key = get_main_config('sms_gateway_key')
try:
sms_gateway_api = get_main_config('sms_gateway_api')
except:
sms_gateway_api = 'frontlinesms'
logger = get_logger()
client_ip = get_client_ip()
if (sms_allowed_ips is not None) and ('*' not in sms_allowed_ips):
if not client_ip in sms_allowed_ips:
logger.info('unallowed sms-reply request from: ' + str(client_ip))
return (json.dumps('client not allowed'), 403, {'Content-Type': 'application/json'})
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {'Content-Type': 'application/json'})
try:
message = data['message']
report_id = data['report_id']
user_id = data['user_id']
except:
return (json.dumps('provided data should contain "message", "report_id", "user_id" parts'), 404, {'Content-Type': 'application/json'})
language = None
sensitive = None
if 'language' in data:
language = data['language']
if 'sensitive' in data:
sensitive = data['sensitive']
if ('control' in data) and (type(data['control']) is dict):
control = data['control']
if ('sms_gateway_url' in control) and control['sms_gateway_url']:
sms_gateway_url = control['sms_gateway_url']
if ('sms_gateway_key' in control) and control['sms_gateway_key']:
sms_gateway_key = control['sms_gateway_key']
res = process.do_post_reply(mongo_dbs.get_db().db, sms_gateway_api, sms_gateway_url, sms_gateway_key, message, report_id, user_id, language, sensitive, client_ip)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
if 2 < len(res):
ret_data['_meta']['reason'] = res[2]
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 404, {'Content-Type': 'application/json'})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default, sort_keys=True), 200, {'Content-Type': 'application/json'})
def feed_sms_send_one_post():
from citizendesk.feeds.sms.send import process
from citizendesk.feeds.config import get_config as get_main_config
sms_allowed_ips = get_main_config('sms_allowed_ips')
sms_gateway_url = get_main_config('sms_gateway_url')
sms_gateway_key = get_main_config('sms_gateway_key')
try:
sms_gateway_api = get_main_config('sms_gateway_api')
except:
sms_gateway_api = 'frontlinesms'
logger = get_logger()
client_ip = get_client_ip()
if (sms_allowed_ips is not None) and ('*' not in sms_allowed_ips):
if not client_ip in sms_allowed_ips:
logger.info('unallowed sms-send request from: ' + str(client_ip))
return (json.dumps('client not allowed'), 403, {
'Content-Type': 'application/json'
})
try:
data = request.get_json(True, False, False)
if type(data) is not dict:
data = None
except:
data = None
if data is None:
try:
data = request.json
if type(data) is not dict:
data = None
except:
data = None
if data is None:
return (json.dumps('provided data are not valid json'), 404, {
'Content-Type': 'application/json'
})
try:
message = data['message']
targets = data['targets']
user_id = data['user_id']
except:
return (json.dumps(
'provided data should contain "message", "targets", "user_id" parts'
), 404, {
'Content-Type': 'application/json'
})
language = None
sensitive = None
if 'language' in data:
language = data['language']
if 'sensitive' in data:
sensitive = data['sensitive']
if ('control' in data) and (type(data['control']) is dict):
control = data['control']
if ('sms_gateway_url' in control) and control['sms_gateway_url']:
sms_gateway_url = control['sms_gateway_url']
if ('sms_gateway_key' in control) and control['sms_gateway_key']:
sms_gateway_key = control['sms_gateway_key']
res = process.do_post_send(mongo_dbs.get_db().db, sms_gateway_api,
sms_gateway_url, sms_gateway_key, message,
targets, user_id, language, sensitive,
client_ip)
if not res[0]:
ret_data = {'_meta': {'schema': process.schema, 'message': res[1]}}
if 2 < len(res):
ret_data['_meta']['reason'] = res[2]
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 404, {
'Content-Type': 'application/json'
})
ret_data = {'_meta': {'schema': process.schema}, '_data': res[1]}
return (json.dumps(ret_data, default=json_util.default,
sort_keys=True), 200, {
'Content-Type': 'application/json'
})
def do_post(db, params, main_config, client_ip):
'''
* assure citizen_alias exists (i.e. create/save if does not yet)
* find if following a previous report (incl. taking its session)
* create and save the report
* if starting a new session and with auto-replies:
* create a reply-report
* send a reply message (via gateway)
'''
logger = get_logger()
# taking given params
feed_type = get_conf('feed_type')
publisher = get_conf('publisher')
feed_name = params['feed']
phone_number = params['phone']
message = params['text']
received = params['time']
timestamp = datetime.datetime.utcnow()
# assuring the citizen
alias_id = None
alias_info = None
alias_res = assure_citizen_alias(db, phone_number)
if alias_res[0]:
alias_info = alias_res[1]
alias_id = alias_info['_id']
# finding the followed report
last_report = get_sms(phone_number)
if last_report:
for key in ['produced', 'session', 'report_id']:
if key not in last_report:
last_report = None
# creating the report
authority = get_conf('authority')
channel_type = get_conf('channel_type')
channel_value_receive = get_conf('channel_value_receive')
sms_filter = None
if feed_name:
sms_filter = {'feed_name': feed_name}
use_identifiers = _create_phone_number_identities(phone_number)
channels = [{'type': channel_type, 'value': channel_value_receive, 'filter': sms_filter, 'request': None, 'reasons': None}]
authors = [{'authority': authority, 'identifiers': use_identifiers}]
endorsers = []
original = {'message': message}
texts = [{'original': message, 'transcript': None}]
tags = []
if message:
tags = _extract_tags(message)
report_id = gen_id(feed_type, channel_type, channel_value_receive, timestamp)
session = report_id
parent_id = None
new_session = True
pinned_id = None
assignments = []
if last_report:
if is_within_session(last_report['produced'], received, main_config):
session = last_report['session']
parent_id = last_report['report_id']
if 'pinned_id' in last_report:
pinned_id = last_report['pinned_id']
if 'assignments' in last_report:
assignments = last_report['assignments']
new_session = False
report = {}
report['report_id'] = report_id
report['parent_id'] = parent_id
report['client_ip'] = client_ip
report['feed_type'] = feed_type
report['produced'] = received
report['session'] = session
report['publisher'] = publisher
report['channels'] = channels
report['authors'] = authors
report['original'] = original
report['texts'] = texts
report['tags'] = tags
report['pinned_id'] = pinned_id
report['assignments'] = assignments
report['proto'] = False
report_doc_id = report_holder.save_report(report)
if not report_doc_id:
return (False, 'can not save SMS report')
report = report_holder.provide_report(feed_type, report_id)
reply_res = ask_sender(db, new_session, report, alias_info, phone_number, main_config)
if not reply_res[0]:
reason = str(reply_res[1])
if 2 < len(reply_res):
reason += ', ' + str(reply_res[2])
logger.warning('Issue during auto-reply SMS: ' + reason)
return (True, str(report_doc_id))