def user_followers():
applog.debug("Listing users with more followers",
extra={
'method': 'GET',
'logger': __name__,
'level': 'debug',
'uuid': current_request_id(),
'endpoint': request.path
})
result = None
try:
result = twittcl.list_users_by_followers('DESC')
except Exception as e:
tb = traceback.format_exc()
applog.error("Error when listing users with more followers",
extra={
'method': 'GET',
'logger': __name__,
'level': 'error',
'uuid': current_request_id(),
'endpoint': request.path,
'trace': tb
})
return '{ "status": "500", "message": "Internal Server Error" }', 500
return jsonify(result)
def general_exception_handler(error: Exception) -> flask.Response:
"""
Function to handle base exceptions generated by the application.
"""
DEFAULT_500_MSG = (
"Sorry, we messed something up. A team of highly trained monkeys " +
"has been dispatched to solve the situation. If you see them, please "
+ f"show your request_id: {current_request_id()}")
# type, value, traceback
tp, vl, tb = sys.exc_info()
log_error = {
"request_id": current_request_id(),
"exception_type": str(tp),
"message": str(vl),
"traceback": str(traceback.format_tb(tb, 10)),
}
logging.error(log_error)
final_msg = {
"your_request_id": current_request_id(),
"status": 500,
"msg": DEFAULT_500_MSG
}
return make_response(jsonify(final_msg), 500)
def posts_hour():
applog.debug("List posts per hour",
extra={
'method': 'GET',
'logger': __name__,
'level': 'debug',
'uuid': current_request_id(),
'endpoint': request.path
})
result = None
try:
result = twittcl.total_posts_by_hour()
except Exception as e:
tb = traceback.format_exc()
applog.error("Error when listing posts by time of publication",
extra={
'method': 'GET',
'logger': __name__,
'level': 'error',
'uuid': current_request_id(),
'endpoint': request.path,
'trace': tb
})
return '{ "status": "500", "message": "Internal Server Error" }', 500
return jsonify(result)
def posts_tags(group):
applog.debug("List posts group %s" % group,
extra={
'method': 'GET',
'logger': __name__,
'level': 'debug',
'uuid': current_request_id(),
'endpoint': request.path
})
result = None
if group == "location" or group == "lang":
applog.info("get total post by tag",
extra={
'logger': __name__,
'level': 'info',
'uuid': current_request_id(),
'endpoint': request.path
})
try:
result = twittcl.total_posts_by_tag(group)
except Exception as e:
tb = traceback.format_exc()
applog.error("Error when listing posts by time of publication",
extra={
'method': 'GET',
'logger': __name__,
'level': 'error',
'uuid': current_request_id(),
'endpoint': request.path,
'trace': tb
})
return '{ "status": "500", "message": "Internal Server Error" }', 500
return jsonify(result)
else:
applog.warning("Invalid group %s" % group,
extra={
'method': 'GET',
'logger': __name__,
'level': 'warning',
'uuid': current_request_id(),
'endpoint': request.path
})
return '{ "status": "400", "message": "Bad request!" }', 400
def after_request(response):
response.headers.add('x-request-id', current_request_id())
applog.info("Request",
extra={
'method': request.method,
'logger': __name__,
'level': 'info',
'uuid': current_request_id(),
'endpoint': request.path,
'status_code': response.status_code
})
return response
def _get_list_score_lang(self, usernames):
pipeline_lang = [{
"$group": {
"_id": "$lang",
"hashtag": {
"$push": "$hashtag"
},
"count": {
"$sum": 1
}
}
}]
list_score_lang = []
for user in usernames:
for tag in self._get_tag_list():
self._applog.debug("Exec pipeline in database",
extra={
'logger': __name__,
'level': 'debug',
'uuid': current_request_id()
})
data = Twitt.objects(
hashtag=tag, username=user['_id']).aggregate(pipeline_lang)
self._applog.debug("Exec pipeline OK",
extra={
'logger': __name__,
'level': 'debug',
'uuid': current_request_id()
})
for d in data:
score_lang = {}
score_lang["username"] = user["_id"]
score_lang["hashtag"] = tag
score_lang["lang"] = d['_id']
score_lang["count"] = d['count']
list_score_lang.append(score_lang)
return list_score_lang
def after_all_response(response: flask.Response) -> flask.Response:
"""
Setups the response object
"""
response.headers["X-Request-ID"] = current_request_id()
return response
def index():
applog.debug("List endpoints",
extra={
'method': 'GET',
'logger': __name__,
'level': 'debug',
'uuid': current_request_id(),
'endpoint': request.path
})
endpoints = {
'info':
'Twitter API',
'version':
'1.0.0',
'endpoints': [{
'url': '/twitter/api/v1/publish/posts',
'method': 'POST'
}, {
'url': '/twitter/api/v1/users/followers',
'method': 'GET'
}, {
'url': '/twitter/api/v1/posts/hour',
'method': 'GET'
}, {
'url': '/twitter/api/v1/posts/tags/location',
'method': 'GET'
}, {
'url': '/twitter/api/v1/posts/tags/lang',
'method': 'GET'
}]
}
return jsonify(endpoints), 200
def __init__(self, engine, encoding=None):
self._engine = engine
self._encoding = encoding or default_encoding
self._request_id = current_request_id()
if IsDeepDebug:
print('--> decoder: %s' % self._request_id)
self._requested_object = None
self._file_id = None
self._client = None
self._file_type = None
self._file_status = None
self._total = 0
# Encodings list
self._encodings = None
# Temp file object name
self._tmp = None
# Image decoded content
self._image = None
# List of application tags
self._tags = None
# XML iterator
self._iter = None
self.has_body = False
def user_id():
log_id = f.request.args.get('user')
user = '******'
if log_id in ['', 'vister', None]:
log_id = str(current_request_id())
user = '******'
return f.jsonify({'user': log_id, 'type': user})
def _init_state(self, **kw):
if IsDeepDebug:
print_to(None, 'FileDecoder.inistate')
self._original = kw.get('original') or ''
self._request_id = current_request_id()
self._tmp = '%s/%s%s' % (self.tmp_folder, str(self._request_id), '%s')
def run_job(_id, history_id=None, **kwargs):
db = Mongo()
record = Job.find_by_id(_id)
if not record:
return False
request_id = str(current_request_id())
username = None if not login_user else login_user.get('username')
params = (_id, request_id, username, history_id)
queue_name = get_queue_by_job(_id)
extra = record.get('extra')
template = record.get('template')
schedule = extra.get('schedule')
ansible_type = record.get('type')
if template.get('run_type') == 'schedule':
existed = db.collection('scheduler_jobs').find_one(
{'_id': record['_id']})
if existed:
return False
scheduler.add_job(func=run_schedule_task,
trigger='cron',
args=params,
coalesce=True,
kwargs=kwargs,
id=str(record.get('_id')),
max_instances=1,
name=record.get('name'),
**schedule)
return True
else:
func = run_playbook_task if ansible_type != 'adhoc' else run_adhoc_task
task = Task(tiger,
func=func,
args=params,
kwargs=kwargs,
queue=queue_name,
unique=True,
lock=True,
lock_key=_id)
task_record = {
'job_id': _id,
'type': 'trigger',
'ansible': ansible_type,
'state': QUEUED,
'queue': queue_name,
'result': '',
'request_id': request_id,
't_id': task.id,
'created_at': time(),
'kwargs': kwargs,
}
result = db.collection('tasks').insert_one(task_record)
task.delay()
return result.inserted_id
def before_request(self):
if current_request_id():
request.request_id = current_request_id()
elif "X-Request-Id" in request.headers:
request.request_id = f'{request.headers["X-Request-Id"]},{str(uuid.uuid4())}'
else:
request.request_id = str(uuid.uuid4())
data = ""
try:
data = request.get_data().decode("utf-8")
except:
pass
self.app.logger.info(f'REQUEST: {request.method} {request.url}')
self.app.logger.info(
f'REQUEST HEADERS: {request.method} {request.url}\n{request.headers}'
)
self.app.logger.info(
f'REQUEST BODY: {request.method} {request.url}\n{data}')
def after_request(response):
app.logger.info('{} {} {} {} {} {}'.format(
request.remote_addr,
request.scheme,
request.method,
request.full_path,
response.status,
response.content_length
))
response.headers.add('X-REQUEST-ID', current_request_id())
return response
def publish_post():
applog.debug("Publish new tweetts..",
extra={
'method': 'POST',
'logger': __name__,
'level': 'debug',
'uuid': current_request_id(),
'endpoint': request.path
})
result = None
data = request.json
try:
if twittcl.publish(data):
retult = "{ 'status': 'success' }"
else:
retult = "{ 'status': 'failed' }"
except Exception as e:
tb = traceback.format_exc()
applog.error("Error when publish new tweets",
extra={
'method': 'POST',
'logger': __name__,
'level': 'error',
'uuid': current_request_id(),
'endpoint': request.path,
'trace': tb
})
return jsonify(
'{ "status": "500", "message": "Internal Server Error" }'), 500
return jsonify(result)
def total_posts_by_hour(self):
list_posts = []
pipeline = [{
"$group": {
"_id": "$post_hour",
'total_post': {
'$sum': 1
}
}
}, {
"$sort": {
"_id": 1
}
}]
self._applog.debug("Exec pipeline in database",
extra={
'logger': __name__,
'level': 'debug',
'uuid': current_request_id()
})
data = Twitt.objects().aggregate(pipeline)
self._applog.debug("Exec pipeline OK",
extra={
'logger': __name__,
'level': 'debug',
'uuid': current_request_id()
})
for d in data:
list_posts.append(d)
return list_posts
def get_orders(user_id):
logger.info('got_request',
extra={'endpoint': '/user/<user_id>/orders'})
headers = {'X-Request-ID': current_request_id()}
response = requests.get(ORDERS_URL.format(user_id=user_id),
headers=headers)
orders = response.json()['result']['orders']
book_ids = get_book_ids_from_orders(orders)
response = requests.get(BOOKS_URL,
params={'book_ids': book_ids},
headers=headers)
books = response.json()['result']
return jsonify({'result': {
'orders': orders,
'books': books,
}})
def run(_id):
book = Book.find_by_id(_id)
if not book:
return jsonify({'message': 'record not found', 'code': 10404}), 404
payload = request.get_json()
options = payload.get('options')
entry = options.get('entry')
args = options.get('args')
req_id = str(current_request_id())
params = {
'username': login_user.get('username'),
'req_id': req_id,
'args': args,
'options': options
}
result = dispatch(_id, entry, params)
if not result:
return jsonify({'message': 'invalid request', 'code': 104008}), 400
return jsonify({'message': 'ok', 'code': 0, 'data': {'taskId': result}})
def append_request_id(response):
response.headers.add('X-REQUEST-ID', current_request_id())
data = request.get_data()
try:
data = data.decode('utf-8')
except Exception as e:
pass
user_agent = request.headers.get('User-Agent', None)
remote_addr = request.environ.get('HTTP_X_REAL_IP',
request.remote_addr)
telemetry(method=request.method,
path=request.path,
args=request.args.to_dict(),
status=response.status,
ip=remote_addr,
user_agent=user_agent,
payload=data)
return response
def publish(self, tweets_list):
for tweet in tweets_list:
twitte = Twitt(id=tweet['id'],
text=tweet['text'],
created_at=tweet['created_at'],
post_hour=tweet['post_hour'],
hashtag=tweet['hashtag'],
lang=tweet['lang'],
author_id=tweet['author_id'],
username=tweet['username'],
name=tweet['name'],
location=tweet['location'],
followers_count=tweet['followers_count']).save()
self._applog.debug("Post save in database.",
extra={
'logger': __name__,
'level': 'debug',
'uuid': current_request_id()
})
return True
def add_adhoc():
payload = request.get_json()
if not payload:
return jsonify({
'message': 'invalid params',
'code': 104001,
}), 400
module = payload.get('module')
args = payload.get('args')
inventory = payload.get('inventory')
private_key = payload.get('private_key')
verbosity = payload.get('verbosity')
name = payload.get('name')
schedule = payload.get('schedule')
check = payload.get('check')
job_id = payload.get('job_id')
extra_options = payload.get('extraOptions')
status = int(payload.get('status', 0))
notification = payload.get('notification')
maintainer = payload.get('maintainer') or []
if maintainer and isinstance(maintainer, list):
users = db.collection('users').find({'username': {'$in': maintainer}})
names = list(map(lambda i: i['username'], users))
maintainer.extend(names)
maintainer.append(login_user.get('username'))
if not job_id:
existed = db.collection('jobs').find_one({'name': name})
if existed and existed.get('status') != -1:
return jsonify({'message': 'name exist', 'code': 104007}), 400
if not module or not inventory or not name:
return jsonify({
'message': 'miss required params',
'code': 104002,
}), 400
# check_module = db.collection('ansible_modules').find_one({
# 'name': module
# })
#
# if not check_module:
# return jsonify({
# 'message': 'invalid module',
# 'code': 104003,
# }), 400
inventory_content = parse_cmdb_inventory(inventory)
if not inventory:
return jsonify({
'message': 'invalid inventory',
'code': 104004,
}), 400
key_text = get_credential_content_by_id(private_key, 'private_key')
if not key_text:
return jsonify({
'message': 'invalid private key',
'code': 104004,
}), 400
options = {}
if extra_options:
options.update(extra_options)
if verbosity:
options['verbosity'] = verbosity
if check:
with NamedTemporaryFile('w+t', delete=True) as fd:
fd.write(key_text)
fd.seek(0)
options['private_key'] = fd.name
tasks = [{'action': {'module': module, 'args': args}}]
hosts = inventory_content
runner = AdHocRunner(hosts, options=options)
runner.run('all', tasks)
result = runner.get_result()
return jsonify({'message': 'ok', 'code': 0, 'data': result})
else:
token = str(base64.b64encode(bytes(current_request_id(), 'utf8')),
'utf8')
data = {
'name': name,
'template': {
'inventory': inventory,
'private_key': private_key,
'verbosity': verbosity,
'module': module,
'args': args,
'extraOptions': extra_options,
},
'extra': {
'schedule': schedule,
'notification': notification,
},
'token': token,
'maintainer': maintainer,
'type': 'adhoc',
'created_at': time.time(),
'status': status,
'add_by': login_user.get('username')
}
if job_id:
record = db.collection('jobs').find_one({'_id': ObjectId(job_id)})
if not record:
return jsonify({
'message': 'record not found',
'code': 104040
}), 404
update = {
'$set': data,
}
db.collection('jobs').update_one({'_id': ObjectId(job_id)},
update=update)
logger.info('update job',
extra={
'record': record,
'changed': data
})
else:
result = db.collection('jobs').insert_one(data)
data['_id'] = result.inserted_id
logger.info('add job', extra={'record': data})
return jsonify({
'message': 'ok',
'code': 0,
})
def add_jobs():
"""
add job
TODO(sang)
:return: json
"""
body = request.get_json()
user = login_user
if not body:
return jsonify({
'message': 'miss required params',
'code': 104000,
}), 400
job_type = body.get('type')
if job_type == 'adhoc':
return add_adhoc()
current_id = body.get('currentId')
record = None
if current_id:
record = Job.find_by_id(current_id)
if not record:
return jsonify({'message': 'job not found', 'code': 104044}), 400
payload = load_ansible_playbook(body)
if payload.get('message') is not 'ok':
return jsonify(payload), 400
data = payload.get('data')
options = data.get('options')
is_check = body.get('check')
private_key = data.get('private_key')
wk = Workspace()
res = wk.load_book_from_db(name=data.get('book_name'),
roles=data.get('roles'))
if not res:
return jsonify({
'message': 'book not found',
'code': 104000,
}), 400
entry = wk.get_book_entry(data.get('book_name'), data.get('entry'))
dry_run = bool(is_check)
options['check'] = dry_run
if dry_run:
with NamedTemporaryFile('w+t', delete=True) as fd:
if private_key:
key_text = get_credential_content_by_id(
private_key, 'private_key')
if not key_text:
return jsonify({
'message': 'invalid private_key',
'code': 104033,
}), 401
fd.write(key_text)
fd.seek(0)
options['private_key'] = fd.name
play = PlayBookRunner(data['inventory'],
options,
callback=CallbackModule())
play.run(entry)
return jsonify({
'message': 'ok',
'code': 0,
'data': {
'result': play.get_result(),
'options': options
}
})
name = data.get('name')
existed = Job.find_one({'name': name})
if existed and not current_id:
return jsonify({
'message': 'name existed',
'code': 104001,
}), 400
token = str(base64.b64encode(bytes(current_request_id(), 'utf8')), 'utf8')
new_record = {
'name': name,
'type': 'playbook',
'token': token,
'description': data.get('description'),
'book_id': data.get('book_id'),
'template': data.get('template'),
'extra': data.get('extra'),
'entry': data['entry'],
'status': data['status'],
'maintainer': [user.get('username')],
'created_at': int(time.time()),
'updated_at': datetime.datetime.now().isoformat(),
}
if record:
Job.update_one({'_id': record['_id']}, update={'$set': new_record})
else:
Job.insert_one(new_record)
return jsonify({
'message': 'ok',
'code': 0,
})
def add(a, b):
"""Simple function to add two numbers that is not aware of the request id"""
logging.info('Called generic_add({}, {}) from request_id: {}'.format(
a, b, current_request_id()))
return a + b
def get_home():
return {"id": current_request_id(), "message": "Hello world"}, 200
def append_request_id(response):
"""
post process append request id
"""
response.headers.add('X-REQUEST-ID', current_request_id())
return response
def get_request_correlation_id():
if request.headers.get('WT_CORRELATION_ID', None) is None:
return current_request_id()
return request.headers.get('WT_CORRELATION_ID', '-')
def run_task():
payload = request.get_json()
if not payload:
return jsonify({
'message': 'invalid params',
'code': 114000,
}), 400
req_id = str(current_request_id())
run_type = payload.get('type')
inventory = payload.get('inventory')
private_key = payload.get('private_key')
module = payload.get('module')
args = payload.get('args')
entry = payload.get('entry')
become_mehtod = payload.get('become_method')
become_user = payload.get('become_user')
verbosity = payload.get('verbosity', 0) or 1
extra_options = payload.get('extraOptions')
hosts = parse_cmdb_inventory(inventory)
if not hosts:
return jsonify({
'message': 'invalid inventory',
'code': 114001,
}), 400
options = {}
if extra_options:
options.update(extra_options)
if verbosity:
options['verbosity'] = verbosity
if become_mehtod:
options['become'] = 'yes'
options['become_method'] = become_mehtod
if become_user:
options['become_user'] = become_user
params = {
'username': login_user.get('username'),
'req_id': req_id,
'options': options,
'inventory': inventory,
'run_type': run_type,
}
if private_key:
params['private_key'] = private_key
if run_type == 'adhoc':
if not module:
return jsonify({
'message': 'invalid module',
'code': 114002,
}), 400
tasks = [{'action': {'module': module, 'args': args}}]
params['tasks'] = tasks
else:
if not entry:
return jsonify({
'message': 'invalid entry',
'code': 114004,
}), 400
params['entry'] = entry
result = dispatch(params)
if not result:
return jsonify({'message': 'invalid request', 'code': 104008}), 400
return jsonify({'message': 'ok', 'code': 0, 'data': {'taskId': result}})
def append_request_id(response):
response.headers.add("X-REQUEST-ID", current_request_id())
return response
