def user_followers(): applog.debug("Listing users with more followers", extra={ 'method': 'GET', 'logger': __name__, 'level': 'debug', 'uuid': current_request_id(), 'endpoint': request.path }) result = None try: result = twittcl.list_users_by_followers('DESC') except Exception as e: tb = traceback.format_exc() applog.error("Error when listing users with more followers", extra={ 'method': 'GET', 'logger': __name__, 'level': 'error', 'uuid': current_request_id(), 'endpoint': request.path, 'trace': tb }) return '{ "status": "500", "message": "Internal Server Error" }', 500 return jsonify(result)
def general_exception_handler(error: Exception) -> flask.Response: """ Function to handle base exceptions generated by the application. """ DEFAULT_500_MSG = ( "Sorry, we messed something up. A team of highly trained monkeys " + "has been dispatched to solve the situation. If you see them, please " + f"show your request_id: {current_request_id()}") # type, value, traceback tp, vl, tb = sys.exc_info() log_error = { "request_id": current_request_id(), "exception_type": str(tp), "message": str(vl), "traceback": str(traceback.format_tb(tb, 10)), } logging.error(log_error) final_msg = { "your_request_id": current_request_id(), "status": 500, "msg": DEFAULT_500_MSG } return make_response(jsonify(final_msg), 500)
def posts_hour(): applog.debug("List posts per hour", extra={ 'method': 'GET', 'logger': __name__, 'level': 'debug', 'uuid': current_request_id(), 'endpoint': request.path }) result = None try: result = twittcl.total_posts_by_hour() except Exception as e: tb = traceback.format_exc() applog.error("Error when listing posts by time of publication", extra={ 'method': 'GET', 'logger': __name__, 'level': 'error', 'uuid': current_request_id(), 'endpoint': request.path, 'trace': tb }) return '{ "status": "500", "message": "Internal Server Error" }', 500 return jsonify(result)
def posts_tags(group): applog.debug("List posts group %s" % group, extra={ 'method': 'GET', 'logger': __name__, 'level': 'debug', 'uuid': current_request_id(), 'endpoint': request.path }) result = None if group == "location" or group == "lang": applog.info("get total post by tag", extra={ 'logger': __name__, 'level': 'info', 'uuid': current_request_id(), 'endpoint': request.path }) try: result = twittcl.total_posts_by_tag(group) except Exception as e: tb = traceback.format_exc() applog.error("Error when listing posts by time of publication", extra={ 'method': 'GET', 'logger': __name__, 'level': 'error', 'uuid': current_request_id(), 'endpoint': request.path, 'trace': tb }) return '{ "status": "500", "message": "Internal Server Error" }', 500 return jsonify(result) else: applog.warning("Invalid group %s" % group, extra={ 'method': 'GET', 'logger': __name__, 'level': 'warning', 'uuid': current_request_id(), 'endpoint': request.path }) return '{ "status": "400", "message": "Bad request!" }', 400
def after_request(response): response.headers.add('x-request-id', current_request_id()) applog.info("Request", extra={ 'method': request.method, 'logger': __name__, 'level': 'info', 'uuid': current_request_id(), 'endpoint': request.path, 'status_code': response.status_code }) return response
def _get_list_score_lang(self, usernames): pipeline_lang = [{ "$group": { "_id": "$lang", "hashtag": { "$push": "$hashtag" }, "count": { "$sum": 1 } } }] list_score_lang = [] for user in usernames: for tag in self._get_tag_list(): self._applog.debug("Exec pipeline in database", extra={ 'logger': __name__, 'level': 'debug', 'uuid': current_request_id() }) data = Twitt.objects( hashtag=tag, username=user['_id']).aggregate(pipeline_lang) self._applog.debug("Exec pipeline OK", extra={ 'logger': __name__, 'level': 'debug', 'uuid': current_request_id() }) for d in data: score_lang = {} score_lang["username"] = user["_id"] score_lang["hashtag"] = tag score_lang["lang"] = d['_id'] score_lang["count"] = d['count'] list_score_lang.append(score_lang) return list_score_lang
def after_all_response(response: flask.Response) -> flask.Response: """ Setups the response object """ response.headers["X-Request-ID"] = current_request_id() return response
def index(): applog.debug("List endpoints", extra={ 'method': 'GET', 'logger': __name__, 'level': 'debug', 'uuid': current_request_id(), 'endpoint': request.path }) endpoints = { 'info': 'Twitter API', 'version': '1.0.0', 'endpoints': [{ 'url': '/twitter/api/v1/publish/posts', 'method': 'POST' }, { 'url': '/twitter/api/v1/users/followers', 'method': 'GET' }, { 'url': '/twitter/api/v1/posts/hour', 'method': 'GET' }, { 'url': '/twitter/api/v1/posts/tags/location', 'method': 'GET' }, { 'url': '/twitter/api/v1/posts/tags/lang', 'method': 'GET' }] } return jsonify(endpoints), 200
def __init__(self, engine, encoding=None): self._engine = engine self._encoding = encoding or default_encoding self._request_id = current_request_id() if IsDeepDebug: print('--> decoder: %s' % self._request_id) self._requested_object = None self._file_id = None self._client = None self._file_type = None self._file_status = None self._total = 0 # Encodings list self._encodings = None # Temp file object name self._tmp = None # Image decoded content self._image = None # List of application tags self._tags = None # XML iterator self._iter = None self.has_body = False
def user_id(): log_id = f.request.args.get('user') user = '******' if log_id in ['', 'vister', None]: log_id = str(current_request_id()) user = '******' return f.jsonify({'user': log_id, 'type': user})
def _init_state(self, **kw): if IsDeepDebug: print_to(None, 'FileDecoder.inistate') self._original = kw.get('original') or '' self._request_id = current_request_id() self._tmp = '%s/%s%s' % (self.tmp_folder, str(self._request_id), '%s')
def run_job(_id, history_id=None, **kwargs): db = Mongo() record = Job.find_by_id(_id) if not record: return False request_id = str(current_request_id()) username = None if not login_user else login_user.get('username') params = (_id, request_id, username, history_id) queue_name = get_queue_by_job(_id) extra = record.get('extra') template = record.get('template') schedule = extra.get('schedule') ansible_type = record.get('type') if template.get('run_type') == 'schedule': existed = db.collection('scheduler_jobs').find_one( {'_id': record['_id']}) if existed: return False scheduler.add_job(func=run_schedule_task, trigger='cron', args=params, coalesce=True, kwargs=kwargs, id=str(record.get('_id')), max_instances=1, name=record.get('name'), **schedule) return True else: func = run_playbook_task if ansible_type != 'adhoc' else run_adhoc_task task = Task(tiger, func=func, args=params, kwargs=kwargs, queue=queue_name, unique=True, lock=True, lock_key=_id) task_record = { 'job_id': _id, 'type': 'trigger', 'ansible': ansible_type, 'state': QUEUED, 'queue': queue_name, 'result': '', 'request_id': request_id, 't_id': task.id, 'created_at': time(), 'kwargs': kwargs, } result = db.collection('tasks').insert_one(task_record) task.delay() return result.inserted_id
def before_request(self): if current_request_id(): request.request_id = current_request_id() elif "X-Request-Id" in request.headers: request.request_id = f'{request.headers["X-Request-Id"]},{str(uuid.uuid4())}' else: request.request_id = str(uuid.uuid4()) data = "" try: data = request.get_data().decode("utf-8") except: pass self.app.logger.info(f'REQUEST: {request.method} {request.url}') self.app.logger.info( f'REQUEST HEADERS: {request.method} {request.url}\n{request.headers}' ) self.app.logger.info( f'REQUEST BODY: {request.method} {request.url}\n{data}')
def after_request(response): app.logger.info('{} {} {} {} {} {}'.format( request.remote_addr, request.scheme, request.method, request.full_path, response.status, response.content_length )) response.headers.add('X-REQUEST-ID', current_request_id()) return response
def publish_post(): applog.debug("Publish new tweetts..", extra={ 'method': 'POST', 'logger': __name__, 'level': 'debug', 'uuid': current_request_id(), 'endpoint': request.path }) result = None data = request.json try: if twittcl.publish(data): retult = "{ 'status': 'success' }" else: retult = "{ 'status': 'failed' }" except Exception as e: tb = traceback.format_exc() applog.error("Error when publish new tweets", extra={ 'method': 'POST', 'logger': __name__, 'level': 'error', 'uuid': current_request_id(), 'endpoint': request.path, 'trace': tb }) return jsonify( '{ "status": "500", "message": "Internal Server Error" }'), 500 return jsonify(result)
def total_posts_by_hour(self): list_posts = [] pipeline = [{ "$group": { "_id": "$post_hour", 'total_post': { '$sum': 1 } } }, { "$sort": { "_id": 1 } }] self._applog.debug("Exec pipeline in database", extra={ 'logger': __name__, 'level': 'debug', 'uuid': current_request_id() }) data = Twitt.objects().aggregate(pipeline) self._applog.debug("Exec pipeline OK", extra={ 'logger': __name__, 'level': 'debug', 'uuid': current_request_id() }) for d in data: list_posts.append(d) return list_posts
def get_orders(user_id): logger.info('got_request', extra={'endpoint': '/user/<user_id>/orders'}) headers = {'X-Request-ID': current_request_id()} response = requests.get(ORDERS_URL.format(user_id=user_id), headers=headers) orders = response.json()['result']['orders'] book_ids = get_book_ids_from_orders(orders) response = requests.get(BOOKS_URL, params={'book_ids': book_ids}, headers=headers) books = response.json()['result'] return jsonify({'result': { 'orders': orders, 'books': books, }})
def run(_id): book = Book.find_by_id(_id) if not book: return jsonify({'message': 'record not found', 'code': 10404}), 404 payload = request.get_json() options = payload.get('options') entry = options.get('entry') args = options.get('args') req_id = str(current_request_id()) params = { 'username': login_user.get('username'), 'req_id': req_id, 'args': args, 'options': options } result = dispatch(_id, entry, params) if not result: return jsonify({'message': 'invalid request', 'code': 104008}), 400 return jsonify({'message': 'ok', 'code': 0, 'data': {'taskId': result}})
def append_request_id(response): response.headers.add('X-REQUEST-ID', current_request_id()) data = request.get_data() try: data = data.decode('utf-8') except Exception as e: pass user_agent = request.headers.get('User-Agent', None) remote_addr = request.environ.get('HTTP_X_REAL_IP', request.remote_addr) telemetry(method=request.method, path=request.path, args=request.args.to_dict(), status=response.status, ip=remote_addr, user_agent=user_agent, payload=data) return response
def publish(self, tweets_list): for tweet in tweets_list: twitte = Twitt(id=tweet['id'], text=tweet['text'], created_at=tweet['created_at'], post_hour=tweet['post_hour'], hashtag=tweet['hashtag'], lang=tweet['lang'], author_id=tweet['author_id'], username=tweet['username'], name=tweet['name'], location=tweet['location'], followers_count=tweet['followers_count']).save() self._applog.debug("Post save in database.", extra={ 'logger': __name__, 'level': 'debug', 'uuid': current_request_id() }) return True
def add_adhoc(): payload = request.get_json() if not payload: return jsonify({ 'message': 'invalid params', 'code': 104001, }), 400 module = payload.get('module') args = payload.get('args') inventory = payload.get('inventory') private_key = payload.get('private_key') verbosity = payload.get('verbosity') name = payload.get('name') schedule = payload.get('schedule') check = payload.get('check') job_id = payload.get('job_id') extra_options = payload.get('extraOptions') status = int(payload.get('status', 0)) notification = payload.get('notification') maintainer = payload.get('maintainer') or [] if maintainer and isinstance(maintainer, list): users = db.collection('users').find({'username': {'$in': maintainer}}) names = list(map(lambda i: i['username'], users)) maintainer.extend(names) maintainer.append(login_user.get('username')) if not job_id: existed = db.collection('jobs').find_one({'name': name}) if existed and existed.get('status') != -1: return jsonify({'message': 'name exist', 'code': 104007}), 400 if not module or not inventory or not name: return jsonify({ 'message': 'miss required params', 'code': 104002, }), 400 # check_module = db.collection('ansible_modules').find_one({ # 'name': module # }) # # if not check_module: # return jsonify({ # 'message': 'invalid module', # 'code': 104003, # }), 400 inventory_content = parse_cmdb_inventory(inventory) if not inventory: return jsonify({ 'message': 'invalid inventory', 'code': 104004, }), 400 key_text = get_credential_content_by_id(private_key, 'private_key') if not key_text: return jsonify({ 'message': 'invalid private key', 'code': 104004, }), 400 options = {} if extra_options: options.update(extra_options) if verbosity: options['verbosity'] = verbosity if check: with NamedTemporaryFile('w+t', delete=True) as fd: fd.write(key_text) fd.seek(0) options['private_key'] = fd.name tasks = [{'action': {'module': module, 'args': args}}] hosts = inventory_content runner = AdHocRunner(hosts, options=options) runner.run('all', tasks) result = runner.get_result() return jsonify({'message': 'ok', 'code': 0, 'data': result}) else: token = str(base64.b64encode(bytes(current_request_id(), 'utf8')), 'utf8') data = { 'name': name, 'template': { 'inventory': inventory, 'private_key': private_key, 'verbosity': verbosity, 'module': module, 'args': args, 'extraOptions': extra_options, }, 'extra': { 'schedule': schedule, 'notification': notification, }, 'token': token, 'maintainer': maintainer, 'type': 'adhoc', 'created_at': time.time(), 'status': status, 'add_by': login_user.get('username') } if job_id: record = db.collection('jobs').find_one({'_id': ObjectId(job_id)}) if not record: return jsonify({ 'message': 'record not found', 'code': 104040 }), 404 update = { '$set': data, } db.collection('jobs').update_one({'_id': ObjectId(job_id)}, update=update) logger.info('update job', extra={ 'record': record, 'changed': data }) else: result = db.collection('jobs').insert_one(data) data['_id'] = result.inserted_id logger.info('add job', extra={'record': data}) return jsonify({ 'message': 'ok', 'code': 0, })
def add_jobs(): """ add job TODO(sang) :return: json """ body = request.get_json() user = login_user if not body: return jsonify({ 'message': 'miss required params', 'code': 104000, }), 400 job_type = body.get('type') if job_type == 'adhoc': return add_adhoc() current_id = body.get('currentId') record = None if current_id: record = Job.find_by_id(current_id) if not record: return jsonify({'message': 'job not found', 'code': 104044}), 400 payload = load_ansible_playbook(body) if payload.get('message') is not 'ok': return jsonify(payload), 400 data = payload.get('data') options = data.get('options') is_check = body.get('check') private_key = data.get('private_key') wk = Workspace() res = wk.load_book_from_db(name=data.get('book_name'), roles=data.get('roles')) if not res: return jsonify({ 'message': 'book not found', 'code': 104000, }), 400 entry = wk.get_book_entry(data.get('book_name'), data.get('entry')) dry_run = bool(is_check) options['check'] = dry_run if dry_run: with NamedTemporaryFile('w+t', delete=True) as fd: if private_key: key_text = get_credential_content_by_id( private_key, 'private_key') if not key_text: return jsonify({ 'message': 'invalid private_key', 'code': 104033, }), 401 fd.write(key_text) fd.seek(0) options['private_key'] = fd.name play = PlayBookRunner(data['inventory'], options, callback=CallbackModule()) play.run(entry) return jsonify({ 'message': 'ok', 'code': 0, 'data': { 'result': play.get_result(), 'options': options } }) name = data.get('name') existed = Job.find_one({'name': name}) if existed and not current_id: return jsonify({ 'message': 'name existed', 'code': 104001, }), 400 token = str(base64.b64encode(bytes(current_request_id(), 'utf8')), 'utf8') new_record = { 'name': name, 'type': 'playbook', 'token': token, 'description': data.get('description'), 'book_id': data.get('book_id'), 'template': data.get('template'), 'extra': data.get('extra'), 'entry': data['entry'], 'status': data['status'], 'maintainer': [user.get('username')], 'created_at': int(time.time()), 'updated_at': datetime.datetime.now().isoformat(), } if record: Job.update_one({'_id': record['_id']}, update={'$set': new_record}) else: Job.insert_one(new_record) return jsonify({ 'message': 'ok', 'code': 0, })
def add(a, b): """Simple function to add two numbers that is not aware of the request id""" logging.info('Called generic_add({}, {}) from request_id: {}'.format( a, b, current_request_id())) return a + b
def get_home(): return {"id": current_request_id(), "message": "Hello world"}, 200
def append_request_id(response): """ post process append request id """ response.headers.add('X-REQUEST-ID', current_request_id()) return response
def get_request_correlation_id(): if request.headers.get('WT_CORRELATION_ID', None) is None: return current_request_id() return request.headers.get('WT_CORRELATION_ID', '-')
def run_task(): payload = request.get_json() if not payload: return jsonify({ 'message': 'invalid params', 'code': 114000, }), 400 req_id = str(current_request_id()) run_type = payload.get('type') inventory = payload.get('inventory') private_key = payload.get('private_key') module = payload.get('module') args = payload.get('args') entry = payload.get('entry') become_mehtod = payload.get('become_method') become_user = payload.get('become_user') verbosity = payload.get('verbosity', 0) or 1 extra_options = payload.get('extraOptions') hosts = parse_cmdb_inventory(inventory) if not hosts: return jsonify({ 'message': 'invalid inventory', 'code': 114001, }), 400 options = {} if extra_options: options.update(extra_options) if verbosity: options['verbosity'] = verbosity if become_mehtod: options['become'] = 'yes' options['become_method'] = become_mehtod if become_user: options['become_user'] = become_user params = { 'username': login_user.get('username'), 'req_id': req_id, 'options': options, 'inventory': inventory, 'run_type': run_type, } if private_key: params['private_key'] = private_key if run_type == 'adhoc': if not module: return jsonify({ 'message': 'invalid module', 'code': 114002, }), 400 tasks = [{'action': {'module': module, 'args': args}}] params['tasks'] = tasks else: if not entry: return jsonify({ 'message': 'invalid entry', 'code': 114004, }), 400 params['entry'] = entry result = dispatch(params) if not result: return jsonify({'message': 'invalid request', 'code': 104008}), 400 return jsonify({'message': 'ok', 'code': 0, 'data': {'taskId': result}})
def append_request_id(response): response.headers.add("X-REQUEST-ID", current_request_id()) return response