def cancel_scheduled_operation(id, app): # Cancel one scheduled operation. If the operation is running, the task # is going to be aborted. # Check the id if id not in [ t['id'] for t in list_scheduled_vacuum(app) + list_scheduled_analyze(app) + list_scheduled_reindex(app) ]: raise HTTPError(404, "Scheduled operation not found") try: # Ask it to the task manager taskmanager.TaskManager.send_message( str(os.path.join(app.config.temboard.home, '.tm.socket')), taskmanager.Message( taskmanager.MSG_TYPE_TASK_CANCEL, dict(task_id=id), ), authkey=None, ) except Exception as e: logger.exception(str(e)) raise HTTPError(500, "Unable to cancel operation") return dict(response="ok")
def post_cancel_backup(http_context, app): task_id = http_context['urlvars'][0] tasks = functions.list_backup_tasks( str(os.path.join(app.config.temboard.home, '.tm.socket'))) task = None for t in tasks: if task_id == t['id']: task = t if task is None: raise HTTPError(404, "Operation not found in current task list") # Cancel or abort the task depending on its status if task.status < taskmanager.TASK_STATUS_DOING: msg = taskmanager.MSG_TYPE_TASK_CANCEL response = "cancel signal sent" elif task.status == taskmanager.TASK_STATUS_DOING: msg = taskmanager.MSG_TYPE_TASK_ABORT response = "abort signal sent" else: # Send a 410 Gone when the task is done or already cancelled or aborted raise HTTPError(410, "Operation has already completed") taskmanager.TaskManager.send_message( str(os.path.join(app.config.temboard.home, '.tm.socket')), taskmanager.Message(msg, dict(task_id=task_id)), authkey=None, ) return {'response': response}
def get_file_versions(filepath): """ Returns a list of version number for a file path. """ filedir = os.path.dirname(filepath) filename = os.path.basename(filepath) if not os.path.isdir(filedir): raise HTTPError( 500, "Unable to list content from directory: %s" % (filedir)) if not os.path.isfile(filepath): raise HTTPError(404, "File %s does not exist." % (filepath)) versions = [] l_filename = len(filename) for f in listdir(filedir): try: if f[:l_filename] == filename \ and check_version_format(f[l_filename + 1:]): """ Let's consider f as one of the previous version of the file if the first part of f's name is equal to original file name and the rest of f's name (-1 for the seperating dot) is a valid version number. """ versions.append(f[l_filename + 1:]) except Exception: pass # Return a sorted versions list. return sorted(versions, reverse=True)
def get_statements(http_context, app): """Return a snapshot of latest statistics of executed SQL statements """ config = app.config dbname = config.statements.dbname assert dbname == "postgres", dbname snapshot_datetime = now() conninfo = dict(config.postgresql, dbname=dbname) try: with Postgres(**conninfo).connect() as conn: data = list(conn.query(query)) except Exception as e: pg_version = app.postgres.fetch_version() if (pg_version < 90600 or 'relation "pg_stat_statements" does not exist' in str(e)): raise HTTPError( 404, "pg_stat_statements not enabled on database %s" % dbname) logger.error( "Failed to get pg_stat_statements data on database %s: %s", dbname, e, ) raise HTTPError(500, e) else: return {"snapshot_datetime": snapshot_datetime, "data": data}
def post_pg_ident(conn, config, http_context): set_logger_name("settings") logger = get_logger(config) if 'content' not in http_context['post']: raise HTTPError(406, "Parameter 'content' not sent.") try: conn.execute( "SELECT setting FROM pg_settings WHERE name = 'ident_file'") pg_ident_file = list(conn.get_rows())[0]['setting'] except error as e: logger.error(str(e.message)) raise HTTPError(500, 'Internal error.') with open(pg_ident_file, 'r') as fd: pg_ident_data = fd.read() fd.close() try: with open(pg_ident_file + ".previous", 'w') as fdp: fdp.write(pg_ident_data) fdp.close() except Exception as e: raise HTTPError(500, 'Internal error.') with open(pg_ident_file, 'w') as fd: fd.write(http_context['post']['content']) fd.close() return {'update': True}
def api_function_wrapper(config, http_context, sessions, module, function_name): """ Simple API function wrapper in charge of: - instanciate a new logger; - check the user session id; - call a function named 'function_name' from 'module_name' module and return its result; """ logger = get_logger(config) logger.info("%s - %s" % ( module.__name__, function_name, )) username = check_sessionid(http_context['headers'], sessions) http_context['username'] = username try: dm = getattr(module, function_name)(config, http_context) return dm except HTTPError as e: logger.error(format_exc()) raise HTTPError(e.code, e.message['error']) except Exception as e: logger.error(format_exc()) raise HTTPError(500, "Internal error.")
def get_discover(http_context, app, sessions): logger.info('Starting discovery.') # Optionnal validation of key. For compatibility, we accept unauthenticated # /discover. But for better reliability, we validate a key sent by HTTP # header. temboard-agent-register sends key to prevent configuration # mismatch. request_key = http_context['headers'].get('X-Temboard-Agent-Key') if request_key and request_key != app.config.temboard['key']: raise HTTPError(401, "Invalid key") discover = dict( hostname=None, cpu=None, memory_size=None, pg_port=app.config.postgresql['port'], pg_version=None, pg_version_summary=None, pg_data=None, plugins=[plugin for plugin in app.config.temboard['plugins']], ) try: # Gather system informations sysinfo = SysInfo() hostname = sysinfo.hostname(app.config.temboard['hostname']) cpu = sysinfo.n_cpu() memory_size = sysinfo.memory_size() except (Exception, HTTPError) as e: logger.exception(str(e)) logger.error('System discovery failed.') # We stop here if system information has not been collected if isinstance(e, HTTPError): raise e else: raise HTTPError(500, "Internal error.") discover.update(hostname=hostname, cpu=cpu, memory_size=memory_size) try: with app.postgres.connect() as conn: pginfo = PgInfo(conn) discover.update(pg_block_size=int(pginfo.setting('block_size')), pg_version=pginfo.version()['full'], pg_version_summary=pginfo.version()['summary'], pg_data=pginfo.setting('data_directory')) except Exception as e: logger.exception(str(e)) logger.error('Postgres discovery failed.') # Do not raise HTTPError, just keeping null values for Postgres # informations. logger.info('Discovery done.') logger.debug(discover) return discover
def api_vacuum(http_context, queue_in = None, config = None, sessions = None, commands = None): set_logger_name("administration") worker = b'vacuum' # Get a new logger. logger = get_logger(config) try: check_sessionid(http_context['headers'], sessions) post = http_context['post'] # Check POST parameters. validate_parameters(post, [ ('database', T_OBJECTNAME, False), ('table', T_OBJECTNAME, False), ('mode', T_VACUUMMODE, False) ]) # Serialize parameters. parameters = base64.b64encode( pickle.dumps({ 'database': post['database'], 'table': post['table'], 'mode': post['mode'] })).decode('utf-8') except (Exception, HTTPError) as e: logger.traceback(get_tb()) logger.error(str(e)) if isinstance(e, HTTPError): raise e else: raise HTTPError(500, "Internal error.") # Check command uniqueness. try: commands.check_uniqueness(worker, parameters) except SharedItem_exists as e: logger.traceback(get_tb()) logger.error(str(e)) raise HTTPError(402, "Vaccum '%s' already running on table '%s'." % (post['mode'], post['table'])) cid = hash_id(worker + b'-' + parameters.encode('utf-8')) command = Command( cid.encode('utf-8'), time.time(), 0, worker, parameters, 0, u'') try: commands.add(command) # Put the Command in the command queue queue_in.put(command) return {"cid": cid} except SharedItem_no_free_slot_left as e: logger.traceback(get_tb()) logger.error(str(e)) raise HTTPError(500, "Internal error.")
def lazy_check(self): if self.connection not in ['host', 'hostssl', 'hostnossl', 'local']: raise HTTPError(406, "Invalid connection: %s" % (self.connection)) if len(self.database) < 1: raise HTTPError(406, "Invalid database: %s" % (self.database)) if len(self.user) < 1: raise HTTPError(406, "Invalid user: %s" % (self.user)) if self.connection != 'local' and \ len(self.address) == 0: raise HTTPError(406, "An address is required for method '%s'." % (self.connection)) if len(self.auth_method) == 0: raise HTTPError(406, "Authentication method must be set.")
def post_hba(conn, config, http_context): new_version = False set_logger_name("settings") logger = get_logger(config) # Push a notification. try: NotificationMgmt.push( config, Notification(username=http_context['username'], message="HBA file updated")) except NotificationError as e: logger.error(e.message) if 'entries' not in http_context['post']: raise HTTPError(406, "Parameter 'entries' not sent.") if http_context and 'new_version' in http_context['post']: # Check parameter 'version' validate_parameters(http_context['post'], [('new_version', T_NEW_VERSION, False)]) if http_context['post']['new_version'] is True: new_version = True hba_file = get_setting(conn, 'hba_file') hba_entries = [] logger.debug(http_context['post']['entries']) for entry in http_context['post']['entries']: if 'comment' in entry and len(entry['connection']) == 0: new_hba_entry = HBAComment() new_hba_entry.comment = entry['comment'] else: new_hba_entry = HBAEntry() try: new_hba_entry.connection = entry[ 'connection'] if 'connection' in entry else '' new_hba_entry.database = entry[ 'database'] if 'database' in entry else '' new_hba_entry.user = entry['user'] if 'user' in entry else '' new_hba_entry.address = entry[ 'address'] if 'address' in entry else '' new_hba_entry.auth_method = entry[ 'auth_method'] if 'auth_method' in entry else '' new_hba_entry.auth_options = entry[ 'auth_options'] if 'auth_options' in entry else '' except Exception as e: logger.error(e.message) raise HTTPError(406, "Invalid HBA entry.") new_hba_entry.lazy_check() hba_entries.append(new_hba_entry) return HBAManager.save_entries(hba_file, hba_entries, new_version)
def get_route(self, method, path): # Returns the right route according to method/path s_path = path.split(b'/')[1:] root = s_path[0] for route in get_routes(): # Check that HTTP method and url root are matching. if not (route['http_method'] == self.http_method and route['root'] == root): continue p = 0 # Check each element in the path. for elt in s_path: try: if type(route['splitpath'][p]) not in (str, bytes): # Then this is a regular expression. res = route['splitpath'][p].match(elt.decode('utf-8')) if not res: break else: if route['splitpath'][p] != elt: break except IndexError: break p += 1 if p == len(s_path) == len(route['splitpath']): return route raise HTTPError(404, 'URL not found.')
def get_discover(http_context, app, sessions): logger.info('Starting discovery.') try: sysinfo = SysInfo() with app.postgres.connect() as conn: pginfo = PgInfo(conn) ret = dict(hostname=sysinfo.hostname( app.config.temboard['hostname']), cpu=sysinfo.n_cpu(), memory_size=sysinfo.memory_size(), pg_port=pginfo.setting('port'), pg_version=pginfo.version()['full'], pg_version_summary=pginfo.version()['summary'], pg_data=pginfo.setting('data_directory'), plugins=[ plugin_name for plugin_name in app.config.temboard['plugins'] ]) logger.info('Discovery done.') return ret except (error, Exception, HTTPError) as e: logger.exception(e) logger.info('Discovery failed.') if isinstance(e, HTTPError): raise e else: raise HTTPError(500, "Internal error.")
def save_file_content(content, filepath, new_version=False): ret = {'last_version': None} if new_version is True and os.path.isfile(filepath): """ When new_version param. is true, we need to save current file as a new version before writing new file content. """ # Build new version's file path. dt_str = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") filepath_version = "%s.%s" % (filepath, dt_str) ret['last_version'] = dt_str # Read current version's content. cur_content = None with open(filepath, 'r') as fd: cur_content = fd.read() # Check if new version's file exists. if os.path.isfile(filepath_version): raise HTTPError( 500, "Unable to create a new version, file %s " "already exists." % (filepath_version)) # Write current version's content in new version's file. with open(filepath_version, 'w') as fd: fd.write(cur_content) else: versions = get_file_versions(filepath) if len(versions) > 0: ret['last_version'] = versions[-1] # Write new file content into current file. with open(filepath, 'w') as fd: fd.write(content) ret['filepath'] = filepath return ret
def api_function_wrapper(config, http_context, sessions, module, function_name): """ API function wrapper in charge of: - instanciate a new logger; - check the user session id; - call a function named 'function_name' from 'module_name' module and return its result; """ logger = get_logger(config) logger.debug("Calling %s.%s()." % (module.__name__, function_name,)) logger.debug(http_context) try: username = check_sessionid(http_context['headers'], sessions) http_context['username'] = username dm = getattr(module, function_name)(config, http_context) logger.debug("Done.") return dm except (Exception, HTTPError) as e: logger.traceback(get_tb()) logger.error(str(e)) logger.debug("Failed.") if isinstance(e, HTTPError): raise e else: raise HTTPError(500, "Internal error.")
def post_hba_raw(conn, config, http_context): new_version = False # Push a notification. try: NotificationMgmt.push( config, Notification(username=http_context['username'], message="HBA file updated")) except NotificationError as e: logger.error(e.message) if 'content' not in http_context['post']: raise HTTPError(406, "Parameter 'content' not sent.") if http_context and 'new_version' in http_context['post']: # Check parameter 'version' validate_parameters(http_context['post'], [('new_version', T_NEW_VERSION, False)]) if http_context['post']['new_version'] is True: new_version = True hba_file = get_setting(conn, 'hba_file') return HBAManager.save_file_content(hba_file, http_context['post']['content'], new_version)
def auth_user(filepath, username, password): """ Hash and compair the given couple username/password. """ (l_username, l_hpasswd) = get_user(filepath, username) n_hpasswd = hash_password(username, password) if n_hpasswd.decode('utf-8') != l_hpasswd: raise HTTPError(404, 'Invalid username/password.')
def get_file_configuration(conn, ): query = """SELECT setting AS config_file FROM pg_settings WHERE name = 'config_file'""" try: conn.execute(query) config_file = list(conn.get_rows())[0]['config_file'] return parse_configuration_file(config_file) except error as e: raise HTTPError(500, "Internal error.")
def monitoring_probe_cpu(http_context, config=None, sessions=None): check_sessionid(http_context['headers'], sessions) try: output = api_run_probe(probe_cpu(config.plugins['monitoring']), config) return output except Exception as e: logger.error(str(e.message)) raise HTTPError(500, "Internal error.")
def get_auto_configuration(conn, ): query = """SELECT setting AS data_dir FROM pg_settings WHERE name = 'data_directory'""" try: conn.execute(query) pg_data = list(conn.get_rows())[0]['data_dir'] pg_auto_conf = "%s/postgresql.auto.conf" % (pg_data, ) return parse_configuration_file(pg_auto_conf) except error as e: raise HTTPError(500, "Internal error.")
def get_user(filepath, username): """ Get a user/passwd form the file. """ for line in read_password_file(filepath): (l_username, l_hpasswd) = line.strip().split(':') if username == l_username: return (l_username, l_hpasswd) raise HTTPError(404, 'Invalid username/password.')
def notifications(http_context, app, sessions): logger.info("Get notifications.") try: notifications = NotificationMgmt.get_last_n(app.config, -1) logger.info("Done.") return list(notifications) except (NotificationError, Exception) as e: logger.exception(e) logger.info("Failed.") raise HTTPError(500, "Internal error.")
def check_version_format(version): """ Checks if a version number is well formed, eg: YYYY-MM-DDTHH:mm:ss """ try: datetime.datetime.strptime(version, "%Y-%m-%dT%H:%M:%S") return True except Exception: raise HTTPError(406, "Bad version format, should be 'YYYY-MM-DDTHH:mm:ss'")
def check_sessionid(http_header, sessions): validate_parameters(http_header, [('X-Session', T_SESSIONID, False)]) try: session = sessions.get_by_sessionid( http_header['X-Session'].encode('utf-8')) session.time = time.time() username = session.username sessions.update(session) return username except SharedItem_not_found: raise HTTPError(401, "Invalid session.")
def schedule_operation(what, when, config, expire=86400): options = {'config': pickle(config)} try: res = taskmanager.schedule_task( what + '_worker', options=options, start=when, listener_addr=str(os.path.join(config.temboard.home, '.tm.socket')), expire=expire, ) except Exception as e: logger.exception(str(e)) raise HTTPError(500, "Unable to schedule {}".format(what)) if res.type == taskmanager.MSG_TYPE_ERROR: logger.error(res.content) raise HTTPError(500, "Unable to schedule {}".format(what)) return res.content
def profile(http_context, queue_in = None, config = None, sessions = None, commands = None): """ @api {get} /profile Get current user name. @apiVersion 0.0.1 @apiName Profile @apiGroup User @apiHeader {String} X-Session Session ID. @apiSuccess {String} username Username. @apiExample {curl} Example usage: curl -k -H "X-Session: fa452548403ac53f2158a65f5eb6db9723d2b07238dd83f5b6d9ca52ce817b63" https://localhost:2345/profile @apiSuccessExample Success-Reponse: HTTP/1.0 200 OK Server: temboard-agent/0.0.1 Python/2.7.8 Date: Wed, 22 Apr 2015 12:33:19 GMT Content-type: application/json { "username": "******" } @apiError (500 error) error Internal error. @apiError (401 error) error Invalid session ID. @apiError (406 error) error Session ID malformed. @apiErrorExample 401 error example HTTP/1.0 401 Unauthorized Server: temboard-agent/0.0.1 Python/2.7.8 Date: Wed, 22 Apr 2015 12:36:33 GMT Content-type: application/json {"error": "Invalid session."} @apiErrorExample 406 error example HTTP/1.0 406 Not Acceptable Server: temboard-agent/0.0.1 Python/2.7.8 Date: Wed, 22 Apr 2015 12:37:23 GMT Content-type: application/json {"error": "Parameter 'X-Session' is malformed."} """ headers = http_context['headers'] set_logger_name("api") logger = get_logger(config) check_sessionid(headers, sessions) logger.info("[profile] User session: %s" % (headers['X-Session'])) try: session = sessions.get_by_sessionid(headers['X-Session'].encode('utf-8')) return {'username': session.username} except SharedItem_not_found: raise HTTPError(401, "Invalid session.")
def delete_file_version(filepath, version): """ Remove a previous version of file. """ check_version_format(version) if version in get_file_versions(filepath): filepath_version = "%s.%s" % (filepath, version) os.remove(filepath_version) return {'version': version, 'deleted': True} else: raise HTTPError( 404, "Version %s of file %s does not exist." % (version, filepath))
def profile(http_context, app, sessions): headers = http_context['headers'] logger.info("Get user profile.") try: xsession = headers['X-Session'].encode('utf-8') session = sessions.get_by_sessionid(xsession) logger.info("Done.") return {'username': session.username.decode('utf-8')} except SharedItem_not_found as e: logger.exception(e) logger.info("Failed.") raise HTTPError(401, "Invalid session.")
def get_discover(http_context, app, sessions): logger.info('Starting discovery.') discover = dict( hostname=None, cpu=None, memory_size=None, pg_port=app.config.postgresql['port'], pg_version=None, pg_version_summary=None, pg_data=None, plugins=[plugin for plugin in app.config.temboard['plugins']], ) try: # Gather system informations sysinfo = SysInfo() hostname = sysinfo.hostname(app.config.temboard['hostname']) cpu = sysinfo.n_cpu() memory_size = sysinfo.memory_size() except (Exception, HTTPError) as e: logger.exception(str(e)) logger.error('System discovery failed.') # We stop here if system information has not been collected if isinstance(e, HTTPError): raise e else: raise HTTPError(500, "Internal error.") discover.update( hostname=hostname, cpu=cpu, memory_size=memory_size ) try: with app.postgres.connect() as conn: pginfo = PgInfo(conn) discover.update( pg_block_size=int(pginfo.setting('block_size')), pg_version=pginfo.version()['full'], pg_version_summary=pginfo.version()['summary'], pg_data=pginfo.setting('data_directory') ) except Exception as e: logger.exception(str(e)) logger.error('Postgres discovery failed.') # Do not raise HTTPError, just keeping null values for Postgres # informations. logger.info('Discovery done.') logger.debug(discover) return discover
def validate_parameters(values, types): """ Verify that each value of dict 'values' is valid. For doing that, we have to loop over all 'types' elements which are tuples: ('values' key, validation regexp, if the value item currently checked is a list of thing to check). If values[key] (or each element of it when it's a list) does not match with the regexp then we trow an error. """ for (key, typ, is_list) in types: try: if not is_list: # If 'typ' is a string, it must be considered as a regexp pattern. if type(typ) == str and re.match(typ, str( values[key])) is None: raise HTTPError(406, "Parameter '%s' is malformed." % (key, )) if type(typ) != str and typ != type(values[key]): raise HTTPError(406, "Parameter '%s' is malformed." % (key, )) if is_list: for value in values[key]: if type(typ) == str and re.match(typ, str(value)) is None: raise HTTPError( 406, "Parameter '%s' is malformed." % (key, )) if type(typ) != str and typ != type(value): raise HTTPError( 406, "Parameter '%s' is malformed." % (key, )) except KeyError as e: raise HTTPError(406, "Parameter '%s' not sent." % (key, )) except Exception as e: raise HTTPError(406, "Parameter '%s' is malformed." % (key, ))
def api_function_wrapper_pg(config, http_context, sessions, module, function_name): """ Simple API function wrapper in charge of: - instanciate a new logger; - check the user session id; - start a new PostgreSQL connexion; - call a function named 'function_name' from 'module_name' module and return its result; - close the PG connexion. """ logger = get_logger(config) logger.info("%s - %s" % ( module.__name__, function_name, )) username = check_sessionid(http_context['headers'], sessions) http_context['username'] = username conn = connector(host=config.postgresql['host'], port=config.postgresql['port'], user=config.postgresql['user'], password=config.postgresql['password'], database=config.postgresql['dbname']) try: conn.connect() dm = getattr(module, function_name)(conn, config, http_context) conn.close() return dm except (error, Exception, HTTPError) as e: logger.error(format_exc()) try: conn.close() except Exception: pass if isinstance(e, HTTPError): raise HTTPError(e.code, e.message['error']) else: raise HTTPError(500, "Internal error.")