def cancel_scheduled_operation(id, app):
# Cancel one scheduled operation. If the operation is running, the task
# is going to be aborted.
# Check the id
if id not in [
t['id'] for t in list_scheduled_vacuum(app) +
list_scheduled_analyze(app) + list_scheduled_reindex(app)
]:
raise HTTPError(404, "Scheduled operation not found")
try:
# Ask it to the task manager
taskmanager.TaskManager.send_message(
str(os.path.join(app.config.temboard.home, '.tm.socket')),
taskmanager.Message(
taskmanager.MSG_TYPE_TASK_CANCEL,
dict(task_id=id),
),
authkey=None,
)
except Exception as e:
logger.exception(str(e))
raise HTTPError(500, "Unable to cancel operation")
return dict(response="ok")
def post_cancel_backup(http_context, app):
task_id = http_context['urlvars'][0]
tasks = functions.list_backup_tasks(
str(os.path.join(app.config.temboard.home, '.tm.socket')))
task = None
for t in tasks:
if task_id == t['id']:
task = t
if task is None:
raise HTTPError(404, "Operation not found in current task list")
# Cancel or abort the task depending on its status
if task.status < taskmanager.TASK_STATUS_DOING:
msg = taskmanager.MSG_TYPE_TASK_CANCEL
response = "cancel signal sent"
elif task.status == taskmanager.TASK_STATUS_DOING:
msg = taskmanager.MSG_TYPE_TASK_ABORT
response = "abort signal sent"
else:
# Send a 410 Gone when the task is done or already cancelled or aborted
raise HTTPError(410, "Operation has already completed")
taskmanager.TaskManager.send_message(
str(os.path.join(app.config.temboard.home, '.tm.socket')),
taskmanager.Message(msg, dict(task_id=task_id)),
authkey=None,
)
return {'response': response}
def get_file_versions(filepath):
"""
Returns a list of version number for a file path.
"""
filedir = os.path.dirname(filepath)
filename = os.path.basename(filepath)
if not os.path.isdir(filedir):
raise HTTPError(
500, "Unable to list content from directory: %s" % (filedir))
if not os.path.isfile(filepath):
raise HTTPError(404, "File %s does not exist." % (filepath))
versions = []
l_filename = len(filename)
for f in listdir(filedir):
try:
if f[:l_filename] == filename \
and check_version_format(f[l_filename + 1:]):
"""
Let's consider f as one of the previous version of the file
if the first part of f's name is equal to original file name
and the rest of f's name (-1 for the seperating dot) is a
valid version number.
"""
versions.append(f[l_filename + 1:])
except Exception:
pass
# Return a sorted versions list.
return sorted(versions, reverse=True)
def get_statements(http_context, app):
"""Return a snapshot of latest statistics of executed SQL statements
"""
config = app.config
dbname = config.statements.dbname
assert dbname == "postgres", dbname
snapshot_datetime = now()
conninfo = dict(config.postgresql, dbname=dbname)
try:
with Postgres(**conninfo).connect() as conn:
data = list(conn.query(query))
except Exception as e:
pg_version = app.postgres.fetch_version()
if (pg_version < 90600
or 'relation "pg_stat_statements" does not exist' in str(e)):
raise HTTPError(
404, "pg_stat_statements not enabled on database %s" % dbname)
logger.error(
"Failed to get pg_stat_statements data on database %s: %s",
dbname,
e,
)
raise HTTPError(500, e)
else:
return {"snapshot_datetime": snapshot_datetime, "data": data}
def post_pg_ident(conn, config, http_context):
set_logger_name("settings")
logger = get_logger(config)
if 'content' not in http_context['post']:
raise HTTPError(406, "Parameter 'content' not sent.")
try:
conn.execute(
"SELECT setting FROM pg_settings WHERE name = 'ident_file'")
pg_ident_file = list(conn.get_rows())[0]['setting']
except error as e:
logger.error(str(e.message))
raise HTTPError(500, 'Internal error.')
with open(pg_ident_file, 'r') as fd:
pg_ident_data = fd.read()
fd.close()
try:
with open(pg_ident_file + ".previous", 'w') as fdp:
fdp.write(pg_ident_data)
fdp.close()
except Exception as e:
raise HTTPError(500, 'Internal error.')
with open(pg_ident_file, 'w') as fd:
fd.write(http_context['post']['content'])
fd.close()
return {'update': True}
def api_function_wrapper(config, http_context, sessions, module,
function_name):
"""
Simple API function wrapper in charge of:
- instanciate a new logger;
- check the user session id;
- call a function named 'function_name' from 'module_name' module and return its result;
"""
logger = get_logger(config)
logger.info("%s - %s" % (
module.__name__,
function_name,
))
username = check_sessionid(http_context['headers'], sessions)
http_context['username'] = username
try:
dm = getattr(module, function_name)(config, http_context)
return dm
except HTTPError as e:
logger.error(format_exc())
raise HTTPError(e.code, e.message['error'])
except Exception as e:
logger.error(format_exc())
raise HTTPError(500, "Internal error.")
def get_discover(http_context, app, sessions):
logger.info('Starting discovery.')
# Optionnal validation of key. For compatibility, we accept unauthenticated
# /discover. But for better reliability, we validate a key sent by HTTP
# header. temboard-agent-register sends key to prevent configuration
# mismatch.
request_key = http_context['headers'].get('X-Temboard-Agent-Key')
if request_key and request_key != app.config.temboard['key']:
raise HTTPError(401, "Invalid key")
discover = dict(
hostname=None,
cpu=None,
memory_size=None,
pg_port=app.config.postgresql['port'],
pg_version=None,
pg_version_summary=None,
pg_data=None,
plugins=[plugin for plugin in app.config.temboard['plugins']],
)
try:
# Gather system informations
sysinfo = SysInfo()
hostname = sysinfo.hostname(app.config.temboard['hostname'])
cpu = sysinfo.n_cpu()
memory_size = sysinfo.memory_size()
except (Exception, HTTPError) as e:
logger.exception(str(e))
logger.error('System discovery failed.')
# We stop here if system information has not been collected
if isinstance(e, HTTPError):
raise e
else:
raise HTTPError(500, "Internal error.")
discover.update(hostname=hostname, cpu=cpu, memory_size=memory_size)
try:
with app.postgres.connect() as conn:
pginfo = PgInfo(conn)
discover.update(pg_block_size=int(pginfo.setting('block_size')),
pg_version=pginfo.version()['full'],
pg_version_summary=pginfo.version()['summary'],
pg_data=pginfo.setting('data_directory'))
except Exception as e:
logger.exception(str(e))
logger.error('Postgres discovery failed.')
# Do not raise HTTPError, just keeping null values for Postgres
# informations.
logger.info('Discovery done.')
logger.debug(discover)
return discover
def api_vacuum(http_context, queue_in = None, config = None, sessions = None, commands = None):
set_logger_name("administration")
worker = b'vacuum'
# Get a new logger.
logger = get_logger(config)
try:
check_sessionid(http_context['headers'], sessions)
post = http_context['post']
# Check POST parameters.
validate_parameters(post, [
('database', T_OBJECTNAME, False),
('table', T_OBJECTNAME, False),
('mode', T_VACUUMMODE, False)
])
# Serialize parameters.
parameters = base64.b64encode(
pickle.dumps({
'database': post['database'],
'table': post['table'],
'mode': post['mode']
})).decode('utf-8')
except (Exception, HTTPError) as e:
logger.traceback(get_tb())
logger.error(str(e))
if isinstance(e, HTTPError):
raise e
else:
raise HTTPError(500, "Internal error.")
# Check command uniqueness.
try:
commands.check_uniqueness(worker, parameters)
except SharedItem_exists as e:
logger.traceback(get_tb())
logger.error(str(e))
raise HTTPError(402, "Vaccum '%s' already running on table '%s'." % (post['mode'], post['table']))
cid = hash_id(worker + b'-' + parameters.encode('utf-8'))
command = Command(
cid.encode('utf-8'),
time.time(),
0,
worker,
parameters,
0,
u'')
try:
commands.add(command)
# Put the Command in the command queue
queue_in.put(command)
return {"cid": cid}
except SharedItem_no_free_slot_left as e:
logger.traceback(get_tb())
logger.error(str(e))
raise HTTPError(500, "Internal error.")
def lazy_check(self):
if self.connection not in ['host', 'hostssl', 'hostnossl', 'local']:
raise HTTPError(406, "Invalid connection: %s" % (self.connection))
if len(self.database) < 1:
raise HTTPError(406, "Invalid database: %s" % (self.database))
if len(self.user) < 1:
raise HTTPError(406, "Invalid user: %s" % (self.user))
if self.connection != 'local' and \
len(self.address) == 0:
raise HTTPError(406, "An address is required for method '%s'." %
(self.connection))
if len(self.auth_method) == 0:
raise HTTPError(406, "Authentication method must be set.")
def post_hba(conn, config, http_context):
new_version = False
set_logger_name("settings")
logger = get_logger(config)
# Push a notification.
try:
NotificationMgmt.push(
config,
Notification(username=http_context['username'],
message="HBA file updated"))
except NotificationError as e:
logger.error(e.message)
if 'entries' not in http_context['post']:
raise HTTPError(406, "Parameter 'entries' not sent.")
if http_context and 'new_version' in http_context['post']:
# Check parameter 'version'
validate_parameters(http_context['post'],
[('new_version', T_NEW_VERSION, False)])
if http_context['post']['new_version'] is True:
new_version = True
hba_file = get_setting(conn, 'hba_file')
hba_entries = []
logger.debug(http_context['post']['entries'])
for entry in http_context['post']['entries']:
if 'comment' in entry and len(entry['connection']) == 0:
new_hba_entry = HBAComment()
new_hba_entry.comment = entry['comment']
else:
new_hba_entry = HBAEntry()
try:
new_hba_entry.connection = entry[
'connection'] if 'connection' in entry else ''
new_hba_entry.database = entry[
'database'] if 'database' in entry else ''
new_hba_entry.user = entry['user'] if 'user' in entry else ''
new_hba_entry.address = entry[
'address'] if 'address' in entry else ''
new_hba_entry.auth_method = entry[
'auth_method'] if 'auth_method' in entry else ''
new_hba_entry.auth_options = entry[
'auth_options'] if 'auth_options' in entry else ''
except Exception as e:
logger.error(e.message)
raise HTTPError(406, "Invalid HBA entry.")
new_hba_entry.lazy_check()
hba_entries.append(new_hba_entry)
return HBAManager.save_entries(hba_file, hba_entries, new_version)
def get_route(self, method, path):
# Returns the right route according to method/path
s_path = path.split(b'/')[1:]
root = s_path[0]
for route in get_routes():
# Check that HTTP method and url root are matching.
if not (route['http_method'] == self.http_method
and route['root'] == root):
continue
p = 0
# Check each element in the path.
for elt in s_path:
try:
if type(route['splitpath'][p]) not in (str, bytes):
# Then this is a regular expression.
res = route['splitpath'][p].match(elt.decode('utf-8'))
if not res:
break
else:
if route['splitpath'][p] != elt:
break
except IndexError:
break
p += 1
if p == len(s_path) == len(route['splitpath']):
return route
raise HTTPError(404, 'URL not found.')
def get_discover(http_context, app, sessions):
logger.info('Starting discovery.')
try:
sysinfo = SysInfo()
with app.postgres.connect() as conn:
pginfo = PgInfo(conn)
ret = dict(hostname=sysinfo.hostname(
app.config.temboard['hostname']),
cpu=sysinfo.n_cpu(),
memory_size=sysinfo.memory_size(),
pg_port=pginfo.setting('port'),
pg_version=pginfo.version()['full'],
pg_version_summary=pginfo.version()['summary'],
pg_data=pginfo.setting('data_directory'),
plugins=[
plugin_name
for plugin_name in app.config.temboard['plugins']
])
logger.info('Discovery done.')
return ret
except (error, Exception, HTTPError) as e:
logger.exception(e)
logger.info('Discovery failed.')
if isinstance(e, HTTPError):
raise e
else:
raise HTTPError(500, "Internal error.")
def save_file_content(content, filepath, new_version=False):
ret = {'last_version': None}
if new_version is True and os.path.isfile(filepath):
"""
When new_version param. is true, we need to save current file as a new
version before writing new file content.
"""
# Build new version's file path.
dt_str = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
filepath_version = "%s.%s" % (filepath, dt_str)
ret['last_version'] = dt_str
# Read current version's content.
cur_content = None
with open(filepath, 'r') as fd:
cur_content = fd.read()
# Check if new version's file exists.
if os.path.isfile(filepath_version):
raise HTTPError(
500, "Unable to create a new version, file %s "
"already exists." % (filepath_version))
# Write current version's content in new version's file.
with open(filepath_version, 'w') as fd:
fd.write(cur_content)
else:
versions = get_file_versions(filepath)
if len(versions) > 0:
ret['last_version'] = versions[-1]
# Write new file content into current file.
with open(filepath, 'w') as fd:
fd.write(content)
ret['filepath'] = filepath
return ret
def api_function_wrapper(config, http_context, sessions, module, function_name):
"""
API function wrapper in charge of:
- instanciate a new logger;
- check the user session id;
- call a function named 'function_name' from 'module_name' module and return its result;
"""
logger = get_logger(config)
logger.debug("Calling %s.%s()." % (module.__name__, function_name,))
logger.debug(http_context)
try:
username = check_sessionid(http_context['headers'], sessions)
http_context['username'] = username
dm = getattr(module, function_name)(config, http_context)
logger.debug("Done.")
return dm
except (Exception, HTTPError) as e:
logger.traceback(get_tb())
logger.error(str(e))
logger.debug("Failed.")
if isinstance(e, HTTPError):
raise e
else:
raise HTTPError(500, "Internal error.")
def post_hba_raw(conn, config, http_context):
new_version = False
# Push a notification.
try:
NotificationMgmt.push(
config,
Notification(username=http_context['username'],
message="HBA file updated"))
except NotificationError as e:
logger.error(e.message)
if 'content' not in http_context['post']:
raise HTTPError(406, "Parameter 'content' not sent.")
if http_context and 'new_version' in http_context['post']:
# Check parameter 'version'
validate_parameters(http_context['post'],
[('new_version', T_NEW_VERSION, False)])
if http_context['post']['new_version'] is True:
new_version = True
hba_file = get_setting(conn, 'hba_file')
return HBAManager.save_file_content(hba_file,
http_context['post']['content'],
new_version)
def auth_user(filepath, username, password):
"""
Hash and compair the given couple username/password.
"""
(l_username, l_hpasswd) = get_user(filepath, username)
n_hpasswd = hash_password(username, password)
if n_hpasswd.decode('utf-8') != l_hpasswd:
raise HTTPError(404, 'Invalid username/password.')
def get_file_configuration(conn, ):
query = """SELECT setting AS config_file FROM pg_settings WHERE name = 'config_file'"""
try:
conn.execute(query)
config_file = list(conn.get_rows())[0]['config_file']
return parse_configuration_file(config_file)
except error as e:
raise HTTPError(500, "Internal error.")
def monitoring_probe_cpu(http_context, config=None, sessions=None):
check_sessionid(http_context['headers'], sessions)
try:
output = api_run_probe(probe_cpu(config.plugins['monitoring']), config)
return output
except Exception as e:
logger.error(str(e.message))
raise HTTPError(500, "Internal error.")
def get_auto_configuration(conn, ):
query = """SELECT setting AS data_dir FROM pg_settings WHERE name = 'data_directory'"""
try:
conn.execute(query)
pg_data = list(conn.get_rows())[0]['data_dir']
pg_auto_conf = "%s/postgresql.auto.conf" % (pg_data, )
return parse_configuration_file(pg_auto_conf)
except error as e:
raise HTTPError(500, "Internal error.")
def get_user(filepath, username):
"""
Get a user/passwd form the file.
"""
for line in read_password_file(filepath):
(l_username, l_hpasswd) = line.strip().split(':')
if username == l_username:
return (l_username, l_hpasswd)
raise HTTPError(404, 'Invalid username/password.')
def notifications(http_context, app, sessions):
logger.info("Get notifications.")
try:
notifications = NotificationMgmt.get_last_n(app.config, -1)
logger.info("Done.")
return list(notifications)
except (NotificationError, Exception) as e:
logger.exception(e)
logger.info("Failed.")
raise HTTPError(500, "Internal error.")
def check_version_format(version):
"""
Checks if a version number is well formed, eg:
YYYY-MM-DDTHH:mm:ss
"""
try:
datetime.datetime.strptime(version, "%Y-%m-%dT%H:%M:%S")
return True
except Exception:
raise HTTPError(406,
"Bad version format, should be 'YYYY-MM-DDTHH:mm:ss'")
def check_sessionid(http_header, sessions):
validate_parameters(http_header, [('X-Session', T_SESSIONID, False)])
try:
session = sessions.get_by_sessionid(
http_header['X-Session'].encode('utf-8'))
session.time = time.time()
username = session.username
sessions.update(session)
return username
except SharedItem_not_found:
raise HTTPError(401, "Invalid session.")
def schedule_operation(what, when, config, expire=86400):
options = {'config': pickle(config)}
try:
res = taskmanager.schedule_task(
what + '_worker',
options=options,
start=when,
listener_addr=str(os.path.join(config.temboard.home,
'.tm.socket')),
expire=expire,
)
except Exception as e:
logger.exception(str(e))
raise HTTPError(500, "Unable to schedule {}".format(what))
if res.type == taskmanager.MSG_TYPE_ERROR:
logger.error(res.content)
raise HTTPError(500, "Unable to schedule {}".format(what))
return res.content
def profile(http_context, queue_in = None, config = None, sessions = None, commands = None):
"""
@api {get} /profile Get current user name.
@apiVersion 0.0.1
@apiName Profile
@apiGroup User
@apiHeader {String} X-Session Session ID.
@apiSuccess {String} username Username.
@apiExample {curl} Example usage:
curl -k -H "X-Session: fa452548403ac53f2158a65f5eb6db9723d2b07238dd83f5b6d9ca52ce817b63" https://localhost:2345/profile
@apiSuccessExample Success-Reponse:
HTTP/1.0 200 OK
Server: temboard-agent/0.0.1 Python/2.7.8
Date: Wed, 22 Apr 2015 12:33:19 GMT
Content-type: application/json
{
"username": "******"
}
@apiError (500 error) error Internal error.
@apiError (401 error) error Invalid session ID.
@apiError (406 error) error Session ID malformed.
@apiErrorExample 401 error example
HTTP/1.0 401 Unauthorized
Server: temboard-agent/0.0.1 Python/2.7.8
Date: Wed, 22 Apr 2015 12:36:33 GMT
Content-type: application/json
{"error": "Invalid session."}
@apiErrorExample 406 error example
HTTP/1.0 406 Not Acceptable
Server: temboard-agent/0.0.1 Python/2.7.8
Date: Wed, 22 Apr 2015 12:37:23 GMT
Content-type: application/json
{"error": "Parameter 'X-Session' is malformed."}
"""
headers = http_context['headers']
set_logger_name("api")
logger = get_logger(config)
check_sessionid(headers, sessions)
logger.info("[profile] User session: %s" % (headers['X-Session']))
try:
session = sessions.get_by_sessionid(headers['X-Session'].encode('utf-8'))
return {'username': session.username}
except SharedItem_not_found:
raise HTTPError(401, "Invalid session.")
def delete_file_version(filepath, version):
"""
Remove a previous version of file.
"""
check_version_format(version)
if version in get_file_versions(filepath):
filepath_version = "%s.%s" % (filepath, version)
os.remove(filepath_version)
return {'version': version, 'deleted': True}
else:
raise HTTPError(
404, "Version %s of file %s does not exist." % (version, filepath))
def profile(http_context, app, sessions):
headers = http_context['headers']
logger.info("Get user profile.")
try:
xsession = headers['X-Session'].encode('utf-8')
session = sessions.get_by_sessionid(xsession)
logger.info("Done.")
return {'username': session.username.decode('utf-8')}
except SharedItem_not_found as e:
logger.exception(e)
logger.info("Failed.")
raise HTTPError(401, "Invalid session.")
def get_discover(http_context, app, sessions):
logger.info('Starting discovery.')
discover = dict(
hostname=None,
cpu=None,
memory_size=None,
pg_port=app.config.postgresql['port'],
pg_version=None,
pg_version_summary=None,
pg_data=None,
plugins=[plugin for plugin in app.config.temboard['plugins']],
)
try:
# Gather system informations
sysinfo = SysInfo()
hostname = sysinfo.hostname(app.config.temboard['hostname'])
cpu = sysinfo.n_cpu()
memory_size = sysinfo.memory_size()
except (Exception, HTTPError) as e:
logger.exception(str(e))
logger.error('System discovery failed.')
# We stop here if system information has not been collected
if isinstance(e, HTTPError):
raise e
else:
raise HTTPError(500, "Internal error.")
discover.update(
hostname=hostname, cpu=cpu, memory_size=memory_size
)
try:
with app.postgres.connect() as conn:
pginfo = PgInfo(conn)
discover.update(
pg_block_size=int(pginfo.setting('block_size')),
pg_version=pginfo.version()['full'],
pg_version_summary=pginfo.version()['summary'],
pg_data=pginfo.setting('data_directory')
)
except Exception as e:
logger.exception(str(e))
logger.error('Postgres discovery failed.')
# Do not raise HTTPError, just keeping null values for Postgres
# informations.
logger.info('Discovery done.')
logger.debug(discover)
return discover
def validate_parameters(values, types):
"""
Verify that each value of dict 'values' is valid. For doing that, we have
to loop over all 'types' elements which are tuples: ('values' key,
validation regexp, if the value item currently checked is a list of thing
to check).
If values[key] (or each element of it when it's a list) does not match
with the regexp then we trow an error.
"""
for (key, typ, is_list) in types:
try:
if not is_list:
# If 'typ' is a string, it must be considered as a regexp pattern.
if type(typ) == str and re.match(typ, str(
values[key])) is None:
raise HTTPError(406,
"Parameter '%s' is malformed." % (key, ))
if type(typ) != str and typ != type(values[key]):
raise HTTPError(406,
"Parameter '%s' is malformed." % (key, ))
if is_list:
for value in values[key]:
if type(typ) == str and re.match(typ, str(value)) is None:
raise HTTPError(
406, "Parameter '%s' is malformed." % (key, ))
if type(typ) != str and typ != type(value):
raise HTTPError(
406, "Parameter '%s' is malformed." % (key, ))
except KeyError as e:
raise HTTPError(406, "Parameter '%s' not sent." % (key, ))
except Exception as e:
raise HTTPError(406, "Parameter '%s' is malformed." % (key, ))
def api_function_wrapper_pg(config, http_context, sessions, module,
function_name):
"""
Simple API function wrapper in charge of:
- instanciate a new logger;
- check the user session id;
- start a new PostgreSQL connexion;
- call a function named 'function_name' from 'module_name' module and return its result;
- close the PG connexion.
"""
logger = get_logger(config)
logger.info("%s - %s" % (
module.__name__,
function_name,
))
username = check_sessionid(http_context['headers'], sessions)
http_context['username'] = username
conn = connector(host=config.postgresql['host'],
port=config.postgresql['port'],
user=config.postgresql['user'],
password=config.postgresql['password'],
database=config.postgresql['dbname'])
try:
conn.connect()
dm = getattr(module, function_name)(conn, config, http_context)
conn.close()
return dm
except (error, Exception, HTTPError) as e:
logger.error(format_exc())
try:
conn.close()
except Exception:
pass
if isinstance(e, HTTPError):
raise HTTPError(e.code, e.message['error'])
else:
raise HTTPError(500, "Internal error.")
