def monetdbd_set(request):
properties = request.get('properties', None)
dbfarm = check_db_farm(request)
# Check if the user input is empty
if is_empty(properties):
raise InvalidUsage('Provide at least one property.', status_code=400)
whitelist = [
'all', 'logfile', 'pidfile', 'sockdir', 'port', 'ipv6', 'listenaddr',
'control', 'discovery', 'discoveryttl', 'dbfarm', 'exittimeout',
'forward', 'snapshotdir', 'snapshotcompression', 'mapisock',
'controlsock', 'status', 'hostname', 'passphrase'
]
for entry in properties:
if entry not in whitelist:
raise InvalidUsage('No such property: ' + entry, status_code=400)
properties = [
'%s=%s' % (key, value) for (key, value) in properties.items()
]
execute_command = ['monetdbd', 'set'] + properties + dbfarm
current_app.logger.debug(execute_command)
try:
result = subprocess.check_output(execute_command,
stderr=subprocess.STDOUT,
shell=False)
result = str(result.decode('ascii'))
except subprocess.CalledProcessError as e:
error_message = str(e.output.decode('ascii')).strip('\n')
raise InvalidUsage(error_message, status_code=400)
return result
def format_dict(request_args, rest_keys='arguments'):
request_dict = {}
request_dict['options'] = {}
request_dict['arguments'] = {}
request_dict['databases'] = []
for key, value in request_args.items():
if key == 'databases':
request_dict['databases'] = request_args.get('databases').split(
',')
elif key == 'host':
request_dict['options']['host'] = value
elif key == 'port':
request_dict['options']['port'] = value
elif key == 'password':
request_dict['options']['password'] = value
# TODO: An extra check would be to have a list of possible arguments
elif key == 'arguments':
request_dict['arguments'][key] = value
elif key == 'properties':
request_dict['properties'] = request_args.get('properties').split(
',')
else:
raise InvalidUsage('Unknown argument: {}'.format(key),
status_code=404)
return request_dict
def arguments_get_request(request, command, allowed_arguments):
arguments_list = []
arguments = request.get('arguments', None)
if arguments is None:
return arguments_list
for key, value in arguments.items():
if key not in allowed_arguments:
raise InvalidUsage(
'Unknown argument "{}" for endpoint "{}"'.format(
str(key), command),
status_code=400)
all = arguments.get('all', None)
long = arguments.get('long', None)
password = arguments.get('pass', None)
force = arguments.get('force', None)
if all and all is True:
arguments_list += ['-a']
if long and long is True:
arguments_list += ['-l']
if force and force is True:
arguments_list += ['-f']
if password:
arguments_list += ['-p', password]
return arguments_list
def serialize_status_long(appstruct):
key_value = appstruct.split("\n")
dict = {}
list_of_dict = []
for v in key_value:
entry = v.split(": ")
if len(entry) > 1:
key = entry[0].replace(' ', '').replace(' ', '_')
dict[key] = entry[1]
else:
list_of_dict.append(dict)
dict = {}
# Remove empty
while {} in list_of_dict:
list_of_dict.remove({})
s = {'status': list_of_dict}
try:
serialized = ExtendedStatus().serialize(s)
except colander.Invalid as e:
errors = e.asdict()
raise InvalidUsage(errors, status_code=400)
return serialized
def monetdb_generic_command(request_model, command, allowed_arguments=[]):
execute_command = ['monetdb']
# Pass the options to the execute_command
execute_command.extend(options_get_request(request_model))
execute_command.append(command)
# Pass the arguments to the execute_command
execute_command.extend(
arguments_get_request(request_model, command, allowed_arguments))
if '-a' not in execute_command:
database = check_db_name(request_model)
execute_command += database
try:
result = subprocess.check_output(execute_command,
stderr=subprocess.STDOUT,
shell=False)
result = str(result.decode('ascii')).replace('\r',
' ').replace('\n', ' ')
except subprocess.CalledProcessError as e:
error_message = str(e.output.decode('ascii')).replace('\r',
' ').replace(
'\n', ' ')
raise InvalidUsage(error_message, status_code=400)
return result
def serialize_status(appstruct):
status = appstruct.split("\n")
list_of_dict = []
for i in range(1, len(status) - 1):
line_status = status[i].split()
dict = {}
# Normal case
if len(line_status) >= 6:
dict['database_name'] = line_status[0]
dict['state'] = line_status[1]
dict['state_time'] = line_status[2]
dict['health'] = line_status[3]
dict['health_time'] = line_status[4]
dict['remarks'] = line_status[5]
# No time case
else:
dict['database_name'] = line_status[0]
dict['state'] = line_status[1]
dict['state_time'] = ''
dict['health'] = ''
dict['health_time'] = ''
dict['remarks'] = ''
list_of_dict.append(dict)
s = {'status': list_of_dict}
try:
serialized = Status().serialize(s)
except colander.Invalid as e:
errors = e.asdict()
raise InvalidUsage(errors, status_code=400)
return serialized
def monetdb_set(request):
request_model = validate_model(request)
execute_command = ['monetdb']
# Pass the options to the execute_command
execute_command.extend(options_get_request(request_model))
execute_command.append('set')
# Properties should be a string (property=value) in the monetdb execute_command
properties = properties_get_request(request_model)
execute_command.append('property_holder')
database = check_db_name(request_model)
execute_command += database
for entry in properties:
# For each entry replace the 'property_holder' with a property
execute_command[-2] = entry
try:
result = subprocess.check_output(execute_command,
stderr=subprocess.STDOUT,
shell=False)
result = str(result.decode('ascii'))
except subprocess.CalledProcessError as e:
error_message = str(e.output.decode('ascii')).replace(
'\r', ' ').replace('\n', ' ')
raise InvalidUsage(error_message, status_code=400)
return result
def monetdb_status(request):
request_model = validate_model(request)
execute_command = ['monetdb']
# Pass the options to the execute_command
execute_command.extend(options_get_request(request_model))
execute_command.append('status')
# Pass the arguments to the execute_command
execute_command.extend(
arguments_get_request(request_model,
'status',
allowed_arguments=['long']))
databases = check_db_name(request_model, allow_empty=True)
if databases:
execute_command.extend(databases)
try:
result = subprocess.check_output(execute_command,
stderr=subprocess.STDOUT,
shell=False)
result = str(result.decode('ascii'))
except subprocess.CalledProcessError as e:
error_message = str(e.output.decode('ascii')).replace('\r',
' ').replace(
'\n', ' ')
raise InvalidUsage(error_message, status_code=400)
if "-l" in execute_command:
result = serialize_status_long(result)
else:
result = serialize_status(result)
return result['status']
def monetdbd_start(request):
"""
Accepts dictionary of dbfarm e.g. {'dbfarm': 'mydbfarm'}
"""
dbfarm = check_db_farm(request)
execute_command = ['monetdbd', 'start'] + dbfarm
try:
if checkIfProcessRunning('monetdbd'):
raise InvalidUsage('another monetdbd is already running',
status_code=400)
subprocess.Popen(execute_command,
stderr=subprocess.STDOUT,
shell=False)
except subprocess.CalledProcessError as e:
error_message = str(e.output.decode('ascii')).strip('\n')
raise InvalidUsage(error_message, status_code=400)
def validate_get_model(dict_object):
try:
model = Gmodel().deserialize(dict_object)
except colander.Invalid as e:
errors = e.asdict()
raise InvalidUsage(errors, status_code=400)
return model
def monetdb_version():
"""
Returns monetdb version as string e.g. "MonetDB Database Server v11.35.20"
"""
try:
result = subprocess.check_output(['monetdbd', 'version'],
shell=False).decode('ascii')
result = str(result).strip('\n')
except subprocess.CalledProcessError:
raise InvalidUsage('Internal Server Error', status_code=500)
return result
def properties_get_request(request):
properties = request.get('properties', None)
# Check if the user input is empty
if is_empty(properties):
raise InvalidUsage('Provide at least one property.', status_code=400)
properties = [
'%s=%s' % (key, value) for (key, value) in properties.items()
]
return properties
def monetdb_version():
try:
result = subprocess.check_output(['monetdb', 'version'],
stderr=subprocess.STDOUT,
shell=False)
result = str(result.decode('ascii')).strip('\n')
except subprocess.CalledProcessError as e:
error_message = str(e.output.decode('ascii')).strip('\n')
raise InvalidUsage(error_message, status_code=400)
return result
def monetdb_get(request):
"""
Example of input:
{'options': {'port': '50000'}, 'databases': ['mydb'], 'properties': ['mclients', 'type']}
"""
request_model = validate_get_model(request)
execute_command = ['monetdb']
# Pass the options to the execute_command
execute_command.extend(options_get_request(request_model))
execute_command.append('get')
properties = request_model.get('properties', None)
# Properties should be a string in the monetdb execute_command
properties = ','.join(properties)
# Check if the user input is empty
if is_empty(properties):
raise InvalidUsage('Provide at least one property.', status_code=400)
if 'all' in properties:
execute_command.append('all')
else:
execute_command.append(properties)
# Get all without db returns everything, while get all db will return all the properties for
# the given database
database = check_db_name(request_model, allow_empty=True)
execute_command += database
try:
result = subprocess.check_output(execute_command,
stderr=subprocess.STDOUT,
shell=False)
result = str(result.decode('ascii'))
except subprocess.CalledProcessError as e:
error_message = str(e.output.decode('ascii')).replace('\r',
' ').replace(
'\n', ' ')
raise InvalidUsage(error_message, status_code=400)
result = serialize_monetdb_get(result)
return result
def monetdbd_get(request):
"""
Accepts dictionary of dbfarm, properties
e.g. {'dbfarm': 'mydbfarm', 'properties': ['hostname', 'status']}
"""
properties = request.get('properties', None)
dbfarm = check_db_farm(request)
# Check if the user input is empty
if is_empty(properties):
raise InvalidUsage('Provide at least one property.', status_code=400)
whitelist = [
'all', 'logfile', 'pidfile', 'sockdir', 'port', 'ipv6', 'listenaddr',
'control', 'discovery', 'discoveryttl', 'dbfarm', 'exittimeout',
'forward', 'snapshotdir', 'snapshotcompression', 'mapisock',
'controlsock', 'status', 'hostname'
]
for entry in properties:
if entry not in whitelist:
raise InvalidUsage('No such property: ' + entry, status_code=400)
if 'all' in properties:
options = ['all']
else:
options = [','.join(map(str, properties))]
execute_command = ['monetdbd', 'get'] + options + dbfarm
try:
result = subprocess.check_output(execute_command,
stderr=subprocess.STDOUT,
shell=False)
result = str(result.decode('ascii'))
result = parse_result(result)
except subprocess.CalledProcessError as e:
error_message = str(e.output.decode('ascii')).strip('\n')
raise InvalidUsage(error_message, status_code=400)
return result
def check_db_farm(request):
"""
Sanitize dbfarm user input. Allow the dbfarm to exist on the current path, no subfolders
"""
if request:
dbfarm = (request.get('dbfarm', None))
else:
raise InvalidUsage('Provide one dbfarm', status_code=400)
if dbfarm:
dbfarm = dbfarm.split(',')
else:
raise InvalidUsage('Provide one dbfarm', status_code=400)
if is_empty(dbfarm) or len(dbfarm) != 1:
raise InvalidUsage('Provide one dbfarm', status_code=400)
# Remove any unwanted charachters
dbfarm[0] = slugify(dbfarm[0])
# Allow dbfarm to be created only in the project's current path. Subject to change.
basedir = os.getcwd()
test_path = Path((basedir + '/' + dbfarm[0])).resolve()
if test_path.parent != Path(basedir).resolve():
raise InvalidUsage('Do not use a path with dbfarm', status_code=400)
return dbfarm
def check_db_name(request_model, allow_empty=False):
"""
Sanitize database user input. Allow multiple databases.
"""
database = None
if request_model:
database = request_model.get('databases', None)
if database is None and allow_empty:
return None
if is_empty(database) and allow_empty is False:
raise InvalidUsage('Provide at least one database', status_code=400)
elif isinstance(database, list):
if not all(isinstance(entry, str) for entry in database):
raise InvalidUsage('List of databases must be in string format',
status_code=400)
else:
raise InvalidUsage(
'Provide a database as a list of strings: {"databases": ["<database>"]}',
status_code=400)
# Remove any unwanted charachters
database = [slugify(entry) for entry in database]
return database
def monetdbd_create(request):
"""
Accepts dictionary of dbfarm e.g. {'dbfarm': 'mydbfarm'}
"""
dbfarm = check_db_farm(request)
execute_command = ['monetdbd', 'create'] + dbfarm
try:
result = subprocess.check_output(execute_command,
stderr=subprocess.STDOUT,
shell=False)
result = str(result.decode('ascii'))
result = parse_result(result)
except subprocess.CalledProcessError as e:
error_message = str(e.output.decode('ascii')).strip('\n')
raise InvalidUsage(error_message, status_code=400)
return result