def load_experiment_history(cls, experiment_id, searchinfo):
rest_proxy = rest_proxy_from_searchinfo(searchinfo)
url_parts = ['mltk', 'experiments', experiment_id, 'history']
url = make_splunk_url(rest_proxy, 'user', extra_url_parts=url_parts)
reply = rest_proxy.make_rest_call('GET', url)
return pd.DataFrame(cls.parse_reply(reply))
def move_model_file_from_staging(model_filename, searchinfo, namespace, f):
rest_proxy = rest_proxy_from_searchinfo(searchinfo)
url = rest_url_util.make_lookup_url(rest_proxy,
namespace=namespace,
lookup_file=model_filename)
payload = {'eai:data': f.name, 'output_mode': 'json'}
# try to update the model
reply = rest_proxy.make_rest_call('POST', url, payload)
# if we fail to update the model because it doesn't exist, try to create it instead
if not reply['success']:
if reply['error_type'] == 'ResourceNotFound':
payload['name'] = model_filename
reply = rest_proxy.make_rest_call('POST', url, payload)
# the redundant-looking check is actually necessary because it prevents this logic from triggering if the update fails but the create succceeds
if not reply['success']:
try:
# if the model save fails, clean up the temp model file
os.unlink(f.name)
# if we somehow fail to clean up the temp model, don't expose the error to the user
except Exception as e:
logger.debug(str(e))
models_util.parse_reply(reply)
def get_lookups_from_splunk(searchinfo, namespace, cb_reply_parser, query_params):
"""
Gets a list of models from Splunk's /lookup-table-files endpoint
Args:
searchinfo (dict): a seachinfo object
namespace (string): which namespace to get lookups from
cb_reply_parser(function): a callback to process the reply from splunk
query_params (list): a list of tuples representing URL params, ie. [(count, -1)]
Returns:
lookup_files (dict): a map from a lookup file's location on disk to info about it
"""
rest_proxy = rest_proxy_from_searchinfo(searchinfo)
# searchinfo can be null, in which case we should fall back to the safe 'nobody' username because we can't get the user
try:
username = rest_proxy.splunk_user
except AttributeError:
username = '******'
query_params_copy = build_lookups_query_params(query_params, username)
url = make_get_lookup_url(rest_proxy, namespace=namespace, lookup_file=None, url_params=query_params_copy)
reply = rest_proxy.make_rest_call('GET', url)
lookup_files = cb_reply_parser(reply)
return lookup_files
def delete_model_with_splunk_rest(model_name, searchinfo=None, namespace=None):
file_name = models_util.model_name_to_filename(model_name)
logger.debug('Deleting model: %s' % file_name)
rest_proxy = rest_proxy_from_searchinfo(searchinfo)
url = rest_url_util.make_get_lookup_url(rest_proxy,
namespace=namespace,
lookup_file=file_name)
reply = rest_proxy.make_rest_call('DELETE', url)
parse_model_reply(reply)
def _delete_models(request, url_parts):
if len(url_parts) == 1:
try:
searchinfo = searchinfo_from_request(request)
rest_proxy = rest_proxy_from_searchinfo(searchinfo)
model_list = get_model_list_by_experiment(
rest_proxy, namespace='user', experiment_id=url_parts[0])
for model_name in model_list:
url = rest_url_util.make_get_lookup_url(
rest_proxy, namespace='user', lookup_file=model_name)
reply = rest_proxy.make_rest_call('DELETE', url)
except Exception as e:
cexc.log_traceback()
pass
def get_lookup_file_from_searchinfo(file_name, searchinfo, namespace):
"""
file a GET request to /lookup-table-files/filename endpoint, return the reply
Args:
file_name (str) : file name of the existing lookup file
searchinfo (dict) : searchinfo
namespace (str) : lookup file namespace
Returns:
reply (dict) : the response from GET
"""
rest_proxy = rest_proxy_from_searchinfo(searchinfo)
url = rest_url_util.make_get_lookup_url(rest_proxy, namespace=namespace, lookup_file=file_name)
return rest_proxy.make_rest_call('GET', url)
def load_collection(cls, collection_name, searchinfo):
"""
Create the output table of KVStore entries.
Args:
collection_name (str): the name of the KVStore collection to load
searchinfo (dict): information required for search
"""
rest_proxy = rest_proxy_from_searchinfo(searchinfo)
kvstore_user_reply = kvstore_util.load_collection_from_rest('user', collection_name, rest_proxy)
kvstore_shared_reply = kvstore_util.load_collection_from_rest('app', collection_name, rest_proxy)
formatted = cls.parse_reply(kvstore_user_reply) + cls.parse_reply(kvstore_shared_reply)
return pd.DataFrame(formatted)
def model_name_to_path_from_splunk_rest(model_name,
searchinfo,
namespace=None):
file_name = model_name_to_filename(model_name)
rest_proxy = rest_proxy_from_searchinfo(searchinfo)
url = rest_url_util.make_get_lookup_url(rest_proxy,
namespace=namespace,
lookup_file=file_name)
reply = rest_proxy.make_rest_call('GET', url)
json_content = models_util.parse_reply(reply)
try:
file_path = json_content['entry'][0]['content']['eai:data']
except Exception as e:
logger.debug(str(e))
logger.debug(json_content)
raise Exception("Please check mlspl.log for more details.")
return file_path
def update_model_file_from_rest(model_filename, searchinfo, namespace,
model_filepath):
"""
update the model file by replacing it with a file from the upload staging area.
Args:
model_filename (str): target model file name
searchinfo (dict): searchinfo
namespace (str): file namespace. 'user', 'app' or 'global'
model_filepath (str): the file path of the source file, it has to be in STAGING AREA.
Returns:
reply (dict): reply from POST request.
"""
rest_proxy = rest_proxy_from_searchinfo(searchinfo)
url = rest_url_util.make_lookup_url(rest_proxy,
namespace=namespace,
lookup_file=model_filename)
payload = {'eai:data': model_filepath, 'output_mode': 'json'}
return rest_proxy.make_rest_call('POST', url, postargs=payload)
def create_model_file_from_rest(model_filename, searchinfo, namespace,
model_filepath):
"""
Create a ml-spl model file by moving the file from the upload staging area into $SPLUNK_HOME.
Args:
model_filename (str): the target model name
searchinfo (dict) :
namespace (str) : file namespace
model_filepath (str) : the file path of the source file, it has to be in STAGING AREA.
Returns:
reply (dict): reply from POST request.
"""
rest_proxy = rest_proxy_from_searchinfo(searchinfo)
url = rest_url_util.make_lookup_url(rest_proxy, namespace=namespace)
payload = {
'eai:data': model_filepath,
'name': model_filename,
'output_mode': 'json'
}
return rest_proxy.make_rest_call('POST', url, postargs=payload)