def _get_training_dataset_rest(training_dataset_id, featurestore_id):
"""
Makes a REST call to hopsworks for getting the metadata of a particular training dataset (including the statistics)
Args:
:training_dataset_id: id of the training_dataset
:featurestore_id: id of the featurestore where the training dataset resides
Returns:
The REST response
Raises:
:RestAPIError: if there was an error in the REST call to Hopsworks
"""
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
method = constants.HTTP_CONFIG.HTTP_GET
connection = util._get_http_connection(https=True)
resource_url = (constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + util.project_id() +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
str(featurestore_id) +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_TRAININGDATASETS_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
str(training_dataset_id))
response = util.send_request(connection,
method,
resource_url,
headers=headers)
resp_body = response.read().decode('utf-8')
response_object = json.loads(resp_body)
try: # for python 3
if (response.code != 200):
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not get the metadata of featuregroup (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.code, response.reason, error_code, error_msg, user_msg))
except: # for python 2
if (response.status != 200):
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not get the metadata of featuregroup (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status, response.reason, error_code, error_msg, user_msg))
return response_object
def _get_featurestore_metadata(featurestore):
"""
Makes a REST call to hopsworks to get all metadata of a featurestore (featuregroups and
training datasets) for the provided featurestore.
Args:
:featurestore: the name of the database, defaults to the project's featurestore
Returns:
JSON response
Raises:
:RestAPIError: if there was an error in the REST call to Hopsworks
"""
method = constants.HTTP_CONFIG.HTTP_GET
connection = util._get_http_connection(https=True)
resource_url = (
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + util.project_id() +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + featurestore +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORE_METADATA_RESOURCE)
response = util.send_request(connection, method, resource_url)
resp_body = response.read().decode('utf-8')
response_object = json.loads(resp_body)
# for python 3
if sys.version_info > (3, 0):
if response.code != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not fetch featurestore metadata for featurestore: {} (url: {}), "
"server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, "
"error msg: {}, user msg: {}".format(
resource_url, featurestore, response.code, response.reason,
error_code, error_msg, user_msg))
else: # for python 2
if response.status != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not fetch featurestore metadata for featurestore: {} (url: {}), "
"server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, "
"error msg: {}, user msg: {}".format(
resource_url, featurestore, response.status, response.reason, error_code, error_msg, user_msg))
return response_object
def get_last_execution_info(job_name):
"""
Get information related to the last execution of `job_name`
Args:
job_name (str): Name of the Hopsworks job to retrieve the last execution
Returns:
str: Json object containing the information related to the last execution
"""
method, endpoint = EXECUTION_STATE
endpoint = endpoint.format(project_id=util.project_id(), job_name=job_name)
return rest_rpc._http(endpoint, method=method)
def launch_job(job_name, args):
"""
Launch a job in Hopsworks given the job name - the job needs to exists in Hopsworks before launching it.
Args:
job_name (str): Name of the job to launch
args (str): Runtime arguments of the job.
Returns:
(str): Json containing Hopsworks response
"""
method, endpoint = RUN_JOB
endpoint = endpoint.format(project_id=util.project_id(), job_name=job_name)
return rest_rpc._http(endpoint, method=method, data=args)
def create_job(job_name):
"""
Create a job in Hopsworks given the job name.
Args:
job_name (str): Name of the job to create
Returns:
(str): Json containing Hopsworks response
"""
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
method, endpoint = CREATE_JOB
endpoint = endpoint.format(project_id=util.project_id(), job_name=job_name)
return rest_rpc._http(endpoint, headers=headers, method=method)
def launch_job(job_name):
"""
Launch a job in Hopsworks given the job name - the job needs to exists in Hopsworks before launching it.
Args:
job_name (str): Name of the job to launch
Returns:
(str): Json containing Hopsworks response
"""
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
method, endpoint = RUN_JOB
endpoint = endpoint.format(project_id=util.project_id(), job_name=job_name)
return rest_rpc._http(endpoint, headers=headers, method=method)
def _get_featurestores():
"""
Sends a REST request to get all featurestores for the project
Returns:
a list of Featurestore JSON DTOs
Raises:
:RestAPIError: if there was an error in the REST call to Hopsworks
"""
method = constants.HTTP_CONFIG.HTTP_GET
connection = util._get_http_connection(https=True)
resource_url = (constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + util.project_id() +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE)
response = util.send_request(connection, method, resource_url)
resp_body = response.read().decode('utf-8')
response_object = json.loads(resp_body)
# for python 3
if sys.version_info > (3, 0):
if response.code != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not fetch feature stores (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.code, response.reason, error_code, error_msg, user_msg))
else: # for python 2
if response.status != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not fetch feature stores (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.code, response.reason, error_code, error_msg, user_msg))
return response_object
def _get_api_project_path():
return _get_api_path() + util.project_id()