def _get_featurestores():
"""
Sends a REST request to get all featurestores for the project
Returns:
a list of Featurestore JSON DTOs
Raises:
:RestAPIError: if there was an error in the REST call to Hopsworks
"""
method = constants.HTTP_CONFIG.HTTP_GET
connection = util._get_http_connection(https=True)
resource_url = (constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER +
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE)
response = util.send_request(connection, method, resource_url)
resp_body = response.read()
response_object = json.loads(resp_body)
# for python 3
if sys.version_info > (3, 0):
if response.code != 200:
error_code, error_msg, user_msg = util._parse_rest_error(response_object)
raise RestAPIError("Could not fetch feature stores (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.code, response.reason, error_code, error_msg, user_msg))
else: # for python 2
if response.status != 200:
error_code, error_msg, user_msg = util._parse_rest_error(response_object)
raise RestAPIError("Could not fetch feature stores (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.code, response.reason, error_code, error_msg, user_msg))
return response_object
def _get_training_dataset_rest(training_dataset_id, featurestore_id):
"""
Makes a REST call to hopsworks for getting the metadata of a particular training dataset (including the statistics)
Args:
:training_dataset_id: id of the training_dataset
:featurestore_id: id of the featurestore where the training dataset resides
Returns:
The REST response
Raises:
:RestAPIError: if there was an error in the REST call to Hopsworks
"""
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
method = constants.HTTP_CONFIG.HTTP_GET
connection = util._get_http_connection(https=True)
resource_url = (constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + util.project_id() +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
str(featurestore_id) +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_TRAININGDATASETS_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
str(training_dataset_id))
response = util.send_request(connection,
method,
resource_url,
headers=headers)
resp_body = response.read().decode('utf-8')
response_object = json.loads(resp_body)
try: # for python 3
if (response.code != 200):
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not get the metadata of featuregroup (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.code, response.reason, error_code, error_msg, user_msg))
except: # for python 2
if (response.status != 200):
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not get the metadata of featuregroup (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status, response.reason, error_code, error_msg, user_msg))
return response_object
def _get_featurestore_metadata(featurestore):
"""
Makes a REST call to hopsworks to get all metadata of a featurestore (featuregroups and
training datasets) for the provided featurestore.
Args:
:featurestore: the name of the database, defaults to the project's featurestore
Returns:
JSON response
Raises:
:RestAPIError: if there was an error in the REST call to Hopsworks
"""
method = constants.HTTP_CONFIG.HTTP_GET
connection = util._get_http_connection(https=True)
resource_url = (
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + util.project_id() +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + featurestore +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORE_METADATA_RESOURCE)
response = util.send_request(connection, method, resource_url)
resp_body = response.read().decode('utf-8')
response_object = json.loads(resp_body)
# for python 3
if sys.version_info > (3, 0):
if response.code != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not fetch featurestore metadata for featurestore: {} (url: {}), "
"server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, "
"error msg: {}, user msg: {}".format(
resource_url, featurestore, response.code, response.reason,
error_code, error_msg, user_msg))
else: # for python 2
if response.status != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not fetch featurestore metadata for featurestore: {} (url: {}), "
"server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, "
"error msg: {}, user msg: {}".format(
resource_url, featurestore, response.status, response.reason, error_code, error_msg, user_msg))
return response_object
def _job_execution_action(name, action):
"""
Manages execution for the given job, start or stop. Submits an http request to the HOPSWORKS REST API.
Returns:
The job status.
"""
method = constants.HTTP_CONFIG.HTTP_POST
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_JOBS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
name + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_EXECUTIONS_RESOURCE + "?action=" + action
response = util.send_request(method, resource_url)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not perform action on job's execution (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
return response_object
def get_authorization_token():
"""
Get the authorization token to interact with elasticsearch.
Args:
Returns:
The authorization token to be used in http header.
"""
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
method = constants.HTTP_CONFIG.HTTP_GET
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_ELASTIC_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_ELASTIC_JWT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id()
response = util.send_request(method, resource_url, headers=headers)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not get authorization token for elastic (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
return "Bearer " + response_object["token"]
def _attach_model_xattr(ml_id, json_data):
"""
Utility method for putting JSON data into elastic search
Args:
:ml_id: the id of the model
:json_data: the data to put
Returns:
None
"""
headers = {'Content-type': 'application/json'}
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_MODELS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
ml_id
response = util.send_request('PUT',
resource_url,
data=json_data,
headers=headers)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not attach model summary to model (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
else:
return response_object
def _http(resource_url,
headers=None,
method=constants.HTTP_CONFIG.HTTP_GET,
data=None):
response = util.send_request(method,
resource_url,
headers=headers,
data=data)
try:
response_object = response.json()
except JSONDecodeError:
response_object = None
if (response.status_code // 100) != 2:
if response_object:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
else:
error_code, error_msg, user_msg = "", "", ""
raise RestAPIError(
"Could not execute HTTP request (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
return response_object
def _get_online_featurestore_jdbc_connector_rest(featurestore_id):
"""
Makes a REST call to Hopsworks to get the JDBC connection to the online feature store
Args:
:featurestore_id: the id of the featurestore
Returns:
the http response
"""
method = constants.HTTP_CONFIG.HTTP_GET
resource_url = (
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + str(featurestore_id) +
constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.
HOPSWORKS_FEATURESTORES_STORAGE_CONNECTORS_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.
HOPSWORKS_ONLINE_FEATURESTORE_STORAGE_CONNECTOR_RESOURCE)
response = util.send_request(method, resource_url)
response_object = response.json()
if response.status_code != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not fetch online featurestore connector (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
return response_object
def get_executions(name, query=""):
"""
Get a list of the currently running executions for this job.
Returns:
The job status.
"""
method = constants.HTTP_CONFIG.HTTP_GET
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_JOBS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
name + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_EXECUTIONS_RESOURCE + query
response = util.send_request(method, resource_url)
response_object = response.json()
if response.status_code >= 500:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not get current job's execution (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
if response.status_code >= 400:
return None
return response_object
def _get_fs_tags():
"""
Makes a REST call to Hopsworks to get tags that can be attached to featuregroups or training datasets
Returns:
List of tags
"""
method = constants.HTTP_CONFIG.HTTP_GET
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
resource_url = (constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORE_TAGS_RESOURCE)
response = util.send_request(method, resource_url, headers=headers)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not get tags (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
results = []
if 'items' in response_object:
for tag in response_object['items']:
results.append(tag['name'])
return results
def remove_xattr(hdfs_path, xattr_name):
"""
Remove an extended attribute attached to an hdfs_path
Args:
:hdfs_path: path of a file or directory
:xattr_name: name of the extended attribute
Returns:
None
"""
hdfs_path = urllib.parse.quote(hdfs._expand_path(hdfs_path))
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
method = constants.HTTP_CONFIG.HTTP_DELETE
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_XATTR_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs_path + constants.DELIMITERS.QUESTION_MARK_DELIMITER + constants.XATTRS.XATTRS_PARAM_NAME + \
constants.DELIMITERS.JDBC_CONNECTION_STRING_VALUE_DELIMITER + xattr_name
response = util.send_request(method, resource_url, headers=headers)
if response.status_code >= 400:
response_object = response.json()
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not remove extened attributes from a path (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
def get_schema(topic):
"""
Gets the Avro schema for a particular Kafka topic.
Args:
:topic: Kafka topic name
Returns:
Avro schema as a string object in JSON format
"""
method = constants.HTTP_CONFIG.HTTP_GET
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_KAFKA_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_TOPICS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
topic + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_SUBJECTS_RESOURCE
response = util.send_request(method, resource_url)
response_object = response.json()
if response.status_code != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not get Avro schema (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
return response_object['schema']
def delete_secret(name):
"""
Delete a secret for this user
>>> from hops import secret
>>> secret.delete_secret('my_secret')
Args:
name: Name of the secret to delete
"""
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_USER_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_SECRETS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
name
method = constants.HTTP_CONFIG.HTTP_DELETE
response = util.send_request(method, resource_url)
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not delete secret (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
def start_job(name, args=None):
"""
Start an execution of the job. Only one execution can be active for a job.
Returns:
The job status.
"""
method = constants.HTTP_CONFIG.HTTP_POST
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_JOBS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
name + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_EXECUTIONS_RESOURCE
response = util.send_request(method, resource_url, args)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not perform action on job's execution (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
return response_object
def _post_fs_schema(schemaName, schemaValue):
"""
Makes a REST call to Hopsworks to create schemas for tags that can be attached to datasets
"""
method = constants.HTTP_CONFIG.HTTP_POST
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
resource_url = (
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORE_TAGS_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + "?name=" + schemaName)
response = util.send_request(method,
resource_url,
headers=headers,
data=schemaValue)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not create tag schema (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
def _get_fs_schema(tagName):
"""
Makes a REST call to Hopsworks to get tag schema that can be attached to featuregroups or training datasets
"""
method = constants.HTTP_CONFIG.HTTP_GET
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
resource_url = (
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORE_TAGS_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + tagName)
response = util.send_request(method, resource_url, headers=headers)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not get tags (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
tag = {}
if ('name' in response_object) and ('value' in response_object):
tag[response_object['name']] = json.loads(response_object['value'])
return tag
def download(self):
with util.send_request(constants.HTTP_CONFIG.HTTP_GET,
resource="/" + self.resource,
stream=True) as response:
if response.status_code // 100 != 2:
error_code, error_msg, user_msg = util._parse_rest_error(
response.json())
raise RestAPIError(
"Could not perform action on job's execution (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(self.resource, response.status_code,
response.reason, error_code, error_msg, user_msg))
with open(self.file, "wb") as f:
downloaded = 0
file_size = response.headers.get('Content-Length')
if not file_size:
print("Downloading file ...", end=" ")
for chunk in response.iter_content(chunk_size=self.chunk_size):
f.write(chunk)
downloaded += len(chunk)
if file_size:
progress = round(downloaded / int(file_size) * 100, 3)
print("Progress: " + str(progress) + "%")
if not file_size:
print("DONE")
def _get_project_info(project_name):
"""
Makes a REST call to hopsworks to get all metadata of a project for the provided project.
Args:
:project_name: the name of the project
Returns:
JSON response
Raises:
:RestAPIError: if there was an error in the REST call to Hopsworks
"""
method = constants.HTTP_CONFIG.HTTP_GET
connection = util._get_http_connection(https=True)
resource_url = (constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_INFO_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + project_name)
response = util.send_request(connection, method, resource_url)
resp_body = response.read().decode('utf-8')
response_object = json.loads(resp_body)
# for python 3
if sys.version_info > (3, 0):
if response.code != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not fetch project metadata for project: {} (url: {}), "
"server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, "
"error msg: {}, user msg: {}".format(
project_name, resource_url, response.code, response.reason,
error_code, error_msg, user_msg))
else: # for python 2
if response.status != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not fetch project metadata for project: {} (url: {}), "
"server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, "
"error msg: {}, user msg: {}".format(
project_name, resource_url, response.status, response.reason, error_code, error_msg, user_msg))
return response_object
def _parse_response(response, url):
if response.ok:
return response.json()
else:
error_code, error_msg, user_msg = util._parse_rest_error(
response.json())
raise RestAPIError(
"Error calling {}. Got status: HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}"
", user msg: {}".format(url, response.status_code, response.reason,
error_code, error_msg, user_msg))
def _delete_table_contents(featuregroup_id, featurestore_id):
"""
Sends a request to clear the contents of a featuregroup by dropping the featuregroup and recreating it with
the same metadata.
Args:
:featuregroup_id: id of the featuregroup
:featurestore_id: id of the featurestore
Returns:
The JSON response
Raises:
:RestAPIError: if there was an error in the REST call to Hopsworks
"""
method = constants.HTTP_CONFIG.HTTP_POST
connection = util._get_http_connection(https=True)
resource_url = (constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER +
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER +
str(featurestore_id) + constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATUREGROUPS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER +
str(featuregroup_id) + constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATUREGROUP_CLEAR_RESOURCE)
response = util.send_request(connection, method, resource_url)
resp_body = response.read()
response_object = json.loads(resp_body)
# for python 3
if sys.version_info > (3, 0):
if response.code != 200:
error_code, error_msg, user_msg = util._parse_rest_error(response_object)
raise RestAPIError("Could not clear featuregroup contents (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.code, response.reason, error_code, error_msg, user_msg))
else: # for python 2
if response.status != 200:
error_code, error_msg, user_msg = util._parse_rest_error(response_object)
raise RestAPIError("Could not clear featuregroup contents (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.code, response.reason, error_code, error_msg, user_msg))
return response_object
def create_secret(name, secret, project_name=None):
"""
Create a secret
Creating a secret for this user
>>> from hops import secret
>>> secret_token = 'DIOK4jmgFdwadjnDDW98'
>>> secret.create_secret('my_secret', secret_token)
Creating a secret and share it with all members of a project
>>> from hops import secret
>>> secret_token = 'DIOK4jmgFdwadjnDDW98'
>>> secret.create_secret('my_secret', secret_token, project_name='someproject')
Args:
name: Name of the secret to create
secret: Value of the secret
project_name: Name of the project to share the secret with
"""
secret_config = {'name': name, 'secret': secret}
if project_name is None:
secret_config['visibility'] = "PRIVATE"
else:
scope_project = project.get_project_info(project_name)
secret_config['scope'] = scope_project['projectId']
secret_config['visibility'] = "PROJECT"
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
method = constants.HTTP_CONFIG.HTTP_POST
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_USER_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_SECRETS_RESOURCE
response = util.send_request(method,
resource_url,
data=json.dumps(secret_config),
headers=headers)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not create secret (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
def _get_tags(featurestore_id, id, resource):
"""
Makes a REST call to Hopsworks to get tags attached to a featuregroup or training dataset
Args:
:featurestore_id: the id of the featurestore
:id: the id of the featuregroup or training dataset
:resource: featuregroup or training dataset resource
Returns:
A dictionary containing the tags
"""
method = constants.HTTP_CONFIG.HTTP_GET
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
resource_url = (constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
str(featurestore_id) +
constants.DELIMITERS.SLASH_DELIMITER + resource +
constants.DELIMITERS.SLASH_DELIMITER + str(id) +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORE_TAGS_RESOURCE)
response = util.send_request(method, resource_url, headers=headers)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not get tags (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
tags = {}
if "items" in response_object:
for tag in response_object["items"]:
tags[tag["name"]] = tag["value"]
return tags
def get_secret(name, owner=None):
"""
Get a secret
Get a secret for this user
>>> from hops import secret
>>> secret.get_secret('my_secret')
Get a secret shared with this project by the secret owner
>>> from hops import secret
>>> secret.get_secret('shared_secret', owner='username')
Args:
name: Name of the secret to get
owner: The username of the user that created the secret
Returns:
The secret
"""
if owner is None:
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_USER_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_SECRETS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
name
else:
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_USER_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_SECRETS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_SHARED + "?name=" + name + "&owner=" + owner
method = constants.HTTP_CONFIG.HTTP_GET
response = util.send_request(method, resource_url)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not get secret (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
return response_object['items'][0]['secret']
def _get_featuregroup_rest(featuregroup_id, featurestore_id):
"""
Makes a REST call to hopsworks for getting the metadata of a particular featuregroup (including the statistics)
Args:
:featuregroup_id: id of the featuregroup
:featurestore_id: id of the featurestore where the featuregroup resides
Returns:
The REST response
Raises:
:RestAPIError: if there was an error in the REST call to Hopsworks
"""
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
method = constants.HTTP_CONFIG.HTTP_GET
resource_url = (constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
str(featurestore_id) +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATUREGROUPS_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
str(featuregroup_id))
response = util.send_request(method, resource_url, headers=headers)
response_object = response.json()
if response.status_code != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not get the metadata of featuregroup (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
return response_object
def _get_featurestore_metadata(featurestore):
"""
Makes a REST call to hopsworks to get all metadata of a featurestore (featuregroups and
training datasets) for the provided featurestore.
Args:
:featurestore: the name of the database, defaults to the project's featurestore
Returns:
JSON response
Raises:
:RestAPIError: if there was an error in the REST call to Hopsworks
"""
method = constants.HTTP_CONFIG.HTTP_GET
resource_url = (
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + featurestore +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORE_METADATA_RESOURCE)
response = util.send_request(method, resource_url)
response_object = response.json()
if response.status_code != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not fetch featurestore metadata for featurestore: {} (url: {}), "
"server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, "
"error msg: {}, user msg: {}".format(resource_url, featurestore,
response.status_code,
response.reason, error_code,
error_msg, user_msg))
return response_object
def get_current_execution(name):
method = constants.HTTP_CONFIG.HTTP_GET
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_JOBS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
name + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_EXECUTIONS_RESOURCE + "?offset=0&limit=1&sort_by=id:desc"
response = util.send_request(method, resource_url)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not get current job's execution (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
return response_object
def _add_tag(featurestore_id, id, tag, value, resource):
"""
Makes a REST call to Hopsworks to attach tags to a featuregroup or training dataset
Args:
:featurestore_id: the id of the featurestore
:id: the id of the featuregroup or training dataset
:tag: name of the tag to attach
:value: value of the tag
:resource: featuregroup or training dataset resource
Returns:
None
"""
method = constants.HTTP_CONFIG.HTTP_PUT
resource_url = (
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + str(featurestore_id) +
constants.DELIMITERS.SLASH_DELIMITER + resource +
constants.DELIMITERS.SLASH_DELIMITER + str(id) +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORE_TAGS_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + tag)
if value is not None:
resource_url = resource_url + "?value=" + value
response = util.send_request(method, resource_url)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not attach tags (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
def _attach_experiment_xattr(app_id, run_id, json_data, xattr):
"""
Utility method for putting JSON data into elastic search
Args:
:project: the project of the user/app
:appid: the YARN appid
:elastic_id: the id in elastic
:json_data: the data to put
Returns:
None
"""
json_data = dumps(json_data)
headers = {'Content-type': 'application/json'}
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_EXPERIMENTS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
app_id + "_" + str(run_id) + "?xattr=" + xattr
response = util.send_request('PUT',
resource_url,
data=json_data,
headers=headers)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not create experiment (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
else:
return response_object
def get_xattr(hdfs_path, xattr_name=None):
"""
Get the extended attribute attached to an hdfs_path.
Args:
:hdfs_path: path of a file or directory
:xattr_name: name of the extended attribute
Returns:
A dictionary with the extended attribute(s) as key value pair(s). If the :xattr_name is None,
the API returns all associated extended attributes.
"""
hdfs_path = urllib.parse.quote(hdfs._expand_path(hdfs_path))
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
method = constants.HTTP_CONFIG.HTTP_GET
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_XATTR_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs_path
if xattr_name is not None:
resource_url += constants.DELIMITERS.QUESTION_MARK_DELIMITER + constants.XATTRS.XATTRS_PARAM_NAME + \
constants.DELIMITERS.JDBC_CONNECTION_STRING_VALUE_DELIMITER + xattr_name
response = util.send_request(method, resource_url, headers=headers)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not get extened attributes attached to a path (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
results = {}
for item in response_object["items"]:
results[item["name"]] = item["value"]
return results
def _attach_experiment_xattr(ml_id, json_data, op_type):
"""
Utility method for putting JSON data into elastic search
Args:
:ml_id: experiment id
:json_data: the experiment json object
:op_type: operation type INIT/MODEL_UPDATE/FULL_UPDATE
Returns:
None
"""
json_data = dumps(json_data)
headers = {'Content-type': 'application/json'}
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_EXPERIMENTS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
ml_id + "?type=" + op_type
response = util.send_request('PUT',
resource_url,
data=json_data,
headers=headers)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not create experiment (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
else:
return response_object
