def _archive(remote_path,
project_name=None,
block=False,
timeout=120,
action='zip'):
"""
Create an archive (zip file) of a file or directory in a Hopsworks dataset.
Args:
:remote_path: the path to the remote file or directory in the dataset.
:action: Allowed values are zip/unzip. Whether to compress/extract respectively.
:block: whether this method should wait for the zipping process to complete before returning.
:project_name: whether this method should wait for the zipping process to complete beefore returning.
:timeout: number of seconds to wait for the action to complete before returning.
Returns:
None
"""
project_id = project.project_id_as_shared(project_name)
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
project_id + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_DATASETS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
remote_path + "?action=" + action
util.send_request('POST', resource_url)
if block is True:
# Wait for zip file to appear. When it does, check that parent dir zipState is not set to CHOWNING
count = 0
while count < timeout:
# Get the status of the zipped file
zip_exists = path_exists(remote_path + ".zip", project_name)
# Get the zipState of the directory being zipped
dir_status = get_path_info(remote_path, project_name)
zip_state = dir_status[
'zipState'] if 'zipState' in dir_status else None
if zip_exists and zip_state == 'NONE':
print("Zipping completed.")
return
else:
print("Zipping...")
time.sleep(1)
count += 1
raise CompressTimeout(
"Timeout of {} seconds exceeded while compressing {}.".format(
timeout, remote_path))
def get_authorization_token():
"""
Get the authorization token to interact with elasticsearch.
Args:
Returns:
The authorization token to be used in http header.
"""
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
method = constants.HTTP_CONFIG.HTTP_GET
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_ELASTIC_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_ELASTIC_JWT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id()
response = util.send_request(method, resource_url, headers=headers)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not get authorization token for elastic (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
return "Bearer " + response_object["token"]
def _attach_model_xattr(ml_id, json_data):
"""
Utility method for putting JSON data into elastic search
Args:
:ml_id: the id of the model
:json_data: the data to put
Returns:
None
"""
headers = {'Content-type': 'application/json'}
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_MODELS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
ml_id
response = util.send_request('PUT',
resource_url,
data=json_data,
headers=headers)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not attach model summary to model (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
else:
return response_object
def _http(resource_url,
headers=None,
method=constants.HTTP_CONFIG.HTTP_GET,
data=None):
response = util.send_request(method,
resource_url,
headers=headers,
data=data)
try:
response_object = response.json()
except JSONDecodeError:
response_object = None
if (response.status_code // 100) != 2:
if response_object:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
else:
error_code, error_msg, user_msg = "", "", ""
raise RestAPIError(
"Could not execute HTTP request (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
return response_object
def start_job(name, args=None):
"""
Start an execution of the job. Only one execution can be active for a job.
Returns:
The job status.
"""
method = constants.HTTP_CONFIG.HTTP_POST
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_JOBS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
name + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_EXECUTIONS_RESOURCE
response = util.send_request(method, resource_url, args)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not perform action on job's execution (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
return response_object
def assume_role(role_arn=None, role_session_name=None, duration_seconds=3600):
"""
Assume a role and sets the temporary credential to the spark context hadoop configuration and environment variables.
Args:
:role_arn: (string) the role arn to be assumed
:role_session_name: (string) use to uniquely identify a session when the same role is assumed by different principals or for different reasons.
:duration_seconds: (int) the duration of the session. Maximum session duration is 3600 seconds.
>>> from hops.credentials_provider import assume_role
>>> assume_role(role_arn="arn:aws:iam::<AccountNumber>:role/analyst")
or
>>> assume_role() # to assume the default role
Returns:
temporary credentials
"""
query = _query_string(role_arn, role_session_name, duration_seconds)
method = constants.HTTP_CONFIG.HTTP_GET
resource_url = _get_cloud_resource(
) + constants.REST_CONFIG.HOPSWORKS_AWS_CLOUD_SESSION_TOKEN_RESOURCE + query
response = util.send_request(method, resource_url)
json_content = _parse_response(response, resource_url)
_set_spark_hadoop_conf(json_content)
_set_envs(json_content)
return json_content
def delete_secret(name):
"""
Delete a secret for this user
>>> from hops import secret
>>> secret.delete_secret('my_secret')
Args:
name: Name of the secret to delete
"""
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_USER_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_SECRETS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
name
method = constants.HTTP_CONFIG.HTTP_DELETE
response = util.send_request(method, resource_url)
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not delete secret (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
def _delete_serving_rest(serving_id):
"""
Makes a REST request to Hopsworks REST API for deleting a serving instance
Args:
:serving_id: id of the serving to delete
Returns:
None
Raises:
:RestAPIError: if there was an error with the REST call to Hopsworks
"""
method = constants.HTTP_CONFIG.HTTP_DELETE
resource_url = (constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_SERVING_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + str(serving_id))
response = util.send_request(method, resource_url)
if response.status_code != 200:
response_object = response.json()
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise exceptions.RestAPIError(
"Could not delete serving with id {} (url: {}), "
"server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, "
"user msg: {}".format(serving_id, resource_url,
response.status_code, response.reason,
error_code, error_msg, user_msg))
def _get_fs_tags():
"""
Makes a REST call to Hopsworks to get tags that can be attached to featuregroups or training datasets
Returns:
List of tags
"""
method = constants.HTTP_CONFIG.HTTP_GET
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
resource_url = (constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORE_TAGS_RESOURCE)
response = util.send_request(method, resource_url, headers=headers)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not get tags (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
results = []
if 'items' in response_object:
for tag in response_object['items']:
results.append(tag['name'])
return results
def remove_xattr(hdfs_path, xattr_name):
"""
Remove an extended attribute attached to an hdfs_path
Args:
:hdfs_path: path of a file or directory
:xattr_name: name of the extended attribute
Returns:
None
"""
hdfs_path = urllib.parse.quote(hdfs._expand_path(hdfs_path))
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
method = constants.HTTP_CONFIG.HTTP_DELETE
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_XATTR_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs_path + constants.DELIMITERS.QUESTION_MARK_DELIMITER + constants.XATTRS.XATTRS_PARAM_NAME + \
constants.DELIMITERS.JDBC_CONNECTION_STRING_VALUE_DELIMITER + xattr_name
response = util.send_request(method, resource_url, headers=headers)
if response.status_code >= 400:
response_object = response.json()
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not remove extened attributes from a path (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
def _start_or_stop_serving_rest(serving_id, action):
"""
Makes a REST request to Hopsworks REST API for starting/stopping a serving instance
Args:
:serving_id: id of the serving to start/stop
:action: the action to perform (start or stop)
Returns:
None
Raises:
:RestAPIError: if there was an error with the REST call to Hopsworks
"""
method = constants.HTTP_CONFIG.HTTP_POST
resource_url = (constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER +
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_SERVING_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER
+ str(serving_id) + constants.MODEL_SERVING.SERVING_START_OR_STOP_PATH_PARAM + action)
response = util.send_request(method, resource_url)
if response.status_code != 200:
response_object = response.json()
error_code, error_msg, user_msg = util._parse_rest_error(response_object)
raise exceptions.RestAPIError("Could not perform action {} on serving with id {} (url: {}), "
"server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, "
"user msg: {}".format(action, serving_id, resource_url, response.status_code,
response.reason, error_code, error_msg, user_msg))
def _get_online_featurestore_jdbc_connector_rest(featurestore_id):
"""
Makes a REST call to Hopsworks to get the JDBC connection to the online feature store
Args:
:featurestore_id: the id of the featurestore
Returns:
the http response
"""
method = constants.HTTP_CONFIG.HTTP_GET
resource_url = (
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + str(featurestore_id) +
constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.
HOPSWORKS_FEATURESTORES_STORAGE_CONNECTORS_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.
HOPSWORKS_ONLINE_FEATURESTORE_STORAGE_CONNECTOR_RESOURCE)
response = util.send_request(method, resource_url)
response_object = response.json()
if response.status_code != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not fetch online featurestore connector (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
return response_object
def _get_fs_schema(tagName):
"""
Makes a REST call to Hopsworks to get tag schema that can be attached to featuregroups or training datasets
"""
method = constants.HTTP_CONFIG.HTTP_GET
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
resource_url = (
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORE_TAGS_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + tagName)
response = util.send_request(method, resource_url, headers=headers)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not get tags (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
tag = {}
if ('name' in response_object) and ('value' in response_object):
tag[response_object['name']] = json.loads(response_object['value'])
return tag
def _post_fs_schema(schemaName, schemaValue):
"""
Makes a REST call to Hopsworks to create schemas for tags that can be attached to datasets
"""
method = constants.HTTP_CONFIG.HTTP_POST
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
resource_url = (
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORE_TAGS_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + "?name=" + schemaName)
response = util.send_request(method,
resource_url,
headers=headers,
data=schemaValue)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not create tag schema (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
def _get_servings_rest():
"""
Makes a REST request to Hopsworks to get a list of all servings in the current project
Returns:
JSON response parsed as a python dict
Raises:
:RestAPIError: if there was an error with the REST call to Hopsworks
"""
method = constants.HTTP_CONFIG.HTTP_GET
resource_url = (constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_SERVING_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER)
response = util.send_request(method, resource_url)
response_object = response.json()
if response.status_code != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise exceptions.RestAPIError(
"Could not fetch list of servings from Hopsworks REST API (url: {}), "
"server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, "
"error msg: {}, user msg: {}".format(resource_url,
response.status_code,
response.reason, error_code,
error_msg, user_msg))
return response_object
def get_executions(name, query=""):
"""
Get a list of the currently running executions for this job.
Returns:
The job status.
"""
method = constants.HTTP_CONFIG.HTTP_GET
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_JOBS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
name + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_EXECUTIONS_RESOURCE + query
response = util.send_request(method, resource_url)
response_object = response.json()
if response.status_code >= 500:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not get current job's execution (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
if response.status_code >= 400:
return None
return response_object
def get_schema(topic):
"""
Gets the Avro schema for a particular Kafka topic.
Args:
:topic: Kafka topic name
Returns:
Avro schema as a string object in JSON format
"""
method = constants.HTTP_CONFIG.HTTP_GET
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_KAFKA_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_TOPICS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
topic + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_SUBJECTS_RESOURCE
response = util.send_request(method, resource_url)
response_object = response.json()
if response.status_code != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not get Avro schema (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
return response_object['schema']
def download(self):
with util.send_request(constants.HTTP_CONFIG.HTTP_GET,
resource="/" + self.resource,
stream=True) as response:
if response.status_code // 100 != 2:
error_code, error_msg, user_msg = util._parse_rest_error(
response.json())
raise RestAPIError(
"Could not perform action on job's execution (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(self.resource, response.status_code,
response.reason, error_code, error_msg, user_msg))
with open(self.file, "wb") as f:
downloaded = 0
file_size = response.headers.get('Content-Length')
if not file_size:
print("Downloading file ...", end=" ")
for chunk in response.iter_content(chunk_size=self.chunk_size):
f.write(chunk)
downloaded += len(chunk)
if file_size:
progress = round(downloaded / int(file_size) * 100, 3)
print("Progress: " + str(progress) + "%")
if not file_size:
print("DONE")
def _job_execution_action(name, action):
"""
Manages execution for the given job, start or stop. Submits an http request to the HOPSWORKS REST API.
Returns:
The job status.
"""
method = constants.HTTP_CONFIG.HTTP_POST
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_JOBS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
name + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_EXECUTIONS_RESOURCE + "?action=" + action
response = util.send_request(method, resource_url)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not perform action on job's execution (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
return response_object
def _make_inference_request_rest(serving_name, data, verb):
"""
Makes a REST request to Hopsworks for submitting an inference request to the serving instance
Args:
:serving_name: name of the model being served
:data: data/json to send to the serving
:verb: type of request (:predict, :classify, or :regress)
Returns:
the JSON response
Raises:
:RestAPIError: if there was an error with the REST call to Hopsworks
"""
json_embeddable = json.dumps(data)
headers = {constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON}
method = constants.HTTP_CONFIG.HTTP_POST
resource_url = (constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER +
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_INFERENCE_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_MODELS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER
+ serving_name + verb)
response = util.send_request(method, resource_url, data=json_embeddable, headers=headers)
response_object = response.json()
error_code, error_msg, user_msg = util._parse_rest_error(response_object)
if response.status_code != 201 and response.status_code != 200:
raise exceptions.RestAPIError("Could not create or update serving (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, "
"user msg: {}".format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
return response_object
def _attach_model_xattr(ml_id, json_data):
"""
Utility method for putting JSON data into elastic search
Args:
:project: the project of the user/app
:appid: the YARN appid
:elastic_id: the id in elastic
:json_data: the data to put
Returns:
None
"""
headers = {'Content-type': 'application/json'}
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_MODELS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
ml_id
resp = util.send_request('PUT',
resource_url,
data=json_data,
headers=headers)
def _get_featurestores():
"""
Sends a REST request to get all featurestores for the project
Returns:
a list of Featurestore JSON DTOs
Raises:
:RestAPIError: if there was an error in the REST call to Hopsworks
"""
method = constants.HTTP_CONFIG.HTTP_GET
resource_url = (constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE)
response = util.send_request(method, resource_url)
response_object = response.json()
if response.status_code != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not fetch feature stores (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
return response_object
def _create_or_update_serving_rest(model_path, model_name, serving_type, model_version,
batching_enabled = None, topic_name=None, num_partitions = None,
num_replicas = None, serving_id = None, instances=1):
"""
Makes a REST request to Hopsworks for creating or updating a model serving instance
Args:
:model_path: path to the model or artifact being served
:model_name: the name of the serving to create
:serving_type: the type of serving
:model_version: version of the serving
:batching_enabled: boolean flag whether to enable batching for inference requests to the serving
:topic_name: name of the kafka topic ("CREATE" to create a new one, or "NONE" to not use kafka topic)
:num_partitions: kafka partitions
:num_replicas: kafka replicas
:serving_id: the id of the serving in case of UPDATE, if serving_id is None, it is a CREATE operation.
:instances: the number of serving instances (the more instances the more inference requests can
be served in parallel)
Returns:
None
Raises:
:RestAPIError: if there was an error with the REST call to Hopsworks
"""
json_contents = {
constants.REST_CONFIG.JSON_SERVING_MODEL_VERSION: model_version,
constants.REST_CONFIG.JSON_SERVING_ARTIFACT_PATH: model_path,
constants.REST_CONFIG.JSON_SERVING_TYPE: serving_type,
constants.REST_CONFIG.JSON_SERVING_NAME: model_name,
constants.REST_CONFIG.JSON_SERVING_KAFKA_TOPIC_DTO: {
constants.REST_CONFIG.JSON_KAFKA_TOPIC_NAME: topic_name,
constants.REST_CONFIG.JSON_KAFKA_NUM_PARTITIONS: num_partitions,
constants.REST_CONFIG.JSON_KAFKA_NUM_REPLICAS: num_replicas
},
constants.REST_CONFIG.JSON_SERVING_REQUESTED_INSTANCES: instances,
}
if serving_id is not None:
json_contents[constants.REST_CONFIG.JSON_SERVING_ID] = serving_id
if serving_type == constants.MODEL_SERVING.SERVING_TYPE_TENSORFLOW:
json_contents[constants.REST_CONFIG.JSON_SERVING_BATCHING_ENABLED] = batching_enabled
json_embeddable = json.dumps(json_contents)
headers = {constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON}
method = constants.HTTP_CONFIG.HTTP_PUT
resource_url = (constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER +
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_SERVING_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER)
response = util.send_request(method, resource_url, data=json_embeddable, headers=headers)
if response.status_code != 201 and response.status_code != 200:
response_object = response.json()
error_code, error_msg, user_msg = util._parse_rest_error(response_object)
raise exceptions.RestAPIError("Could not create or update serving (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, "
"user msg: {}".format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
def _get_roles(role_id=None):
by_id = ""
if role_id:
by_id = constants.DELIMITERS.SLASH_DELIMITER + str(role_id)
method = constants.HTTP_CONFIG.HTTP_GET
resource_url = _get_cloud_resource(
) + constants.REST_CONFIG.HOPSWORKS_CLOUD_ROLE_MAPPINGS_RESOURCE + by_id
response = util.send_request(method, resource_url)
return _parse_response(response, resource_url)
def _get_training_dataset_rest(training_dataset_id, featurestore_id):
"""
Makes a REST call to hopsworks for getting the metadata of a particular training dataset (including the statistics)
Args:
:training_dataset_id: id of the training_dataset
:featurestore_id: id of the featurestore where the training dataset resides
Returns:
The REST response
Raises:
:RestAPIError: if there was an error in the REST call to Hopsworks
"""
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
method = constants.HTTP_CONFIG.HTTP_GET
connection = util._get_http_connection(https=True)
resource_url = (constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + util.project_id() +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
str(featurestore_id) +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_TRAININGDATASETS_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
str(training_dataset_id))
response = util.send_request(connection,
method,
resource_url,
headers=headers)
resp_body = response.read().decode('utf-8')
response_object = json.loads(resp_body)
try: # for python 3
if (response.code != 200):
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not get the metadata of featuregroup (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.code, response.reason, error_code, error_msg, user_msg))
except: # for python 2
if (response.status != 200):
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not get the metadata of featuregroup (url: {}), server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format(
resource_url, response.status, response.reason, error_code, error_msg, user_msg))
return response_object
def get_best_model(name, metric, direction):
"""
Get the best model version by sorting on attached metadata such as model accuracy.
For example if you run this:
>>> from hops import model
>>> from hops.model import Metric
>>> model.get_best_model('mnist', 'accuracy', Metric.MAX)
It will return the mnist version where the 'accuracy' is the highest.
Args:
:name: name of the model
:metric: name of the metric to compare
:direction: whether metric should be maximized or minimized to find the best model
Returns:
The best model
Raises:
:ModelNotFound: if the model was not found
"""
if direction.upper() == Metric.MAX:
direction = "desc"
elif direction.upper() == Metric.MIN:
direction = "asc"
else:
raise Exception(
"Invalid direction, should be Metric.MAX or Metric.MIN")
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_MODELS_RESOURCE + \
"?filter_by=name_eq:" + name + "&sort_by=" + metric + ":" + direction + "&limit=1"
response_object = util.send_request('GET', resource_url, headers=headers)
if not response_object.ok or 'items' not in json.loads(
response_object.content.decode("UTF-8")):
raise ModelNotFound(
"No model with name {} and metric {} could be found.".format(
name, metric))
return json.loads(response_object.content.decode("UTF-8"))['items'][0]
def create_secret(name, secret, project_name=None):
"""
Create a secret
Creating a secret for this user
>>> from hops import secret
>>> secret_token = 'DIOK4jmgFdwadjnDDW98'
>>> secret.create_secret('my_secret', secret_token)
Creating a secret and share it with all members of a project
>>> from hops import secret
>>> secret_token = 'DIOK4jmgFdwadjnDDW98'
>>> secret.create_secret('my_secret', secret_token, project_name='someproject')
Args:
name: Name of the secret to create
secret: Value of the secret
project_name: Name of the project to share the secret with
"""
secret_config = {'name': name, 'secret': secret}
if project_name is None:
secret_config['visibility'] = "PRIVATE"
else:
scope_project = project.get_project_info(project_name)
secret_config['scope'] = scope_project['projectId']
secret_config['visibility'] = "PROJECT"
headers = {
constants.HTTP_CONFIG.HTTP_CONTENT_TYPE:
constants.HTTP_CONFIG.HTTP_APPLICATION_JSON
}
method = constants.HTTP_CONFIG.HTTP_POST
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_USER_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_SECRETS_RESOURCE
response = util.send_request(method,
resource_url,
data=json.dumps(secret_config),
headers=headers)
response_object = response.json()
if response.status_code >= 400:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not create secret (url: {}), server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}"
.format(resource_url, response.status_code, response.reason,
error_code, error_msg, user_msg))
def delete(remote_path, project_name=None, block=True, timeout=30):
"""
Delete the dir or file in Hopsworks, specified by the remote_path.
Example usage:
>>> from hops import dataset
>>> dataset.delete("Projects/project_name/Resources/myremotefile.txt")
Args:
:remote_path: the path to the remote file or directory in the dataset
:project_name: whether this method should wait for the zipping process to complete before returning.
:block: whether to wait for the deletion to complete or not.
:timeout: number of seconds to wait for the deletion to complete before returning.
Returns:
None
"""
project_id = project.project_id_as_shared(project_name)
resource_url = constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
project_id + constants.DELIMITERS.SLASH_DELIMITER + \
constants.REST_CONFIG.HOPSWORKS_DATASETS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \
remote_path
util.send_request('DELETE', resource_url)
# Check if path is indeed deleted as REST API is asynchronous
if block:
count = 0
while count < timeout and path_exists(remote_path, project_name):
print("Waiting for deletion...")
count += 1
time.sleep(1)
if count >= timeout:
raise DeletionTimeout(
"Timeout of {} seconds exceeded while deleting path {}.".
format(timeout, remote_path))
def _get_featurestore_metadata(featurestore):
"""
Makes a REST call to hopsworks to get all metadata of a featurestore (featuregroups and
training datasets) for the provided featurestore.
Args:
:featurestore: the name of the database, defaults to the project's featurestore
Returns:
JSON response
Raises:
:RestAPIError: if there was an error in the REST call to Hopsworks
"""
method = constants.HTTP_CONFIG.HTTP_GET
connection = util._get_http_connection(https=True)
resource_url = (
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + util.project_id() +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE +
constants.DELIMITERS.SLASH_DELIMITER + featurestore +
constants.DELIMITERS.SLASH_DELIMITER +
constants.REST_CONFIG.HOPSWORKS_FEATURESTORE_METADATA_RESOURCE)
response = util.send_request(connection, method, resource_url)
resp_body = response.read().decode('utf-8')
response_object = json.loads(resp_body)
# for python 3
if sys.version_info > (3, 0):
if response.code != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError(
"Could not fetch featurestore metadata for featurestore: {} (url: {}), "
"server response: \n "
"HTTP code: {}, HTTP reason: {}, error code: {}, "
"error msg: {}, user msg: {}".format(
resource_url, featurestore, response.code, response.reason,
error_code, error_msg, user_msg))
else: # for python 2
if response.status != 200:
error_code, error_msg, user_msg = util._parse_rest_error(
response_object)
raise RestAPIError("Could not fetch featurestore metadata for featurestore: {} (url: {}), "
"server response: \n " \
"HTTP code: {}, HTTP reason: {}, error code: {}, "
"error msg: {}, user msg: {}".format(
resource_url, featurestore, response.status, response.reason, error_code, error_msg, user_msg))
return response_object
def send_request(self,
method,
resource,
data=None,
headers=None,
stream=False,
files=None):
return hopsutil.send_request(method,
resource,
data=data,
headers=headers,
stream=stream,
files=files)
