def remove_xattr(hdfs_path, xattr_name): """ Remove an extended attribute attached to an hdfs_path Args: :hdfs_path: path of a file or directory :xattr_name: name of the extended attribute Returns: None """ hdfs_path = urllib.parse.quote(hdfs._expand_path(hdfs_path)) headers = { constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON } method = constants.HTTP_CONFIG.HTTP_DELETE resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_XATTR_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs_path + constants.DELIMITERS.QUESTION_MARK_DELIMITER + constants.XATTRS.XATTRS_PARAM_NAME + \ constants.DELIMITERS.JDBC_CONNECTION_STRING_VALUE_DELIMITER + xattr_name response = util.send_request(method, resource_url, headers=headers) if response.status_code >= 400: response_object = response.json() error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise RestAPIError("Could not remove extened attributes from a path (url: {}), server response: \n " \ "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
def _get_online_featurestore_jdbc_connector_rest(featurestore_id): """ Makes a REST call to Hopsworks to get the JDBC connection to the online feature store Args: :featurestore_id: the id of the featurestore Returns: the http response """ method = constants.HTTP_CONFIG.HTTP_GET resource_url = ( constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + str(featurestore_id) + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG. HOPSWORKS_FEATURESTORES_STORAGE_CONNECTORS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG. HOPSWORKS_ONLINE_FEATURESTORE_STORAGE_CONNECTOR_RESOURCE) response = util.send_request(method, resource_url) response_object = response.json() if response.status_code != 200: error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise RestAPIError("Could not fetch online featurestore connector (url: {}), server response: \n " \ "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) return response_object
def _delete_serving_rest(serving_id): """ Makes a REST request to Hopsworks REST API for deleting a serving instance Args: :serving_id: id of the serving to delete Returns: None Raises: :RestAPIError: if there was an error with the REST call to Hopsworks """ method = constants.HTTP_CONFIG.HTTP_DELETE resource_url = (constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_SERVING_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + str(serving_id)) response = util.send_request(method, resource_url) if response.status_code != 200: response_object = response.json() error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise exceptions.RestAPIError( "Could not delete serving with id {} (url: {}), " "server response: \n " "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, " "user msg: {}".format(serving_id, resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
def _attach_model_xattr(ml_id, json_data): """ Utility method for putting JSON data into elastic search Args: :project: the project of the user/app :appid: the YARN appid :elastic_id: the id in elastic :json_data: the data to put Returns: None """ headers = {'Content-type': 'application/json'} resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_MODELS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ ml_id resp = util.send_request('PUT', resource_url, data=json_data, headers=headers)
def _get_servings_rest(): """ Makes a REST request to Hopsworks to get a list of all servings in the current project Returns: JSON response parsed as a python dict Raises: :RestAPIError: if there was an error with the REST call to Hopsworks """ method = constants.HTTP_CONFIG.HTTP_GET resource_url = (constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_SERVING_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER) response = util.send_request(method, resource_url) response_object = response.json() if response.status_code != 200: error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise exceptions.RestAPIError( "Could not fetch list of servings from Hopsworks REST API (url: {}), " "server response: \n " "HTTP code: {}, HTTP reason: {}, error code: {}, " "error msg: {}, user msg: {}".format(resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) return response_object
def _job_execution_action(name, action): """ Manages execution for the given job, start or stop. Submits an http request to the HOPSWORKS REST API. Returns: The job status. """ method = constants.HTTP_CONFIG.HTTP_POST resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_JOBS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ name + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_EXECUTIONS_RESOURCE + "?action=" + action response = util.send_request(method, resource_url) response_object = response.json() if response.status_code >= 400: error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise RestAPIError( "Could not perform action on job's execution (url: {}), server response: \n " "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}" .format(resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) return response_object
def get_authorization_token(): """ Get the authorization token to interact with elasticsearch. Args: Returns: The authorization token to be used in http header. """ headers = { constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON } method = constants.HTTP_CONFIG.HTTP_GET resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_ELASTIC_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_ELASTIC_JWT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs.project_id() response = util.send_request(method, resource_url, headers=headers) response_object = response.json() if response.status_code >= 400: error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise RestAPIError("Could not get authorization token for elastic (url: {}), server response: \n " \ "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) return "Bearer " + response_object["token"]
def _make_inference_request_rest(serving_name, data, verb): """ Makes a REST request to Hopsworks for submitting an inference request to the serving instance Args: :serving_name: name of the model being served :data: data/json to send to the serving :verb: type of request (:predict, :classify, or :regress) Returns: the JSON response Raises: :RestAPIError: if there was an error with the REST call to Hopsworks """ json_embeddable = json.dumps(data) headers = {constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON} method = constants.HTTP_CONFIG.HTTP_POST resource_url = (constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_INFERENCE_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_MODELS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + serving_name + verb) response = util.send_request(method, resource_url, data=json_embeddable, headers=headers) response_object = response.json() error_code, error_msg, user_msg = util._parse_rest_error(response_object) if response.status_code != 201 and response.status_code != 200: raise exceptions.RestAPIError("Could not create or update serving (url: {}), server response: \n " \ "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, " "user msg: {}".format(resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) return response_object
def _start_or_stop_serving_rest(serving_id, action): """ Makes a REST request to Hopsworks REST API for starting/stopping a serving instance Args: :serving_id: id of the serving to start/stop :action: the action to perform (start or stop) Returns: None Raises: :RestAPIError: if there was an error with the REST call to Hopsworks """ method = constants.HTTP_CONFIG.HTTP_POST resource_url = (constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_SERVING_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + str(serving_id) + constants.MODEL_SERVING.SERVING_START_OR_STOP_PATH_PARAM + action) response = util.send_request(method, resource_url) if response.status_code != 200: response_object = response.json() error_code, error_msg, user_msg = util._parse_rest_error(response_object) raise exceptions.RestAPIError("Could not perform action {} on serving with id {} (url: {}), " "server response: \n " "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, " "user msg: {}".format(action, serving_id, resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
def _attach_model_xattr(ml_id, json_data): """ Utility method for putting JSON data into elastic search Args: :ml_id: the id of the model :json_data: the data to put Returns: None """ headers = {'Content-type': 'application/json'} resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_MODELS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ ml_id response = util.send_request('PUT', resource_url, data=json_data, headers=headers) response_object = response.json() if response.status_code >= 400: error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise RestAPIError( "Could not attach model summary to model (url: {}), server response: \n " "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}" .format(resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) else: return response_object
def start_job(name, args=None): """ Start an execution of the job. Only one execution can be active for a job. Returns: The job status. """ method = constants.HTTP_CONFIG.HTTP_POST resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_JOBS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ name + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_EXECUTIONS_RESOURCE response = util.send_request(method, resource_url, args) response_object = response.json() if response.status_code >= 400: error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise RestAPIError( "Could not perform action on job's execution (url: {}), server response: \n " "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}" .format(resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) return response_object
def get_executions(name, query=""): """ Get a list of the currently running executions for this job. Returns: The job status. """ method = constants.HTTP_CONFIG.HTTP_GET resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_JOBS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ name + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_EXECUTIONS_RESOURCE + query response = util.send_request(method, resource_url) response_object = response.json() if response.status_code >= 500: error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise RestAPIError( "Could not get current job's execution (url: {}), server response: \n " "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}" .format(resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) if response.status_code >= 400: return None return response_object
def _get_featurestores(): """ Sends a REST request to get all featurestores for the project Returns: a list of Featurestore JSON DTOs Raises: :RestAPIError: if there was an error in the REST call to Hopsworks """ method = constants.HTTP_CONFIG.HTTP_GET resource_url = (constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE) response = util.send_request(method, resource_url) response_object = response.json() if response.status_code != 200: error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise RestAPIError("Could not fetch feature stores (url: {}), server response: \n " \ "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) return response_object
def get_schema(topic): """ Gets the Avro schema for a particular Kafka topic. Args: :topic: Kafka topic name Returns: Avro schema as a string object in JSON format """ method = constants.HTTP_CONFIG.HTTP_GET resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_KAFKA_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_TOPICS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ topic + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_SUBJECTS_RESOURCE response = util.send_request(method, resource_url) response_object = response.json() if response.status_code != 200: error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise RestAPIError( "Could not get Avro schema (url: {}), server response: \n " "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}" .format(resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) return response_object['schema']
def _create_or_update_serving_rest(model_path, model_name, serving_type, model_version, batching_enabled = None, topic_name=None, num_partitions = None, num_replicas = None, serving_id = None, instances=1): """ Makes a REST request to Hopsworks for creating or updating a model serving instance Args: :model_path: path to the model or artifact being served :model_name: the name of the serving to create :serving_type: the type of serving :model_version: version of the serving :batching_enabled: boolean flag whether to enable batching for inference requests to the serving :topic_name: name of the kafka topic ("CREATE" to create a new one, or "NONE" to not use kafka topic) :num_partitions: kafka partitions :num_replicas: kafka replicas :serving_id: the id of the serving in case of UPDATE, if serving_id is None, it is a CREATE operation. :instances: the number of serving instances (the more instances the more inference requests can be served in parallel) Returns: None Raises: :RestAPIError: if there was an error with the REST call to Hopsworks """ json_contents = { constants.REST_CONFIG.JSON_SERVING_MODEL_VERSION: model_version, constants.REST_CONFIG.JSON_SERVING_ARTIFACT_PATH: model_path, constants.REST_CONFIG.JSON_SERVING_TYPE: serving_type, constants.REST_CONFIG.JSON_SERVING_NAME: model_name, constants.REST_CONFIG.JSON_SERVING_KAFKA_TOPIC_DTO: { constants.REST_CONFIG.JSON_KAFKA_TOPIC_NAME: topic_name, constants.REST_CONFIG.JSON_KAFKA_NUM_PARTITIONS: num_partitions, constants.REST_CONFIG.JSON_KAFKA_NUM_REPLICAS: num_replicas }, constants.REST_CONFIG.JSON_SERVING_REQUESTED_INSTANCES: instances, } if serving_id is not None: json_contents[constants.REST_CONFIG.JSON_SERVING_ID] = serving_id if serving_type == constants.MODEL_SERVING.SERVING_TYPE_TENSORFLOW: json_contents[constants.REST_CONFIG.JSON_SERVING_BATCHING_ENABLED] = batching_enabled json_embeddable = json.dumps(json_contents) headers = {constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON} method = constants.HTTP_CONFIG.HTTP_PUT resource_url = (constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_SERVING_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER) response = util.send_request(method, resource_url, data=json_embeddable, headers=headers) if response.status_code != 201 and response.status_code != 200: response_object = response.json() error_code, error_msg, user_msg = util._parse_rest_error(response_object) raise exceptions.RestAPIError("Could not create or update serving (url: {}), server response: \n " \ "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, " "user msg: {}".format(resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
def get_best_model(name, metric, direction): """ Get the best model version by sorting on attached metadata such as model accuracy. For example if you run this: >>> from hops import model >>> from hops.model import Metric >>> model.get_best_model('mnist', 'accuracy', Metric.MAX) It will return the mnist version where the 'accuracy' is the highest. Args: :name: name of the model :metric: name of the metric to compare :direction: whether metric should be maximized or minimized to find the best model Returns: The best model Raises: :ModelNotFound: if the model was not found """ if direction.upper() == Metric.MAX: direction = "desc" elif direction.upper() == Metric.MIN: direction = "asc" else: raise Exception( "Invalid direction, should be Metric.MAX or Metric.MIN") headers = { constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON } resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_MODELS_RESOURCE + \ "?filter_by=name_eq:" + name + "&sort_by=" + metric + ":" + direction + "&limit=1" response_object = util.send_request('GET', resource_url, headers=headers) if not response_object.ok or 'items' not in json.loads( response_object.content.decode("UTF-8")): raise ModelNotFound( "No model with name {} and metric {} could be found.".format( name, metric)) return json.loads(response_object.content.decode("UTF-8"))['items'][0]
def _get_tags(featurestore_id, id, resource): """ Makes a REST call to Hopsworks to get tags attached to a featuregroup or training dataset Args: :featurestore_id: the id of the featurestore :id: the id of the featuregroup or training dataset :resource: featuregroup or training dataset resource Returns: A dictionary containing the tags """ method = constants.HTTP_CONFIG.HTTP_GET headers = { constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON } resource_url = (constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + str(featurestore_id) + constants.DELIMITERS.SLASH_DELIMITER + resource + constants.DELIMITERS.SLASH_DELIMITER + str(id) + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_FEATURESTORE_TAGS_RESOURCE) response = util.send_request(method, resource_url, headers=headers) response_object = response.json() if response.status_code >= 400: error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise RestAPIError("Could not get tags (url: {}), server response: \n " \ "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) tags = {} if "items" in response_object: for tag in response_object["items"]: tags[tag["name"]] = tag["value"] return tags
def get_model(name, version, project_name=None): """ Get a specific model version given a model name and a version. For example if you run this: >>> from hops import model >>> model.get_model('mnist', 1) You will get version 1 of the model 'mnist' Args: :name: name of the model :version: version of the model :project_name name of the project parent of the model. By default, this project is the current project running the experiment Returns: The specified model version Raises: :ModelNotFound: if the model was not found """ headers = { constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON } project_id = project.project_id_as_shared(project_name) resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_MODELS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ str(name) + "_" + str(version) + "?filter_by=endpoint_id:" + project_id print("get model:" + resource_url) response_object = util.send_request('GET', resource_url, headers=headers) if response_object.ok: return response_object raise ModelNotFound( "No model with name: {} and version {} could be found".format( name, version))
def _get_featurestore_metadata(featurestore): """ Makes a REST call to hopsworks to get all metadata of a featurestore (featuregroups and training datasets) for the provided featurestore. Args: :featurestore: the name of the database, defaults to the project's featurestore Returns: JSON response Raises: :RestAPIError: if there was an error in the REST call to Hopsworks """ method = constants.HTTP_CONFIG.HTTP_GET connection = util._get_http_connection(https=True) resource_url = (constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + featurestore + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_FEATURESTORE_METADATA_RESOURCE) response = util.send_request(connection, method, resource_url) resp_body = response.read() response_object = json.loads(resp_body) # for python 3 if sys.version_info > (3, 0): if response.code != 200: error_code, error_msg, user_msg = util._parse_rest_error(response_object) raise RestAPIError("Could not fetch featurestore metadata for featurestore: {} (url: {}), " "server response: \n " "HTTP code: {}, HTTP reason: {}, error code: {}, " "error msg: {}, user msg: {}".format( resource_url, featurestore, response.code, response.reason, error_code, error_msg, user_msg)) else: # for python 2 if response.status != 200: error_code, error_msg, user_msg = util._parse_rest_error(response_object) raise RestAPIError("Could not fetch featurestore metadata for featurestore: {} (url: {}), " "server response: \n " \ "HTTP code: {}, HTTP reason: {}, error code: {}, " "error msg: {}, user msg: {}".format( resource_url, featurestore, response.status, response.reason, error_code, error_msg, user_msg)) return response_object
def _delete_table_contents(featuregroup_id, featurestore_id): """ Sends a request to clear the contents of a featuregroup by dropping the featuregroup and recreating it with the same metadata. Args: :featuregroup_id: id of the featuregroup :featurestore_id: id of the featurestore Returns: The JSON response Raises: :RestAPIError: if there was an error in the REST call to Hopsworks """ method = constants.HTTP_CONFIG.HTTP_POST connection = util._get_http_connection(https=True) resource_url = (constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + str(featurestore_id) + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_FEATUREGROUPS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + str(featuregroup_id) + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_FEATUREGROUP_CLEAR_RESOURCE) response = util.send_request(connection, method, resource_url) resp_body = response.read() response_object = json.loads(resp_body) # for python 3 if sys.version_info > (3, 0): if response.code != 200: error_code, error_msg, user_msg = util._parse_rest_error(response_object) raise RestAPIError("Could not clear featuregroup contents (url: {}), server response: \n " "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( resource_url, response.code, response.reason, error_code, error_msg, user_msg)) else: # for python 2 if response.status != 200: error_code, error_msg, user_msg = util._parse_rest_error(response_object) raise RestAPIError("Could not clear featuregroup contents (url: {}), server response: \n " \ "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( resource_url, response.code, response.reason, error_code, error_msg, user_msg)) return response_object
def _get_training_dataset_rest(training_dataset_id, featurestore_id): """ Makes a REST call to hopsworks for getting the metadata of a particular training dataset (including the statistics) Args: :training_dataset_id: id of the training_dataset :featurestore_id: id of the featurestore where the training dataset resides Returns: The REST response Raises: :RestAPIError: if there was an error in the REST call to Hopsworks """ headers = { constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON } method = constants.HTTP_CONFIG.HTTP_GET resource_url = (constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + str(featurestore_id) + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_TRAININGDATASETS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + str(training_dataset_id)) response = util.send_request(method, resource_url, headers=headers) response_object = response.json() if response.status_code != 200: error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise RestAPIError("Could not get the metadata of featuregroup (url: {}), server response: \n " \ "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) return response_object
def _get_featuregroup_rest(featuregroup_id, featurestore_id): """ Makes a REST call to hopsworks for getting the metadata of a particular featuregroup (including the statistics) Args: :featuregroup_id: id of the featuregroup :featurestore_id: id of the featurestore where the featuregroup resides Returns: The REST response Raises: :RestAPIError: if there was an error in the REST call to Hopsworks """ headers = {constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON} method = constants.HTTP_CONFIG.HTTP_GET connection = util._get_http_connection(https=True) resource_url = (constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + str(featurestore_id) + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_FEATUREGROUPS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + str(featuregroup_id)) response = util.send_request(connection, method, resource_url, headers=headers) resp_body = response.read() response_object = json.loads(resp_body) try: # for python 3 if (response.code != 200): error_code, error_msg, user_msg = util._parse_rest_error(response_object) raise RestAPIError("Could not get the metadata of featuregroup (url: {}), server response: \n " \ "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( resource_url, response.code, response.reason, error_code, error_msg, user_msg)) except: # for python 2 if (response.status != 200): error_code, error_msg, user_msg = util._parse_rest_error(response_object) raise RestAPIError("Could not get the metadata of featuregroup (url: {}), server response: \n " \ "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( resource_url, response.status, response.reason, error_code, error_msg, user_msg)) return response_object
def get_current_execution(name): method = constants.HTTP_CONFIG.HTTP_GET resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_JOBS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ name + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_EXECUTIONS_RESOURCE + "?offset=0&limit=1&sort_by=id:desc" response = util.send_request(method, resource_url) response_object = response.json() if response.status_code >= 400: error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise RestAPIError( "Could not get current job's execution (url: {}), server response: \n " "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}" .format(resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) return response_object
def _attach_experiment_xattr(app_id, run_id, json_data, xattr): """ Utility method for putting JSON data into elastic search Args: :project: the project of the user/app :appid: the YARN appid :elastic_id: the id in elastic :json_data: the data to put Returns: None """ json_data = dumps(json_data) headers = {'Content-type': 'application/json'} resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_EXPERIMENTS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ app_id + "_" + str(run_id) + "?xattr=" + xattr response = util.send_request('PUT', resource_url, data=json_data, headers=headers) response_object = response.json() if response.status_code >= 400: error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise RestAPIError( "Could not create experiment (url: {}), server response: \n " "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}" .format(resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) else: return response_object
def get_xattr(hdfs_path, xattr_name=None): """ Get the extended attribute attached to an hdfs_path. Args: :hdfs_path: path of a file or directory :xattr_name: name of the extended attribute Returns: A dictionary with the extended attribute(s) as key value pair(s). If the :xattr_name is None, the API returns all associated extended attributes. """ hdfs_path = urllib.parse.quote(hdfs._expand_path(hdfs_path)) headers = { constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON } method = constants.HTTP_CONFIG.HTTP_GET resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_XATTR_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs_path if xattr_name is not None: resource_url += constants.DELIMITERS.QUESTION_MARK_DELIMITER + constants.XATTRS.XATTRS_PARAM_NAME + \ constants.DELIMITERS.JDBC_CONNECTION_STRING_VALUE_DELIMITER + xattr_name response = util.send_request(method, resource_url, headers=headers) response_object = response.json() if response.status_code >= 400: error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise RestAPIError("Could not get extened attributes attached to a path (url: {}), server response: \n " \ "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) results = {} for item in response_object["items"]: results[item["name"]] = item["value"] return results
def _add_tag(featurestore_id, id, tag, value, resource): """ Makes a REST call to Hopsworks to attach tags to a featuregroup or training dataset Args: :featurestore_id: the id of the featurestore :id: the id of the featuregroup or training dataset :tag: name of the tag to attach :value: value of the tag :resource: featuregroup or training dataset resource Returns: None """ method = constants.HTTP_CONFIG.HTTP_PUT resource_url = ( constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_FEATURESTORES_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + str(featurestore_id) + constants.DELIMITERS.SLASH_DELIMITER + resource + constants.DELIMITERS.SLASH_DELIMITER + str(id) + constants.DELIMITERS.SLASH_DELIMITER + constants.REST_CONFIG.HOPSWORKS_FEATURESTORE_TAGS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + tag) if value is not None: resource_url = resource_url + "?value=" + value response = util.send_request(method, resource_url) response_object = response.json() if response.status_code >= 400: error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise RestAPIError("Could not attach tags (url: {}), server response: \n " \ "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}".format( resource_url, response.status_code, response.reason, error_code, error_msg, user_msg))
def _attach_experiment_xattr(ml_id, json_data, op_type): """ Utility method for putting JSON data into elastic search Args: :ml_id: experiment id :json_data: the experiment json object :op_type: operation type INIT/MODEL_UPDATE/FULL_UPDATE Returns: None """ json_data = dumps(json_data) headers = {'Content-type': 'application/json'} resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_EXPERIMENTS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ ml_id + "?type=" + op_type response = util.send_request('PUT', resource_url, data=json_data, headers=headers) response_object = response.json() if response.status_code >= 400: error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise RestAPIError( "Could not create experiment (url: {}), server response: \n " "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}" .format(resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) else: return response_object
def _attach_model_link_xattr(ml_id, model): """ Utility method for putting JSON data into elastic search Args: :project: the project of the user/app :appid: the YARN appid :elastic_id: the id in elastic :json_data: the data to put Returns: None """ resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_EXPERIMENTS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ ml_id + "?model=" + model + '&xattr=CREATE' resp = util.send_request('PUT', resource_url)
def create_job(name, job_config): """ Create a job in Hopsworks Args: name: Name of the job to be created. job_config: A dictionary representing the job configuration Returns: HTTP(S)Connection """ headers = { constants.HTTP_CONFIG.HTTP_CONTENT_TYPE: constants.HTTP_CONFIG.HTTP_APPLICATION_JSON } job_config["appName"] = name method = constants.HTTP_CONFIG.HTTP_PUT resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_JOBS_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ name response = util.send_request(method, resource_url, data=json.dumps(job_config), headers=headers) response_object = response.json() if response.status_code >= 400: error_code, error_msg, user_msg = util._parse_rest_error( response_object) raise RestAPIError( "Could not create job (url: {}), server response: \n " "HTTP code: {}, HTTP reason: {}, error code: {}, error msg: {}, user msg: {}" .format(resource_url, response.status_code, response.reason, error_code, error_msg, user_msg)) return response_object
def get_schema(topic): """ Gets the Avro schema for a particular Kafka topic. Args: :topic: Kafka topic name Returns: Avro schema as a string object in JSON format """ method = constants.HTTP_CONFIG.HTTP_GET connection = util._get_http_connection(https=True) resource_url = constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_REST_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_PROJECT_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ hdfs.project_id() + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_KAFKA_RESOURCE + constants.DELIMITERS.SLASH_DELIMITER + \ topic + constants.DELIMITERS.SLASH_DELIMITER + \ constants.REST_CONFIG.HOPSWORKS_SCHEMA_RESOURCE response = util.send_request(connection, method, resource_url) resp_body = response.read() response_object = json.loads(resp_body) return response_object