def ask_for_raw_or_aggregated(self, step): """ ask for raw or aggregated with queries parameters :param step: paginated parameters: [OPTIONAL] the format of the table is: | parameter | value | | hLimit | 10 | | hOffset | 3 | | dateFrom | 2015-02-14T00:00:00 | | dateTo | 2015-12-31T00:00:00 | """ self.resolution = EMPTY self.method = EMPTY self.last_n = EMPTY self.h_limit = EMPTY self.h_offset = EMPTY self.resp = None initial = True params = EMPTY for line in step.hashes: # get step parameters. they are optional if initial: initial = False else: params = params + "&" if line["parameter"] == u'aggrMethod': self.method = line["value"] if line["parameter"] == u'aggrPeriod': self.resolution = line["value"] if line["parameter"] == u'lastN': self.last_n = line["value"] if line["parameter"] == u'hLimit': self.h_limit = line["value"] if line["parameter"] == u'hOffset': self.h_offset = line["value"] params = params + "%s=%s" % (line["parameter"], line["value"]) # ex: ?hLimit=30&hOffset=0&dateFrom=2015-02-14T00:00:00&dateTo=2015-12-31T00:00:00 url = "%s://%s:%s/%s/type/%s/id/%s/attributes/%s_%s?%s" % (self.protocol, self.sth_host, self.sth_port, URL_PATH, self.entity_type, self.entity_id, self.attributes_name, "0", params) self.resp = http_utils.request(http_utils.GET, url=url, headers=self.__create_headers())
def verify_sth_version(self): """ verify sth version """ if self.sth_verify_version.lower().find("true") >= 0: resp=http_utils.request(http_utils.GET, url=self.sth_url+VERSION) assert resp.status_code == 200, " ERROR - verifying sth version, error in status code. \n code: %s\n body: %s " % (str(resp.status_code), str(resp.text)) dict_json = general_utils.convert_str_to_dict(resp.text, general_utils.JSON) assert dict_json[VERSION] == self.sth_version, " ERROR -- the sth version mismatch...\n received: %s \n expected: %s" % (dict_json[VERSION], self.sth_version)
def verify_version (self): """ Verify if ckan is installed and that version is the expected, default version is 2.0 """ if self.ckan_verify_version.lower() == "true": resp= http_utils.request(http_utils.GET, url=self.__create_url(VERSION), headers=self.__create_headers()) body_dict = general_utils.convert_str_to_dict(resp.text, general_utils.JSON) assert self.version == str(body_dict[VERSION]), \ "Wrong ckan version verified: %s. Expected: %s. \n\nBody content: %s" % (str(body_dict[VERSION]), str(self.version), str(resp.text)) return True
def __create_datastore_in_resource (self, resource_id, fields): """ create a datastore in a resource :param resource_id: resource id :param fields: field in datastore """ payload = general_utils.convert_dict_to_str({RESOURCE_ID: resource_id, FIELD:fields, FORCE: TRUE}, general_utils.JSON) resp = http_utils.request(http_utils.POST, url=self.__create_url(DATASTORE_CREATE), headers=self.__create_headers(), data=payload) http_utils.assert_status_code(http_utils.status_codes[http_utils.OK], resp, "ERROR - Creating datastore in resource id: %s" % (resource_id))
def verify_if_dataset_exist(self, name): """ Verify if the dataset exist :param name: dataset name :return: return True if de dataset does not exist, False if it does exist """ resp = http_utils.request(http_utils.GET, url=self.__create_url(PACKAGE_SHOW, name), headers=self.__create_headers()) if resp.status_code == http_utils.status_codes[http_utils.OK]: bodyDict= general_utils.convert_str_to_dict(resp.text, general_utils.JSON) self.dataset_id = bodyDict[RESULT][ID] return self.dataset_id return False
def create_organization (self, name): """ Create a new organization if it does not exist :param name: organization name """ self.organization = name if not(self.verify_if_organization_exist(name)): payload = general_utils.convert_dict_to_str({NAME: name}, general_utils.JSON) resp= http_utils.request(http_utils.POST, url=self.__create_url(ORGANIZATION_CREATE), headers=self.__create_headers(), data=payload) http_utils.assert_status_code(http_utils.status_codes[http_utils.OK], resp, "ERROR - creating organization: %s ..." % (name)) return True return False
def verify_if_organization_exist(self, name): """ Verify if the organization exist :param name: organization name :return: return True if de organization does not exist, False if it does exist """ resp = http_utils.request(http_utils.GET, url=self.__create_url(ORGANIZATION_LIST, name), headers=self.__create_headers()) http_utils.assert_status_code(http_utils.status_codes[http_utils.OK], resp, "ERROR - list of the names of the site's organizations...") body_dict = general_utils.convert_str_to_dict(resp.text, general_utils.JSON) for i in range(len(body_dict[RESULT])): if body_dict[RESULT][i] == name: return True return False
def create_resource(self, name, dataset_name, fields=[]): self.resource = name if not(self.verify_if_resource_exist(name, dataset_name)): payload = general_utils.convert_dict_to_str({NAME: self.resource, URL: URL_EXAMPLE, PACKAGE_ID: self.dataset_id}, general_utils.JSON) resp= http_utils.request(http_utils.POST, url=self.__create_url(RESOURCE_CREATE), headers=self.__create_headers(), data=payload) http_utils.assert_status_code(http_utils.status_codes[http_utils.OK], resp, "ERROR - creating resource: %s ..." % (name)) bodyDict= general_utils.convert_str_to_dict(resp.text, general_utils.JSON) self.resource_id = bodyDict[RESULT][ID] self.__create_datastore_in_resource (self.resource_id, fields) return self.resource_id return False
def verify_version(self): """ Verify if ckan is installed and that version is the expected, default version is 2.0 """ if self.ckan_verify_version.lower() == "true": resp = http_utils.request(http_utils.GET, url=self.__create_url(VERSION), headers=self.__create_headers()) body_dict = general_utils.convert_str_to_dict( resp.text, general_utils.JSON) assert self.version == str(body_dict[VERSION]), \ "Wrong ckan version verified: %s. Expected: %s. \n\nBody content: %s" % (str(body_dict[VERSION]), str(self.version), str(resp.text)) return True
def verify_sth_version(self): """ verify sth version """ if self.sth_verify_version.lower().find("true") >= 0: resp = http_utils.request(http_utils.GET, url=self.sth_url + VERSION) assert resp.status_code == 200, " ERROR - verifying sth version, error in status code. \n code: %s\n body: %s " % ( str(resp.status_code), str(resp.text)) dict_json = general_utils.convert_str_to_dict( resp.text, general_utils.JSON) assert dict_json[ VERSION] == self.sth_version, " ERROR -- the sth version mismatch...\n received: %s \n expected: %s" % ( dict_json[VERSION], self.sth_version)
def datastore_search_last_sql (self, rows, resource_name, dataset_name): """ get last record in a resource :param name: resource name :param dataset_name: dataset name :return: record dict """ resource_id = self.verify_if_resource_exist(resource_name, dataset_name) if resource_id != False: sql = 'SELECT * from "' + resource_id + '" ORDER BY 1 DESC LIMIT '+str (rows) resp= http_utils.request(http_utils.POST, url=self.__create_url(DATASTORE_SEARCH_SQL, sql), headers=self.__create_headers(), data=EMPTY) http_utils.assert_status_code(http_utils.status_codes[http_utils.OK], resp, "ERROR - creating resource: %s ..." % (resource_name)) return resp return resource_id
def verify_if_dataset_exist(self, name): """ Verify if the dataset exist :param name: dataset name :return: return True if de dataset does not exist, False if it does exist """ resp = http_utils.request(http_utils.GET, url=self.__create_url(PACKAGE_SHOW, name), headers=self.__create_headers()) if resp.status_code == http_utils.status_codes[http_utils.OK]: bodyDict = general_utils.convert_str_to_dict( resp.text, general_utils.JSON) self.dataset_id = bodyDict[RESULT][ID] return self.dataset_id return False
def create_dataset (self, name): """ Create a new dataset if it does not exist :param name: dataset name """ self.dataset = name if not(self.verify_if_dataset_exist( name)): payload = general_utils.convert_dict_to_str({NAME: self.dataset, OWNER_ORG: self.organization}, general_utils.JSON) resp= http_utils.request(http_utils.POST, url=self.__create_url(PACKAGE_CREATE), headers=self.__create_headers(), data=payload) http_utils.assert_status_code(http_utils.status_codes[http_utils.OK], resp, "ERROR - creating dataset: %s ..." % (name)) bodyDict= general_utils.convert_str_to_dict(resp.text, general_utils.JSON) self.dataset_id = bodyDict[RESULT][ID] return bodyDict[RESULT][ID] return False
def verify_if_resource_exist(self, name, dataset_name): """ Verify if the resource exist in a dataset :param name: resource name :param dataset_id: :return: return True if de resource does not exist, False if it does exist """ resp = http_utils.request(http_utils.GET, url=self.__create_url(PACKAGE_SHOW, dataset_name), headers=self.__create_headers()) if resp.status_code == http_utils.status_codes[http_utils.OK]: body_dict = general_utils.convert_str_to_dict(resp.text, general_utils.JSON) for i in range(len(body_dict[RESULT][RESOURCES])): if body_dict[RESULT][RESOURCES][i][NAME] == name: self.resource_id = body_dict[RESULT][RESOURCES][i][ID] return self.resource_id return False
def send_notification(self, identity_id, identity_type): """ send a new notification Note: remmenber to create some attribute :param identity_id: identityId use in notification request :param identity_type: identityType use in notification request :param attrs: attributes list :return: response (code status, headers and body) """ assert len(self.attrs) != 0, \ " ERROR - It is necessary to create the attributes previously. See create_attributes() method... " self.identity_id = identity_id self.identity_type = identity_type self.__insert_identity_id_and_identity_type (self.identity_id, self.identity_type) payload = self.__create_payload() return http_utils.request(http_utils.POST, url=self.endpoint_url, headers=self.__create_headers(), data=payload)
def verify_if_organization_exist(self, name): """ Verify if the organization exist :param name: organization name :return: return True if de organization does not exist, False if it does exist """ resp = http_utils.request(http_utils.GET, url=self.__create_url( ORGANIZATION_LIST, name), headers=self.__create_headers()) http_utils.assert_status_code( http_utils.status_codes[http_utils.OK], resp, "ERROR - list of the names of the site's organizations...") body_dict = general_utils.convert_str_to_dict(resp.text, general_utils.JSON) for i in range(len(body_dict[RESULT])): if body_dict[RESULT][i] == name: return True return False
def create_organization(self, name): """ Create a new organization if it does not exist :param name: organization name """ self.organization = name if not (self.verify_if_organization_exist(name)): payload = general_utils.convert_dict_to_str({NAME: name}, general_utils.JSON) resp = http_utils.request( http_utils.POST, url=self.__create_url(ORGANIZATION_CREATE), headers=self.__create_headers(), data=payload) http_utils.assert_status_code( http_utils.status_codes[http_utils.OK], resp, "ERROR - creating organization: %s ..." % (name)) return True return False
def __create_datastore_in_resource(self, resource_id, fields): """ create a datastore in a resource :param resource_id: resource id :param fields: field in datastore """ payload = general_utils.convert_dict_to_str( { RESOURCE_ID: resource_id, FIELD: fields, FORCE: TRUE }, general_utils.JSON) resp = http_utils.request(http_utils.POST, url=self.__create_url(DATASTORE_CREATE), headers=self.__create_headers(), data=payload) http_utils.assert_status_code( http_utils.status_codes[http_utils.OK], resp, "ERROR - Creating datastore in resource id: %s" % (resource_id))
def verify_if_resource_exist(self, name, dataset_name): """ Verify if the resource exist in a dataset :param name: resource name :param dataset_id: :return: return True if de resource does not exist, False if it does exist """ resp = http_utils.request(http_utils.GET, url=self.__create_url( PACKAGE_SHOW, dataset_name), headers=self.__create_headers()) if resp.status_code == http_utils.status_codes[http_utils.OK]: body_dict = general_utils.convert_str_to_dict( resp.text, general_utils.JSON) for i in range(len(body_dict[RESULT][RESOURCES])): if body_dict[RESULT][RESOURCES][i][NAME] == name: self.resource_id = body_dict[RESULT][RESOURCES][i][ID] return self.resource_id return False
def ask_for_raw_or_aggregated(self, step): """ ask for raw or aggregated with queries parameters :param step: paginated parameters: [OPTIONAL] the format of the table is: | parameter | value | | hLimit | 10 | | hOffset | 3 | | dateFrom | 2015-02-14T00:00:00 | | dateTo | 2015-12-31T00:00:00 | """ self.resolution = EMPTY self.method = EMPTY self.last_n = EMPTY self.h_limit = EMPTY self.h_offset = EMPTY self.resp = None initial = True params = EMPTY for line in step.hashes: # get step parameters. they are optional if initial: initial = False else: params = params + "&" if line["parameter"] == u'aggrMethod': self.method = line["value"] if line["parameter"] == u'aggrPeriod': self.resolution = line["value"] if line["parameter"] == u'lastN': self.last_n = line["value"] if line["parameter"] == u'hLimit': self.h_limit = line["value"] if line["parameter"] == u'hOffset': self.h_offset = line["value"] params = params + "%s=%s" % ( line["parameter"], line["value"] ) # ex: ?hLimit=30&hOffset=0&dateFrom=2015-02-14T00:00:00&dateTo=2015-12-31T00:00:00 url = "%s://%s:%s/%s/type/%s/id/%s/attributes/%s_%s?%s" % ( self.protocol, self.sth_host, self.sth_port, URL_PATH, self.entity_type, self.entity_id, self.attributes_name, "0", params) self.resp = http_utils.request(http_utils.GET, url=url, headers=self.__create_headers())
def send_notification(self, identity_id, identity_type): """ send a new notification Note: remmenber to create some attribute :param identity_id: identityId use in notification request :param identity_type: identityType use in notification request :param attrs: attributes list :return: response (code status, headers and body) """ assert len(self.attrs) != 0, \ " ERROR - It is necessary to create the attributes previously. See create_attributes() method... " self.identity_id = identity_id self.identity_type = identity_type self.__insert_identity_id_and_identity_type(self.identity_id, self.identity_type) payload = self.__create_payload() return http_utils.request(http_utils.POST, url=self.endpoint_url, headers=self.__create_headers(), data=payload)
def datastore_search_last_sql(self, rows, resource_name, dataset_name): """ get last record in a resource :param name: resource name :param dataset_name: dataset name :return: record dict """ resource_id = self.verify_if_resource_exist(resource_name, dataset_name) if resource_id != False: sql = 'SELECT * from "' + resource_id + '" ORDER BY 1 DESC LIMIT ' + str( rows) resp = http_utils.request(http_utils.POST, url=self.__create_url( DATASTORE_SEARCH_SQL, sql), headers=self.__create_headers(), data=EMPTY) http_utils.assert_status_code( http_utils.status_codes[http_utils.OK], resp, "ERROR - creating resource: %s ..." % (resource_name)) return resp return resource_id
def create_resource(self, name, dataset_name, fields=[]): self.resource = name if not (self.verify_if_resource_exist(name, dataset_name)): payload = general_utils.convert_dict_to_str( { NAME: self.resource, URL: URL_EXAMPLE, PACKAGE_ID: self.dataset_id }, general_utils.JSON) resp = http_utils.request(http_utils.POST, url=self.__create_url(RESOURCE_CREATE), headers=self.__create_headers(), data=payload) http_utils.assert_status_code( http_utils.status_codes[http_utils.OK], resp, "ERROR - creating resource: %s ..." % (name)) bodyDict = general_utils.convert_str_to_dict( resp.text, general_utils.JSON) self.resource_id = bodyDict[RESULT][ID] self.__create_datastore_in_resource(self.resource_id, fields) return self.resource_id return False
def create_dataset(self, name): """ Create a new dataset if it does not exist :param name: dataset name """ self.dataset = name if not (self.verify_if_dataset_exist(name)): payload = general_utils.convert_dict_to_str( { NAME: self.dataset, OWNER_ORG: self.organization }, general_utils.JSON) resp = http_utils.request(http_utils.POST, url=self.__create_url(PACKAGE_CREATE), headers=self.__create_headers(), data=payload) http_utils.assert_status_code( http_utils.status_codes[http_utils.OK], resp, "ERROR - creating dataset: %s ..." % (name)) bodyDict = general_utils.convert_str_to_dict( resp.text, general_utils.JSON) self.dataset_id = bodyDict[RESULT][ID] return bodyDict[RESULT][ID] return False