def ask_for_raw_or_aggregated(self, step):
"""
ask for raw or aggregated with queries parameters
:param step: paginated parameters: [OPTIONAL]
the format of the table is:
| parameter | value |
| hLimit | 10 |
| hOffset | 3 |
| dateFrom | 2015-02-14T00:00:00 |
| dateTo | 2015-12-31T00:00:00 |
"""
self.resolution = EMPTY
self.method = EMPTY
self.last_n = EMPTY
self.h_limit = EMPTY
self.h_offset = EMPTY
self.resp = None
initial = True
params = EMPTY
for line in step.hashes: # get step parameters. they are optional
if initial:
initial = False
else:
params = params + "&"
if line["parameter"] == u'aggrMethod': self.method = line["value"]
if line["parameter"] == u'aggrPeriod': self.resolution = line["value"]
if line["parameter"] == u'lastN': self.last_n = line["value"]
if line["parameter"] == u'hLimit': self.h_limit = line["value"]
if line["parameter"] == u'hOffset': self.h_offset = line["value"]
params = params + "%s=%s" % (line["parameter"], line["value"]) # ex: ?hLimit=30&hOffset=0&dateFrom=2015-02-14T00:00:00&dateTo=2015-12-31T00:00:00
url = "%s://%s:%s/%s/type/%s/id/%s/attributes/%s_%s?%s" % (self.protocol, self.sth_host, self.sth_port, URL_PATH, self.entity_type, self.entity_id, self.attributes_name, "0", params)
self.resp = http_utils.request(http_utils.GET, url=url, headers=self.__create_headers())
def verify_sth_version(self):
"""
verify sth version
"""
if self.sth_verify_version.lower().find("true") >= 0:
resp=http_utils.request(http_utils.GET, url=self.sth_url+VERSION)
assert resp.status_code == 200, " ERROR - verifying sth version, error in status code. \n code: %s\n body: %s " % (str(resp.status_code), str(resp.text))
dict_json = general_utils.convert_str_to_dict(resp.text, general_utils.JSON)
assert dict_json[VERSION] == self.sth_version, " ERROR -- the sth version mismatch...\n received: %s \n expected: %s" % (dict_json[VERSION], self.sth_version)
def verify_version (self):
"""
Verify if ckan is installed and that version is the expected, default version is 2.0
"""
if self.ckan_verify_version.lower() == "true":
resp= http_utils.request(http_utils.GET, url=self.__create_url(VERSION), headers=self.__create_headers())
body_dict = general_utils.convert_str_to_dict(resp.text, general_utils.JSON)
assert self.version == str(body_dict[VERSION]), \
"Wrong ckan version verified: %s. Expected: %s. \n\nBody content: %s" % (str(body_dict[VERSION]), str(self.version), str(resp.text))
return True
def __create_datastore_in_resource (self, resource_id, fields):
"""
create a datastore in a resource
:param resource_id: resource id
:param fields: field in datastore
"""
payload = general_utils.convert_dict_to_str({RESOURCE_ID: resource_id,
FIELD:fields,
FORCE: TRUE}, general_utils.JSON)
resp = http_utils.request(http_utils.POST, url=self.__create_url(DATASTORE_CREATE), headers=self.__create_headers(), data=payload)
http_utils.assert_status_code(http_utils.status_codes[http_utils.OK], resp, "ERROR - Creating datastore in resource id: %s" % (resource_id))
def verify_if_dataset_exist(self, name):
"""
Verify if the dataset exist
:param name: dataset name
:return: return True if de dataset does not exist, False if it does exist
"""
resp = http_utils.request(http_utils.GET, url=self.__create_url(PACKAGE_SHOW, name), headers=self.__create_headers())
if resp.status_code == http_utils.status_codes[http_utils.OK]:
bodyDict= general_utils.convert_str_to_dict(resp.text, general_utils.JSON)
self.dataset_id = bodyDict[RESULT][ID]
return self.dataset_id
return False
def create_organization (self, name):
"""
Create a new organization if it does not exist
:param name: organization name
"""
self.organization = name
if not(self.verify_if_organization_exist(name)):
payload = general_utils.convert_dict_to_str({NAME: name}, general_utils.JSON)
resp= http_utils.request(http_utils.POST, url=self.__create_url(ORGANIZATION_CREATE), headers=self.__create_headers(), data=payload)
http_utils.assert_status_code(http_utils.status_codes[http_utils.OK], resp, "ERROR - creating organization: %s ..." % (name))
return True
return False
def verify_if_organization_exist(self, name):
"""
Verify if the organization exist
:param name: organization name
:return: return True if de organization does not exist, False if it does exist
"""
resp = http_utils.request(http_utils.GET, url=self.__create_url(ORGANIZATION_LIST, name), headers=self.__create_headers())
http_utils.assert_status_code(http_utils.status_codes[http_utils.OK], resp, "ERROR - list of the names of the site's organizations...")
body_dict = general_utils.convert_str_to_dict(resp.text, general_utils.JSON)
for i in range(len(body_dict[RESULT])):
if body_dict[RESULT][i] == name: return True
return False
def create_resource(self, name, dataset_name, fields=[]):
self.resource = name
if not(self.verify_if_resource_exist(name, dataset_name)):
payload = general_utils.convert_dict_to_str({NAME: self.resource,
URL: URL_EXAMPLE,
PACKAGE_ID: self.dataset_id}, general_utils.JSON)
resp= http_utils.request(http_utils.POST, url=self.__create_url(RESOURCE_CREATE), headers=self.__create_headers(), data=payload)
http_utils.assert_status_code(http_utils.status_codes[http_utils.OK], resp, "ERROR - creating resource: %s ..." % (name))
bodyDict= general_utils.convert_str_to_dict(resp.text, general_utils.JSON)
self.resource_id = bodyDict[RESULT][ID]
self.__create_datastore_in_resource (self.resource_id, fields)
return self.resource_id
return False
def verify_version(self):
"""
Verify if ckan is installed and that version is the expected, default version is 2.0
"""
if self.ckan_verify_version.lower() == "true":
resp = http_utils.request(http_utils.GET,
url=self.__create_url(VERSION),
headers=self.__create_headers())
body_dict = general_utils.convert_str_to_dict(
resp.text, general_utils.JSON)
assert self.version == str(body_dict[VERSION]), \
"Wrong ckan version verified: %s. Expected: %s. \n\nBody content: %s" % (str(body_dict[VERSION]), str(self.version), str(resp.text))
return True
def verify_sth_version(self):
"""
verify sth version
"""
if self.sth_verify_version.lower().find("true") >= 0:
resp = http_utils.request(http_utils.GET,
url=self.sth_url + VERSION)
assert resp.status_code == 200, " ERROR - verifying sth version, error in status code. \n code: %s\n body: %s " % (
str(resp.status_code), str(resp.text))
dict_json = general_utils.convert_str_to_dict(
resp.text, general_utils.JSON)
assert dict_json[
VERSION] == self.sth_version, " ERROR -- the sth version mismatch...\n received: %s \n expected: %s" % (
dict_json[VERSION], self.sth_version)
def datastore_search_last_sql (self, rows, resource_name, dataset_name):
"""
get last record in a resource
:param name: resource name
:param dataset_name: dataset name
:return: record dict
"""
resource_id = self.verify_if_resource_exist(resource_name, dataset_name)
if resource_id != False:
sql = 'SELECT * from "' + resource_id + '" ORDER BY 1 DESC LIMIT '+str (rows)
resp= http_utils.request(http_utils.POST, url=self.__create_url(DATASTORE_SEARCH_SQL, sql), headers=self.__create_headers(), data=EMPTY)
http_utils.assert_status_code(http_utils.status_codes[http_utils.OK], resp, "ERROR - creating resource: %s ..." % (resource_name))
return resp
return resource_id
def verify_if_dataset_exist(self, name):
"""
Verify if the dataset exist
:param name: dataset name
:return: return True if de dataset does not exist, False if it does exist
"""
resp = http_utils.request(http_utils.GET,
url=self.__create_url(PACKAGE_SHOW, name),
headers=self.__create_headers())
if resp.status_code == http_utils.status_codes[http_utils.OK]:
bodyDict = general_utils.convert_str_to_dict(
resp.text, general_utils.JSON)
self.dataset_id = bodyDict[RESULT][ID]
return self.dataset_id
return False
def create_dataset (self, name):
"""
Create a new dataset if it does not exist
:param name: dataset name
"""
self.dataset = name
if not(self.verify_if_dataset_exist( name)):
payload = general_utils.convert_dict_to_str({NAME: self.dataset,
OWNER_ORG: self.organization}, general_utils.JSON)
resp= http_utils.request(http_utils.POST, url=self.__create_url(PACKAGE_CREATE), headers=self.__create_headers(), data=payload)
http_utils.assert_status_code(http_utils.status_codes[http_utils.OK], resp, "ERROR - creating dataset: %s ..." % (name))
bodyDict= general_utils.convert_str_to_dict(resp.text, general_utils.JSON)
self.dataset_id = bodyDict[RESULT][ID]
return bodyDict[RESULT][ID]
return False
def verify_if_resource_exist(self, name, dataset_name):
"""
Verify if the resource exist in a dataset
:param name: resource name
:param dataset_id:
:return: return True if de resource does not exist, False if it does exist
"""
resp = http_utils.request(http_utils.GET, url=self.__create_url(PACKAGE_SHOW, dataset_name), headers=self.__create_headers())
if resp.status_code == http_utils.status_codes[http_utils.OK]:
body_dict = general_utils.convert_str_to_dict(resp.text, general_utils.JSON)
for i in range(len(body_dict[RESULT][RESOURCES])):
if body_dict[RESULT][RESOURCES][i][NAME] == name:
self.resource_id = body_dict[RESULT][RESOURCES][i][ID]
return self.resource_id
return False
def send_notification(self, identity_id, identity_type):
"""
send a new notification
Note: remmenber to create some attribute
:param identity_id: identityId use in notification request
:param identity_type: identityType use in notification request
:param attrs: attributes list
:return: response (code status, headers and body)
"""
assert len(self.attrs) != 0, \
" ERROR - It is necessary to create the attributes previously. See create_attributes() method... "
self.identity_id = identity_id
self.identity_type = identity_type
self.__insert_identity_id_and_identity_type (self.identity_id, self.identity_type)
payload = self.__create_payload()
return http_utils.request(http_utils.POST, url=self.endpoint_url, headers=self.__create_headers(), data=payload)
def verify_if_organization_exist(self, name):
"""
Verify if the organization exist
:param name: organization name
:return: return True if de organization does not exist, False if it does exist
"""
resp = http_utils.request(http_utils.GET,
url=self.__create_url(
ORGANIZATION_LIST, name),
headers=self.__create_headers())
http_utils.assert_status_code(
http_utils.status_codes[http_utils.OK], resp,
"ERROR - list of the names of the site's organizations...")
body_dict = general_utils.convert_str_to_dict(resp.text,
general_utils.JSON)
for i in range(len(body_dict[RESULT])):
if body_dict[RESULT][i] == name: return True
return False
def create_organization(self, name):
"""
Create a new organization if it does not exist
:param name: organization name
"""
self.organization = name
if not (self.verify_if_organization_exist(name)):
payload = general_utils.convert_dict_to_str({NAME: name},
general_utils.JSON)
resp = http_utils.request(
http_utils.POST,
url=self.__create_url(ORGANIZATION_CREATE),
headers=self.__create_headers(),
data=payload)
http_utils.assert_status_code(
http_utils.status_codes[http_utils.OK], resp,
"ERROR - creating organization: %s ..." % (name))
return True
return False
def __create_datastore_in_resource(self, resource_id, fields):
"""
create a datastore in a resource
:param resource_id: resource id
:param fields: field in datastore
"""
payload = general_utils.convert_dict_to_str(
{
RESOURCE_ID: resource_id,
FIELD: fields,
FORCE: TRUE
}, general_utils.JSON)
resp = http_utils.request(http_utils.POST,
url=self.__create_url(DATASTORE_CREATE),
headers=self.__create_headers(),
data=payload)
http_utils.assert_status_code(
http_utils.status_codes[http_utils.OK], resp,
"ERROR - Creating datastore in resource id: %s" % (resource_id))
def verify_if_resource_exist(self, name, dataset_name):
"""
Verify if the resource exist in a dataset
:param name: resource name
:param dataset_id:
:return: return True if de resource does not exist, False if it does exist
"""
resp = http_utils.request(http_utils.GET,
url=self.__create_url(
PACKAGE_SHOW, dataset_name),
headers=self.__create_headers())
if resp.status_code == http_utils.status_codes[http_utils.OK]:
body_dict = general_utils.convert_str_to_dict(
resp.text, general_utils.JSON)
for i in range(len(body_dict[RESULT][RESOURCES])):
if body_dict[RESULT][RESOURCES][i][NAME] == name:
self.resource_id = body_dict[RESULT][RESOURCES][i][ID]
return self.resource_id
return False
def ask_for_raw_or_aggregated(self, step):
"""
ask for raw or aggregated with queries parameters
:param step: paginated parameters: [OPTIONAL]
the format of the table is:
| parameter | value |
| hLimit | 10 |
| hOffset | 3 |
| dateFrom | 2015-02-14T00:00:00 |
| dateTo | 2015-12-31T00:00:00 |
"""
self.resolution = EMPTY
self.method = EMPTY
self.last_n = EMPTY
self.h_limit = EMPTY
self.h_offset = EMPTY
self.resp = None
initial = True
params = EMPTY
for line in step.hashes: # get step parameters. they are optional
if initial:
initial = False
else:
params = params + "&"
if line["parameter"] == u'aggrMethod': self.method = line["value"]
if line["parameter"] == u'aggrPeriod':
self.resolution = line["value"]
if line["parameter"] == u'lastN': self.last_n = line["value"]
if line["parameter"] == u'hLimit': self.h_limit = line["value"]
if line["parameter"] == u'hOffset': self.h_offset = line["value"]
params = params + "%s=%s" % (
line["parameter"], line["value"]
) # ex: ?hLimit=30&hOffset=0&dateFrom=2015-02-14T00:00:00&dateTo=2015-12-31T00:00:00
url = "%s://%s:%s/%s/type/%s/id/%s/attributes/%s_%s?%s" % (
self.protocol, self.sth_host, self.sth_port, URL_PATH,
self.entity_type, self.entity_id, self.attributes_name, "0",
params)
self.resp = http_utils.request(http_utils.GET,
url=url,
headers=self.__create_headers())
def send_notification(self, identity_id, identity_type):
"""
send a new notification
Note: remmenber to create some attribute
:param identity_id: identityId use in notification request
:param identity_type: identityType use in notification request
:param attrs: attributes list
:return: response (code status, headers and body)
"""
assert len(self.attrs) != 0, \
" ERROR - It is necessary to create the attributes previously. See create_attributes() method... "
self.identity_id = identity_id
self.identity_type = identity_type
self.__insert_identity_id_and_identity_type(self.identity_id,
self.identity_type)
payload = self.__create_payload()
return http_utils.request(http_utils.POST,
url=self.endpoint_url,
headers=self.__create_headers(),
data=payload)
def datastore_search_last_sql(self, rows, resource_name, dataset_name):
"""
get last record in a resource
:param name: resource name
:param dataset_name: dataset name
:return: record dict
"""
resource_id = self.verify_if_resource_exist(resource_name,
dataset_name)
if resource_id != False:
sql = 'SELECT * from "' + resource_id + '" ORDER BY 1 DESC LIMIT ' + str(
rows)
resp = http_utils.request(http_utils.POST,
url=self.__create_url(
DATASTORE_SEARCH_SQL, sql),
headers=self.__create_headers(),
data=EMPTY)
http_utils.assert_status_code(
http_utils.status_codes[http_utils.OK], resp,
"ERROR - creating resource: %s ..." % (resource_name))
return resp
return resource_id
def create_resource(self, name, dataset_name, fields=[]):
self.resource = name
if not (self.verify_if_resource_exist(name, dataset_name)):
payload = general_utils.convert_dict_to_str(
{
NAME: self.resource,
URL: URL_EXAMPLE,
PACKAGE_ID: self.dataset_id
}, general_utils.JSON)
resp = http_utils.request(http_utils.POST,
url=self.__create_url(RESOURCE_CREATE),
headers=self.__create_headers(),
data=payload)
http_utils.assert_status_code(
http_utils.status_codes[http_utils.OK], resp,
"ERROR - creating resource: %s ..." % (name))
bodyDict = general_utils.convert_str_to_dict(
resp.text, general_utils.JSON)
self.resource_id = bodyDict[RESULT][ID]
self.__create_datastore_in_resource(self.resource_id, fields)
return self.resource_id
return False
def create_dataset(self, name):
"""
Create a new dataset if it does not exist
:param name: dataset name
"""
self.dataset = name
if not (self.verify_if_dataset_exist(name)):
payload = general_utils.convert_dict_to_str(
{
NAME: self.dataset,
OWNER_ORG: self.organization
}, general_utils.JSON)
resp = http_utils.request(http_utils.POST,
url=self.__create_url(PACKAGE_CREATE),
headers=self.__create_headers(),
data=payload)
http_utils.assert_status_code(
http_utils.status_codes[http_utils.OK], resp,
"ERROR - creating dataset: %s ..." % (name))
bodyDict = general_utils.convert_str_to_dict(
resp.text, general_utils.JSON)
self.dataset_id = bodyDict[RESULT][ID]
return bodyDict[RESULT][ID]
return False