def setup_logging_no_config(loglevel, quiet):
log_level = aprsd_config.LOG_LEVELS[loglevel]
LOG.setLevel(log_level)
log_format = aprsd_config.DEFAULT_LOG_FORMAT
date_format = aprsd_config.DEFAULT_DATE_FORMAT
log_formatter = logging.Formatter(fmt=log_format, datefmt=date_format)
fh = NullHandler()
fh.setFormatter(log_formatter)
LOG.addHandler(fh)
if not quiet:
sh = logging.StreamHandler(sys.stdout)
sh.setFormatter(log_formatter)
LOG.addHandler(sh)
def create_logger(robot, path=None, debug=False):
logger = logging.getLogger(robot)
logger.setLevel(logging.DEBUG if debug else logging.INFO)
if path is None:
handler = NullHandler()
else:
logFile = os.path.join(path, robot + '.log')
handler = TimedRotatingFileHandler(logFile, when='midnight')
formatter = Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
return logger
class SnowRestSession(object):
"""
This class behaves very similarly to a requests.Session object.
It can be configured by loading a .yaml config file,
or by setting attributes.
"""
def __init__(self):
self.instance = None
self.auth_type = None
self.sso_method = None
self.basic_auth_user = None
self.basic_auth_password = None
self.oauth_client_id = None
self.oauth_client_secret = None
self.session_cookie_file_path = None
self.session_cookie = None
self.oauth_token_file_path = None
self.token_dic = None
self.fresh_cookie = False
self.fresh_token = False
self.store_cookie = True
self.store_token = True
self.session = requests.Session()
self._log_enabled = False
self._logger_name = 'snow-client.session_' + uuid.uuid4(
).hex[:-24] # used to get a Logger
self._logger = logging.getLogger(self._logger_name)
self._log_handler = NullHandler()
self._logger.addHandler(self._log_handler)
self._log_file_path = None
self._log_level = 'INFO'
self._log_format = "%(asctime)s [%(name)s] [%(levelname)s] %(message)s"
self._log_file_size_bytes = 1000000
self._log_file_rotations = 10
self._log_file_encoding = 'utf-8'
def load_config_file(self, config_file_path):
"""
Load a .yaml configuration file
Args:
config_file_path (str): the path to the configuration file
Raises:
SnowRestSessionException: if no file is found at the specified path, or some inconsistency is found
in the file.
Exception: if some other error happens when opening or parsing the file
"""
if not config_file_path:
raise SnowRestSessionException(
"SnowRestSession.load_config_file: the parameter config_file_path must have a value"
)
try:
with open(config_file_path) as f:
config_file = yaml.safe_load(f)
except Exception as e:
sys.stderr.write(
"SnowRestSession.load_config_file: Issue when opening the config file at %s.\n"
% config_file_path)
raise e
if 'instance' in config_file:
self.set_instance(config_file['instance'])
if 'auth' in config_file:
if 'type' in config_file['auth']:
self.auth_type = config_file['auth']['type']
if self.auth_type == 'sso_oauth':
if 'sso_method' in config_file['auth']:
self.sso_method = config_file['auth']['sso_method']
if 'oauth_client_id' in config_file['auth']:
self.oauth_client_id = config_file['auth']['oauth_client_id']
if 'oauth_client_secret' in config_file['auth']:
self.oauth_client_secret = config_file['auth'][
'oauth_client_secret']
if 'session' in config_file:
if 'cookie_file' in config_file['session']:
self.session_cookie_file_path = config_file['session'][
'cookie_file']
if 'oauth_tokens_file' in config_file['session']:
self.oauth_token_file_path = config_file['session'][
'oauth_tokens_file']
elif self.auth_type == 'basic':
if 'user' in config_file['auth']:
self.basic_auth_user = config_file['auth']['user']
if 'password' in config_file['auth']:
self.basic_auth_password = config_file['auth']['password']
if 'session' in config_file:
if 'cookie_file' in config_file['session']:
self.session_cookie_file_path = config_file['session'][
'cookie_file']
else:
raise SnowRestSessionException(
"SnowRestSession.load_config_file: the property \"auth_type\" "
"must have a value of \"sso_auth\" or \"basic\"")
if 'log' in config_file:
if 'log_enabled' in config_file['log'] and config_file['log'][
'log_enabled']:
self._log_enabled = True
if 'log_level' in config_file['log']:
self.set_log_level(config_file['log']['log_level'])
if 'log_file_path' in config_file['log']:
self._log_file_path = config_file['log']['log_file_path']
if 'log_format' in config_file['log']:
self._log_format = config_file['log']['log_format']
if 'log_file_size_bytes' in config_file['log']:
self._log_file_size_bytes = config_file['log'][
'log_file_size_bytes']
if 'log_file_rotations' in config_file['log']:
self._log_file_rotations = config_file['log'][
'log_file_rotations']
if 'log_file_encoding' in config_file['log']:
self._log_file_encoding = config_file['log'][
'log_file_encoding']
self._configure_handler()
def set_instance(self, instance):
"""
Args:
instance (str): the ServiceNow instance to use. It is optional to prefix with "https://".
Examples:
>>> s = SnowRestSession()
>>> s.set_instance("cern.service-now.com")
>>>
>>> s2 = SnowRestSession()
>>> s2.set_instance("https://cerntest.service-now.com")
"""
if not instance.startswith('https://'):
self.instance = 'https://' + instance
else:
self.instance = instance
def set_auth_type(self, auth_type):
"""
Args:
auth_type (str): the authentication type to use. Either 'sso_auth' or 'basic'
Raises:
SnowRestSessionException: if auth_type is not 'sso_auth' or 'basic'
"""
if auth_type != 'basic' and auth_type != 'sso_auth':
raise SnowRestSessionException(
"SnowRestSession.set_auth_type: the parameter \"auth_type\" "
"must have a value of \"sso_auth\" or \"basic\"")
self.auth_type = auth_type
def set_sso_method(self, sso_method):
"""
Args:
sso_method (str): the Single Sign On method to use. Either 'kerberos' or 'certificate'.
Needs set_auth_type('sso_auth') to have an effect.
Raises:
SnowRestSessionException: if sso_method is not 'kerberos' or 'certificate'
Note: certificate support needs to be implemented
"""
if sso_method != 'kerberos' and sso_method != 'certificate':
raise SnowRestSessionException(
"SnowRestSession.set_sso_method: the parameter \"sso_method\" "
"must have a value of \"kerberos\" or \"certificate\"")
# TODO: implement certificate support. Update the comment above
if sso_method == 'certificate':
raise SnowRestSessionException(
"SnowRestSession.set_sso_method: certificate support is not yet implemented"
)
self.sso_method = sso_method
def set_oauth_client_id(self, oauth_client_id):
"""
Args:
oauth_client_id (str): the OAuth client id provided to you by your instance's ServiceNow administrator.
Needs set_auth_type('sso_auth') to have an effect.
"""
self.oauth_client_id = oauth_client_id
def set_oauth_client_secret(self, oauth_client_secret):
"""
Args:
oauth_client_secret (str): the OAuth client secret provided to you by your instance's
ServiceNow administrator.
Needs set_auth_type('sso_auth') to have an effect.
An OAuth client secret is sensitive information; please store it as securely as possible.
"""
self.oauth_client_secret = oauth_client_secret
def set_basic_auth_user(self, basic_auth_user):
"""
Args:
basic_auth_user (str): the name of the local ServiceNow account that will be used for authentication.
Needs set_auth_type('basic') to have an effect.
"""
self.basic_auth_user = basic_auth_user
def set_basic_auth_password(self, basic_auth_password):
"""
Args:
basic_auth_password (str): the password of the local ServiceNow account that will be used for
authentication.
Needs set_auth_type('basic') to have an effect.
"""
self.basic_auth_password = basic_auth_password
def set_session_cookie_file_path(self, session_cookie_file_path):
"""
Args:
session_cookie_file_path (str): the path to the file where the session cookies will be persisted.
If not provided, the cookies will not be persisted.
"""
self.session_cookie_file_path = session_cookie_file_path
def set_oauth_token_file_path(self, oauth_token_file_path):
"""
Args:
oauth_token_file_path (str): the path to the file where the OAuth refresh and access tokens
will be persisted. If not provided, the OAuth tokens will not be persisted.
"""
self.oauth_token_file_path = oauth_token_file_path
def set_log_enabled(self, log_enabled):
self._log_enabled = log_enabled
def set_log_level(self, log_level):
self._log_level = log_level
self._logger.setLevel(self._log_level)
def set_log_file_path(self, log_file_path):
self._log_file_path = log_file_path
self._configure_handler()
def set_log_format(self, log_format):
self._log_format = log_format
self._configure_handler()
def set_log_file_size_bytes(self, log_file_size_bytes):
self._log_file_size_bytes = log_file_size_bytes
self._configure_handler()
def set_log_file_rotations(self, log_file_rotations):
self._log_file_rotations = log_file_rotations
self._configure_handler()
def set_log_file_encoding(self, log_file_encoding):
self._log_file_encoding = log_file_encoding
self._configure_handler()
def is_logging_enabled(self):
return self._log_enabled
def get_logger_name(self):
return self._logger_name
def get_logger(self):
return self._logger
def get_log_handler(self):
return self._log_handler
def get(self, url, headers=None, params=None):
"""
Executes a raw GET operation and returns the result.
Used to read or retrieve information.
The authentication method, and session cookie / OAuth tokens persistance will depend on how the session
has been configured.
Args:
url (str): a relative URL, such as ``/api/now/v2/table/incident/c1c535ba85f45540adf94de5b835cd43``.
An absolute URL such as ``https://cerntest.service-now.com/api/now/v2/table/incident/c1c535ba85f45540adf94de5b835cd43`` can also
be used, but the instance should match with the "instance" parameter in the configuration file,
or the value set with the ``.set_instance()`` method.
headers (:obj:`dict`, optional): any additional headers to be be passed. If not set, the 'Accept' header will be set to
'application/json'
params (:obj:`dict`, optional): any additional URL parameters to be be passed
Returns:
requests.Response : If the status code is not 401, a ``requests.Response`` object is returned.
The structure of the result inside the 'text' attribute might vary depending on the REST endpoint
called and the way that the query is configured. ServiceNow may also set various HTTP status codes
or headers in the response.
Raises:
SnowRestSessionException : if the operation could not be performed due to an authentication issue
Examples:
Getting an incident by sys_id with the ServiceNow REST Table API:
>>> s = SnowRestSession()
>>> s.load_config_file('config.yaml')
>>> response = s.get('/api/now/v2/table/incident/c1c535ba85f45540adf94de5b835cd43')
>>> if response.status_code == 200:
>>> import json
>>> result = json.loads(response.text)
>>> if 'result' in result:
>>> incident = result['result'] # querying by sys_id returns a single object
>>> print incident['short_description'] # will print "Test" (as a unicode object)
Getting an incident by number with the ServiceNow REST Table API:
>>> s = SnowRestSession()
>>> s.load_config_file('config.yaml')
>>> response = s.get('/api/now/v2/table/incident?sysparm_query=number=INC0426232')
>>> if response.status_code == 200:
>>> import json
>>> result = json.loads(response.text)
>>> if 'result' in result:
>>> incident_array = result['result'] # querying via an encoded query returns an array of objects
>>> if len(incident_array) > 0:
>>> incident = incident_array[0]
>>> print incident['short_description'] # will print "Test" (as a unicode object)
"""
result = self.__operation(operation='get',
url=url,
headers=headers,
params=params)
return result
def post(self, url, headers=None, params=None, data=None):
"""
Executes a raw POST operation and returns the result.
Used to insert records or other information.
The authentication method, and session cookie / OAuth tokens persistance will depend on how the session
has been configured.
Args:
url (str): a relative URL, such as ``/api/now/v2/table/incident``.
An absolute URL such as ``https://cerntest.service-now.com/api/now/v2/table/incident`` can also
be used, but the instance should match with the "instance" parameter in the configuration file,
or the value set with the ``.set_instance()`` method.
headers (:obj:`dict`, optional): any additional headers to be be passed. If not set, the 'Accept' header
will be set to 'application/json', and 'Content' will be set to 'application/json'
params (:obj:`dict`, optional): any additional URL parameters to be be passed
data (str): the data that will be submitted via POST. Typically, a dictionary
of keys and values, turned into a string with import json; json.dumps()
Returns:
requests.Response : If the status code is not 401, a ``requests.Response`` object is returned.
The structure of the result inside the 'text' attribute might vary depending on the REST endpoint
called, but typically represents the record resulting from the insert. ServiceNow may also set various
HTTP status codes or headers in the response.
Raises:
SnowRestSessionException : if the operation could not be performed due to an authentication issue
Examples:
Inserting an incident with the ServiceNow REST Table API:
>>> s = SnowRestSession()
>>> s.load_config_file('config.yaml')
>>> incident_attributes = {
>>> 'short_description' : "New incident",
>>> 'u_business_service' : 'e85a3f3b0a0a8c0a006a2912f2f352d1', # Service Element "ServiceNow"
>>> 'u_functional_element' : '579fb3d90a0a8c08017ac8a1137c8ee6', # Functional Element "ServiceNow"
>>> 'comments' : "Initial description"
>>> }
>>> import json
>>> data = json.dumps(incident_attributes)
>>> response = s.post('/api/now/v2/table/incident', data=data)
>>> if response.status_code == 201:
>>> result = json.loads(response.text)
>>> if 'result' in result:
>>> incident = result['result']
>>> print incident['number'] # will print the number of the newly created incident
"""
result = self.__operation(operation='post',
url=url,
headers=headers,
params=params,
data=data)
return result
def put(self, url, headers=None, params=None, data=None):
"""
Executes a raw PUT operation and returns the result.
Used to update records or other information.
The authentication method, and session cookie / OAuth tokens persistance will depend on how the session
has been configured.
Args:
url (str): a relative URL, such as ``/api/now/v2/table/incident/c1c535ba85f45540adf94de5b835cd43``.
An absolute URL such as ``https://cerntest.service-now.com/api/now/v2/table/incident/c1c535ba85f45540adf94de5b835cd43`` can also
be used, but the instance should match with the "instance" parameter in the configuration file,
or the value set with the ``.set_instance()`` method.
headers (:obj:`dict`, optional): any additional headers to be be passed. If not set, the 'Accept' header will be set to
'application/json', and 'Content' will be set to 'application/json'
params (:obj:`dict`, optional): any additional URL parameters to be be passed
data (str): the data that will be submitted via PUT. Typically, a dictionary
of keys and values, turned into a string with import json; json.dumps()
Returns:
requests.Response : If the status code is not 401, a ``requests.Response`` object is returned.
The structure of the result inside the 'text' attribute might vary depending on the REST endpoint
called, but typically represents the state of the record after the update.
ServiceNow may also set various HTTP status codes or headers in the response.
Raises:
SnowRestSessionException : if the operation could not be performed due to an authentication issue
Examples:
Updating an incident with the ServiceNow REST Table API:
>>> s = SnowRestSession()
>>> s.load_config_file('config.yaml')
>>> incident_attributes_to_update = {
>>> 'watch_list' : '*****@*****.**' # will completely replace the previous value
>>> }
>>> import json
>>> data = json.dumps(incident_attributes_to_update)
>>> response = s.put('/api/now/v2/table/incident/ca37ed334f56830015d3bc511310c7a8', data=data)
>>> if response.status_code == 200:
>>> result = json.loads(response.text)
>>> if 'result' in result:
>>> incident = result['result']
>>> print incident['number'] # will print the number of the updated incident
>>> print incident['watch_list'] # will print the new value of the watch_list field
"""
result = self.__operation(operation='put',
url=url,
headers=headers,
params=params,
data=data)
return result
def __call_cern_get_sso_cookie(self):
"""
Executes the cern-get-sso-cookie command (available in CERN Linux environments)
in order to perform a Single Sign-On and produce a cookie file.
"""
args = [
"cern-get-sso-cookie", "--reprocess", "--url", self.instance,
"--outfile", self.session_cookie_file_path
]
p = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p.wait()
def __good_cookie(self):
cookie_file = open(self.session_cookie_file_path)
cookie_file_contents = cookie_file.read()
if (cookie_file_contents.find('glide_user_activity') != -1
and cookie_file_contents.find('glide_session_store') != -1
and cookie_file_contents.find('glide_user_route') != -1
and cookie_file_contents.find('JSESSIONID') != -1
and cookie_file_contents.find('BIGipServerpool_cern') != -1):
return True
return False
def __cern_get_sso_cookie(self):
"""
Loads and checks an existing cookie file, in order to avoid logging into ServiceNow and the
CERN Single-Sign-On if possible. If needed, logs in again into both to produce the cookie file.
The cookie file is loaded into memory.
If not needed to persist the cookie file, it is deleted.
Raises:
SnowRestSessionException: if the Single Sign On login is not succesful. This is determined by parsing
the cookie file after it is produced.
"""
if not os.path.exists(
self.session_cookie_file_path
): # cookie file not present. We perform Single Sign On
self.__call_cern_get_sso_cookie()
self.fresh_cookie = True
else: # cookie file present. We check if it has the correct format
self.fresh_cookie = False
if not self.__good_cookie(
): # the format is bad. We create a new cookie file
self.__call_cern_get_sso_cookie()
self.fresh_cookie = True
# we load the cookie file
self.session_cookie = cookielib.MozillaCookieJar(
self.session_cookie_file_path)
self.session_cookie.load()
# we check if the log in was sucessful
if not self.__good_cookie():
raise SnowRestSessionException(
"SnowRestSession.__cern_get_sso_cookie: The current account has failed to perform "
"a Single-Sign-On login in to ServiceNow")
# we load the cookies into the requests.Session session
self.session.cookies = self.session_cookie
# if not needed to persist the cookie file, we delete it
if not self.store_cookie:
os.remove(self.session_cookie_file_path)
def __obtain_tokens(self):
"""
Loads the OAuth access and refresh tokens from the token file.
If the token file is not present, new OAuth access and refresh tokens are obtained.
Raises:
SnowRestSessionException: if the OAuth tokens could not be retrieved from ServiceNow.
IOError, ValueError: if the token file could not be opened
"""
if not os.path.exists(
self.oauth_token_file_path
): # tokens file not present. We obtain new tokens
self.fresh_token = True
token_request = self.session.post(self.instance +
'/oauth_token.do',
data={
'grant_type':
'password',
'client_id':
self.oauth_client_id,
'client_secret':
self.oauth_client_secret
})
if token_request.status_code == 200: # token request was successful
self.token_dic = json.loads(token_request.text)
if self.store_token:
with open(self.oauth_token_file_path, 'w') as token_file:
json.dump(self.token_dic, token_file)
else: # token request failed. Possibly, an existing cookie file contained a timed out session
if self.fresh_cookie: # we just performed a Single-Sign-On. There is a problem with token retrieval.
raise SnowRestSessionException(
"SnowRestSession.__obtain_tokens: OAuth tokens could not be retrieved from ServiceNow. "
"Please check the OAuth client id and OAuth client secret."
)
else: # we may need to perform Single-Sign-On again, as the session may have timed out
self.__cern_get_sso_cookie()
token_request = self.session.post(
self.instance + '/oauth_token.do',
data={
'grant_type': 'password',
'client_id': self.oauth_client_id,
'client_secret': self.oauth_client_secret
})
self.token_dic = None
if token_request.status_code == 200:
self.token_dic = json.loads(token_request.text)
if self.store_token:
with open(self.oauth_token_file_path,
'w') as token_file:
json.dump(self.token_dic, token_file)
else:
raise SnowRestSessionException(
"SnowRestSession.__obtain_tokens: OAuth tokens could not be retrieved from ServiceNow. "
"Please check the OAuth client id and OAuth client secret."
)
else: # the tokens file is present. We try to load it
try:
with open(self.oauth_token_file_path, 'r') as token_file:
self.token_dic = json.load(token_file)
except (IOError, ValueError) as e:
sys.stderr.write(
"SnowRestSession.__obtain_tokens: Issue when opening "
"the token file at %s.\n" % self.oauth_token_file_path)
raise e
def __initiate_session(self):
"""
Initiates a session via CERN Single-Sign-On + OAuth, or basic Authentication
Raises:
SnowRestSessionException: if auth_type is not 'sso_auth' nor 'basic'
"""
if self.auth_type == 'sso_oauth':
self.__cern_get_sso_cookie()
self.__obtain_tokens()
elif self.auth_type == 'basic':
self.session.auth = (self.basic_auth_user,
self.basic_auth_password)
self.session.cookies = cookielib.MozillaCookieJar()
if os.path.exists(self.session_cookie_file_path):
self.session.cookies.load(self.session_cookie_file_path,
ignore_discard=True,
ignore_expires=True)
else:
raise SnowRestSessionException(
"SnowRestSession.__initiate_session: self.auth_type "
"has a value different from \"basic\" and \"sso_auth\"")
def __refresh_token(self):
"""
Requests an OAuth access token via the OAUTH refresh token.
Saves the resulting access token in the token file.
Raises:
SnowRestSessionException: if the access token could not be obtained
"""
token_request = self.session.post(self.instance + '/oauth_token.do',
data={
'grant_type':
'refresh_token',
'client_id':
self.oauth_client_id,
'client_secret':
self.oauth_client_secret,
'refresh_token':
self.token_dic['refresh_token']
})
if token_request.status_code == 200:
self.token_dic = json.loads(token_request.text)
with open(self.oauth_token_file_path, 'w') as token_file:
json.dump(self.token_dic, token_file)
else:
token_request = self.session.post(self.instance +
'/oauth_token.do',
data={
'grant_type':
'password',
'client_id':
self.oauth_client_id,
'client_secret':
self.oauth_client_secret
})
if token_request.status_code == 200:
self.token_dic = json.loads(token_request.text)
with open(self.oauth_token_file_path, 'w') as token_file:
json.dump(self.token_dic, token_file)
else:
if self.fresh_cookie:
raise SnowRestSessionException(
"SnowRestSession.__refresh_token: the OAuth client id and OAuth secret might not be valid"
)
def __save_cookie_basic(self):
"""
Persists the basic authentication cookie file.
"""
self.fresh_cookie = True
self.session.cookies.save(self.session_cookie_file_path,
ignore_discard=True,
ignore_expires=True)
@staticmethod
def __library_user_agent():
user_agent_header = 'cern-snow-client/' + __version__
default_headers = requests.utils.default_headers()
if 'User-Agent' in default_headers:
user_agent_header = user_agent_header + ' ' + default_headers[
'User-Agent']
return user_agent_header
def __execute(self, operation, url, headers=None, params=None, data=None):
"""
Executes directly a REST Operation and returns the result
Args:
operation (str): either 'get', 'post' or 'put'
url (str): a relative URL, such as "/api/now/v2/table/incident".
An absolute URL such as "https://cerntest.service-now.com/api/now/v2/table/incident" can also
be used, but the instance should match with the "instance" parameter in the configuration file,
or the value set with the ``.set_instance()`` method.
headers (:obj:`dict`, optional): any additional headers to be be passed. This method will add
Accept:application/json and Content-Type:application/json if not specified.
It will also add the Authorization header if the authentication type is 'sso_auth',
by setting it to the OAuth access header.
params (:obj:`dict`, optional): any additional URL parameters to be be passed
data (object): the data to be sent in a post or put operation
Returns:
requests.Response
Raises:
SnowRestSessionException: if the operation parameter is not 'get', 'post' or 'put'
"""
if not operation:
raise SnowRestSessionException(
"SnowRestSession.__execute: the operation paramater is mandatory"
)
if not url:
raise SnowRestSessionException(
"SnowRestSession.__execute: the url paramater is mandatory")
if not url.startswith('https://'):
url = self.instance + url
if not url.startswith('https://'):
url = self.instance + url
if not headers:
headers = {}
if 'User-Agent' not in headers:
headers['User-Agent'] = self.__library_user_agent()
if 'Accept' not in headers:
headers['Accept'] = 'application/json'
if (operation == 'post'
or operation == 'put') and 'Content-Type' not in headers:
headers['Content-Type'] = 'application/json'
if self.auth_type == 'sso_oauth':
headers[
'Authorization'] = 'Bearer ' + self.token_dic['access_token']
if operation == 'get':
result = self.session.get(url, headers=headers, params=params)
elif operation == 'post':
result = self.session.post(url,
headers=headers,
params=params,
data=data)
elif operation == 'put':
result = self.session.put(url,
headers=headers,
params=params,
data=data)
else:
raise SnowRestSessionException(
"SnowRestSession.__execute: the operation paramater "
"needs to be either \"get\", \"post\" or \"put\"")
return result
def __operation(self,
operation,
url,
headers=None,
params=None,
data=None):
"""
Executes a REST Operation, taking care of reauthenticating if needed, and returns the result
Args:
operation (str): either 'get', 'post' or 'put'
url (str): a relative URL, such as "/api/now/v2/table/incident".
An absolute URL such as "https://cerntest.service-now.com/api/now/v2/table/incident" can also
be used, but the instance should match with the "instance" parameter in the configuration file,
or the value set with the ``.set_instance()`` method.
headers (:obj:`dict`, optional): any additional headers to be be passed. If not set, some headers will be set by
default (see __execute method)
params (:obj:`dict`, optional): any additional URL parameters to be be passed
data (object): the data to be sent in a post or put operation
Returns:
requests.Response : if the status code is not 401, a requests.Response object is returned
Raises:
SnowRestSessionException : if the operation could not be performed due to an authentication issue
"""
self.__initiate_session()
result = self.__execute(operation,
url,
headers=headers,
params=params,
data=data)
if result.status_code != 401:
if self.auth_type == 'basic':
self.__save_cookie_basic()
return result
else:
if self.auth_type == 'basic':
if not self.fresh_cookie:
self.session.auth = (self.basic_auth_user,
self.basic_auth_password)
result = self.__execute(operation,
url,
headers=headers,
params=params,
data=data)
if result.status_code != 401:
self.__save_cookie_basic()
return result
else:
raise SnowRestSessionException(
"SnowRestSession.__operation: Your basic authentication "
"user and password might not be valid")
else:
raise SnowRestSessionException(
"SnowRestSession.__operation: Your basic authentication "
"user and password might not be valid")
elif self.auth_type == 'sso_auth':
if self.fresh_token:
raise SnowRestSessionException(
"SnowRestSession.__operation: failed to perform the operation. The current account might not "
"be able to log in to ServiceNow or the OAuth client id and secret might not be valid"
)
else:
self.__refresh_token()
token_request = self.session.post(
'post',
self.instance + '/oauth_token.do',
data={
'grant_type': 'password',
'client_id': self.oauth_client_id,
'client_secret': self.oauth_client_secret
})
if token_request.status_code == 200:
self.token_dic = json.loads(token_request.text)
with open(self.oauth_token_file_path,
'w') as token_file:
json.dump(self.token_dic, token_file)
headers['Authorization'] = 'Bearer ' + self.token_dic[
'access_token']
result = self.__execute(operation,
url,
headers=headers,
params=params,
data=data)
if result.status_code != 401:
return result
else:
raise SnowRestSessionException(
"SnowRestSession.__operation: failed to perform the operation. "
"The current account might not be able to log in to ServiceNow or "
"the OAuth client id and secret might not be valid"
)
else:
if self.fresh_cookie:
raise SnowRestSessionException(
"SnowRestSession.__operation: OAuth tokens could not be retrieved from ServiceNow. "
"Please check the OAuth client id and OAuth client secret."
)
else:
os.remove(self.session_cookie_file_path)
self.__cern_get_sso_cookie()
token_request = self.session.post(
'post',
self.instance + '/oauth_token.do',
data={
'grant_type': 'password',
'client_id': self.oauth_client_id,
'client_secret': self.oauth_client_secret
})
if token_request.status_code == 401:
raise SnowRestSessionException(
"SnowRestSession.__operation: OAuth tokens could not be retrieved from ServiceNow. "
"Please check the OAuth client id and OAuth client secret."
)
else:
self.token_dic = json.loads(token_request.text)
with open(self.oauth_token_file_path,
'w') as token_file:
json.dump(self.token_dic, token_file)
headers[
'Authorization'] = 'Bearer ' + self.token_dic[
'access_token']
result = self.__execute(operation,
url,
headers=headers,
params=params,
data=data)
if result.status_code != 401:
return result
else:
raise SnowRestSessionException(
"SnowRestSession.__operation: failed to perform the operation. "
"The current account might not be able to log in to ServiceNow or "
"the OAuth client id and secret might not be valid"
)
else:
raise SnowRestSessionException(
"SnowRestSession.__operation: self.auth_type has a value different from \"basic\" and \"sso_auth\""
)
def _configure_handler(self):
self._logger.removeHandler(self._log_handler)
file_handler_attributes_known = bool(self._log_file_path
and self._log_file_size_bytes
and self._log_file_rotations
and self._log_file_encoding
and self._log_format)
if file_handler_attributes_known:
self._log_handler = RotatingFileHandler(
self._log_file_path,
maxBytes=self._log_file_size_bytes,
backupCount=self._log_file_rotations - 1,
encoding=self._log_file_encoding)
self._log_handler.setFormatter(logging.Formatter(self._log_format))
else:
self._log_handler = NullHandler()
self._logger.addHandler(self._log_handler)
def _debug(self, message):
self._log(logging.DEBUG, message)
def _info(self, message):
self._log(logging.INFO, message)
def _warning(self, message):
self._log(logging.WARNING, message)
def _error(self, message):
self._log(logging.ERROR, message)
def _critical(self, message):
self._log(logging.CRITICAL, message)
def _log(self, level, message):
if self._log_enabled:
self._logger.log(level, message)
class DexcellRestApi(object):
"""
class with all the utils API calls available group by
from deployment calls, location calls and devide calls.
"""
def __init__(self, endpoint, token, logger_name="dexcell_rest_api"):
self.endpoint = endpoint
self.token = token
self.logger = logging.getLogger(logger_name)
if len(self.logger.handlers) == 0:
self.logger.setLevel(logging.INFO)
self.handler = NullHandler()
h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
self.handler.setFormatter(logging.Formatter(h_format))
self.logger.addHandler(self.handler)
def _json_date_handler(self, obj):
return obj.isoformat() if hasattr(obj, 'isoformat') else obj
def _datetime_parser(self, dct):
DATE_FORMAT = "%Y-%m-%dT%H:%M:%S"
for k, v in dct.items():
if isinstance(v, basestring) and re.search(
"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", v):
try:
dct[k] = datetime.strptime(v, DATE_FORMAT)
except:
pass
return dct
def dxdate(self, dt):
""" convert datetime into default date string format used in dexcell api calls """
return dt.strftime("%Y%m%d%H%M%S")
def _call_rest(self, url, payload=None, parse_response=True):
url = self.endpoint + url
self.logger.info('url:%s token:%s' % (url, self.token))
if payload is None:
req = urllib2.Request(url, headers={'x-dexcell-token': self.token})
else:
req = urllib2.Request(url,
payload,
headers={'x-dexcell-token': self.token})
try:
response = urllib2.urlopen(req, timeout=600.0)
data = response.read()
if parse_response:
return json.loads(data)
else:
return data
except urllib2.HTTPError as httperror:
info = json.loads(httperror.read())
if httperror.code == 404:
self.logger.error('error: not found')
raise DexcellRestApiError('NOTFOUND', info['description'],
info['moreInfo'])
elif httperror.code == 401:
self.logger.error('error: not authorized')
raise DexcellRestApiError('INVALIDTOKEN', info['description'],
info['moreInfo'])
else:
raise DexcellRestApiError('UNKNOWN', info['description'],
info['moreInfo'])
self.logger.error('error: %s' % (str(httperror.code)))
def get_deployment(self, dep_id):
""" return dict with basic information from deployment number dep_id"""
url = "/deployments/%i.json" % dep_id
deployment = self._call_rest(url)
return deployment
def get_deployment_locations(self, dep_id):
""" return array with locations information from deployment number dep_id"""
url = "/deployments/%i/locations.json" % dep_id
location_list = self._call_rest(url)
return location_list
def get_deployment_devices(self, dep_id):
""" return array with devices information from deployment number dep_id"""
url = "/deployments/%i/devices.json" % dep_id
device_list = self._call_rest(url)
return device_list
def get_deployment_parameters(self, dep_id):
""" return array with parameters {freq, name, id, i18m, units}
from deployment number dep_id
"""
url = "/deployments/%i/parameters.json" % dep_id
param_list = self._call_rest(url)
return param_list
def get_deployment_supplies(self, dep_id):
""" return array with supplies {pod, name, id}
from deployment number dep_id
"""
url = "/deployments/%i/supplies.json" % dep_id
supply_list = self._call_rest(url)
return supply_list
def get_deployment_notices(self, dep_id, start, end):
""" return array with alerts information from deployment number dep_id
from the interval selected
"""
start = self.dxdate(start)
end = self.dxdate(end)
url = "/deployments/%i/notices.json?start=%s&end=%s" % (dep_id, start,
end)
notice_list = self._call_rest(url)
return notice_list
def get_deployment_parameter_devices(self, dep_id, param_nid):
""" return array with parameters {id, name, networkid}
from deployment number dep_id
"""
url = "/deployments/%i/parameters/%s/devices.json" % (dep_id,
str(param_nid))
device_list = self._call_rest(url)
return device_list
def set_deployment_thing(self, dep_id, key, value):
""" update dict of information saved by the user"""
url = "/deployments/%i/things/set/%s.json" % (dep_id, key)
payload = json.dumps(value, default=self._json_date_handler)
data = self._call_rest(url, payload=payload, parse_response=False)
return data
def get_deployment_thing(self, dep_id, key):
""" return dict of information saved by the user"""
url = "/deployments/%i/things/get/%s.json" % (dep_id, key)
data = self._call_rest(url, parse_response=False)
self.logger.info('dep_thing:%s' % str(data))
data = json.loads(data, object_hook=self._datetime_parser)
return data
def get_location(self, loc_id):
""" return dict with basic information from locat"""
url = "/locations/%i.json" % loc_id
location = self._call_rest(url)
return location
def get_location_parameters(self, loc_id):
""" return array with parameters {freq, name, id, i18m, units}
from location number loc_id
"""
url = "/locations/%i/parameters.json" % loc_id
param_list = self._call_rest(url)
return param_list
def get_location_notices(self, loc_id, start, end):
""" return array with alerts information for location number loc_id
from the interval selected
"""
start = self.dxdate(start)
end = self.dxdate(end)
url = "/locations/%i/notices.json?start=%s&end=%s" % (loc_id, start,
end)
notice_list = self._call_rest(url)
return notice_list
def get_location_comments(self, loc_id, start, end):
""" return array with comments information for location number loc_id
from the interval selected
"""
start = self.dxdate(start)
end = self.dxdate(end)
url = "/locations/%i/comments.json?start=%s&end=%s" % (loc_id, start,
end)
comments = self._call_rest(url)
return comments
def get_location_parameter_devices(self, loc_id, param_nid):
""" return array with parameters {id, name, networkid}
from location number loc_id
"""
url = "/locations/%i/parameters/%i/devices.json" % (loc_id, param_nid)
device_list = self._call_rest(url)
return device_list
def get_location_supplies(self, loc_id):
""" return array with supplies {pod, name, id}
from location number loc_id
"""
url = "/locations/%i/supplies.json" % loc_id
supply_list = self._call_rest(url)
return supply_list
def get_location_devices(self, loc_id):
""" return array with the devices from the location """
url = "/locations/%i/devices.json" % loc_id
device_list = self._call_rest(url)
return device_list
def get_device(self, dev_id):
""" return dict with information for the device """
url = "/devices/%i.json" % dev_id
device = self._call_rest(url)
return device
def get_device_parameters(self, dev_id):
""" return array with parameters {freq, name, id, i18m, units}
from device number dev_id
"""
param_list = self._call_rest("/devices/" + str(dev_id) +
"/parameters.json")
return param_list
def get_simulated_bill(self,
dev_id,
start,
end,
type_param="ELECTRICAL",
parameters="AAANNN",
pod=None,
time='HOUR'):
''' returns bill generated from data in dexcell
Parameters
A : RAW + SUMMARY
R: RAW set of datas returned by time mesure, default hour
S: SUMMARY resum of data: totals, periods...
N: nothing
type_param can be ELECTRICAL, WATER, GAS
'''
new_pod = ''
if pod is not None:
new_pod = "&pod=" + pod
start = start.strftime("%Y%m%d%H%M%S")
end = end.strftime("%Y%m%d%H%M%S")
url = ["/cost/%i/%s.json?start=%s" % (dev_id, type_param, start)]
url.append("&end=%s&applyPattern=%s&period=%s%s" %
(end, parameters, time, new_pod))
url = "".join(url)
bill = self._call_rest(url)
return bill
def get_supply_bills(self,
sup_id,
start,
end,
type_param='ELECTRICAL',
parameters="AAANNN",
pod=None,
time='HOUR'):
''' returns bills updated by the customer
Parameters
A : RAW + SUMMARY
R: RAW set of datas returned by time mesure, default hour
S: SUMMARY resum of data: totals, periods...
N: nothing
type_param can be ELECTRICAL, WATER, GAS
'''
new_pod = ''
if pod is not None:
new_pod = "&pod=" + pod
start = start.strftime("%Y%m%d%H%M%S")
end = end.strftime("%Y%m%d%H%M%S")
url = ["/cost/%i/bills/%s.json?start=%s" % (sup_id, type_param, start)]
url.append("&end=%s&applyPattern=%s&period=%s%s" %
(end, parameters, time, new_pod))
url = "".join(url)
bills = self._call_rest(url)
return bills
def get_session(self, session_id):
""" return the session for an app with a concret session_id"""
url = "/session/%s.json" % session_id
response = self._call_rest(url)
self.logger.info('get session: ' + str(response))
return response
def get_readings(self, dev_id, s_nid, start, end):
""" return array dict with {values, timestamp} """
start = self.dxdate(start)
end = self.dxdate(end)
url = "/devices/%i/%i/readings.json?start=%s&end=%s" % (dev_id, s_nid,
start, end)
readings = self._call_rest(url)
for i in range(0, len(readings)):
try:
readings[i]['ts'] = datetime.strptime(readings[i]['ts'],
"%Y-%m-%d %H:%M:%S")
readings[i]['tsutc'] = datetime.strptime(
readings[i]['tsutc'], "%Y-%m-%d %H:%M:%S")
except KeyError:
pass
return readings
def get_readings_new(self, dev_id, param, frequency, operation, start,
end):
""" returns array of dict of values from the device dev_id with
parameter param with a frequency in the interval start - end.
"""
start = self.dxdate(start)
end = self.dxdate(end)
url = ["/devices/%i/%s/readings.json?" % (dev_id, str(param))]
url.append("start=%s&end=%s&frequency=%s&operation=%s" %
(start, end, str(frequency), str(operation)))
url = "".join(url)
readings = self._call_rest(url)
for i in range(0, len(readings)):
try:
readings[i]['ts'] = datetime.strptime(readings[i]['ts'],
"%Y-%m-%d %H:%M:%S")
readings[i]['tsutc'] = datetime.strptime(
readings[i]['tsutc'], "%Y-%m-%d %H:%M:%S")
except KeyError:
pass
return readings
def get_cost(self,
nid,
start,
end,
energy_type='ELECTRICAL',
period='HOUR',
grouped=False):
""" return array from cost and consumption with timestamp"""
str_grouped = 'TRUE'
if not grouped:
str_grouped = 'FALSE'
start = self.dxdate(start)
end = self.dxdate(end)
url = ["/devices/%i/%s/cost.json?" % (nid, energy_type)]
url.append("start=%s&end=%s&period=%s&grouped=%s" %
(start, end, str(period), str_grouped))
url = "".join(url)
raw_response = self._call_rest(url)
try:
readings = raw_response['readings']
for i in range(0, len(readings)):
readings[i]['ts'] = datetime.strptime(readings[i]['ts'],
"%Y/%m/%d %H:%M:%S")
periods = raw_response['periods']
return readings, periods
except KeyError:
return []
class DexcellRestApiAuth(object):
"""
Class for authentification in Dexcell
software.
"""
def __init__(self,
endpoint,
hash_dexma,
secret,
logger_name="dexcell_rest_api_auth"):
self.endpoint = endpoint
self.hash = hash_dexma
self.secret = secret
self.logger = logging.getLogger(logger_name)
if len(self.logger.handlers) == 0:
self.logger.setLevel(logging.INFO)
self.handler = NullHandler()
h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
self.handler.setFormatter(logging.Formatter(h_format))
self.logger.addHandler(self.handler)
def _json_date_handler(self, obj):
return obj.isoformat() if hasattr(obj, 'isoformat') else obj
def _datetime_parser(self, dct):
DATE_FORMAT = "%Y-%m-%dT%H:%M:%S"
strp = datetime.strptime
for k, v in dct.items():
if isinstance(v, basestring) and re.search(
"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", v):
try:
dct[k] = strp(v, DATE_FORMAT)
except ValueError:
pass
return dct
def _call_rest(self, url):
url = self.endpoint + url
req = urllib2.Request(url)
response = urllib2.urlopen(req)
data = response.read()
self.logger.info(data)
return data
def perm_token(self, temp_token):
''' obtain permanent token for oauth authentication'''
url = "/oauth/accesstoken?temp_token=%s&secret=%s&idclient=%s" % (
str(temp_token), self.secret, self.hash)
response = self._call_rest(url)
return response
def set_key_value(self, key, value):
"Set this key with this value in the key-value data store"
url = self.endpoint + "/things/set/" + key
req = urllib2.Request(url,
json.dumps(value,
default=self._json_date_handler),
headers={'x-dexcell-secret': self.secret})
self.logger.info('storing key: %s with secret: %s' %
(key, self.secret))
response = urllib2.urlopen(req)
data = response.read()
return data
def get_key(self, key):
"Get this key from the key-value data store"
url = "%s/things/get/%s" % (self.endpoint, key)
req = urllib2.Request(url, headers={'x-dexcell-secret': self.secret})
response = urllib2.urlopen(req)
data = response.read()
data = json.loads(data, object_hook=self._datetime_parser)
result = json.loads(data['result'], object_hook=self._datetime_parser)
return result
class DexcellSender(object):
DEFAULT_SERVER = 'insert.dexcell.com'
DEFAULT_URL = '/insert-json.htm'
DEFAULT_LOGFILE = '/var/log/dexma/DexcellSender.log'
DEFAULT_LOGLEVEL = logging.INFO
DEFAULT_GATEWAY = 'None'
DEFAULT_LOGGERNAME = 'DexcellSender'
def __init__(self,
gateway=DEFAULT_GATEWAY,
loggerName=DEFAULT_LOGGERNAME,
logfile=DEFAULT_LOGFILE,
loglevel=DEFAULT_LOGLEVEL,
server=DEFAULT_SERVER,
url=DEFAULT_URL,
https=True,
timeout=30.0):
self.__https = https
self.__server = server
self.__url = url
self.__timeout = timeout
self.__gateway = gateway
self.__logger = logging.getLogger(loggerName)
if len(self.__logger.handlers) == 0:
self.__logger.setLevel(loglevel)
self.handler = NullHandler()
h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
self.handler.setFormatter(logging.Formatter(h_format))
self.__logger.addHandler(self.handler)
def setup(self,
gateway=DEFAULT_GATEWAY,
loggerName=DEFAULT_LOGGERNAME,
logfile=DEFAULT_LOGFILE,
loglevel=DEFAULT_LOGLEVEL,
server=DEFAULT_SERVER,
url=DEFAULT_URL):
"""Setup the Dexcell Sender Object
"""
self.__server = server
self.__url = url
self.__gateway = gateway
self.__logger = logging.getLogger(loggerName)
if len(self.__logger.handlers) == 0:
self.__logger.setLevel(loglevel)
self.handler = NullHandler()
h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
self.handler.setFormatter(logging.Formatter(h_format))
self.__logger.addHandler(self.handler)
def changeGateway(self, gateway):
"""Change the gateway mac that will be sent
"""
self.__gateway = gateway
def __insertRawJSONData(self, data):
"""Insert the raw data string to the server
"""
params = 'data=' + data
headers = {
"Content-type": "application/x-www-form-urlencoded",
"Accept": "text/plain"
}
if self.__https:
conn = httplib.HTTPSConnection(self.__server,
timeout=self.__timeout)
else:
conn = httplib.HTTPConnection(self.__server,
timeout=self.__timeout)
conn.request("POST", self.__url, params, headers)
error = True
maxerror = 0
while error:
try:
response = conn.getresponse()
error = False
except:
print self.__logger.exception("Error inserting data")
maxerror = maxerror + 1
time.sleep(1)
if maxerror > 10:
return (-1, 'FAIL')
logger_msg_wo_params = "Insert from %s with status %s and result %s"
logger_params = (self.__gateway, str(response.status),
str(response.getheader('data')))
logger_message = logger_msg_wo_params % logger_params
self.__logger.debug(logger_message)
return response.status, response.getheader('data')
def insertDexcellServiceMessage(self,
serviceMessage,
timezone='UTC',
extraparams={}):
'''Insert a single DexcellServiceMessage
'''
reading = {
'nodeNetworkId':
str(serviceMessage.node),
'serviceNetworkId':
int(serviceMessage.service),
'value':
float(serviceMessage.value),
'seqNum':
int(serviceMessage.seqnum),
'timeStamp':
time.strftime("%Y-%m-%dT%H:%M:%S.000 " + timezone,
serviceMessage.timestamp)
}
data = {'gatewayId': self.__gateway, 'service': [reading]}
for key in extraparams.keys():
data[key] = extraparams[key]
result = self.__insertRawJSONData(json.dumps(data))
return result
def insertDexcellServiceMessages(self,
serviceMessageIterator,
timezone='UTC',
extraparams={}):
""" Insert many DexcellServiceMessages at once
"""
readings = []
for serviceMessage in serviceMessageIterator:
reading = {
'nodeNetworkId':
str(serviceMessage.node),
'serviceNetworkId':
int(serviceMessage.service),
'value':
float(serviceMessage.value),
'seqNum':
int(serviceMessage.seqnum),
'timeStamp':
time.strftime("%Y-%m-%dT%H:%M:%S.000 " + timezone,
serviceMessage.timestamp)
}
readings.append(reading)
data = {'gatewayId': self.__gateway, 'service': readings}
for key in extraparams.keys():
data[key] = extraparams[key]
result = self.__insertRawJSONData(json.dumps(data))
return result
class DexcellRestApi(object):
"""
class with all the utils API calls available group by
from deployment calls, location calls and devide calls.
"""
def __init__(self, endpoint, token, logger_name="dexcell_rest_api"):
self.endpoint = endpoint
self.token = token
self.logger = logging.getLogger(logger_name)
if len(self.logger.handlers) == 0:
self.logger.setLevel(logging.INFO)
self.handler = NullHandler()
h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
self.handler.setFormatter(logging.Formatter(h_format))
self.logger.addHandler(self.handler)
def _json_date_handler(self, obj):
return obj.isoformat() if hasattr(obj, "isoformat") else obj
def _datetime_parser(self, dct):
DATE_FORMAT = "%Y-%m-%dT%H:%M:%S"
for k, v in dct.items():
if isinstance(v, basestring) and re.search("\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", v):
try:
dct[k] = datetime.strptime(v, DATE_FORMAT)
except:
pass
return dct
def dxdate(self, dt):
""" convert datetime into default date string format used in dexcell api calls """
return dt.strftime("%Y%m%d%H%M%S")
def _call_rest(self, url, payload=None, parse_response=True):
url = self.endpoint + url
self.logger.info("url:%s token:%s" % (url, self.token))
if payload is None:
req = urllib2.Request(url, headers={"x-dexcell-token": self.token})
else:
req = urllib2.Request(url, payload, headers={"x-dexcell-token": self.token})
try:
response = urllib2.urlopen(req, timeout=600.0)
data = response.read()
if parse_response:
return json.loads(data)
else:
return data
except urllib2.HTTPError as httperror:
info = json.loads(httperror.read())
if httperror.code == 404:
self.logger.error("error: not found")
raise DexcellRestApiError("NOTFOUND", info["description"], info["moreInfo"])
elif httperror.code == 401:
self.logger.error("error: not authorized")
raise DexcellRestApiError("INVALIDTOKEN", info["description"], info["moreInfo"])
else:
raise DexcellRestApiError("UNKNOWN", info["description"], info["moreInfo"])
self.logger.error("error: %s" % (str(httperror.code)))
def get_deployment(self, dep_id):
""" return dict with basic information from deployment number dep_id"""
url = "/deployments/%i.json" % dep_id
deployment = self._call_rest(url)
return deployment
def get_deployment_locations(self, dep_id):
""" return array with locations information from deployment number dep_id"""
url = "/deployments/%i/locations.json" % dep_id
location_list = self._call_rest(url)
return location_list
def get_deployment_devices(self, dep_id):
""" return array with devices information from deployment number dep_id"""
url = "/deployments/%i/devices.json" % dep_id
device_list = self._call_rest(url)
return device_list
def get_deployment_parameters(self, dep_id):
""" return array with parameters {freq, name, id, i18m, units}
from deployment number dep_id
"""
url = "/deployments/%i/parameters.json" % dep_id
param_list = self._call_rest(url)
return param_list
def get_deployment_supplies(self, dep_id):
""" return array with supplies {pod, name, id}
from deployment number dep_id
"""
url = "/deployments/%i/supplies.json" % dep_id
supply_list = self._call_rest(url)
return supply_list
def get_deployment_notices(self, dep_id, start, end):
""" return array with alerts information from deployment number dep_id
from the interval selected
"""
start = self.dxdate(start)
end = self.dxdate(end)
url = "/deployments/%i/notices.json?start=%s&end=%s" % (dep_id, start, end)
notice_list = self._call_rest(url)
return notice_list
def get_deployment_parameter_devices(self, dep_id, param_nid):
""" return array with parameters {id, name, networkid}
from deployment number dep_id
"""
url = "/deployments/%i/parameters/%s/devices.json" % (dep_id, str(param_nid))
device_list = self._call_rest(url)
return device_list
def set_deployment_thing(self, dep_id, key, value):
""" update dict of information saved by the user"""
url = "/deployments/%i/things/set/%s.json" % (dep_id, key)
payload = json.dumps(value, default=self._json_date_handler)
data = self._call_rest(url, payload=payload, parse_response=False)
return data
def get_deployment_thing(self, dep_id, key):
""" return dict of information saved by the user"""
url = "/deployments/%i/things/get/%s.json" % (dep_id, key)
data = self._call_rest(url, parse_response=False)
self.logger.info("dep_thing:%s" % str(data))
data = json.loads(data, object_hook=self._datetime_parser)
return data
def get_location(self, loc_id):
""" return dict with basic information from locat"""
url = "/locations/%i.json" % loc_id
location = self._call_rest(url)
return location
def get_location_parameters(self, loc_id):
""" return array with parameters {freq, name, id, i18m, units}
from location number loc_id
"""
url = "/locations/%i/parameters.json" % loc_id
param_list = self._call_rest(url)
return param_list
def get_location_notices(self, loc_id, start, end):
""" return array with alerts information for location number loc_id
from the interval selected
"""
start = self.dxdate(start)
end = self.dxdate(end)
url = "/locations/%i/notices.json?start=%s&end=%s" % (loc_id, start, end)
notice_list = self._call_rest(url)
return notice_list
def get_location_comments(self, loc_id, start, end):
""" return array with comments information for location number loc_id
from the interval selected
"""
start = self.dxdate(start)
end = self.dxdate(end)
url = "/locations/%i/comments.json?start=%s&end=%s" % (loc_id, start, end)
comments = self._call_rest(url)
return comments
def get_location_parameter_devices(self, loc_id, param_nid):
""" return array with parameters {id, name, networkid}
from location number loc_id
"""
url = "/locations/%i/parameters/%i/devices.json" % (loc_id, param_nid)
device_list = self._call_rest(url)
return device_list
def get_location_supplies(self, loc_id):
""" return array with supplies {pod, name, id}
from location number loc_id
"""
url = "/locations/%i/supplies.json" % loc_id
supply_list = self._call_rest(url)
return supply_list
def get_location_devices(self, loc_id):
""" return array with the devices from the location """
url = "/locations/%i/devices.json" % loc_id
device_list = self._call_rest(url)
return device_list
def get_device(self, dev_id):
""" return dict with information for the device """
url = "/devices/%i.json" % dev_id
device = self._call_rest(url)
return device
def get_device_parameters(self, dev_id):
""" return array with parameters {freq, name, id, i18m, units}
from device number dev_id
"""
param_list = self._call_rest("/devices/" + str(dev_id) + "/parameters.json")
return param_list
def get_simulated_bill(
self, dev_id, start, end, type_param="ELECTRICAL", parameters="AAANNN", pod=None, time="HOUR"
):
""" returns bill generated from data in dexcell
Parameters
A : RAW + SUMMARY
R: RAW set of datas returned by time mesure, default hour
S: SUMMARY resum of data: totals, periods...
N: nothing
type_param can be ELECTRICAL, WATER, GAS
"""
new_pod = ""
if pod is not None:
new_pod = "&pod=" + pod
start = start.strftime("%Y%m%d%H%M%S")
end = end.strftime("%Y%m%d%H%M%S")
url = ["/cost/%i/%s.json?start=%s" % (dev_id, type_param, start)]
url.append("&end=%s&applyPattern=%s&period=%s%s" % (end, parameters, time, new_pod))
url = "".join(url)
bill = self._call_rest(url)
return bill
def get_supply_bills(self, sup_id, start, end, type_param="ELECTRICAL", parameters="AAANNN", pod=None, time="HOUR"):
""" returns bills updated by the customer
Parameters
A : RAW + SUMMARY
R: RAW set of datas returned by time mesure, default hour
S: SUMMARY resum of data: totals, periods...
N: nothing
type_param can be ELECTRICAL, WATER, GAS
"""
new_pod = ""
if pod is not None:
new_pod = "&pod=" + pod
start = start.strftime("%Y%m%d%H%M%S")
end = end.strftime("%Y%m%d%H%M%S")
url = ["/cost/%i/bills/%s.json?start=%s" % (sup_id, type_param, start)]
url.append("&end=%s&applyPattern=%s&period=%s%s" % (end, parameters, time, new_pod))
url = "".join(url)
bills = self._call_rest(url)
return bills
def get_session(self, session_id):
""" return the session for an app with a concret session_id"""
url = "/session/%s.json" % session_id
response = self._call_rest(url)
self.logger.info("get session: " + str(response))
return response
def get_readings(self, dev_id, s_nid, start, end):
""" return array dict with {values, timestamp} """
start = self.dxdate(start)
end = self.dxdate(end)
url = "/devices/%i/%i/readings.json?start=%s&end=%s" % (dev_id, s_nid, start, end)
readings = self._call_rest(url)
for i in range(0, len(readings)):
try:
readings[i]["ts"] = datetime.strptime(readings[i]["ts"], "%Y-%m-%d %H:%M:%S")
readings[i]["tsutc"] = datetime.strptime(readings[i]["tsutc"], "%Y-%m-%d %H:%M:%S")
except KeyError:
pass
return readings
def get_readings_new(self, dev_id, param, frequency, operation, start, end):
""" returns array of dict of values from the device dev_id with
parameter param with a frequency in the interval start - end.
"""
start = self.dxdate(start)
end = self.dxdate(end)
url = ["/devices/%i/%s/readings.json?" % (dev_id, str(param))]
url.append("start=%s&end=%s&frequency=%s&operation=%s" % (start, end, str(frequency), str(operation)))
url = "".join(url)
readings = self._call_rest(url)
for i in range(0, len(readings)):
try:
readings[i]["ts"] = datetime.strptime(readings[i]["ts"], "%Y-%m-%d %H:%M:%S")
readings[i]["tsutc"] = datetime.strptime(readings[i]["tsutc"], "%Y-%m-%d %H:%M:%S")
except KeyError:
pass
return readings
def get_cost(self, nid, start, end, energy_type="ELECTRICAL", period="HOUR", grouped=False):
""" return array from cost and consumption with timestamp"""
str_grouped = "TRUE"
if not grouped:
str_grouped = "FALSE"
start = self.dxdate(start)
end = self.dxdate(end)
url = ["/devices/%i/%s/cost.json?" % (nid, energy_type)]
url.append("start=%s&end=%s&period=%s&grouped=%s" % (start, end, str(period), str_grouped))
url = "".join(url)
raw_response = self._call_rest(url)
try:
readings = raw_response["readings"]
for i in range(0, len(readings)):
readings[i]["ts"] = datetime.strptime(readings[i]["ts"], "%Y/%m/%d %H:%M:%S")
periods = raw_response["periods"]
return readings, periods
except KeyError:
return []
class DexcellRestApiAuth(object):
"""
Class for authentification in Dexcell
software.
"""
def __init__(self, endpoint, hash_dexma, secret, logger_name="dexcell_rest_api_auth"):
self.endpoint = endpoint
self.hash = hash_dexma
self.secret = secret
self.logger = logging.getLogger(logger_name)
if len(self.logger.handlers) == 0:
self.logger.setLevel(logging.INFO)
self.handler = NullHandler()
h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
self.handler.setFormatter(logging.Formatter(h_format))
self.logger.addHandler(self.handler)
def _json_date_handler(self, obj):
return obj.isoformat() if hasattr(obj, "isoformat") else obj
def _datetime_parser(self, dct):
DATE_FORMAT = "%Y-%m-%dT%H:%M:%S"
strp = datetime.strptime
for k, v in dct.items():
if isinstance(v, basestring) and re.search("\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", v):
try:
dct[k] = strp(v, DATE_FORMAT)
except ValueError:
pass
return dct
def _call_rest(self, url):
url = self.endpoint + url
req = urllib2.Request(url)
response = urllib2.urlopen(req)
data = response.read()
self.logger.info(data)
return data
def perm_token(self, temp_token):
""" obtain permanent token for oauth authentication"""
url = "/oauth/accesstoken?temp_token=%s&secret=%s&idclient=%s" % (str(temp_token), self.secret, self.hash)
response = self._call_rest(url)
return response
def set_key_value(self, key, value):
"Set this key with this value in the key-value data store"
url = self.endpoint + "/things/set/" + key
req = urllib2.Request(
url, json.dumps(value, default=self._json_date_handler), headers={"x-dexcell-secret": self.secret}
)
self.logger.info("storing key: %s with secret: %s" % (key, self.secret))
response = urllib2.urlopen(req)
data = response.read()
return data
def get_key(self, key):
"Get this key from the key-value data store"
url = "%s/things/get/%s" % (self.endpoint, key)
req = urllib2.Request(url, headers={"x-dexcell-secret": self.secret})
response = urllib2.urlopen(req)
data = response.read()
data = json.loads(data, object_hook=self._datetime_parser)
result = json.loads(data["result"], object_hook=self._datetime_parser)
return result
class DexcellSender(object):
DEFAULT_SERVER = "insert.dexcell.com"
DEFAULT_URL = "/insert-json.htm"
DEFAULT_LOGFILE = "/var/log/dexma/DexcellSender.log"
DEFAULT_LOGLEVEL = logging.INFO
DEFAULT_GATEWAY = "None"
DEFAULT_LOGGERNAME = "DexcellSender"
def __init__(
self,
gateway=DEFAULT_GATEWAY,
loggerName=DEFAULT_LOGGERNAME,
logfile=DEFAULT_LOGFILE,
loglevel=DEFAULT_LOGLEVEL,
server=DEFAULT_SERVER,
url=DEFAULT_URL,
https=True,
timeout=30.0,
):
self.__https = https
self.__server = server
self.__url = url
self.__timeout = timeout
self.__gateway = gateway
self.__logger = logging.getLogger(loggerName)
if len(self.__logger.handlers) == 0:
self.__logger.setLevel(loglevel)
self.handler = NullHandler()
h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
self.handler.setFormatter(logging.Formatter(h_format))
self.__logger.addHandler(self.handler)
def setup(
self,
gateway=DEFAULT_GATEWAY,
loggerName=DEFAULT_LOGGERNAME,
logfile=DEFAULT_LOGFILE,
loglevel=DEFAULT_LOGLEVEL,
server=DEFAULT_SERVER,
url=DEFAULT_URL,
):
"""Setup the Dexcell Sender Object
"""
self.__server = server
self.__url = url
self.__gateway = gateway
self.__logger = logging.getLogger(loggerName)
if len(self.__logger.handlers) == 0:
self.__logger.setLevel(loglevel)
self.handler = NullHandler()
h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
self.handler.setFormatter(logging.Formatter(h_format))
self.__logger.addHandler(self.handler)
def changeGateway(self, gateway):
"""Change the gateway mac that will be sent
"""
self.__gateway = gateway
def __insertRawJSONData(self, data):
"""Insert the raw data string to the server
"""
params = "data=" + data
headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}
if self.__https:
conn = httplib.HTTPSConnection(self.__server, timeout=self.__timeout)
else:
conn = httplib.HTTPConnection(self.__server, timeout=self.__timeout)
conn.request("POST", self.__url, params, headers)
error = True
maxerror = 0
while error:
try:
response = conn.getresponse()
error = False
except:
print self.__logger.exception("Error inserting data")
maxerror = maxerror + 1
time.sleep(1)
if maxerror > 10:
return (-1, "FAIL")
logger_msg_wo_params = "Insert from %s with status %s and result %s"
logger_params = (self.__gateway, str(response.status), str(response.getheader("data")))
logger_message = logger_msg_wo_params % logger_params
self.__logger.debug(logger_message)
return response.status, response.getheader("data")
def insertDexcellServiceMessage(self, serviceMessage, timezone="UTC", extraparams={}):
"""Insert a single DexcellServiceMessage
"""
reading = {
"nodeNetworkId": str(serviceMessage.node),
"serviceNetworkId": int(serviceMessage.service),
"value": float(serviceMessage.value),
"seqNum": int(serviceMessage.seqnum),
"timeStamp": time.strftime("%Y-%m-%dT%H:%M:%S.000 " + timezone, serviceMessage.timestamp),
}
data = {"gatewayId": self.__gateway, "service": [reading]}
for key in extraparams.keys():
data[key] = extraparams[key]
result = self.__insertRawJSONData(json.dumps(data))
return result
def insertDexcellServiceMessages(self, serviceMessageIterator, timezone="UTC", extraparams={}):
""" Insert many DexcellServiceMessages at once
"""
readings = []
for serviceMessage in serviceMessageIterator:
reading = {
"nodeNetworkId": str(serviceMessage.node),
"serviceNetworkId": int(serviceMessage.service),
"value": float(serviceMessage.value),
"seqNum": int(serviceMessage.seqnum),
"timeStamp": time.strftime("%Y-%m-%dT%H:%M:%S.000 " + timezone, serviceMessage.timestamp),
}
readings.append(reading)
data = {"gatewayId": self.__gateway, "service": readings}
for key in extraparams.keys():
data[key] = extraparams[key]
result = self.__insertRawJSONData(json.dumps(data))
return result
from logging import getLogger, NullHandler, StreamHandler, Formatter
from logging import DEBUG, INFO, WARNING, ERROR
import sys
rpkg = getLogger('rpkg')
dgt = getLogger('dgt')
cli = getLogger('cli')
handler = NullHandler()
dgt.addHandler(handler)
cli.addHandler(handler)
# Add a default handler if there are no real handlers set, yet (not counting NullHandler)
if len(rpkg.handlers) <= 1:
handler = StreamHandler(sys.stderr)
formatter = Formatter('[%(name)s] %(levelname)s: %(message)s')
handler.setFormatter(formatter)
rpkg.addHandler(handler)
if len(dgt.handlers) <= 1:
handler = StreamHandler(sys.stderr)
formatter = Formatter('[%(name)s] %(levelname)s: %(message)s')
handler.setFormatter(formatter)
dgt.addHandler(handler)
if len(cli.handlers) <= 1:
handler = StreamHandler(sys.stdout)
formatter = Formatter('%(message)s')
handler.setFormatter(formatter)
cli.addHandler(handler)
def test_null_handler(response):
nh = NullHandler()
formatter = color_debug.ColorFormatter()
nh.setFormatter(formatter)
def test_null_handler(response):
nh = NullHandler()
formatter = color_bucket_logger.ColorFormatter()
nh.setFormatter(formatter)