def setup_logging_no_config(loglevel, quiet): log_level = aprsd_config.LOG_LEVELS[loglevel] LOG.setLevel(log_level) log_format = aprsd_config.DEFAULT_LOG_FORMAT date_format = aprsd_config.DEFAULT_DATE_FORMAT log_formatter = logging.Formatter(fmt=log_format, datefmt=date_format) fh = NullHandler() fh.setFormatter(log_formatter) LOG.addHandler(fh) if not quiet: sh = logging.StreamHandler(sys.stdout) sh.setFormatter(log_formatter) LOG.addHandler(sh)
def create_logger(robot, path=None, debug=False): logger = logging.getLogger(robot) logger.setLevel(logging.DEBUG if debug else logging.INFO) if path is None: handler = NullHandler() else: logFile = os.path.join(path, robot + '.log') handler = TimedRotatingFileHandler(logFile, when='midnight') formatter = Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) logger.addHandler(handler) return logger
class SnowRestSession(object): """ This class behaves very similarly to a requests.Session object. It can be configured by loading a .yaml config file, or by setting attributes. """ def __init__(self): self.instance = None self.auth_type = None self.sso_method = None self.basic_auth_user = None self.basic_auth_password = None self.oauth_client_id = None self.oauth_client_secret = None self.session_cookie_file_path = None self.session_cookie = None self.oauth_token_file_path = None self.token_dic = None self.fresh_cookie = False self.fresh_token = False self.store_cookie = True self.store_token = True self.session = requests.Session() self._log_enabled = False self._logger_name = 'snow-client.session_' + uuid.uuid4( ).hex[:-24] # used to get a Logger self._logger = logging.getLogger(self._logger_name) self._log_handler = NullHandler() self._logger.addHandler(self._log_handler) self._log_file_path = None self._log_level = 'INFO' self._log_format = "%(asctime)s [%(name)s] [%(levelname)s] %(message)s" self._log_file_size_bytes = 1000000 self._log_file_rotations = 10 self._log_file_encoding = 'utf-8' def load_config_file(self, config_file_path): """ Load a .yaml configuration file Args: config_file_path (str): the path to the configuration file Raises: SnowRestSessionException: if no file is found at the specified path, or some inconsistency is found in the file. Exception: if some other error happens when opening or parsing the file """ if not config_file_path: raise SnowRestSessionException( "SnowRestSession.load_config_file: the parameter config_file_path must have a value" ) try: with open(config_file_path) as f: config_file = yaml.safe_load(f) except Exception as e: sys.stderr.write( "SnowRestSession.load_config_file: Issue when opening the config file at %s.\n" % config_file_path) raise e if 'instance' in config_file: self.set_instance(config_file['instance']) if 'auth' in config_file: if 'type' in config_file['auth']: self.auth_type = config_file['auth']['type'] if self.auth_type == 'sso_oauth': if 'sso_method' in config_file['auth']: self.sso_method = config_file['auth']['sso_method'] if 'oauth_client_id' in config_file['auth']: self.oauth_client_id = config_file['auth']['oauth_client_id'] if 'oauth_client_secret' in config_file['auth']: self.oauth_client_secret = config_file['auth'][ 'oauth_client_secret'] if 'session' in config_file: if 'cookie_file' in config_file['session']: self.session_cookie_file_path = config_file['session'][ 'cookie_file'] if 'oauth_tokens_file' in config_file['session']: self.oauth_token_file_path = config_file['session'][ 'oauth_tokens_file'] elif self.auth_type == 'basic': if 'user' in config_file['auth']: self.basic_auth_user = config_file['auth']['user'] if 'password' in config_file['auth']: self.basic_auth_password = config_file['auth']['password'] if 'session' in config_file: if 'cookie_file' in config_file['session']: self.session_cookie_file_path = config_file['session'][ 'cookie_file'] else: raise SnowRestSessionException( "SnowRestSession.load_config_file: the property \"auth_type\" " "must have a value of \"sso_auth\" or \"basic\"") if 'log' in config_file: if 'log_enabled' in config_file['log'] and config_file['log'][ 'log_enabled']: self._log_enabled = True if 'log_level' in config_file['log']: self.set_log_level(config_file['log']['log_level']) if 'log_file_path' in config_file['log']: self._log_file_path = config_file['log']['log_file_path'] if 'log_format' in config_file['log']: self._log_format = config_file['log']['log_format'] if 'log_file_size_bytes' in config_file['log']: self._log_file_size_bytes = config_file['log'][ 'log_file_size_bytes'] if 'log_file_rotations' in config_file['log']: self._log_file_rotations = config_file['log'][ 'log_file_rotations'] if 'log_file_encoding' in config_file['log']: self._log_file_encoding = config_file['log'][ 'log_file_encoding'] self._configure_handler() def set_instance(self, instance): """ Args: instance (str): the ServiceNow instance to use. It is optional to prefix with "https://". Examples: >>> s = SnowRestSession() >>> s.set_instance("cern.service-now.com") >>> >>> s2 = SnowRestSession() >>> s2.set_instance("https://cerntest.service-now.com") """ if not instance.startswith('https://'): self.instance = 'https://' + instance else: self.instance = instance def set_auth_type(self, auth_type): """ Args: auth_type (str): the authentication type to use. Either 'sso_auth' or 'basic' Raises: SnowRestSessionException: if auth_type is not 'sso_auth' or 'basic' """ if auth_type != 'basic' and auth_type != 'sso_auth': raise SnowRestSessionException( "SnowRestSession.set_auth_type: the parameter \"auth_type\" " "must have a value of \"sso_auth\" or \"basic\"") self.auth_type = auth_type def set_sso_method(self, sso_method): """ Args: sso_method (str): the Single Sign On method to use. Either 'kerberos' or 'certificate'. Needs set_auth_type('sso_auth') to have an effect. Raises: SnowRestSessionException: if sso_method is not 'kerberos' or 'certificate' Note: certificate support needs to be implemented """ if sso_method != 'kerberos' and sso_method != 'certificate': raise SnowRestSessionException( "SnowRestSession.set_sso_method: the parameter \"sso_method\" " "must have a value of \"kerberos\" or \"certificate\"") # TODO: implement certificate support. Update the comment above if sso_method == 'certificate': raise SnowRestSessionException( "SnowRestSession.set_sso_method: certificate support is not yet implemented" ) self.sso_method = sso_method def set_oauth_client_id(self, oauth_client_id): """ Args: oauth_client_id (str): the OAuth client id provided to you by your instance's ServiceNow administrator. Needs set_auth_type('sso_auth') to have an effect. """ self.oauth_client_id = oauth_client_id def set_oauth_client_secret(self, oauth_client_secret): """ Args: oauth_client_secret (str): the OAuth client secret provided to you by your instance's ServiceNow administrator. Needs set_auth_type('sso_auth') to have an effect. An OAuth client secret is sensitive information; please store it as securely as possible. """ self.oauth_client_secret = oauth_client_secret def set_basic_auth_user(self, basic_auth_user): """ Args: basic_auth_user (str): the name of the local ServiceNow account that will be used for authentication. Needs set_auth_type('basic') to have an effect. """ self.basic_auth_user = basic_auth_user def set_basic_auth_password(self, basic_auth_password): """ Args: basic_auth_password (str): the password of the local ServiceNow account that will be used for authentication. Needs set_auth_type('basic') to have an effect. """ self.basic_auth_password = basic_auth_password def set_session_cookie_file_path(self, session_cookie_file_path): """ Args: session_cookie_file_path (str): the path to the file where the session cookies will be persisted. If not provided, the cookies will not be persisted. """ self.session_cookie_file_path = session_cookie_file_path def set_oauth_token_file_path(self, oauth_token_file_path): """ Args: oauth_token_file_path (str): the path to the file where the OAuth refresh and access tokens will be persisted. If not provided, the OAuth tokens will not be persisted. """ self.oauth_token_file_path = oauth_token_file_path def set_log_enabled(self, log_enabled): self._log_enabled = log_enabled def set_log_level(self, log_level): self._log_level = log_level self._logger.setLevel(self._log_level) def set_log_file_path(self, log_file_path): self._log_file_path = log_file_path self._configure_handler() def set_log_format(self, log_format): self._log_format = log_format self._configure_handler() def set_log_file_size_bytes(self, log_file_size_bytes): self._log_file_size_bytes = log_file_size_bytes self._configure_handler() def set_log_file_rotations(self, log_file_rotations): self._log_file_rotations = log_file_rotations self._configure_handler() def set_log_file_encoding(self, log_file_encoding): self._log_file_encoding = log_file_encoding self._configure_handler() def is_logging_enabled(self): return self._log_enabled def get_logger_name(self): return self._logger_name def get_logger(self): return self._logger def get_log_handler(self): return self._log_handler def get(self, url, headers=None, params=None): """ Executes a raw GET operation and returns the result. Used to read or retrieve information. The authentication method, and session cookie / OAuth tokens persistance will depend on how the session has been configured. Args: url (str): a relative URL, such as ``/api/now/v2/table/incident/c1c535ba85f45540adf94de5b835cd43``. An absolute URL such as ``https://cerntest.service-now.com/api/now/v2/table/incident/c1c535ba85f45540adf94de5b835cd43`` can also be used, but the instance should match with the "instance" parameter in the configuration file, or the value set with the ``.set_instance()`` method. headers (:obj:`dict`, optional): any additional headers to be be passed. If not set, the 'Accept' header will be set to 'application/json' params (:obj:`dict`, optional): any additional URL parameters to be be passed Returns: requests.Response : If the status code is not 401, a ``requests.Response`` object is returned. The structure of the result inside the 'text' attribute might vary depending on the REST endpoint called and the way that the query is configured. ServiceNow may also set various HTTP status codes or headers in the response. Raises: SnowRestSessionException : if the operation could not be performed due to an authentication issue Examples: Getting an incident by sys_id with the ServiceNow REST Table API: >>> s = SnowRestSession() >>> s.load_config_file('config.yaml') >>> response = s.get('/api/now/v2/table/incident/c1c535ba85f45540adf94de5b835cd43') >>> if response.status_code == 200: >>> import json >>> result = json.loads(response.text) >>> if 'result' in result: >>> incident = result['result'] # querying by sys_id returns a single object >>> print incident['short_description'] # will print "Test" (as a unicode object) Getting an incident by number with the ServiceNow REST Table API: >>> s = SnowRestSession() >>> s.load_config_file('config.yaml') >>> response = s.get('/api/now/v2/table/incident?sysparm_query=number=INC0426232') >>> if response.status_code == 200: >>> import json >>> result = json.loads(response.text) >>> if 'result' in result: >>> incident_array = result['result'] # querying via an encoded query returns an array of objects >>> if len(incident_array) > 0: >>> incident = incident_array[0] >>> print incident['short_description'] # will print "Test" (as a unicode object) """ result = self.__operation(operation='get', url=url, headers=headers, params=params) return result def post(self, url, headers=None, params=None, data=None): """ Executes a raw POST operation and returns the result. Used to insert records or other information. The authentication method, and session cookie / OAuth tokens persistance will depend on how the session has been configured. Args: url (str): a relative URL, such as ``/api/now/v2/table/incident``. An absolute URL such as ``https://cerntest.service-now.com/api/now/v2/table/incident`` can also be used, but the instance should match with the "instance" parameter in the configuration file, or the value set with the ``.set_instance()`` method. headers (:obj:`dict`, optional): any additional headers to be be passed. If not set, the 'Accept' header will be set to 'application/json', and 'Content' will be set to 'application/json' params (:obj:`dict`, optional): any additional URL parameters to be be passed data (str): the data that will be submitted via POST. Typically, a dictionary of keys and values, turned into a string with import json; json.dumps() Returns: requests.Response : If the status code is not 401, a ``requests.Response`` object is returned. The structure of the result inside the 'text' attribute might vary depending on the REST endpoint called, but typically represents the record resulting from the insert. ServiceNow may also set various HTTP status codes or headers in the response. Raises: SnowRestSessionException : if the operation could not be performed due to an authentication issue Examples: Inserting an incident with the ServiceNow REST Table API: >>> s = SnowRestSession() >>> s.load_config_file('config.yaml') >>> incident_attributes = { >>> 'short_description' : "New incident", >>> 'u_business_service' : 'e85a3f3b0a0a8c0a006a2912f2f352d1', # Service Element "ServiceNow" >>> 'u_functional_element' : '579fb3d90a0a8c08017ac8a1137c8ee6', # Functional Element "ServiceNow" >>> 'comments' : "Initial description" >>> } >>> import json >>> data = json.dumps(incident_attributes) >>> response = s.post('/api/now/v2/table/incident', data=data) >>> if response.status_code == 201: >>> result = json.loads(response.text) >>> if 'result' in result: >>> incident = result['result'] >>> print incident['number'] # will print the number of the newly created incident """ result = self.__operation(operation='post', url=url, headers=headers, params=params, data=data) return result def put(self, url, headers=None, params=None, data=None): """ Executes a raw PUT operation and returns the result. Used to update records or other information. The authentication method, and session cookie / OAuth tokens persistance will depend on how the session has been configured. Args: url (str): a relative URL, such as ``/api/now/v2/table/incident/c1c535ba85f45540adf94de5b835cd43``. An absolute URL such as ``https://cerntest.service-now.com/api/now/v2/table/incident/c1c535ba85f45540adf94de5b835cd43`` can also be used, but the instance should match with the "instance" parameter in the configuration file, or the value set with the ``.set_instance()`` method. headers (:obj:`dict`, optional): any additional headers to be be passed. If not set, the 'Accept' header will be set to 'application/json', and 'Content' will be set to 'application/json' params (:obj:`dict`, optional): any additional URL parameters to be be passed data (str): the data that will be submitted via PUT. Typically, a dictionary of keys and values, turned into a string with import json; json.dumps() Returns: requests.Response : If the status code is not 401, a ``requests.Response`` object is returned. The structure of the result inside the 'text' attribute might vary depending on the REST endpoint called, but typically represents the state of the record after the update. ServiceNow may also set various HTTP status codes or headers in the response. Raises: SnowRestSessionException : if the operation could not be performed due to an authentication issue Examples: Updating an incident with the ServiceNow REST Table API: >>> s = SnowRestSession() >>> s.load_config_file('config.yaml') >>> incident_attributes_to_update = { >>> 'watch_list' : '*****@*****.**' # will completely replace the previous value >>> } >>> import json >>> data = json.dumps(incident_attributes_to_update) >>> response = s.put('/api/now/v2/table/incident/ca37ed334f56830015d3bc511310c7a8', data=data) >>> if response.status_code == 200: >>> result = json.loads(response.text) >>> if 'result' in result: >>> incident = result['result'] >>> print incident['number'] # will print the number of the updated incident >>> print incident['watch_list'] # will print the new value of the watch_list field """ result = self.__operation(operation='put', url=url, headers=headers, params=params, data=data) return result def __call_cern_get_sso_cookie(self): """ Executes the cern-get-sso-cookie command (available in CERN Linux environments) in order to perform a Single Sign-On and produce a cookie file. """ args = [ "cern-get-sso-cookie", "--reprocess", "--url", self.instance, "--outfile", self.session_cookie_file_path ] p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.wait() def __good_cookie(self): cookie_file = open(self.session_cookie_file_path) cookie_file_contents = cookie_file.read() if (cookie_file_contents.find('glide_user_activity') != -1 and cookie_file_contents.find('glide_session_store') != -1 and cookie_file_contents.find('glide_user_route') != -1 and cookie_file_contents.find('JSESSIONID') != -1 and cookie_file_contents.find('BIGipServerpool_cern') != -1): return True return False def __cern_get_sso_cookie(self): """ Loads and checks an existing cookie file, in order to avoid logging into ServiceNow and the CERN Single-Sign-On if possible. If needed, logs in again into both to produce the cookie file. The cookie file is loaded into memory. If not needed to persist the cookie file, it is deleted. Raises: SnowRestSessionException: if the Single Sign On login is not succesful. This is determined by parsing the cookie file after it is produced. """ if not os.path.exists( self.session_cookie_file_path ): # cookie file not present. We perform Single Sign On self.__call_cern_get_sso_cookie() self.fresh_cookie = True else: # cookie file present. We check if it has the correct format self.fresh_cookie = False if not self.__good_cookie( ): # the format is bad. We create a new cookie file self.__call_cern_get_sso_cookie() self.fresh_cookie = True # we load the cookie file self.session_cookie = cookielib.MozillaCookieJar( self.session_cookie_file_path) self.session_cookie.load() # we check if the log in was sucessful if not self.__good_cookie(): raise SnowRestSessionException( "SnowRestSession.__cern_get_sso_cookie: The current account has failed to perform " "a Single-Sign-On login in to ServiceNow") # we load the cookies into the requests.Session session self.session.cookies = self.session_cookie # if not needed to persist the cookie file, we delete it if not self.store_cookie: os.remove(self.session_cookie_file_path) def __obtain_tokens(self): """ Loads the OAuth access and refresh tokens from the token file. If the token file is not present, new OAuth access and refresh tokens are obtained. Raises: SnowRestSessionException: if the OAuth tokens could not be retrieved from ServiceNow. IOError, ValueError: if the token file could not be opened """ if not os.path.exists( self.oauth_token_file_path ): # tokens file not present. We obtain new tokens self.fresh_token = True token_request = self.session.post(self.instance + '/oauth_token.do', data={ 'grant_type': 'password', 'client_id': self.oauth_client_id, 'client_secret': self.oauth_client_secret }) if token_request.status_code == 200: # token request was successful self.token_dic = json.loads(token_request.text) if self.store_token: with open(self.oauth_token_file_path, 'w') as token_file: json.dump(self.token_dic, token_file) else: # token request failed. Possibly, an existing cookie file contained a timed out session if self.fresh_cookie: # we just performed a Single-Sign-On. There is a problem with token retrieval. raise SnowRestSessionException( "SnowRestSession.__obtain_tokens: OAuth tokens could not be retrieved from ServiceNow. " "Please check the OAuth client id and OAuth client secret." ) else: # we may need to perform Single-Sign-On again, as the session may have timed out self.__cern_get_sso_cookie() token_request = self.session.post( self.instance + '/oauth_token.do', data={ 'grant_type': 'password', 'client_id': self.oauth_client_id, 'client_secret': self.oauth_client_secret }) self.token_dic = None if token_request.status_code == 200: self.token_dic = json.loads(token_request.text) if self.store_token: with open(self.oauth_token_file_path, 'w') as token_file: json.dump(self.token_dic, token_file) else: raise SnowRestSessionException( "SnowRestSession.__obtain_tokens: OAuth tokens could not be retrieved from ServiceNow. " "Please check the OAuth client id and OAuth client secret." ) else: # the tokens file is present. We try to load it try: with open(self.oauth_token_file_path, 'r') as token_file: self.token_dic = json.load(token_file) except (IOError, ValueError) as e: sys.stderr.write( "SnowRestSession.__obtain_tokens: Issue when opening " "the token file at %s.\n" % self.oauth_token_file_path) raise e def __initiate_session(self): """ Initiates a session via CERN Single-Sign-On + OAuth, or basic Authentication Raises: SnowRestSessionException: if auth_type is not 'sso_auth' nor 'basic' """ if self.auth_type == 'sso_oauth': self.__cern_get_sso_cookie() self.__obtain_tokens() elif self.auth_type == 'basic': self.session.auth = (self.basic_auth_user, self.basic_auth_password) self.session.cookies = cookielib.MozillaCookieJar() if os.path.exists(self.session_cookie_file_path): self.session.cookies.load(self.session_cookie_file_path, ignore_discard=True, ignore_expires=True) else: raise SnowRestSessionException( "SnowRestSession.__initiate_session: self.auth_type " "has a value different from \"basic\" and \"sso_auth\"") def __refresh_token(self): """ Requests an OAuth access token via the OAUTH refresh token. Saves the resulting access token in the token file. Raises: SnowRestSessionException: if the access token could not be obtained """ token_request = self.session.post(self.instance + '/oauth_token.do', data={ 'grant_type': 'refresh_token', 'client_id': self.oauth_client_id, 'client_secret': self.oauth_client_secret, 'refresh_token': self.token_dic['refresh_token'] }) if token_request.status_code == 200: self.token_dic = json.loads(token_request.text) with open(self.oauth_token_file_path, 'w') as token_file: json.dump(self.token_dic, token_file) else: token_request = self.session.post(self.instance + '/oauth_token.do', data={ 'grant_type': 'password', 'client_id': self.oauth_client_id, 'client_secret': self.oauth_client_secret }) if token_request.status_code == 200: self.token_dic = json.loads(token_request.text) with open(self.oauth_token_file_path, 'w') as token_file: json.dump(self.token_dic, token_file) else: if self.fresh_cookie: raise SnowRestSessionException( "SnowRestSession.__refresh_token: the OAuth client id and OAuth secret might not be valid" ) def __save_cookie_basic(self): """ Persists the basic authentication cookie file. """ self.fresh_cookie = True self.session.cookies.save(self.session_cookie_file_path, ignore_discard=True, ignore_expires=True) @staticmethod def __library_user_agent(): user_agent_header = 'cern-snow-client/' + __version__ default_headers = requests.utils.default_headers() if 'User-Agent' in default_headers: user_agent_header = user_agent_header + ' ' + default_headers[ 'User-Agent'] return user_agent_header def __execute(self, operation, url, headers=None, params=None, data=None): """ Executes directly a REST Operation and returns the result Args: operation (str): either 'get', 'post' or 'put' url (str): a relative URL, such as "/api/now/v2/table/incident". An absolute URL such as "https://cerntest.service-now.com/api/now/v2/table/incident" can also be used, but the instance should match with the "instance" parameter in the configuration file, or the value set with the ``.set_instance()`` method. headers (:obj:`dict`, optional): any additional headers to be be passed. This method will add Accept:application/json and Content-Type:application/json if not specified. It will also add the Authorization header if the authentication type is 'sso_auth', by setting it to the OAuth access header. params (:obj:`dict`, optional): any additional URL parameters to be be passed data (object): the data to be sent in a post or put operation Returns: requests.Response Raises: SnowRestSessionException: if the operation parameter is not 'get', 'post' or 'put' """ if not operation: raise SnowRestSessionException( "SnowRestSession.__execute: the operation paramater is mandatory" ) if not url: raise SnowRestSessionException( "SnowRestSession.__execute: the url paramater is mandatory") if not url.startswith('https://'): url = self.instance + url if not url.startswith('https://'): url = self.instance + url if not headers: headers = {} if 'User-Agent' not in headers: headers['User-Agent'] = self.__library_user_agent() if 'Accept' not in headers: headers['Accept'] = 'application/json' if (operation == 'post' or operation == 'put') and 'Content-Type' not in headers: headers['Content-Type'] = 'application/json' if self.auth_type == 'sso_oauth': headers[ 'Authorization'] = 'Bearer ' + self.token_dic['access_token'] if operation == 'get': result = self.session.get(url, headers=headers, params=params) elif operation == 'post': result = self.session.post(url, headers=headers, params=params, data=data) elif operation == 'put': result = self.session.put(url, headers=headers, params=params, data=data) else: raise SnowRestSessionException( "SnowRestSession.__execute: the operation paramater " "needs to be either \"get\", \"post\" or \"put\"") return result def __operation(self, operation, url, headers=None, params=None, data=None): """ Executes a REST Operation, taking care of reauthenticating if needed, and returns the result Args: operation (str): either 'get', 'post' or 'put' url (str): a relative URL, such as "/api/now/v2/table/incident". An absolute URL such as "https://cerntest.service-now.com/api/now/v2/table/incident" can also be used, but the instance should match with the "instance" parameter in the configuration file, or the value set with the ``.set_instance()`` method. headers (:obj:`dict`, optional): any additional headers to be be passed. If not set, some headers will be set by default (see __execute method) params (:obj:`dict`, optional): any additional URL parameters to be be passed data (object): the data to be sent in a post or put operation Returns: requests.Response : if the status code is not 401, a requests.Response object is returned Raises: SnowRestSessionException : if the operation could not be performed due to an authentication issue """ self.__initiate_session() result = self.__execute(operation, url, headers=headers, params=params, data=data) if result.status_code != 401: if self.auth_type == 'basic': self.__save_cookie_basic() return result else: if self.auth_type == 'basic': if not self.fresh_cookie: self.session.auth = (self.basic_auth_user, self.basic_auth_password) result = self.__execute(operation, url, headers=headers, params=params, data=data) if result.status_code != 401: self.__save_cookie_basic() return result else: raise SnowRestSessionException( "SnowRestSession.__operation: Your basic authentication " "user and password might not be valid") else: raise SnowRestSessionException( "SnowRestSession.__operation: Your basic authentication " "user and password might not be valid") elif self.auth_type == 'sso_auth': if self.fresh_token: raise SnowRestSessionException( "SnowRestSession.__operation: failed to perform the operation. The current account might not " "be able to log in to ServiceNow or the OAuth client id and secret might not be valid" ) else: self.__refresh_token() token_request = self.session.post( 'post', self.instance + '/oauth_token.do', data={ 'grant_type': 'password', 'client_id': self.oauth_client_id, 'client_secret': self.oauth_client_secret }) if token_request.status_code == 200: self.token_dic = json.loads(token_request.text) with open(self.oauth_token_file_path, 'w') as token_file: json.dump(self.token_dic, token_file) headers['Authorization'] = 'Bearer ' + self.token_dic[ 'access_token'] result = self.__execute(operation, url, headers=headers, params=params, data=data) if result.status_code != 401: return result else: raise SnowRestSessionException( "SnowRestSession.__operation: failed to perform the operation. " "The current account might not be able to log in to ServiceNow or " "the OAuth client id and secret might not be valid" ) else: if self.fresh_cookie: raise SnowRestSessionException( "SnowRestSession.__operation: OAuth tokens could not be retrieved from ServiceNow. " "Please check the OAuth client id and OAuth client secret." ) else: os.remove(self.session_cookie_file_path) self.__cern_get_sso_cookie() token_request = self.session.post( 'post', self.instance + '/oauth_token.do', data={ 'grant_type': 'password', 'client_id': self.oauth_client_id, 'client_secret': self.oauth_client_secret }) if token_request.status_code == 401: raise SnowRestSessionException( "SnowRestSession.__operation: OAuth tokens could not be retrieved from ServiceNow. " "Please check the OAuth client id and OAuth client secret." ) else: self.token_dic = json.loads(token_request.text) with open(self.oauth_token_file_path, 'w') as token_file: json.dump(self.token_dic, token_file) headers[ 'Authorization'] = 'Bearer ' + self.token_dic[ 'access_token'] result = self.__execute(operation, url, headers=headers, params=params, data=data) if result.status_code != 401: return result else: raise SnowRestSessionException( "SnowRestSession.__operation: failed to perform the operation. " "The current account might not be able to log in to ServiceNow or " "the OAuth client id and secret might not be valid" ) else: raise SnowRestSessionException( "SnowRestSession.__operation: self.auth_type has a value different from \"basic\" and \"sso_auth\"" ) def _configure_handler(self): self._logger.removeHandler(self._log_handler) file_handler_attributes_known = bool(self._log_file_path and self._log_file_size_bytes and self._log_file_rotations and self._log_file_encoding and self._log_format) if file_handler_attributes_known: self._log_handler = RotatingFileHandler( self._log_file_path, maxBytes=self._log_file_size_bytes, backupCount=self._log_file_rotations - 1, encoding=self._log_file_encoding) self._log_handler.setFormatter(logging.Formatter(self._log_format)) else: self._log_handler = NullHandler() self._logger.addHandler(self._log_handler) def _debug(self, message): self._log(logging.DEBUG, message) def _info(self, message): self._log(logging.INFO, message) def _warning(self, message): self._log(logging.WARNING, message) def _error(self, message): self._log(logging.ERROR, message) def _critical(self, message): self._log(logging.CRITICAL, message) def _log(self, level, message): if self._log_enabled: self._logger.log(level, message)
class DexcellRestApi(object): """ class with all the utils API calls available group by from deployment calls, location calls and devide calls. """ def __init__(self, endpoint, token, logger_name="dexcell_rest_api"): self.endpoint = endpoint self.token = token self.logger = logging.getLogger(logger_name) if len(self.logger.handlers) == 0: self.logger.setLevel(logging.INFO) self.handler = NullHandler() h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" self.handler.setFormatter(logging.Formatter(h_format)) self.logger.addHandler(self.handler) def _json_date_handler(self, obj): return obj.isoformat() if hasattr(obj, 'isoformat') else obj def _datetime_parser(self, dct): DATE_FORMAT = "%Y-%m-%dT%H:%M:%S" for k, v in dct.items(): if isinstance(v, basestring) and re.search( "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", v): try: dct[k] = datetime.strptime(v, DATE_FORMAT) except: pass return dct def dxdate(self, dt): """ convert datetime into default date string format used in dexcell api calls """ return dt.strftime("%Y%m%d%H%M%S") def _call_rest(self, url, payload=None, parse_response=True): url = self.endpoint + url self.logger.info('url:%s token:%s' % (url, self.token)) if payload is None: req = urllib2.Request(url, headers={'x-dexcell-token': self.token}) else: req = urllib2.Request(url, payload, headers={'x-dexcell-token': self.token}) try: response = urllib2.urlopen(req, timeout=600.0) data = response.read() if parse_response: return json.loads(data) else: return data except urllib2.HTTPError as httperror: info = json.loads(httperror.read()) if httperror.code == 404: self.logger.error('error: not found') raise DexcellRestApiError('NOTFOUND', info['description'], info['moreInfo']) elif httperror.code == 401: self.logger.error('error: not authorized') raise DexcellRestApiError('INVALIDTOKEN', info['description'], info['moreInfo']) else: raise DexcellRestApiError('UNKNOWN', info['description'], info['moreInfo']) self.logger.error('error: %s' % (str(httperror.code))) def get_deployment(self, dep_id): """ return dict with basic information from deployment number dep_id""" url = "/deployments/%i.json" % dep_id deployment = self._call_rest(url) return deployment def get_deployment_locations(self, dep_id): """ return array with locations information from deployment number dep_id""" url = "/deployments/%i/locations.json" % dep_id location_list = self._call_rest(url) return location_list def get_deployment_devices(self, dep_id): """ return array with devices information from deployment number dep_id""" url = "/deployments/%i/devices.json" % dep_id device_list = self._call_rest(url) return device_list def get_deployment_parameters(self, dep_id): """ return array with parameters {freq, name, id, i18m, units} from deployment number dep_id """ url = "/deployments/%i/parameters.json" % dep_id param_list = self._call_rest(url) return param_list def get_deployment_supplies(self, dep_id): """ return array with supplies {pod, name, id} from deployment number dep_id """ url = "/deployments/%i/supplies.json" % dep_id supply_list = self._call_rest(url) return supply_list def get_deployment_notices(self, dep_id, start, end): """ return array with alerts information from deployment number dep_id from the interval selected """ start = self.dxdate(start) end = self.dxdate(end) url = "/deployments/%i/notices.json?start=%s&end=%s" % (dep_id, start, end) notice_list = self._call_rest(url) return notice_list def get_deployment_parameter_devices(self, dep_id, param_nid): """ return array with parameters {id, name, networkid} from deployment number dep_id """ url = "/deployments/%i/parameters/%s/devices.json" % (dep_id, str(param_nid)) device_list = self._call_rest(url) return device_list def set_deployment_thing(self, dep_id, key, value): """ update dict of information saved by the user""" url = "/deployments/%i/things/set/%s.json" % (dep_id, key) payload = json.dumps(value, default=self._json_date_handler) data = self._call_rest(url, payload=payload, parse_response=False) return data def get_deployment_thing(self, dep_id, key): """ return dict of information saved by the user""" url = "/deployments/%i/things/get/%s.json" % (dep_id, key) data = self._call_rest(url, parse_response=False) self.logger.info('dep_thing:%s' % str(data)) data = json.loads(data, object_hook=self._datetime_parser) return data def get_location(self, loc_id): """ return dict with basic information from locat""" url = "/locations/%i.json" % loc_id location = self._call_rest(url) return location def get_location_parameters(self, loc_id): """ return array with parameters {freq, name, id, i18m, units} from location number loc_id """ url = "/locations/%i/parameters.json" % loc_id param_list = self._call_rest(url) return param_list def get_location_notices(self, loc_id, start, end): """ return array with alerts information for location number loc_id from the interval selected """ start = self.dxdate(start) end = self.dxdate(end) url = "/locations/%i/notices.json?start=%s&end=%s" % (loc_id, start, end) notice_list = self._call_rest(url) return notice_list def get_location_comments(self, loc_id, start, end): """ return array with comments information for location number loc_id from the interval selected """ start = self.dxdate(start) end = self.dxdate(end) url = "/locations/%i/comments.json?start=%s&end=%s" % (loc_id, start, end) comments = self._call_rest(url) return comments def get_location_parameter_devices(self, loc_id, param_nid): """ return array with parameters {id, name, networkid} from location number loc_id """ url = "/locations/%i/parameters/%i/devices.json" % (loc_id, param_nid) device_list = self._call_rest(url) return device_list def get_location_supplies(self, loc_id): """ return array with supplies {pod, name, id} from location number loc_id """ url = "/locations/%i/supplies.json" % loc_id supply_list = self._call_rest(url) return supply_list def get_location_devices(self, loc_id): """ return array with the devices from the location """ url = "/locations/%i/devices.json" % loc_id device_list = self._call_rest(url) return device_list def get_device(self, dev_id): """ return dict with information for the device """ url = "/devices/%i.json" % dev_id device = self._call_rest(url) return device def get_device_parameters(self, dev_id): """ return array with parameters {freq, name, id, i18m, units} from device number dev_id """ param_list = self._call_rest("/devices/" + str(dev_id) + "/parameters.json") return param_list def get_simulated_bill(self, dev_id, start, end, type_param="ELECTRICAL", parameters="AAANNN", pod=None, time='HOUR'): ''' returns bill generated from data in dexcell Parameters A : RAW + SUMMARY R: RAW set of datas returned by time mesure, default hour S: SUMMARY resum of data: totals, periods... N: nothing type_param can be ELECTRICAL, WATER, GAS ''' new_pod = '' if pod is not None: new_pod = "&pod=" + pod start = start.strftime("%Y%m%d%H%M%S") end = end.strftime("%Y%m%d%H%M%S") url = ["/cost/%i/%s.json?start=%s" % (dev_id, type_param, start)] url.append("&end=%s&applyPattern=%s&period=%s%s" % (end, parameters, time, new_pod)) url = "".join(url) bill = self._call_rest(url) return bill def get_supply_bills(self, sup_id, start, end, type_param='ELECTRICAL', parameters="AAANNN", pod=None, time='HOUR'): ''' returns bills updated by the customer Parameters A : RAW + SUMMARY R: RAW set of datas returned by time mesure, default hour S: SUMMARY resum of data: totals, periods... N: nothing type_param can be ELECTRICAL, WATER, GAS ''' new_pod = '' if pod is not None: new_pod = "&pod=" + pod start = start.strftime("%Y%m%d%H%M%S") end = end.strftime("%Y%m%d%H%M%S") url = ["/cost/%i/bills/%s.json?start=%s" % (sup_id, type_param, start)] url.append("&end=%s&applyPattern=%s&period=%s%s" % (end, parameters, time, new_pod)) url = "".join(url) bills = self._call_rest(url) return bills def get_session(self, session_id): """ return the session for an app with a concret session_id""" url = "/session/%s.json" % session_id response = self._call_rest(url) self.logger.info('get session: ' + str(response)) return response def get_readings(self, dev_id, s_nid, start, end): """ return array dict with {values, timestamp} """ start = self.dxdate(start) end = self.dxdate(end) url = "/devices/%i/%i/readings.json?start=%s&end=%s" % (dev_id, s_nid, start, end) readings = self._call_rest(url) for i in range(0, len(readings)): try: readings[i]['ts'] = datetime.strptime(readings[i]['ts'], "%Y-%m-%d %H:%M:%S") readings[i]['tsutc'] = datetime.strptime( readings[i]['tsutc'], "%Y-%m-%d %H:%M:%S") except KeyError: pass return readings def get_readings_new(self, dev_id, param, frequency, operation, start, end): """ returns array of dict of values from the device dev_id with parameter param with a frequency in the interval start - end. """ start = self.dxdate(start) end = self.dxdate(end) url = ["/devices/%i/%s/readings.json?" % (dev_id, str(param))] url.append("start=%s&end=%s&frequency=%s&operation=%s" % (start, end, str(frequency), str(operation))) url = "".join(url) readings = self._call_rest(url) for i in range(0, len(readings)): try: readings[i]['ts'] = datetime.strptime(readings[i]['ts'], "%Y-%m-%d %H:%M:%S") readings[i]['tsutc'] = datetime.strptime( readings[i]['tsutc'], "%Y-%m-%d %H:%M:%S") except KeyError: pass return readings def get_cost(self, nid, start, end, energy_type='ELECTRICAL', period='HOUR', grouped=False): """ return array from cost and consumption with timestamp""" str_grouped = 'TRUE' if not grouped: str_grouped = 'FALSE' start = self.dxdate(start) end = self.dxdate(end) url = ["/devices/%i/%s/cost.json?" % (nid, energy_type)] url.append("start=%s&end=%s&period=%s&grouped=%s" % (start, end, str(period), str_grouped)) url = "".join(url) raw_response = self._call_rest(url) try: readings = raw_response['readings'] for i in range(0, len(readings)): readings[i]['ts'] = datetime.strptime(readings[i]['ts'], "%Y/%m/%d %H:%M:%S") periods = raw_response['periods'] return readings, periods except KeyError: return []
class DexcellRestApiAuth(object): """ Class for authentification in Dexcell software. """ def __init__(self, endpoint, hash_dexma, secret, logger_name="dexcell_rest_api_auth"): self.endpoint = endpoint self.hash = hash_dexma self.secret = secret self.logger = logging.getLogger(logger_name) if len(self.logger.handlers) == 0: self.logger.setLevel(logging.INFO) self.handler = NullHandler() h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" self.handler.setFormatter(logging.Formatter(h_format)) self.logger.addHandler(self.handler) def _json_date_handler(self, obj): return obj.isoformat() if hasattr(obj, 'isoformat') else obj def _datetime_parser(self, dct): DATE_FORMAT = "%Y-%m-%dT%H:%M:%S" strp = datetime.strptime for k, v in dct.items(): if isinstance(v, basestring) and re.search( "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", v): try: dct[k] = strp(v, DATE_FORMAT) except ValueError: pass return dct def _call_rest(self, url): url = self.endpoint + url req = urllib2.Request(url) response = urllib2.urlopen(req) data = response.read() self.logger.info(data) return data def perm_token(self, temp_token): ''' obtain permanent token for oauth authentication''' url = "/oauth/accesstoken?temp_token=%s&secret=%s&idclient=%s" % ( str(temp_token), self.secret, self.hash) response = self._call_rest(url) return response def set_key_value(self, key, value): "Set this key with this value in the key-value data store" url = self.endpoint + "/things/set/" + key req = urllib2.Request(url, json.dumps(value, default=self._json_date_handler), headers={'x-dexcell-secret': self.secret}) self.logger.info('storing key: %s with secret: %s' % (key, self.secret)) response = urllib2.urlopen(req) data = response.read() return data def get_key(self, key): "Get this key from the key-value data store" url = "%s/things/get/%s" % (self.endpoint, key) req = urllib2.Request(url, headers={'x-dexcell-secret': self.secret}) response = urllib2.urlopen(req) data = response.read() data = json.loads(data, object_hook=self._datetime_parser) result = json.loads(data['result'], object_hook=self._datetime_parser) return result
class DexcellSender(object): DEFAULT_SERVER = 'insert.dexcell.com' DEFAULT_URL = '/insert-json.htm' DEFAULT_LOGFILE = '/var/log/dexma/DexcellSender.log' DEFAULT_LOGLEVEL = logging.INFO DEFAULT_GATEWAY = 'None' DEFAULT_LOGGERNAME = 'DexcellSender' def __init__(self, gateway=DEFAULT_GATEWAY, loggerName=DEFAULT_LOGGERNAME, logfile=DEFAULT_LOGFILE, loglevel=DEFAULT_LOGLEVEL, server=DEFAULT_SERVER, url=DEFAULT_URL, https=True, timeout=30.0): self.__https = https self.__server = server self.__url = url self.__timeout = timeout self.__gateway = gateway self.__logger = logging.getLogger(loggerName) if len(self.__logger.handlers) == 0: self.__logger.setLevel(loglevel) self.handler = NullHandler() h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" self.handler.setFormatter(logging.Formatter(h_format)) self.__logger.addHandler(self.handler) def setup(self, gateway=DEFAULT_GATEWAY, loggerName=DEFAULT_LOGGERNAME, logfile=DEFAULT_LOGFILE, loglevel=DEFAULT_LOGLEVEL, server=DEFAULT_SERVER, url=DEFAULT_URL): """Setup the Dexcell Sender Object """ self.__server = server self.__url = url self.__gateway = gateway self.__logger = logging.getLogger(loggerName) if len(self.__logger.handlers) == 0: self.__logger.setLevel(loglevel) self.handler = NullHandler() h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" self.handler.setFormatter(logging.Formatter(h_format)) self.__logger.addHandler(self.handler) def changeGateway(self, gateway): """Change the gateway mac that will be sent """ self.__gateway = gateway def __insertRawJSONData(self, data): """Insert the raw data string to the server """ params = 'data=' + data headers = { "Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain" } if self.__https: conn = httplib.HTTPSConnection(self.__server, timeout=self.__timeout) else: conn = httplib.HTTPConnection(self.__server, timeout=self.__timeout) conn.request("POST", self.__url, params, headers) error = True maxerror = 0 while error: try: response = conn.getresponse() error = False except: print self.__logger.exception("Error inserting data") maxerror = maxerror + 1 time.sleep(1) if maxerror > 10: return (-1, 'FAIL') logger_msg_wo_params = "Insert from %s with status %s and result %s" logger_params = (self.__gateway, str(response.status), str(response.getheader('data'))) logger_message = logger_msg_wo_params % logger_params self.__logger.debug(logger_message) return response.status, response.getheader('data') def insertDexcellServiceMessage(self, serviceMessage, timezone='UTC', extraparams={}): '''Insert a single DexcellServiceMessage ''' reading = { 'nodeNetworkId': str(serviceMessage.node), 'serviceNetworkId': int(serviceMessage.service), 'value': float(serviceMessage.value), 'seqNum': int(serviceMessage.seqnum), 'timeStamp': time.strftime("%Y-%m-%dT%H:%M:%S.000 " + timezone, serviceMessage.timestamp) } data = {'gatewayId': self.__gateway, 'service': [reading]} for key in extraparams.keys(): data[key] = extraparams[key] result = self.__insertRawJSONData(json.dumps(data)) return result def insertDexcellServiceMessages(self, serviceMessageIterator, timezone='UTC', extraparams={}): """ Insert many DexcellServiceMessages at once """ readings = [] for serviceMessage in serviceMessageIterator: reading = { 'nodeNetworkId': str(serviceMessage.node), 'serviceNetworkId': int(serviceMessage.service), 'value': float(serviceMessage.value), 'seqNum': int(serviceMessage.seqnum), 'timeStamp': time.strftime("%Y-%m-%dT%H:%M:%S.000 " + timezone, serviceMessage.timestamp) } readings.append(reading) data = {'gatewayId': self.__gateway, 'service': readings} for key in extraparams.keys(): data[key] = extraparams[key] result = self.__insertRawJSONData(json.dumps(data)) return result
class DexcellRestApi(object): """ class with all the utils API calls available group by from deployment calls, location calls and devide calls. """ def __init__(self, endpoint, token, logger_name="dexcell_rest_api"): self.endpoint = endpoint self.token = token self.logger = logging.getLogger(logger_name) if len(self.logger.handlers) == 0: self.logger.setLevel(logging.INFO) self.handler = NullHandler() h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" self.handler.setFormatter(logging.Formatter(h_format)) self.logger.addHandler(self.handler) def _json_date_handler(self, obj): return obj.isoformat() if hasattr(obj, "isoformat") else obj def _datetime_parser(self, dct): DATE_FORMAT = "%Y-%m-%dT%H:%M:%S" for k, v in dct.items(): if isinstance(v, basestring) and re.search("\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", v): try: dct[k] = datetime.strptime(v, DATE_FORMAT) except: pass return dct def dxdate(self, dt): """ convert datetime into default date string format used in dexcell api calls """ return dt.strftime("%Y%m%d%H%M%S") def _call_rest(self, url, payload=None, parse_response=True): url = self.endpoint + url self.logger.info("url:%s token:%s" % (url, self.token)) if payload is None: req = urllib2.Request(url, headers={"x-dexcell-token": self.token}) else: req = urllib2.Request(url, payload, headers={"x-dexcell-token": self.token}) try: response = urllib2.urlopen(req, timeout=600.0) data = response.read() if parse_response: return json.loads(data) else: return data except urllib2.HTTPError as httperror: info = json.loads(httperror.read()) if httperror.code == 404: self.logger.error("error: not found") raise DexcellRestApiError("NOTFOUND", info["description"], info["moreInfo"]) elif httperror.code == 401: self.logger.error("error: not authorized") raise DexcellRestApiError("INVALIDTOKEN", info["description"], info["moreInfo"]) else: raise DexcellRestApiError("UNKNOWN", info["description"], info["moreInfo"]) self.logger.error("error: %s" % (str(httperror.code))) def get_deployment(self, dep_id): """ return dict with basic information from deployment number dep_id""" url = "/deployments/%i.json" % dep_id deployment = self._call_rest(url) return deployment def get_deployment_locations(self, dep_id): """ return array with locations information from deployment number dep_id""" url = "/deployments/%i/locations.json" % dep_id location_list = self._call_rest(url) return location_list def get_deployment_devices(self, dep_id): """ return array with devices information from deployment number dep_id""" url = "/deployments/%i/devices.json" % dep_id device_list = self._call_rest(url) return device_list def get_deployment_parameters(self, dep_id): """ return array with parameters {freq, name, id, i18m, units} from deployment number dep_id """ url = "/deployments/%i/parameters.json" % dep_id param_list = self._call_rest(url) return param_list def get_deployment_supplies(self, dep_id): """ return array with supplies {pod, name, id} from deployment number dep_id """ url = "/deployments/%i/supplies.json" % dep_id supply_list = self._call_rest(url) return supply_list def get_deployment_notices(self, dep_id, start, end): """ return array with alerts information from deployment number dep_id from the interval selected """ start = self.dxdate(start) end = self.dxdate(end) url = "/deployments/%i/notices.json?start=%s&end=%s" % (dep_id, start, end) notice_list = self._call_rest(url) return notice_list def get_deployment_parameter_devices(self, dep_id, param_nid): """ return array with parameters {id, name, networkid} from deployment number dep_id """ url = "/deployments/%i/parameters/%s/devices.json" % (dep_id, str(param_nid)) device_list = self._call_rest(url) return device_list def set_deployment_thing(self, dep_id, key, value): """ update dict of information saved by the user""" url = "/deployments/%i/things/set/%s.json" % (dep_id, key) payload = json.dumps(value, default=self._json_date_handler) data = self._call_rest(url, payload=payload, parse_response=False) return data def get_deployment_thing(self, dep_id, key): """ return dict of information saved by the user""" url = "/deployments/%i/things/get/%s.json" % (dep_id, key) data = self._call_rest(url, parse_response=False) self.logger.info("dep_thing:%s" % str(data)) data = json.loads(data, object_hook=self._datetime_parser) return data def get_location(self, loc_id): """ return dict with basic information from locat""" url = "/locations/%i.json" % loc_id location = self._call_rest(url) return location def get_location_parameters(self, loc_id): """ return array with parameters {freq, name, id, i18m, units} from location number loc_id """ url = "/locations/%i/parameters.json" % loc_id param_list = self._call_rest(url) return param_list def get_location_notices(self, loc_id, start, end): """ return array with alerts information for location number loc_id from the interval selected """ start = self.dxdate(start) end = self.dxdate(end) url = "/locations/%i/notices.json?start=%s&end=%s" % (loc_id, start, end) notice_list = self._call_rest(url) return notice_list def get_location_comments(self, loc_id, start, end): """ return array with comments information for location number loc_id from the interval selected """ start = self.dxdate(start) end = self.dxdate(end) url = "/locations/%i/comments.json?start=%s&end=%s" % (loc_id, start, end) comments = self._call_rest(url) return comments def get_location_parameter_devices(self, loc_id, param_nid): """ return array with parameters {id, name, networkid} from location number loc_id """ url = "/locations/%i/parameters/%i/devices.json" % (loc_id, param_nid) device_list = self._call_rest(url) return device_list def get_location_supplies(self, loc_id): """ return array with supplies {pod, name, id} from location number loc_id """ url = "/locations/%i/supplies.json" % loc_id supply_list = self._call_rest(url) return supply_list def get_location_devices(self, loc_id): """ return array with the devices from the location """ url = "/locations/%i/devices.json" % loc_id device_list = self._call_rest(url) return device_list def get_device(self, dev_id): """ return dict with information for the device """ url = "/devices/%i.json" % dev_id device = self._call_rest(url) return device def get_device_parameters(self, dev_id): """ return array with parameters {freq, name, id, i18m, units} from device number dev_id """ param_list = self._call_rest("/devices/" + str(dev_id) + "/parameters.json") return param_list def get_simulated_bill( self, dev_id, start, end, type_param="ELECTRICAL", parameters="AAANNN", pod=None, time="HOUR" ): """ returns bill generated from data in dexcell Parameters A : RAW + SUMMARY R: RAW set of datas returned by time mesure, default hour S: SUMMARY resum of data: totals, periods... N: nothing type_param can be ELECTRICAL, WATER, GAS """ new_pod = "" if pod is not None: new_pod = "&pod=" + pod start = start.strftime("%Y%m%d%H%M%S") end = end.strftime("%Y%m%d%H%M%S") url = ["/cost/%i/%s.json?start=%s" % (dev_id, type_param, start)] url.append("&end=%s&applyPattern=%s&period=%s%s" % (end, parameters, time, new_pod)) url = "".join(url) bill = self._call_rest(url) return bill def get_supply_bills(self, sup_id, start, end, type_param="ELECTRICAL", parameters="AAANNN", pod=None, time="HOUR"): """ returns bills updated by the customer Parameters A : RAW + SUMMARY R: RAW set of datas returned by time mesure, default hour S: SUMMARY resum of data: totals, periods... N: nothing type_param can be ELECTRICAL, WATER, GAS """ new_pod = "" if pod is not None: new_pod = "&pod=" + pod start = start.strftime("%Y%m%d%H%M%S") end = end.strftime("%Y%m%d%H%M%S") url = ["/cost/%i/bills/%s.json?start=%s" % (sup_id, type_param, start)] url.append("&end=%s&applyPattern=%s&period=%s%s" % (end, parameters, time, new_pod)) url = "".join(url) bills = self._call_rest(url) return bills def get_session(self, session_id): """ return the session for an app with a concret session_id""" url = "/session/%s.json" % session_id response = self._call_rest(url) self.logger.info("get session: " + str(response)) return response def get_readings(self, dev_id, s_nid, start, end): """ return array dict with {values, timestamp} """ start = self.dxdate(start) end = self.dxdate(end) url = "/devices/%i/%i/readings.json?start=%s&end=%s" % (dev_id, s_nid, start, end) readings = self._call_rest(url) for i in range(0, len(readings)): try: readings[i]["ts"] = datetime.strptime(readings[i]["ts"], "%Y-%m-%d %H:%M:%S") readings[i]["tsutc"] = datetime.strptime(readings[i]["tsutc"], "%Y-%m-%d %H:%M:%S") except KeyError: pass return readings def get_readings_new(self, dev_id, param, frequency, operation, start, end): """ returns array of dict of values from the device dev_id with parameter param with a frequency in the interval start - end. """ start = self.dxdate(start) end = self.dxdate(end) url = ["/devices/%i/%s/readings.json?" % (dev_id, str(param))] url.append("start=%s&end=%s&frequency=%s&operation=%s" % (start, end, str(frequency), str(operation))) url = "".join(url) readings = self._call_rest(url) for i in range(0, len(readings)): try: readings[i]["ts"] = datetime.strptime(readings[i]["ts"], "%Y-%m-%d %H:%M:%S") readings[i]["tsutc"] = datetime.strptime(readings[i]["tsutc"], "%Y-%m-%d %H:%M:%S") except KeyError: pass return readings def get_cost(self, nid, start, end, energy_type="ELECTRICAL", period="HOUR", grouped=False): """ return array from cost and consumption with timestamp""" str_grouped = "TRUE" if not grouped: str_grouped = "FALSE" start = self.dxdate(start) end = self.dxdate(end) url = ["/devices/%i/%s/cost.json?" % (nid, energy_type)] url.append("start=%s&end=%s&period=%s&grouped=%s" % (start, end, str(period), str_grouped)) url = "".join(url) raw_response = self._call_rest(url) try: readings = raw_response["readings"] for i in range(0, len(readings)): readings[i]["ts"] = datetime.strptime(readings[i]["ts"], "%Y/%m/%d %H:%M:%S") periods = raw_response["periods"] return readings, periods except KeyError: return []
class DexcellRestApiAuth(object): """ Class for authentification in Dexcell software. """ def __init__(self, endpoint, hash_dexma, secret, logger_name="dexcell_rest_api_auth"): self.endpoint = endpoint self.hash = hash_dexma self.secret = secret self.logger = logging.getLogger(logger_name) if len(self.logger.handlers) == 0: self.logger.setLevel(logging.INFO) self.handler = NullHandler() h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" self.handler.setFormatter(logging.Formatter(h_format)) self.logger.addHandler(self.handler) def _json_date_handler(self, obj): return obj.isoformat() if hasattr(obj, "isoformat") else obj def _datetime_parser(self, dct): DATE_FORMAT = "%Y-%m-%dT%H:%M:%S" strp = datetime.strptime for k, v in dct.items(): if isinstance(v, basestring) and re.search("\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", v): try: dct[k] = strp(v, DATE_FORMAT) except ValueError: pass return dct def _call_rest(self, url): url = self.endpoint + url req = urllib2.Request(url) response = urllib2.urlopen(req) data = response.read() self.logger.info(data) return data def perm_token(self, temp_token): """ obtain permanent token for oauth authentication""" url = "/oauth/accesstoken?temp_token=%s&secret=%s&idclient=%s" % (str(temp_token), self.secret, self.hash) response = self._call_rest(url) return response def set_key_value(self, key, value): "Set this key with this value in the key-value data store" url = self.endpoint + "/things/set/" + key req = urllib2.Request( url, json.dumps(value, default=self._json_date_handler), headers={"x-dexcell-secret": self.secret} ) self.logger.info("storing key: %s with secret: %s" % (key, self.secret)) response = urllib2.urlopen(req) data = response.read() return data def get_key(self, key): "Get this key from the key-value data store" url = "%s/things/get/%s" % (self.endpoint, key) req = urllib2.Request(url, headers={"x-dexcell-secret": self.secret}) response = urllib2.urlopen(req) data = response.read() data = json.loads(data, object_hook=self._datetime_parser) result = json.loads(data["result"], object_hook=self._datetime_parser) return result
class DexcellSender(object): DEFAULT_SERVER = "insert.dexcell.com" DEFAULT_URL = "/insert-json.htm" DEFAULT_LOGFILE = "/var/log/dexma/DexcellSender.log" DEFAULT_LOGLEVEL = logging.INFO DEFAULT_GATEWAY = "None" DEFAULT_LOGGERNAME = "DexcellSender" def __init__( self, gateway=DEFAULT_GATEWAY, loggerName=DEFAULT_LOGGERNAME, logfile=DEFAULT_LOGFILE, loglevel=DEFAULT_LOGLEVEL, server=DEFAULT_SERVER, url=DEFAULT_URL, https=True, timeout=30.0, ): self.__https = https self.__server = server self.__url = url self.__timeout = timeout self.__gateway = gateway self.__logger = logging.getLogger(loggerName) if len(self.__logger.handlers) == 0: self.__logger.setLevel(loglevel) self.handler = NullHandler() h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" self.handler.setFormatter(logging.Formatter(h_format)) self.__logger.addHandler(self.handler) def setup( self, gateway=DEFAULT_GATEWAY, loggerName=DEFAULT_LOGGERNAME, logfile=DEFAULT_LOGFILE, loglevel=DEFAULT_LOGLEVEL, server=DEFAULT_SERVER, url=DEFAULT_URL, ): """Setup the Dexcell Sender Object """ self.__server = server self.__url = url self.__gateway = gateway self.__logger = logging.getLogger(loggerName) if len(self.__logger.handlers) == 0: self.__logger.setLevel(loglevel) self.handler = NullHandler() h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" self.handler.setFormatter(logging.Formatter(h_format)) self.__logger.addHandler(self.handler) def changeGateway(self, gateway): """Change the gateway mac that will be sent """ self.__gateway = gateway def __insertRawJSONData(self, data): """Insert the raw data string to the server """ params = "data=" + data headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"} if self.__https: conn = httplib.HTTPSConnection(self.__server, timeout=self.__timeout) else: conn = httplib.HTTPConnection(self.__server, timeout=self.__timeout) conn.request("POST", self.__url, params, headers) error = True maxerror = 0 while error: try: response = conn.getresponse() error = False except: print self.__logger.exception("Error inserting data") maxerror = maxerror + 1 time.sleep(1) if maxerror > 10: return (-1, "FAIL") logger_msg_wo_params = "Insert from %s with status %s and result %s" logger_params = (self.__gateway, str(response.status), str(response.getheader("data"))) logger_message = logger_msg_wo_params % logger_params self.__logger.debug(logger_message) return response.status, response.getheader("data") def insertDexcellServiceMessage(self, serviceMessage, timezone="UTC", extraparams={}): """Insert a single DexcellServiceMessage """ reading = { "nodeNetworkId": str(serviceMessage.node), "serviceNetworkId": int(serviceMessage.service), "value": float(serviceMessage.value), "seqNum": int(serviceMessage.seqnum), "timeStamp": time.strftime("%Y-%m-%dT%H:%M:%S.000 " + timezone, serviceMessage.timestamp), } data = {"gatewayId": self.__gateway, "service": [reading]} for key in extraparams.keys(): data[key] = extraparams[key] result = self.__insertRawJSONData(json.dumps(data)) return result def insertDexcellServiceMessages(self, serviceMessageIterator, timezone="UTC", extraparams={}): """ Insert many DexcellServiceMessages at once """ readings = [] for serviceMessage in serviceMessageIterator: reading = { "nodeNetworkId": str(serviceMessage.node), "serviceNetworkId": int(serviceMessage.service), "value": float(serviceMessage.value), "seqNum": int(serviceMessage.seqnum), "timeStamp": time.strftime("%Y-%m-%dT%H:%M:%S.000 " + timezone, serviceMessage.timestamp), } readings.append(reading) data = {"gatewayId": self.__gateway, "service": readings} for key in extraparams.keys(): data[key] = extraparams[key] result = self.__insertRawJSONData(json.dumps(data)) return result
from logging import getLogger, NullHandler, StreamHandler, Formatter from logging import DEBUG, INFO, WARNING, ERROR import sys rpkg = getLogger('rpkg') dgt = getLogger('dgt') cli = getLogger('cli') handler = NullHandler() dgt.addHandler(handler) cli.addHandler(handler) # Add a default handler if there are no real handlers set, yet (not counting NullHandler) if len(rpkg.handlers) <= 1: handler = StreamHandler(sys.stderr) formatter = Formatter('[%(name)s] %(levelname)s: %(message)s') handler.setFormatter(formatter) rpkg.addHandler(handler) if len(dgt.handlers) <= 1: handler = StreamHandler(sys.stderr) formatter = Formatter('[%(name)s] %(levelname)s: %(message)s') handler.setFormatter(formatter) dgt.addHandler(handler) if len(cli.handlers) <= 1: handler = StreamHandler(sys.stdout) formatter = Formatter('%(message)s') handler.setFormatter(formatter) cli.addHandler(handler)
def test_null_handler(response): nh = NullHandler() formatter = color_debug.ColorFormatter() nh.setFormatter(formatter)
def test_null_handler(response): nh = NullHandler() formatter = color_bucket_logger.ColorFormatter() nh.setFormatter(formatter)