Beispiel #1
0
def setup_logging_no_config(loglevel, quiet):
    log_level = aprsd_config.LOG_LEVELS[loglevel]
    LOG.setLevel(log_level)
    log_format = aprsd_config.DEFAULT_LOG_FORMAT
    date_format = aprsd_config.DEFAULT_DATE_FORMAT
    log_formatter = logging.Formatter(fmt=log_format, datefmt=date_format)
    fh = NullHandler()

    fh.setFormatter(log_formatter)
    LOG.addHandler(fh)

    if not quiet:
        sh = logging.StreamHandler(sys.stdout)
        sh.setFormatter(log_formatter)
        LOG.addHandler(sh)
Beispiel #2
0
def create_logger(robot, path=None, debug=False):

    logger = logging.getLogger(robot)
    logger.setLevel(logging.DEBUG if debug else logging.INFO)

    if path is None:
        handler = NullHandler()
    else:
        logFile = os.path.join(path, robot + '.log')
        handler = TimedRotatingFileHandler(logFile, when='midnight')

    formatter = Formatter(
        '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    handler.setFormatter(formatter)

    logger.addHandler(handler)

    return logger
Beispiel #3
0
class SnowRestSession(object):
    """
    This class behaves very similarly to a requests.Session object.
    It can be configured by loading a .yaml config file,
    or by setting attributes.
    """
    def __init__(self):
        self.instance = None
        self.auth_type = None
        self.sso_method = None
        self.basic_auth_user = None
        self.basic_auth_password = None
        self.oauth_client_id = None
        self.oauth_client_secret = None
        self.session_cookie_file_path = None
        self.session_cookie = None
        self.oauth_token_file_path = None
        self.token_dic = None
        self.fresh_cookie = False
        self.fresh_token = False
        self.store_cookie = True
        self.store_token = True

        self.session = requests.Session()

        self._log_enabled = False
        self._logger_name = 'snow-client.session_' + uuid.uuid4(
        ).hex[:-24]  # used to get a Logger
        self._logger = logging.getLogger(self._logger_name)
        self._log_handler = NullHandler()
        self._logger.addHandler(self._log_handler)

        self._log_file_path = None
        self._log_level = 'INFO'
        self._log_format = "%(asctime)s [%(name)s] [%(levelname)s] %(message)s"
        self._log_file_size_bytes = 1000000
        self._log_file_rotations = 10
        self._log_file_encoding = 'utf-8'

    def load_config_file(self, config_file_path):
        """
        Load a .yaml configuration file

        Args:
            config_file_path (str): the path to the configuration file

        Raises:
            SnowRestSessionException: if no file is found at the specified path, or some inconsistency is found
                in the file.
            Exception: if some other error happens when opening or parsing the file
        """
        if not config_file_path:
            raise SnowRestSessionException(
                "SnowRestSession.load_config_file: the parameter config_file_path must have a value"
            )

        try:
            with open(config_file_path) as f:
                config_file = yaml.safe_load(f)
        except Exception as e:
            sys.stderr.write(
                "SnowRestSession.load_config_file: Issue when opening the config file at %s.\n"
                % config_file_path)
            raise e

        if 'instance' in config_file:
            self.set_instance(config_file['instance'])

        if 'auth' in config_file:
            if 'type' in config_file['auth']:
                self.auth_type = config_file['auth']['type']

        if self.auth_type == 'sso_oauth':
            if 'sso_method' in config_file['auth']:
                self.sso_method = config_file['auth']['sso_method']

            if 'oauth_client_id' in config_file['auth']:
                self.oauth_client_id = config_file['auth']['oauth_client_id']

            if 'oauth_client_secret' in config_file['auth']:
                self.oauth_client_secret = config_file['auth'][
                    'oauth_client_secret']

            if 'session' in config_file:
                if 'cookie_file' in config_file['session']:
                    self.session_cookie_file_path = config_file['session'][
                        'cookie_file']

            if 'oauth_tokens_file' in config_file['session']:
                self.oauth_token_file_path = config_file['session'][
                    'oauth_tokens_file']

        elif self.auth_type == 'basic':
            if 'user' in config_file['auth']:
                self.basic_auth_user = config_file['auth']['user']

            if 'password' in config_file['auth']:
                self.basic_auth_password = config_file['auth']['password']

            if 'session' in config_file:
                if 'cookie_file' in config_file['session']:
                    self.session_cookie_file_path = config_file['session'][
                        'cookie_file']
        else:
            raise SnowRestSessionException(
                "SnowRestSession.load_config_file: the property \"auth_type\" "
                "must have a value of \"sso_auth\" or \"basic\"")

        if 'log' in config_file:
            if 'log_enabled' in config_file['log'] and config_file['log'][
                    'log_enabled']:
                self._log_enabled = True
            if 'log_level' in config_file['log']:
                self.set_log_level(config_file['log']['log_level'])
            if 'log_file_path' in config_file['log']:
                self._log_file_path = config_file['log']['log_file_path']
            if 'log_format' in config_file['log']:
                self._log_format = config_file['log']['log_format']
            if 'log_file_size_bytes' in config_file['log']:
                self._log_file_size_bytes = config_file['log'][
                    'log_file_size_bytes']
            if 'log_file_rotations' in config_file['log']:
                self._log_file_rotations = config_file['log'][
                    'log_file_rotations']
            if 'log_file_encoding' in config_file['log']:
                self._log_file_encoding = config_file['log'][
                    'log_file_encoding']
            self._configure_handler()

    def set_instance(self, instance):
        """
        Args:
            instance (str): the ServiceNow instance to use. It is optional to prefix with "https://".
        Examples:
            >>> s = SnowRestSession()
            >>> s.set_instance("cern.service-now.com")
            >>>
            >>> s2 = SnowRestSession()
            >>> s2.set_instance("https://cerntest.service-now.com")
        """
        if not instance.startswith('https://'):
            self.instance = 'https://' + instance
        else:
            self.instance = instance

    def set_auth_type(self, auth_type):
        """
        Args:
            auth_type (str): the authentication type to use. Either 'sso_auth' or 'basic'

        Raises:
            SnowRestSessionException: if auth_type is not 'sso_auth' or 'basic'
        """
        if auth_type != 'basic' and auth_type != 'sso_auth':
            raise SnowRestSessionException(
                "SnowRestSession.set_auth_type: the parameter \"auth_type\" "
                "must have a value of \"sso_auth\" or \"basic\"")
        self.auth_type = auth_type

    def set_sso_method(self, sso_method):
        """
        Args:
            sso_method (str): the Single Sign On method to use. Either 'kerberos' or 'certificate'.
            Needs set_auth_type('sso_auth') to have an effect.

        Raises:
            SnowRestSessionException: if sso_method is not 'kerberos' or 'certificate'

        Note: certificate support needs to be implemented
        """
        if sso_method != 'kerberos' and sso_method != 'certificate':
            raise SnowRestSessionException(
                "SnowRestSession.set_sso_method: the parameter \"sso_method\" "
                "must have a value of \"kerberos\" or \"certificate\"")

        # TODO: implement certificate support. Update the comment above
        if sso_method == 'certificate':
            raise SnowRestSessionException(
                "SnowRestSession.set_sso_method: certificate support is not yet implemented"
            )
        self.sso_method = sso_method

    def set_oauth_client_id(self, oauth_client_id):
        """
        Args:
            oauth_client_id (str): the OAuth client id provided to you by your instance's ServiceNow administrator.
            Needs set_auth_type('sso_auth') to have an effect.
        """
        self.oauth_client_id = oauth_client_id

    def set_oauth_client_secret(self, oauth_client_secret):
        """
        Args:
            oauth_client_secret (str): the OAuth client secret provided to you by your instance's
            ServiceNow administrator.
            Needs set_auth_type('sso_auth') to have an effect.
            An OAuth client secret is sensitive information; please store it as securely as possible.
        """
        self.oauth_client_secret = oauth_client_secret

    def set_basic_auth_user(self, basic_auth_user):
        """
        Args:
            basic_auth_user (str): the name of the local ServiceNow account that will be used for authentication.
            Needs set_auth_type('basic') to have an effect.
        """
        self.basic_auth_user = basic_auth_user

    def set_basic_auth_password(self, basic_auth_password):
        """
        Args:
            basic_auth_password (str): the password of the local ServiceNow account that will be used for
            authentication.
            Needs set_auth_type('basic') to have an effect.
        """
        self.basic_auth_password = basic_auth_password

    def set_session_cookie_file_path(self, session_cookie_file_path):
        """
        Args:
            session_cookie_file_path (str): the path to the file where the session cookies will be persisted.
            If not provided, the cookies will not be persisted.
        """
        self.session_cookie_file_path = session_cookie_file_path

    def set_oauth_token_file_path(self, oauth_token_file_path):
        """
        Args:
            oauth_token_file_path (str): the path to the file where the OAuth refresh and access tokens
            will be persisted. If not provided, the OAuth tokens will not be persisted.
        """
        self.oauth_token_file_path = oauth_token_file_path

    def set_log_enabled(self, log_enabled):
        self._log_enabled = log_enabled

    def set_log_level(self, log_level):
        self._log_level = log_level
        self._logger.setLevel(self._log_level)

    def set_log_file_path(self, log_file_path):
        self._log_file_path = log_file_path
        self._configure_handler()

    def set_log_format(self, log_format):
        self._log_format = log_format
        self._configure_handler()

    def set_log_file_size_bytes(self, log_file_size_bytes):
        self._log_file_size_bytes = log_file_size_bytes
        self._configure_handler()

    def set_log_file_rotations(self, log_file_rotations):
        self._log_file_rotations = log_file_rotations
        self._configure_handler()

    def set_log_file_encoding(self, log_file_encoding):
        self._log_file_encoding = log_file_encoding
        self._configure_handler()

    def is_logging_enabled(self):
        return self._log_enabled

    def get_logger_name(self):
        return self._logger_name

    def get_logger(self):
        return self._logger

    def get_log_handler(self):
        return self._log_handler

    def get(self, url, headers=None, params=None):
        """
        Executes a raw GET operation and returns the result.
        Used to read or retrieve information.
        The authentication method, and session cookie / OAuth tokens persistance will depend on how the session
        has been configured.

        Args:
            url (str): a relative URL, such as ``/api/now/v2/table/incident/c1c535ba85f45540adf94de5b835cd43``.
                An absolute URL such as ``https://cerntest.service-now.com/api/now/v2/table/incident/c1c535ba85f45540adf94de5b835cd43`` can also
                be used, but the instance should match with the "instance" parameter in the configuration file,
                or the value set with the ``.set_instance()`` method.
            headers (:obj:`dict`, optional): any additional headers to be be passed. If not set, the 'Accept' header will be set to
                'application/json'
            params (:obj:`dict`, optional): any additional URL parameters to be be passed

        Returns:
            requests.Response : If the status code is not 401, a ``requests.Response`` object is returned.
            The structure of the result inside the 'text' attribute might vary depending on the REST endpoint
            called and the way that the query is configured. ServiceNow may also set various HTTP status codes
            or headers in the response.

        Raises:
            SnowRestSessionException : if the operation could not be performed due to an authentication issue

        Examples:
            Getting an incident by sys_id with the ServiceNow REST Table API:

            >>> s = SnowRestSession()
            >>> s.load_config_file('config.yaml')
            >>> response = s.get('/api/now/v2/table/incident/c1c535ba85f45540adf94de5b835cd43')
            >>> if response.status_code == 200:
            >>>     import json
            >>>     result = json.loads(response.text)
            >>>     if 'result' in result:
            >>>         incident = result['result']  # querying by sys_id returns a single object
            >>>         print incident['short_description']  # will print "Test" (as a unicode object)

            Getting an incident by number with the ServiceNow REST Table API:

            >>> s = SnowRestSession()
            >>> s.load_config_file('config.yaml')
            >>> response = s.get('/api/now/v2/table/incident?sysparm_query=number=INC0426232')
            >>> if response.status_code == 200:
            >>>     import json
            >>>     result = json.loads(response.text)
            >>>     if 'result' in result:
            >>>         incident_array = result['result']  # querying via an encoded query returns an array of objects
            >>>         if len(incident_array) > 0:
            >>>             incident = incident_array[0]
            >>>             print incident['short_description']  # will print "Test" (as a unicode object)
        """
        result = self.__operation(operation='get',
                                  url=url,
                                  headers=headers,
                                  params=params)
        return result

    def post(self, url, headers=None, params=None, data=None):
        """
        Executes a raw POST operation and returns the result.
        Used to insert records or other information.
        The authentication method, and session cookie / OAuth tokens persistance will depend on how the session
        has been configured.

        Args:
            url (str): a relative URL, such as ``/api/now/v2/table/incident``.
                An absolute URL such as ``https://cerntest.service-now.com/api/now/v2/table/incident`` can also
                be used, but the instance should match with the "instance" parameter in the configuration file,
                or the value set with the ``.set_instance()`` method.
            headers (:obj:`dict`, optional): any additional headers to be be passed. If not set, the 'Accept' header
                will be set to 'application/json', and 'Content' will be set to 'application/json'
            params (:obj:`dict`, optional): any additional URL parameters to be be passed
            data (str): the data that will be submitted via POST. Typically, a dictionary
                of keys and values, turned into a string with import json; json.dumps()

        Returns:
            requests.Response : If the status code is not 401, a ``requests.Response`` object is returned.
            The structure of the result inside the 'text' attribute might vary depending on the REST endpoint
            called, but typically represents the record resulting from the insert. ServiceNow may also set various
            HTTP status codes or headers in the response.

        Raises:
            SnowRestSessionException : if the operation could not be performed due to an authentication issue

        Examples:
            Inserting an incident with the ServiceNow REST Table API:

            >>> s = SnowRestSession()
            >>> s.load_config_file('config.yaml')
            >>> incident_attributes = {
            >>>    'short_description' : "New incident",
            >>>    'u_business_service' : 'e85a3f3b0a0a8c0a006a2912f2f352d1',  # Service Element "ServiceNow"
            >>>    'u_functional_element' : '579fb3d90a0a8c08017ac8a1137c8ee6',  # Functional Element "ServiceNow"
            >>>    'comments' : "Initial description"
            >>> }
            >>> import json
            >>> data = json.dumps(incident_attributes)
            >>> response = s.post('/api/now/v2/table/incident', data=data)
            >>> if response.status_code == 201:
            >>>     result = json.loads(response.text)
            >>>     if 'result' in result:
            >>>         incident = result['result']
            >>>         print incident['number']  # will print the number of the newly created incident
        """
        result = self.__operation(operation='post',
                                  url=url,
                                  headers=headers,
                                  params=params,
                                  data=data)
        return result

    def put(self, url, headers=None, params=None, data=None):
        """
        Executes a raw PUT operation and returns the result.
        Used to update records or other information.
        The authentication method, and session cookie / OAuth tokens persistance will depend on how the session
        has been configured.

        Args:
            url (str): a relative URL, such as ``/api/now/v2/table/incident/c1c535ba85f45540adf94de5b835cd43``.
                An absolute URL such as ``https://cerntest.service-now.com/api/now/v2/table/incident/c1c535ba85f45540adf94de5b835cd43`` can also
                be used, but the instance should match with the "instance" parameter in the configuration file,
                or the value set with the ``.set_instance()`` method.
            headers (:obj:`dict`, optional): any additional headers to be be passed. If not set, the 'Accept' header will be set to
                'application/json', and 'Content' will be set to 'application/json'
            params (:obj:`dict`, optional): any additional URL parameters to be be passed
            data (str): the data that will be submitted via PUT. Typically, a dictionary
                of keys and values, turned into a string with import json; json.dumps()

        Returns:
            requests.Response : If the status code is not 401, a ``requests.Response`` object is returned.
            The structure of the result inside the 'text' attribute might vary depending on the REST endpoint
            called, but typically represents the state of the record after the update.
            ServiceNow may also set various HTTP status codes or headers in the response.

        Raises:
            SnowRestSessionException : if the operation could not be performed due to an authentication issue

        Examples:
            Updating an incident with the ServiceNow REST Table API:

            >>> s = SnowRestSession()
            >>> s.load_config_file('config.yaml')
            >>> incident_attributes_to_update = {
            >>>    'watch_list' : '*****@*****.**'  # will completely replace the previous value
            >>> }
            >>> import json
            >>> data = json.dumps(incident_attributes_to_update)
            >>> response = s.put('/api/now/v2/table/incident/ca37ed334f56830015d3bc511310c7a8', data=data)
            >>> if response.status_code == 200:
            >>>     result = json.loads(response.text)
            >>>     if 'result' in result:
            >>>         incident = result['result']
            >>>         print incident['number']  # will print the number of the updated incident
            >>>         print incident['watch_list']  # will print the new value of the watch_list field
        """
        result = self.__operation(operation='put',
                                  url=url,
                                  headers=headers,
                                  params=params,
                                  data=data)
        return result

    def __call_cern_get_sso_cookie(self):
        """
        Executes the cern-get-sso-cookie command (available in CERN Linux environments)
        in order to perform a Single Sign-On and produce a cookie file.
        """
        args = [
            "cern-get-sso-cookie", "--reprocess", "--url", self.instance,
            "--outfile", self.session_cookie_file_path
        ]
        p = subprocess.Popen(args,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        p.wait()

    def __good_cookie(self):
        cookie_file = open(self.session_cookie_file_path)
        cookie_file_contents = cookie_file.read()

        if (cookie_file_contents.find('glide_user_activity') != -1
                and cookie_file_contents.find('glide_session_store') != -1
                and cookie_file_contents.find('glide_user_route') != -1
                and cookie_file_contents.find('JSESSIONID') != -1
                and cookie_file_contents.find('BIGipServerpool_cern') != -1):
            return True

        return False

    def __cern_get_sso_cookie(self):
        """
        Loads and checks an existing cookie file, in order to avoid logging into ServiceNow and the
        CERN Single-Sign-On if possible. If needed, logs in again into both to produce the cookie file.
        The cookie file is loaded into memory.
        If not needed to persist the cookie file, it is deleted.

        Raises:
            SnowRestSessionException: if the Single Sign On login is not succesful. This is determined by parsing
                the cookie file after it is produced.
        """

        if not os.path.exists(
                self.session_cookie_file_path
        ):  # cookie file not present. We perform Single Sign On
            self.__call_cern_get_sso_cookie()
            self.fresh_cookie = True

        else:  # cookie file present. We check if it has the correct format
            self.fresh_cookie = False
            if not self.__good_cookie(
            ):  # the format is bad. We create a new cookie file
                self.__call_cern_get_sso_cookie()
                self.fresh_cookie = True

        # we load the cookie file
        self.session_cookie = cookielib.MozillaCookieJar(
            self.session_cookie_file_path)
        self.session_cookie.load()

        # we check if the log in was sucessful
        if not self.__good_cookie():
            raise SnowRestSessionException(
                "SnowRestSession.__cern_get_sso_cookie: The current account has failed to perform "
                "a Single-Sign-On login in to ServiceNow")

        # we load the cookies into the requests.Session session
        self.session.cookies = self.session_cookie

        # if not needed to persist the cookie file, we delete it
        if not self.store_cookie:
            os.remove(self.session_cookie_file_path)

    def __obtain_tokens(self):
        """
        Loads the OAuth access and refresh tokens from the token file.
        If the token file is not present, new OAuth access and refresh tokens are obtained.

        Raises:
            SnowRestSessionException: if the OAuth tokens could not be retrieved from ServiceNow.
            IOError, ValueError: if the token file could not be opened
        """

        if not os.path.exists(
                self.oauth_token_file_path
        ):  # tokens file not present. We obtain new tokens
            self.fresh_token = True
            token_request = self.session.post(self.instance +
                                              '/oauth_token.do',
                                              data={
                                                  'grant_type':
                                                  'password',
                                                  'client_id':
                                                  self.oauth_client_id,
                                                  'client_secret':
                                                  self.oauth_client_secret
                                              })

            if token_request.status_code == 200:  # token request was successful
                self.token_dic = json.loads(token_request.text)
                if self.store_token:
                    with open(self.oauth_token_file_path, 'w') as token_file:
                        json.dump(self.token_dic, token_file)

            else:  # token request failed. Possibly, an existing cookie file contained a timed out session
                if self.fresh_cookie:  # we just performed a Single-Sign-On. There is a problem with token retrieval.
                    raise SnowRestSessionException(
                        "SnowRestSession.__obtain_tokens: OAuth tokens could not be retrieved from ServiceNow. "
                        "Please check the OAuth client id and OAuth client secret."
                    )

                else:  # we may need to perform Single-Sign-On again, as the session may have timed out
                    self.__cern_get_sso_cookie()
                    token_request = self.session.post(
                        self.instance + '/oauth_token.do',
                        data={
                            'grant_type': 'password',
                            'client_id': self.oauth_client_id,
                            'client_secret': self.oauth_client_secret
                        })
                    self.token_dic = None
                    if token_request.status_code == 200:
                        self.token_dic = json.loads(token_request.text)
                        if self.store_token:
                            with open(self.oauth_token_file_path,
                                      'w') as token_file:
                                json.dump(self.token_dic, token_file)
                        else:
                            raise SnowRestSessionException(
                                "SnowRestSession.__obtain_tokens: OAuth tokens could not be retrieved from ServiceNow. "
                                "Please check the OAuth client id and OAuth client secret."
                            )

        else:  # the tokens file is present. We try to load it
            try:
                with open(self.oauth_token_file_path, 'r') as token_file:
                    self.token_dic = json.load(token_file)
            except (IOError, ValueError) as e:
                sys.stderr.write(
                    "SnowRestSession.__obtain_tokens: Issue when opening "
                    "the token file at %s.\n" % self.oauth_token_file_path)
                raise e

    def __initiate_session(self):
        """
        Initiates a session via CERN Single-Sign-On + OAuth, or basic Authentication

        Raises:
            SnowRestSessionException: if auth_type is not 'sso_auth' nor 'basic'
        """
        if self.auth_type == 'sso_oauth':
            self.__cern_get_sso_cookie()
            self.__obtain_tokens()

        elif self.auth_type == 'basic':
            self.session.auth = (self.basic_auth_user,
                                 self.basic_auth_password)
            self.session.cookies = cookielib.MozillaCookieJar()
            if os.path.exists(self.session_cookie_file_path):
                self.session.cookies.load(self.session_cookie_file_path,
                                          ignore_discard=True,
                                          ignore_expires=True)

        else:
            raise SnowRestSessionException(
                "SnowRestSession.__initiate_session: self.auth_type "
                "has a value different from \"basic\" and \"sso_auth\"")

    def __refresh_token(self):
        """
        Requests an OAuth access token via the OAUTH refresh token.
        Saves the resulting access token in the token file.

        Raises:
            SnowRestSessionException: if the access token could not be obtained
        """
        token_request = self.session.post(self.instance + '/oauth_token.do',
                                          data={
                                              'grant_type':
                                              'refresh_token',
                                              'client_id':
                                              self.oauth_client_id,
                                              'client_secret':
                                              self.oauth_client_secret,
                                              'refresh_token':
                                              self.token_dic['refresh_token']
                                          })

        if token_request.status_code == 200:
            self.token_dic = json.loads(token_request.text)
            with open(self.oauth_token_file_path, 'w') as token_file:
                json.dump(self.token_dic, token_file)

        else:
            token_request = self.session.post(self.instance +
                                              '/oauth_token.do',
                                              data={
                                                  'grant_type':
                                                  'password',
                                                  'client_id':
                                                  self.oauth_client_id,
                                                  'client_secret':
                                                  self.oauth_client_secret
                                              })
            if token_request.status_code == 200:
                self.token_dic = json.loads(token_request.text)
                with open(self.oauth_token_file_path, 'w') as token_file:
                    json.dump(self.token_dic, token_file)
            else:
                if self.fresh_cookie:
                    raise SnowRestSessionException(
                        "SnowRestSession.__refresh_token: the OAuth client id and OAuth secret might not be valid"
                    )

    def __save_cookie_basic(self):
        """
        Persists the basic authentication cookie file.
        """
        self.fresh_cookie = True
        self.session.cookies.save(self.session_cookie_file_path,
                                  ignore_discard=True,
                                  ignore_expires=True)

    @staticmethod
    def __library_user_agent():
        user_agent_header = 'cern-snow-client/' + __version__
        default_headers = requests.utils.default_headers()
        if 'User-Agent' in default_headers:
            user_agent_header = user_agent_header + ' ' + default_headers[
                'User-Agent']
        return user_agent_header

    def __execute(self, operation, url, headers=None, params=None, data=None):
        """
        Executes directly a REST Operation and returns the result

        Args:
            operation (str): either 'get', 'post' or 'put'
            url (str): a relative URL, such as "/api/now/v2/table/incident".
                An absolute URL such as "https://cerntest.service-now.com/api/now/v2/table/incident" can also
                be used, but the instance should match with the "instance" parameter in the configuration file,
                or the value set with the ``.set_instance()`` method.
            headers (:obj:`dict`, optional): any additional headers to be be passed. This method will add
                Accept:application/json and Content-Type:application/json if not specified.
                It will also add the Authorization header if the authentication type is 'sso_auth',
                by setting it to the OAuth access header.
            params (:obj:`dict`, optional): any additional URL parameters to be be passed
            data (object): the data to be sent in a post or put operation

        Returns:
            requests.Response

        Raises:
            SnowRestSessionException: if the operation parameter is not 'get', 'post' or 'put'

        """
        if not operation:
            raise SnowRestSessionException(
                "SnowRestSession.__execute: the operation paramater is mandatory"
            )
        if not url:
            raise SnowRestSessionException(
                "SnowRestSession.__execute: the url paramater is mandatory")
        if not url.startswith('https://'):
            url = self.instance + url

        if not url.startswith('https://'):
            url = self.instance + url

        if not headers:
            headers = {}
        if 'User-Agent' not in headers:
            headers['User-Agent'] = self.__library_user_agent()
        if 'Accept' not in headers:
            headers['Accept'] = 'application/json'
        if (operation == 'post'
                or operation == 'put') and 'Content-Type' not in headers:
            headers['Content-Type'] = 'application/json'
        if self.auth_type == 'sso_oauth':
            headers[
                'Authorization'] = 'Bearer ' + self.token_dic['access_token']

        if operation == 'get':
            result = self.session.get(url, headers=headers, params=params)
        elif operation == 'post':
            result = self.session.post(url,
                                       headers=headers,
                                       params=params,
                                       data=data)
        elif operation == 'put':
            result = self.session.put(url,
                                      headers=headers,
                                      params=params,
                                      data=data)
        else:
            raise SnowRestSessionException(
                "SnowRestSession.__execute: the operation paramater "
                "needs to be either \"get\", \"post\" or \"put\"")

        return result

    def __operation(self,
                    operation,
                    url,
                    headers=None,
                    params=None,
                    data=None):
        """
        Executes a REST Operation, taking care of reauthenticating if needed, and returns the result

        Args:
            operation (str): either 'get', 'post' or 'put'
            url (str): a relative URL, such as "/api/now/v2/table/incident".
                An absolute URL such as "https://cerntest.service-now.com/api/now/v2/table/incident" can also
                be used, but the instance should match with the "instance" parameter in the configuration file,
                or the value set with the ``.set_instance()`` method.
            headers (:obj:`dict`, optional): any additional headers to be be passed. If not set, some headers will be set by
                default (see __execute method)
            params (:obj:`dict`, optional): any additional URL parameters to be be passed
            data (object): the data to be sent in a post or put operation

        Returns:
            requests.Response : if the status code is not 401, a requests.Response object is returned

        Raises:
            SnowRestSessionException : if the operation could not be performed due to an authentication issue
        """
        self.__initiate_session()

        result = self.__execute(operation,
                                url,
                                headers=headers,
                                params=params,
                                data=data)

        if result.status_code != 401:
            if self.auth_type == 'basic':
                self.__save_cookie_basic()
            return result

        else:
            if self.auth_type == 'basic':
                if not self.fresh_cookie:
                    self.session.auth = (self.basic_auth_user,
                                         self.basic_auth_password)
                    result = self.__execute(operation,
                                            url,
                                            headers=headers,
                                            params=params,
                                            data=data)
                    if result.status_code != 401:
                        self.__save_cookie_basic()
                        return result
                    else:
                        raise SnowRestSessionException(
                            "SnowRestSession.__operation: Your basic authentication "
                            "user and password might not be valid")
                else:
                    raise SnowRestSessionException(
                        "SnowRestSession.__operation: Your basic authentication "
                        "user and password might not be valid")

            elif self.auth_type == 'sso_auth':

                if self.fresh_token:
                    raise SnowRestSessionException(
                        "SnowRestSession.__operation: failed to perform the operation. The current account might not "
                        "be able to log in to ServiceNow or the OAuth client id and secret might not be valid"
                    )

                else:
                    self.__refresh_token()
                    token_request = self.session.post(
                        'post',
                        self.instance + '/oauth_token.do',
                        data={
                            'grant_type': 'password',
                            'client_id': self.oauth_client_id,
                            'client_secret': self.oauth_client_secret
                        })
                    if token_request.status_code == 200:
                        self.token_dic = json.loads(token_request.text)
                        with open(self.oauth_token_file_path,
                                  'w') as token_file:
                            json.dump(self.token_dic, token_file)
                        headers['Authorization'] = 'Bearer ' + self.token_dic[
                            'access_token']
                        result = self.__execute(operation,
                                                url,
                                                headers=headers,
                                                params=params,
                                                data=data)
                        if result.status_code != 401:
                            return result
                        else:
                            raise SnowRestSessionException(
                                "SnowRestSession.__operation: failed to perform the operation. "
                                "The current account might not be able to log in to ServiceNow or "
                                "the OAuth client id and secret might not be valid"
                            )

                    else:
                        if self.fresh_cookie:
                            raise SnowRestSessionException(
                                "SnowRestSession.__operation: OAuth tokens could not be retrieved from ServiceNow. "
                                "Please check the OAuth client id and OAuth client secret."
                            )
                        else:
                            os.remove(self.session_cookie_file_path)
                            self.__cern_get_sso_cookie()
                            token_request = self.session.post(
                                'post',
                                self.instance + '/oauth_token.do',
                                data={
                                    'grant_type': 'password',
                                    'client_id': self.oauth_client_id,
                                    'client_secret': self.oauth_client_secret
                                })
                            if token_request.status_code == 401:
                                raise SnowRestSessionException(
                                    "SnowRestSession.__operation: OAuth tokens could not be retrieved from ServiceNow. "
                                    "Please check the OAuth client id and OAuth client secret."
                                )
                            else:
                                self.token_dic = json.loads(token_request.text)
                                with open(self.oauth_token_file_path,
                                          'w') as token_file:
                                    json.dump(self.token_dic, token_file)
                                headers[
                                    'Authorization'] = 'Bearer ' + self.token_dic[
                                        'access_token']
                                result = self.__execute(operation,
                                                        url,
                                                        headers=headers,
                                                        params=params,
                                                        data=data)
                                if result.status_code != 401:
                                    return result
                                else:
                                    raise SnowRestSessionException(
                                        "SnowRestSession.__operation: failed to perform the operation. "
                                        "The current account might not be able to log in to ServiceNow or "
                                        "the OAuth client id and secret might not be valid"
                                    )
            else:
                raise SnowRestSessionException(
                    "SnowRestSession.__operation: self.auth_type has a value different from \"basic\" and \"sso_auth\""
                )

    def _configure_handler(self):
        self._logger.removeHandler(self._log_handler)

        file_handler_attributes_known = bool(self._log_file_path
                                             and self._log_file_size_bytes
                                             and self._log_file_rotations
                                             and self._log_file_encoding
                                             and self._log_format)

        if file_handler_attributes_known:
            self._log_handler = RotatingFileHandler(
                self._log_file_path,
                maxBytes=self._log_file_size_bytes,
                backupCount=self._log_file_rotations - 1,
                encoding=self._log_file_encoding)
            self._log_handler.setFormatter(logging.Formatter(self._log_format))

        else:
            self._log_handler = NullHandler()

        self._logger.addHandler(self._log_handler)

    def _debug(self, message):
        self._log(logging.DEBUG, message)

    def _info(self, message):
        self._log(logging.INFO, message)

    def _warning(self, message):
        self._log(logging.WARNING, message)

    def _error(self, message):
        self._log(logging.ERROR, message)

    def _critical(self, message):
        self._log(logging.CRITICAL, message)

    def _log(self, level, message):
        if self._log_enabled:
            self._logger.log(level, message)
Beispiel #4
0
class DexcellRestApi(object):
    """
        class with all the utils API calls available group by
        from deployment calls, location calls and devide calls.
    """
    def __init__(self, endpoint, token, logger_name="dexcell_rest_api"):
        self.endpoint = endpoint
        self.token = token
        self.logger = logging.getLogger(logger_name)
        if len(self.logger.handlers) == 0:
            self.logger.setLevel(logging.INFO)
            self.handler = NullHandler()
            h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
            self.handler.setFormatter(logging.Formatter(h_format))
            self.logger.addHandler(self.handler)

    def _json_date_handler(self, obj):
        return obj.isoformat() if hasattr(obj, 'isoformat') else obj

    def _datetime_parser(self, dct):
        DATE_FORMAT = "%Y-%m-%dT%H:%M:%S"
        for k, v in dct.items():
            if isinstance(v, basestring) and re.search(
                    "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", v):
                try:
                    dct[k] = datetime.strptime(v, DATE_FORMAT)
                except:
                    pass
        return dct

    def dxdate(self, dt):
        """ convert datetime into default date string format used in dexcell api calls """
        return dt.strftime("%Y%m%d%H%M%S")

    def _call_rest(self, url, payload=None, parse_response=True):
        url = self.endpoint + url
        self.logger.info('url:%s token:%s' % (url, self.token))
        if payload is None:
            req = urllib2.Request(url, headers={'x-dexcell-token': self.token})
        else:
            req = urllib2.Request(url,
                                  payload,
                                  headers={'x-dexcell-token': self.token})
        try:
            response = urllib2.urlopen(req, timeout=600.0)
            data = response.read()
            if parse_response:
                return json.loads(data)
            else:
                return data
        except urllib2.HTTPError as httperror:

            info = json.loads(httperror.read())

            if httperror.code == 404:
                self.logger.error('error: not found')
                raise DexcellRestApiError('NOTFOUND', info['description'],
                                          info['moreInfo'])
            elif httperror.code == 401:
                self.logger.error('error: not authorized')
                raise DexcellRestApiError('INVALIDTOKEN', info['description'],
                                          info['moreInfo'])
            else:
                raise DexcellRestApiError('UNKNOWN', info['description'],
                                          info['moreInfo'])
                self.logger.error('error: %s' % (str(httperror.code)))

    def get_deployment(self, dep_id):
        """ return dict with basic information from deployment number dep_id"""
        url = "/deployments/%i.json" % dep_id
        deployment = self._call_rest(url)
        return deployment

    def get_deployment_locations(self, dep_id):
        """ return array with locations information from deployment number dep_id"""
        url = "/deployments/%i/locations.json" % dep_id
        location_list = self._call_rest(url)
        return location_list

    def get_deployment_devices(self, dep_id):
        """ return array with devices information from deployment number dep_id"""
        url = "/deployments/%i/devices.json" % dep_id
        device_list = self._call_rest(url)
        return device_list

    def get_deployment_parameters(self, dep_id):
        """ return array with parameters {freq, name, id, i18m, units}
            from deployment number dep_id
        """
        url = "/deployments/%i/parameters.json" % dep_id
        param_list = self._call_rest(url)
        return param_list

    def get_deployment_supplies(self, dep_id):
        """ return array with supplies {pod, name, id}
            from deployment number dep_id
        """
        url = "/deployments/%i/supplies.json" % dep_id
        supply_list = self._call_rest(url)
        return supply_list

    def get_deployment_notices(self, dep_id, start, end):
        """ return array with alerts information from deployment number dep_id
            from the interval selected
        """
        start = self.dxdate(start)
        end = self.dxdate(end)
        url = "/deployments/%i/notices.json?start=%s&end=%s" % (dep_id, start,
                                                                end)
        notice_list = self._call_rest(url)
        return notice_list

    def get_deployment_parameter_devices(self, dep_id, param_nid):
        """ return array with parameters {id, name, networkid}
            from deployment number dep_id
        """
        url = "/deployments/%i/parameters/%s/devices.json" % (dep_id,
                                                              str(param_nid))
        device_list = self._call_rest(url)
        return device_list

    def set_deployment_thing(self, dep_id, key, value):
        """ update dict of information saved by the user"""
        url = "/deployments/%i/things/set/%s.json" % (dep_id, key)
        payload = json.dumps(value, default=self._json_date_handler)
        data = self._call_rest(url, payload=payload, parse_response=False)
        return data

    def get_deployment_thing(self, dep_id, key):
        """ return dict of information saved by the user"""
        url = "/deployments/%i/things/get/%s.json" % (dep_id, key)
        data = self._call_rest(url, parse_response=False)
        self.logger.info('dep_thing:%s' % str(data))
        data = json.loads(data, object_hook=self._datetime_parser)
        return data

    def get_location(self, loc_id):
        """ return dict with basic information from locat"""
        url = "/locations/%i.json" % loc_id
        location = self._call_rest(url)
        return location

    def get_location_parameters(self, loc_id):
        """ return array with parameters {freq, name, id, i18m, units}
            from location number loc_id
        """
        url = "/locations/%i/parameters.json" % loc_id
        param_list = self._call_rest(url)
        return param_list

    def get_location_notices(self, loc_id, start, end):
        """ return array with alerts information for location number loc_id
            from the interval selected
        """
        start = self.dxdate(start)
        end = self.dxdate(end)
        url = "/locations/%i/notices.json?start=%s&end=%s" % (loc_id, start,
                                                              end)
        notice_list = self._call_rest(url)
        return notice_list

    def get_location_comments(self, loc_id, start, end):
        """ return array with comments information for location number loc_id
            from the interval selected
        """
        start = self.dxdate(start)
        end = self.dxdate(end)
        url = "/locations/%i/comments.json?start=%s&end=%s" % (loc_id, start,
                                                               end)
        comments = self._call_rest(url)
        return comments

    def get_location_parameter_devices(self, loc_id, param_nid):
        """ return array with parameters {id, name, networkid}
            from location number loc_id
        """
        url = "/locations/%i/parameters/%i/devices.json" % (loc_id, param_nid)
        device_list = self._call_rest(url)
        return device_list

    def get_location_supplies(self, loc_id):
        """ return array with supplies {pod, name, id}
            from location number loc_id
        """
        url = "/locations/%i/supplies.json" % loc_id
        supply_list = self._call_rest(url)
        return supply_list

    def get_location_devices(self, loc_id):
        """ return array with the devices from the location """
        url = "/locations/%i/devices.json" % loc_id
        device_list = self._call_rest(url)
        return device_list

    def get_device(self, dev_id):
        """ return dict with information for the device """
        url = "/devices/%i.json" % dev_id
        device = self._call_rest(url)
        return device

    def get_device_parameters(self, dev_id):
        """ return array with parameters {freq, name, id, i18m, units}
            from device number dev_id
        """
        param_list = self._call_rest("/devices/" + str(dev_id) +
                                     "/parameters.json")
        return param_list

    def get_simulated_bill(self,
                           dev_id,
                           start,
                           end,
                           type_param="ELECTRICAL",
                           parameters="AAANNN",
                           pod=None,
                           time='HOUR'):
        ''' returns bill generated from data in dexcell
            Parameters
                A : RAW + SUMMARY
                R: RAW set of datas returned by time mesure, default hour
                S: SUMMARY resum of data: totals, periods...
                N: nothing
            type_param can be ELECTRICAL, WATER, GAS
        '''
        new_pod = ''
        if pod is not None:
            new_pod = "&pod=" + pod
        start = start.strftime("%Y%m%d%H%M%S")
        end = end.strftime("%Y%m%d%H%M%S")
        url = ["/cost/%i/%s.json?start=%s" % (dev_id, type_param, start)]
        url.append("&end=%s&applyPattern=%s&period=%s%s" %
                   (end, parameters, time, new_pod))
        url = "".join(url)
        bill = self._call_rest(url)
        return bill

    def get_supply_bills(self,
                         sup_id,
                         start,
                         end,
                         type_param='ELECTRICAL',
                         parameters="AAANNN",
                         pod=None,
                         time='HOUR'):
        ''' returns bills updated by the customer
            Parameters
                A : RAW + SUMMARY
                R: RAW set of datas returned by time mesure, default hour
                S: SUMMARY resum of data: totals, periods...
                N: nothing
            type_param can be ELECTRICAL, WATER, GAS
        '''
        new_pod = ''
        if pod is not None:
            new_pod = "&pod=" + pod
        start = start.strftime("%Y%m%d%H%M%S")
        end = end.strftime("%Y%m%d%H%M%S")
        url = ["/cost/%i/bills/%s.json?start=%s" % (sup_id, type_param, start)]
        url.append("&end=%s&applyPattern=%s&period=%s%s" %
                   (end, parameters, time, new_pod))
        url = "".join(url)
        bills = self._call_rest(url)
        return bills

    def get_session(self, session_id):
        """ return the session for an app with a concret session_id"""
        url = "/session/%s.json" % session_id
        response = self._call_rest(url)
        self.logger.info('get session: ' + str(response))
        return response

    def get_readings(self, dev_id, s_nid, start, end):
        """ return array dict with {values, timestamp} """
        start = self.dxdate(start)
        end = self.dxdate(end)
        url = "/devices/%i/%i/readings.json?start=%s&end=%s" % (dev_id, s_nid,
                                                                start, end)
        readings = self._call_rest(url)
        for i in range(0, len(readings)):
            try:
                readings[i]['ts'] = datetime.strptime(readings[i]['ts'],
                                                      "%Y-%m-%d %H:%M:%S")
                readings[i]['tsutc'] = datetime.strptime(
                    readings[i]['tsutc'], "%Y-%m-%d %H:%M:%S")
            except KeyError:
                pass
        return readings

    def get_readings_new(self, dev_id, param, frequency, operation, start,
                         end):
        """ returns array of dict of values from the device dev_id with
            parameter param with a frequency in the interval start - end.
        """
        start = self.dxdate(start)
        end = self.dxdate(end)
        url = ["/devices/%i/%s/readings.json?" % (dev_id, str(param))]
        url.append("start=%s&end=%s&frequency=%s&operation=%s" %
                   (start, end, str(frequency), str(operation)))
        url = "".join(url)
        readings = self._call_rest(url)
        for i in range(0, len(readings)):
            try:
                readings[i]['ts'] = datetime.strptime(readings[i]['ts'],
                                                      "%Y-%m-%d %H:%M:%S")
                readings[i]['tsutc'] = datetime.strptime(
                    readings[i]['tsutc'], "%Y-%m-%d %H:%M:%S")
            except KeyError:
                pass
        return readings

    def get_cost(self,
                 nid,
                 start,
                 end,
                 energy_type='ELECTRICAL',
                 period='HOUR',
                 grouped=False):
        """ return array from cost and consumption with timestamp"""
        str_grouped = 'TRUE'
        if not grouped:
            str_grouped = 'FALSE'
        start = self.dxdate(start)
        end = self.dxdate(end)
        url = ["/devices/%i/%s/cost.json?" % (nid, energy_type)]
        url.append("start=%s&end=%s&period=%s&grouped=%s" %
                   (start, end, str(period), str_grouped))
        url = "".join(url)
        raw_response = self._call_rest(url)
        try:
            readings = raw_response['readings']
            for i in range(0, len(readings)):
                readings[i]['ts'] = datetime.strptime(readings[i]['ts'],
                                                      "%Y/%m/%d %H:%M:%S")
            periods = raw_response['periods']
            return readings, periods
        except KeyError:
            return []
Beispiel #5
0
class DexcellRestApiAuth(object):
    """
        Class for authentification in Dexcell
        software.
    """
    def __init__(self,
                 endpoint,
                 hash_dexma,
                 secret,
                 logger_name="dexcell_rest_api_auth"):
        self.endpoint = endpoint
        self.hash = hash_dexma
        self.secret = secret
        self.logger = logging.getLogger(logger_name)
        if len(self.logger.handlers) == 0:
            self.logger.setLevel(logging.INFO)
            self.handler = NullHandler()
            h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
            self.handler.setFormatter(logging.Formatter(h_format))
            self.logger.addHandler(self.handler)

    def _json_date_handler(self, obj):
        return obj.isoformat() if hasattr(obj, 'isoformat') else obj

    def _datetime_parser(self, dct):
        DATE_FORMAT = "%Y-%m-%dT%H:%M:%S"
        strp = datetime.strptime
        for k, v in dct.items():
            if isinstance(v, basestring) and re.search(
                    "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", v):
                try:
                    dct[k] = strp(v, DATE_FORMAT)
                except ValueError:
                    pass
        return dct

    def _call_rest(self, url):
        url = self.endpoint + url
        req = urllib2.Request(url)
        response = urllib2.urlopen(req)
        data = response.read()
        self.logger.info(data)
        return data

    def perm_token(self, temp_token):
        ''' obtain permanent token for oauth authentication'''
        url = "/oauth/accesstoken?temp_token=%s&secret=%s&idclient=%s" % (
            str(temp_token), self.secret, self.hash)
        response = self._call_rest(url)
        return response

    def set_key_value(self, key, value):
        "Set this key with this value in the key-value data store"
        url = self.endpoint + "/things/set/" + key
        req = urllib2.Request(url,
                              json.dumps(value,
                                         default=self._json_date_handler),
                              headers={'x-dexcell-secret': self.secret})
        self.logger.info('storing key: %s with secret: %s' %
                         (key, self.secret))
        response = urllib2.urlopen(req)
        data = response.read()
        return data

    def get_key(self, key):
        "Get this key from the key-value data store"
        url = "%s/things/get/%s" % (self.endpoint, key)
        req = urllib2.Request(url, headers={'x-dexcell-secret': self.secret})
        response = urllib2.urlopen(req)
        data = response.read()
        data = json.loads(data, object_hook=self._datetime_parser)
        result = json.loads(data['result'], object_hook=self._datetime_parser)
        return result
Beispiel #6
0
class DexcellSender(object):

    DEFAULT_SERVER = 'insert.dexcell.com'
    DEFAULT_URL = '/insert-json.htm'
    DEFAULT_LOGFILE = '/var/log/dexma/DexcellSender.log'
    DEFAULT_LOGLEVEL = logging.INFO
    DEFAULT_GATEWAY = 'None'
    DEFAULT_LOGGERNAME = 'DexcellSender'

    def __init__(self,
                 gateway=DEFAULT_GATEWAY,
                 loggerName=DEFAULT_LOGGERNAME,
                 logfile=DEFAULT_LOGFILE,
                 loglevel=DEFAULT_LOGLEVEL,
                 server=DEFAULT_SERVER,
                 url=DEFAULT_URL,
                 https=True,
                 timeout=30.0):
        self.__https = https
        self.__server = server
        self.__url = url
        self.__timeout = timeout
        self.__gateway = gateway
        self.__logger = logging.getLogger(loggerName)
        if len(self.__logger.handlers) == 0:
            self.__logger.setLevel(loglevel)
            self.handler = NullHandler()
            h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
            self.handler.setFormatter(logging.Formatter(h_format))
            self.__logger.addHandler(self.handler)

    def setup(self,
              gateway=DEFAULT_GATEWAY,
              loggerName=DEFAULT_LOGGERNAME,
              logfile=DEFAULT_LOGFILE,
              loglevel=DEFAULT_LOGLEVEL,
              server=DEFAULT_SERVER,
              url=DEFAULT_URL):
        """Setup the Dexcell Sender Object
        """
        self.__server = server
        self.__url = url
        self.__gateway = gateway
        self.__logger = logging.getLogger(loggerName)
        if len(self.__logger.handlers) == 0:
            self.__logger.setLevel(loglevel)
            self.handler = NullHandler()
            h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
            self.handler.setFormatter(logging.Formatter(h_format))
            self.__logger.addHandler(self.handler)

    def changeGateway(self, gateway):
        """Change the gateway mac that will be sent
        """
        self.__gateway = gateway

    def __insertRawJSONData(self, data):
        """Insert the raw data string to the server
        """
        params = 'data=' + data
        headers = {
            "Content-type": "application/x-www-form-urlencoded",
            "Accept": "text/plain"
        }
        if self.__https:
            conn = httplib.HTTPSConnection(self.__server,
                                           timeout=self.__timeout)
        else:
            conn = httplib.HTTPConnection(self.__server,
                                          timeout=self.__timeout)
        conn.request("POST", self.__url, params, headers)
        error = True
        maxerror = 0
        while error:
            try:
                response = conn.getresponse()
                error = False
            except:
                print self.__logger.exception("Error inserting data")
                maxerror = maxerror + 1
                time.sleep(1)
                if maxerror > 10:
                    return (-1, 'FAIL')
        logger_msg_wo_params = "Insert from %s with status %s and result %s"
        logger_params = (self.__gateway, str(response.status),
                         str(response.getheader('data')))
        logger_message = logger_msg_wo_params % logger_params
        self.__logger.debug(logger_message)
        return response.status, response.getheader('data')

    def insertDexcellServiceMessage(self,
                                    serviceMessage,
                                    timezone='UTC',
                                    extraparams={}):
        '''Insert a single DexcellServiceMessage
        '''
        reading = {
            'nodeNetworkId':
            str(serviceMessage.node),
            'serviceNetworkId':
            int(serviceMessage.service),
            'value':
            float(serviceMessage.value),
            'seqNum':
            int(serviceMessage.seqnum),
            'timeStamp':
            time.strftime("%Y-%m-%dT%H:%M:%S.000 " + timezone,
                          serviceMessage.timestamp)
        }
        data = {'gatewayId': self.__gateway, 'service': [reading]}
        for key in extraparams.keys():
            data[key] = extraparams[key]
        result = self.__insertRawJSONData(json.dumps(data))
        return result

    def insertDexcellServiceMessages(self,
                                     serviceMessageIterator,
                                     timezone='UTC',
                                     extraparams={}):
        """ Insert many DexcellServiceMessages at once
        """
        readings = []
        for serviceMessage in serviceMessageIterator:
            reading = {
                'nodeNetworkId':
                str(serviceMessage.node),
                'serviceNetworkId':
                int(serviceMessage.service),
                'value':
                float(serviceMessage.value),
                'seqNum':
                int(serviceMessage.seqnum),
                'timeStamp':
                time.strftime("%Y-%m-%dT%H:%M:%S.000 " + timezone,
                              serviceMessage.timestamp)
            }
            readings.append(reading)
        data = {'gatewayId': self.__gateway, 'service': readings}
        for key in extraparams.keys():
            data[key] = extraparams[key]
        result = self.__insertRawJSONData(json.dumps(data))
        return result
Beispiel #7
0
class DexcellRestApi(object):

    """
        class with all the utils API calls available group by
        from deployment calls, location calls and devide calls.
    """

    def __init__(self, endpoint, token, logger_name="dexcell_rest_api"):
        self.endpoint = endpoint
        self.token = token
        self.logger = logging.getLogger(logger_name)
        if len(self.logger.handlers) == 0:
            self.logger.setLevel(logging.INFO)
            self.handler = NullHandler()
            h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
            self.handler.setFormatter(logging.Formatter(h_format))
            self.logger.addHandler(self.handler)

    def _json_date_handler(self, obj):
        return obj.isoformat() if hasattr(obj, "isoformat") else obj

    def _datetime_parser(self, dct):
        DATE_FORMAT = "%Y-%m-%dT%H:%M:%S"
        for k, v in dct.items():
            if isinstance(v, basestring) and re.search("\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", v):
                try:
                    dct[k] = datetime.strptime(v, DATE_FORMAT)
                except:
                    pass
        return dct

    def dxdate(self, dt):
        """ convert datetime into default date string format used in dexcell api calls """
        return dt.strftime("%Y%m%d%H%M%S")

    def _call_rest(self, url, payload=None, parse_response=True):
        url = self.endpoint + url
        self.logger.info("url:%s token:%s" % (url, self.token))
        if payload is None:
            req = urllib2.Request(url, headers={"x-dexcell-token": self.token})
        else:
            req = urllib2.Request(url, payload, headers={"x-dexcell-token": self.token})
        try:
            response = urllib2.urlopen(req, timeout=600.0)
            data = response.read()
            if parse_response:
                return json.loads(data)
            else:
                return data
        except urllib2.HTTPError as httperror:

            info = json.loads(httperror.read())

            if httperror.code == 404:
                self.logger.error("error: not found")
                raise DexcellRestApiError("NOTFOUND", info["description"], info["moreInfo"])
            elif httperror.code == 401:
                self.logger.error("error: not authorized")
                raise DexcellRestApiError("INVALIDTOKEN", info["description"], info["moreInfo"])
            else:
                raise DexcellRestApiError("UNKNOWN", info["description"], info["moreInfo"])
                self.logger.error("error: %s" % (str(httperror.code)))

    def get_deployment(self, dep_id):
        """ return dict with basic information from deployment number dep_id"""
        url = "/deployments/%i.json" % dep_id
        deployment = self._call_rest(url)
        return deployment

    def get_deployment_locations(self, dep_id):
        """ return array with locations information from deployment number dep_id"""
        url = "/deployments/%i/locations.json" % dep_id
        location_list = self._call_rest(url)
        return location_list

    def get_deployment_devices(self, dep_id):
        """ return array with devices information from deployment number dep_id"""
        url = "/deployments/%i/devices.json" % dep_id
        device_list = self._call_rest(url)
        return device_list

    def get_deployment_parameters(self, dep_id):
        """ return array with parameters {freq, name, id, i18m, units}
            from deployment number dep_id
        """
        url = "/deployments/%i/parameters.json" % dep_id
        param_list = self._call_rest(url)
        return param_list

    def get_deployment_supplies(self, dep_id):
        """ return array with supplies {pod, name, id}
            from deployment number dep_id
        """
        url = "/deployments/%i/supplies.json" % dep_id
        supply_list = self._call_rest(url)
        return supply_list

    def get_deployment_notices(self, dep_id, start, end):
        """ return array with alerts information from deployment number dep_id
            from the interval selected
        """
        start = self.dxdate(start)
        end = self.dxdate(end)
        url = "/deployments/%i/notices.json?start=%s&end=%s" % (dep_id, start, end)
        notice_list = self._call_rest(url)
        return notice_list

    def get_deployment_parameter_devices(self, dep_id, param_nid):
        """ return array with parameters {id, name, networkid}
            from deployment number dep_id
        """
        url = "/deployments/%i/parameters/%s/devices.json" % (dep_id, str(param_nid))
        device_list = self._call_rest(url)
        return device_list

    def set_deployment_thing(self, dep_id, key, value):
        """ update dict of information saved by the user"""
        url = "/deployments/%i/things/set/%s.json" % (dep_id, key)
        payload = json.dumps(value, default=self._json_date_handler)
        data = self._call_rest(url, payload=payload, parse_response=False)
        return data

    def get_deployment_thing(self, dep_id, key):
        """ return dict of information saved by the user"""
        url = "/deployments/%i/things/get/%s.json" % (dep_id, key)
        data = self._call_rest(url, parse_response=False)
        self.logger.info("dep_thing:%s" % str(data))
        data = json.loads(data, object_hook=self._datetime_parser)
        return data

    def get_location(self, loc_id):
        """ return dict with basic information from locat"""
        url = "/locations/%i.json" % loc_id
        location = self._call_rest(url)
        return location

    def get_location_parameters(self, loc_id):
        """ return array with parameters {freq, name, id, i18m, units}
            from location number loc_id
        """
        url = "/locations/%i/parameters.json" % loc_id
        param_list = self._call_rest(url)
        return param_list

    def get_location_notices(self, loc_id, start, end):
        """ return array with alerts information for location number loc_id
            from the interval selected
        """
        start = self.dxdate(start)
        end = self.dxdate(end)
        url = "/locations/%i/notices.json?start=%s&end=%s" % (loc_id, start, end)
        notice_list = self._call_rest(url)
        return notice_list

    def get_location_comments(self, loc_id, start, end):
        """ return array with comments information for location number loc_id
            from the interval selected
        """
        start = self.dxdate(start)
        end = self.dxdate(end)
        url = "/locations/%i/comments.json?start=%s&end=%s" % (loc_id, start, end)
        comments = self._call_rest(url)
        return comments

    def get_location_parameter_devices(self, loc_id, param_nid):
        """ return array with parameters {id, name, networkid}
            from location number loc_id
        """
        url = "/locations/%i/parameters/%i/devices.json" % (loc_id, param_nid)
        device_list = self._call_rest(url)
        return device_list

    def get_location_supplies(self, loc_id):
        """ return array with supplies {pod, name, id}
            from location number loc_id
        """
        url = "/locations/%i/supplies.json" % loc_id
        supply_list = self._call_rest(url)
        return supply_list

    def get_location_devices(self, loc_id):
        """ return array with the devices from the location """
        url = "/locations/%i/devices.json" % loc_id
        device_list = self._call_rest(url)
        return device_list

    def get_device(self, dev_id):
        """ return dict with information for the device """
        url = "/devices/%i.json" % dev_id
        device = self._call_rest(url)
        return device

    def get_device_parameters(self, dev_id):
        """ return array with parameters {freq, name, id, i18m, units}
            from device number dev_id
        """
        param_list = self._call_rest("/devices/" + str(dev_id) + "/parameters.json")
        return param_list

    def get_simulated_bill(
        self, dev_id, start, end, type_param="ELECTRICAL", parameters="AAANNN", pod=None, time="HOUR"
    ):
        """ returns bill generated from data in dexcell
            Parameters
                A : RAW + SUMMARY
                R: RAW set of datas returned by time mesure, default hour
                S: SUMMARY resum of data: totals, periods...
                N: nothing
            type_param can be ELECTRICAL, WATER, GAS
        """
        new_pod = ""
        if pod is not None:
            new_pod = "&pod=" + pod
        start = start.strftime("%Y%m%d%H%M%S")
        end = end.strftime("%Y%m%d%H%M%S")
        url = ["/cost/%i/%s.json?start=%s" % (dev_id, type_param, start)]
        url.append("&end=%s&applyPattern=%s&period=%s%s" % (end, parameters, time, new_pod))
        url = "".join(url)
        bill = self._call_rest(url)
        return bill

    def get_supply_bills(self, sup_id, start, end, type_param="ELECTRICAL", parameters="AAANNN", pod=None, time="HOUR"):
        """ returns bills updated by the customer
            Parameters
                A : RAW + SUMMARY
                R: RAW set of datas returned by time mesure, default hour
                S: SUMMARY resum of data: totals, periods...
                N: nothing
            type_param can be ELECTRICAL, WATER, GAS
        """
        new_pod = ""
        if pod is not None:
            new_pod = "&pod=" + pod
        start = start.strftime("%Y%m%d%H%M%S")
        end = end.strftime("%Y%m%d%H%M%S")
        url = ["/cost/%i/bills/%s.json?start=%s" % (sup_id, type_param, start)]
        url.append("&end=%s&applyPattern=%s&period=%s%s" % (end, parameters, time, new_pod))
        url = "".join(url)
        bills = self._call_rest(url)
        return bills

    def get_session(self, session_id):
        """ return the session for an app with a concret session_id"""
        url = "/session/%s.json" % session_id
        response = self._call_rest(url)
        self.logger.info("get session: " + str(response))
        return response

    def get_readings(self, dev_id, s_nid, start, end):
        """ return array dict with {values, timestamp} """
        start = self.dxdate(start)
        end = self.dxdate(end)
        url = "/devices/%i/%i/readings.json?start=%s&end=%s" % (dev_id, s_nid, start, end)
        readings = self._call_rest(url)
        for i in range(0, len(readings)):
            try:
                readings[i]["ts"] = datetime.strptime(readings[i]["ts"], "%Y-%m-%d %H:%M:%S")
                readings[i]["tsutc"] = datetime.strptime(readings[i]["tsutc"], "%Y-%m-%d %H:%M:%S")
            except KeyError:
                pass
        return readings

    def get_readings_new(self, dev_id, param, frequency, operation, start, end):
        """ returns array of dict of values from the device dev_id with
            parameter param with a frequency in the interval start - end.
        """
        start = self.dxdate(start)
        end = self.dxdate(end)
        url = ["/devices/%i/%s/readings.json?" % (dev_id, str(param))]
        url.append("start=%s&end=%s&frequency=%s&operation=%s" % (start, end, str(frequency), str(operation)))
        url = "".join(url)
        readings = self._call_rest(url)
        for i in range(0, len(readings)):
            try:
                readings[i]["ts"] = datetime.strptime(readings[i]["ts"], "%Y-%m-%d %H:%M:%S")
                readings[i]["tsutc"] = datetime.strptime(readings[i]["tsutc"], "%Y-%m-%d %H:%M:%S")
            except KeyError:
                pass
        return readings

    def get_cost(self, nid, start, end, energy_type="ELECTRICAL", period="HOUR", grouped=False):
        """ return array from cost and consumption with timestamp"""
        str_grouped = "TRUE"
        if not grouped:
            str_grouped = "FALSE"
        start = self.dxdate(start)
        end = self.dxdate(end)
        url = ["/devices/%i/%s/cost.json?" % (nid, energy_type)]
        url.append("start=%s&end=%s&period=%s&grouped=%s" % (start, end, str(period), str_grouped))
        url = "".join(url)
        raw_response = self._call_rest(url)
        try:
            readings = raw_response["readings"]
            for i in range(0, len(readings)):
                readings[i]["ts"] = datetime.strptime(readings[i]["ts"], "%Y/%m/%d %H:%M:%S")
            periods = raw_response["periods"]
            return readings, periods
        except KeyError:
            return []
Beispiel #8
0
class DexcellRestApiAuth(object):
    """
        Class for authentification in Dexcell
        software.
    """

    def __init__(self, endpoint, hash_dexma, secret, logger_name="dexcell_rest_api_auth"):
        self.endpoint = endpoint
        self.hash = hash_dexma
        self.secret = secret
        self.logger = logging.getLogger(logger_name)
        if len(self.logger.handlers) == 0:
            self.logger.setLevel(logging.INFO)
            self.handler = NullHandler()
            h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
            self.handler.setFormatter(logging.Formatter(h_format))
            self.logger.addHandler(self.handler)

    def _json_date_handler(self, obj):
        return obj.isoformat() if hasattr(obj, "isoformat") else obj

    def _datetime_parser(self, dct):
        DATE_FORMAT = "%Y-%m-%dT%H:%M:%S"
        strp = datetime.strptime
        for k, v in dct.items():
            if isinstance(v, basestring) and re.search("\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", v):
                try:
                    dct[k] = strp(v, DATE_FORMAT)
                except ValueError:
                    pass
        return dct

    def _call_rest(self, url):
        url = self.endpoint + url
        req = urllib2.Request(url)
        response = urllib2.urlopen(req)
        data = response.read()
        self.logger.info(data)
        return data

    def perm_token(self, temp_token):
        """ obtain permanent token for oauth authentication"""
        url = "/oauth/accesstoken?temp_token=%s&secret=%s&idclient=%s" % (str(temp_token), self.secret, self.hash)
        response = self._call_rest(url)
        return response

    def set_key_value(self, key, value):
        "Set this key with this value in the key-value data store"
        url = self.endpoint + "/things/set/" + key
        req = urllib2.Request(
            url, json.dumps(value, default=self._json_date_handler), headers={"x-dexcell-secret": self.secret}
        )
        self.logger.info("storing key: %s with secret: %s" % (key, self.secret))
        response = urllib2.urlopen(req)
        data = response.read()
        return data

    def get_key(self, key):
        "Get this key from the key-value data store"
        url = "%s/things/get/%s" % (self.endpoint, key)
        req = urllib2.Request(url, headers={"x-dexcell-secret": self.secret})
        response = urllib2.urlopen(req)
        data = response.read()
        data = json.loads(data, object_hook=self._datetime_parser)
        result = json.loads(data["result"], object_hook=self._datetime_parser)
        return result
Beispiel #9
0
class DexcellSender(object):

    DEFAULT_SERVER = "insert.dexcell.com"
    DEFAULT_URL = "/insert-json.htm"
    DEFAULT_LOGFILE = "/var/log/dexma/DexcellSender.log"
    DEFAULT_LOGLEVEL = logging.INFO
    DEFAULT_GATEWAY = "None"
    DEFAULT_LOGGERNAME = "DexcellSender"

    def __init__(
        self,
        gateway=DEFAULT_GATEWAY,
        loggerName=DEFAULT_LOGGERNAME,
        logfile=DEFAULT_LOGFILE,
        loglevel=DEFAULT_LOGLEVEL,
        server=DEFAULT_SERVER,
        url=DEFAULT_URL,
        https=True,
        timeout=30.0,
    ):
        self.__https = https
        self.__server = server
        self.__url = url
        self.__timeout = timeout
        self.__gateway = gateway
        self.__logger = logging.getLogger(loggerName)
        if len(self.__logger.handlers) == 0:
            self.__logger.setLevel(loglevel)
            self.handler = NullHandler()
            h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
            self.handler.setFormatter(logging.Formatter(h_format))
            self.__logger.addHandler(self.handler)

    def setup(
        self,
        gateway=DEFAULT_GATEWAY,
        loggerName=DEFAULT_LOGGERNAME,
        logfile=DEFAULT_LOGFILE,
        loglevel=DEFAULT_LOGLEVEL,
        server=DEFAULT_SERVER,
        url=DEFAULT_URL,
    ):
        """Setup the Dexcell Sender Object
        """
        self.__server = server
        self.__url = url
        self.__gateway = gateway
        self.__logger = logging.getLogger(loggerName)
        if len(self.__logger.handlers) == 0:
            self.__logger.setLevel(loglevel)
            self.handler = NullHandler()
            h_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
            self.handler.setFormatter(logging.Formatter(h_format))
            self.__logger.addHandler(self.handler)

    def changeGateway(self, gateway):
        """Change the gateway mac that will be sent
        """
        self.__gateway = gateway

    def __insertRawJSONData(self, data):
        """Insert the raw data string to the server
        """
        params = "data=" + data
        headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}
        if self.__https:
            conn = httplib.HTTPSConnection(self.__server, timeout=self.__timeout)
        else:
            conn = httplib.HTTPConnection(self.__server, timeout=self.__timeout)
        conn.request("POST", self.__url, params, headers)
        error = True
        maxerror = 0
        while error:
            try:
                response = conn.getresponse()
                error = False
            except:
                print self.__logger.exception("Error inserting data")
                maxerror = maxerror + 1
                time.sleep(1)
                if maxerror > 10:
                    return (-1, "FAIL")
        logger_msg_wo_params = "Insert from %s with status %s and result %s"
        logger_params = (self.__gateway, str(response.status), str(response.getheader("data")))
        logger_message = logger_msg_wo_params % logger_params
        self.__logger.debug(logger_message)
        return response.status, response.getheader("data")

    def insertDexcellServiceMessage(self, serviceMessage, timezone="UTC", extraparams={}):
        """Insert a single DexcellServiceMessage
        """
        reading = {
            "nodeNetworkId": str(serviceMessage.node),
            "serviceNetworkId": int(serviceMessage.service),
            "value": float(serviceMessage.value),
            "seqNum": int(serviceMessage.seqnum),
            "timeStamp": time.strftime("%Y-%m-%dT%H:%M:%S.000 " + timezone, serviceMessage.timestamp),
        }
        data = {"gatewayId": self.__gateway, "service": [reading]}
        for key in extraparams.keys():
            data[key] = extraparams[key]
        result = self.__insertRawJSONData(json.dumps(data))
        return result

    def insertDexcellServiceMessages(self, serviceMessageIterator, timezone="UTC", extraparams={}):
        """ Insert many DexcellServiceMessages at once
        """
        readings = []
        for serviceMessage in serviceMessageIterator:
            reading = {
                "nodeNetworkId": str(serviceMessage.node),
                "serviceNetworkId": int(serviceMessage.service),
                "value": float(serviceMessage.value),
                "seqNum": int(serviceMessage.seqnum),
                "timeStamp": time.strftime("%Y-%m-%dT%H:%M:%S.000 " + timezone, serviceMessage.timestamp),
            }
            readings.append(reading)
        data = {"gatewayId": self.__gateway, "service": readings}
        for key in extraparams.keys():
            data[key] = extraparams[key]
        result = self.__insertRawJSONData(json.dumps(data))
        return result
Beispiel #10
0
from logging import getLogger, NullHandler, StreamHandler, Formatter
from logging import DEBUG, INFO, WARNING, ERROR
import sys

rpkg = getLogger('rpkg')

dgt = getLogger('dgt')

cli = getLogger('cli')

handler = NullHandler()
dgt.addHandler(handler)
cli.addHandler(handler)

# Add a default handler if there are no real handlers set, yet (not counting NullHandler)
if len(rpkg.handlers) <= 1:
    handler = StreamHandler(sys.stderr)
    formatter = Formatter('[%(name)s] %(levelname)s: %(message)s')
    handler.setFormatter(formatter)
    rpkg.addHandler(handler)
if len(dgt.handlers) <= 1:
    handler = StreamHandler(sys.stderr)
    formatter = Formatter('[%(name)s] %(levelname)s: %(message)s')
    handler.setFormatter(formatter)
    dgt.addHandler(handler)
if len(cli.handlers) <= 1:
    handler = StreamHandler(sys.stdout)
    formatter = Formatter('%(message)s')
    handler.setFormatter(formatter)
    cli.addHandler(handler)
Beispiel #11
0
def test_null_handler(response):
    nh = NullHandler()
    formatter = color_debug.ColorFormatter()
    nh.setFormatter(formatter)
Beispiel #12
0
def test_null_handler(response):
    nh = NullHandler()
    formatter = color_bucket_logger.ColorFormatter()
    nh.setFormatter(formatter)