예제 #1
0
    def test_data(self):
        """Testing method / function data."""
        # Initialize key variables
        _data = []
        expected = []
        checksum = data.hashstring(str(random()))
        pattoo_key = data.hashstring(str(random()))
        agent_id = data.hashstring(str(random()))
        polling_interval = 300 * 1000
        data_type = DATA_FLOAT
        _pattoo_value = 27
        _timestamp = int(time.time() * 1000)
        ts_start = _timestamp

        for count in range(0, 10):
            timestamp = _timestamp + (polling_interval * count)
            ts_stop = timestamp
            pattoo_value = _pattoo_value * count
            insert = PattooDBrecord(
                pattoo_checksum=checksum,
                pattoo_key=pattoo_key,
                pattoo_agent_id=agent_id,
                pattoo_agent_polling_interval=polling_interval,
                pattoo_timestamp=timestamp,
                pattoo_data_type=data_type,
                pattoo_value=pattoo_value * count,
                pattoo_agent_polled_target='pattoo_agent_polled_target',
                pattoo_agent_program='pattoo_agent_program',
                pattoo_agent_hostname='pattoo_agent_hostname',
                pattoo_metadata=[])

            # Create checksum entry in the DB, then update the data table
            idx_datapoint = datapoint.idx_datapoint(insert)
            _data.append(
                IDXTimestampValue(idx_datapoint=idx_datapoint,
                                  polling_interval=polling_interval,
                                  timestamp=timestamp,
                                  value=pattoo_value))

            # Append to expected results
            expected.append({
                'timestamp':
                times.normalized_timestamp(polling_interval, timestamp),
                'value':
                pattoo_value
            })

        # Insert rows of new data
        lib_data.insert_rows(_data)

        # Test
        obj = DataPoint(idx_datapoint)
        result = obj.data(ts_start, ts_stop)
        self.assertEqual(result, expected)
예제 #2
0
    def test_route_graphql(self):
        """Testing method / function add_url_rule (graphql)."""
        # Initialize key variables
        _data = []
        pattoo_checksum = data.hashstring(str(random()))
        pattoo_key = data.hashstring(str(random()))
        agent_id = data.hashstring(str(random()))
        pattoo_agent_polled_target = data.hashstring(str(random()))
        _pi = 300 * 1000
        data_type = DATA_FLOAT
        timestamp = int(time.time()) * 1000
        pattoo_value = round(uniform(1, 100), 5)

        insert = PattooDBrecord(
            pattoo_checksum=pattoo_checksum,
            pattoo_key=pattoo_key,
            pattoo_agent_id=agent_id,
            pattoo_agent_polling_interval=_pi,
            pattoo_timestamp=timestamp,
            pattoo_data_type=data_type,
            pattoo_value=pattoo_value,
            pattoo_agent_polled_target=pattoo_agent_polled_target,
            pattoo_agent_program='pattoo_agent_program',
            pattoo_agent_hostname='pattoo_agent_hostname',
            pattoo_metadata=[])

        # Create checksum entry in the DB, then update the data table
        idx_datapoint = datapoint.idx_datapoint(insert)
        _data.append(
            IDXTimestampValue(idx_datapoint=idx_datapoint,
                              polling_interval=_pi,
                              timestamp=timestamp,
                              value=pattoo_value))

        # Insert rows of new data
        lib_data.insert_rows(_data)

        # Test
        query = ('''\
{
  allDatapoints(idxDatapoint: "IDX") {
    edges {
      node {
        checksum
      }
    }
  }
}
'''.replace('IDX', str(idx_datapoint)))

        # Test
        graphql_result = _get(query)
        result = graphql_result['data']['allDatapoints']['edges'][0]['node']
        self.assertEqual(result['checksum'], pattoo_checksum)
예제 #3
0
    def test_insert_rows(self):
        """Testing method / function insert_rows."""
        # Initialize key variables
        checksum = lib_data.hashstring(str(random()))
        agent_id = lib_data.hashstring(str(random()))
        data_type = DATA_FLOAT
        polling_interval = 10
        pattoo_value = 27
        pattoo_key = lib_data.hashstring(str(random()))
        timestamp = int(time.time() * 1000)

        insert = PattooDBrecord(
            pattoo_checksum=checksum,
            pattoo_key=pattoo_key,
            pattoo_agent_id=agent_id,
            pattoo_agent_polling_interval=polling_interval,
            pattoo_timestamp=timestamp,
            pattoo_data_type=data_type,
            pattoo_value=pattoo_value,
            pattoo_agent_polled_target='pattoo_agent_polled_target',
            pattoo_agent_program='pattoo_agent_program',
            pattoo_agent_hostname='pattoo_agent_hostname',
            pattoo_metadata=[]
        )

        # Create checksum entry in the DB, then update the data table
        idx_datapoint = datapoint.idx_datapoint(insert)
        _data = [IDXTimestampValue(
            idx_datapoint=idx_datapoint,
            polling_interval=polling_interval,
            timestamp=timestamp,
            value=pattoo_value)]
        data.insert_rows(_data)

        # Verify that the data is there
        with db.db_query(20015) as session:
            rows = session.query(
                Data.value).filter(and_(
                    Data.idx_datapoint == idx_datapoint,
                    Data.timestamp == timestamp))
        for row in rows:
            self.assertEqual(row.value, pattoo_value)
예제 #4
0
    def test_route_data(self):
        """Testing method / function route_data."""
        # Initialize key variables
        secondsago = 3600
        ts_start = uri.chart_timestamp_args(secondsago)

        # Initialize key variables
        _data = []
        checksum = data.hashstring(str(random()))
        pattoo_key = data.hashstring(str(random()))
        agent_id = data.hashstring(str(random()))
        _pi = 300 * 1000
        data_type = DATA_FLOAT
        now = int(time.time()) * 1000
        count = 0

        for timestamp in range(ts_start, now, _pi):
            insert = PattooDBrecord(
                pattoo_checksum=checksum,
                pattoo_key=pattoo_key,
                pattoo_agent_id=agent_id,
                pattoo_agent_polling_interval=_pi,
                pattoo_timestamp=timestamp,
                pattoo_data_type=data_type,
                pattoo_value=count,
                pattoo_agent_polled_target='pattoo_agent_polled_target',
                pattoo_agent_program='pattoo_agent_program',
                pattoo_agent_hostname='pattoo_agent_hostname',
                pattoo_metadata=[])
            count += 1

            # Create checksum entry in the DB, then update the data table
            idx_datapoint = datapoint.idx_datapoint(insert)
            _data.append(
                IDXTimestampValue(idx_datapoint=idx_datapoint,
                                  polling_interval=_pi,
                                  timestamp=timestamp,
                                  value=count))

        # Insert rows of new data
        lib_data.insert_rows(_data)

        # Test
        obj = DataPoint(idx_datapoint)
        ts_stop = obj.last_timestamp()
        expected = obj.data(ts_start, ts_stop)

        # Create URL
        config = WebConfig()
        url = ('{}/{}'.format(config.web_api_server_url(graphql=False),
                              idx_datapoint))

        # Check response
        with requests.get(url) as response:
            result = response.json()

        count = 0
        for item in result:
            ts_norm = times.normalized_timestamp(_pi, ts_start)
            if item['timestamp'] < ts_norm:
                self.assertIsNone(item['value'])
            else:
                self.assertEqual(item, expected[count])
                count += 1
예제 #5
0
def process_db_records(pattoo_db_records):
    """Insert all data values for an agent into database.

    Args:
        pattoo_db_records: List of dicts read from cache files.

    Returns:
        None

    Method:
        1) Get all the idx_datapoint and idx_pair values that exist in the
           PattooDBrecord data from the database. All the records MUST be
           from the same source.
        2) Add these idx values to tracking memory variables for speedy lookup
        3) Ignore non numeric data values sent
        4) Add data to the database. If new checksum values are found in the
           PattooDBrecord data, then create the new index values to the
           database, update the tracking memory variables before hand.

    """
    # Initialize key variables
    _data = {}

    # Return if there is nothint to process
    if bool(pattoo_db_records) is False:
        return

    # Get DataPoint.idx_datapoint and idx_pair values from db. This is used to
    # speed up the process by reducing the need for future database access.
    agent_id = pattoo_db_records[0].pattoo_agent_id
    checksum_table = misc.agent_checksums(agent_id)

    # Process data
    for pdbr in pattoo_db_records:
        # We only want to insert non-string, non-None values
        if pdbr.pattoo_data_type in [DATA_NONE, DATA_STRING]:
            continue

        # Try to make the value a float for insertion into the database
        try:
            float_value = float(pdbr.pattoo_value)
        except:
            continue

        # Get the idx_datapoint value for the PattooDBrecord
        if pdbr.pattoo_checksum in checksum_table:
            # Get last_timestamp for existing idx_datapoint entry
            idx_datapoint = checksum_table[pdbr.pattoo_checksum].idx_datapoint
        else:
            # Entry not in database. Update the database and get the
            # required idx_datapoint
            idx_datapoint = datapoint.idx_datapoint(pdbr)
            if bool(idx_datapoint) is True:
                # Update the lookup table
                checksum_table[pdbr.pattoo_checksum] = ChecksumLookup(
                    idx_datapoint=idx_datapoint,
                    polling_interval=int(pdbr.pattoo_agent_polling_interval),
                    last_timestamp=1)

                # Update the Glue table
                idx_pairs = get.pairs(pdbr)
                glue.insert_rows(idx_datapoint, idx_pairs)
            else:
                continue

        # Append item to items
        if pdbr.pattoo_timestamp > checksum_table[
                pdbr.pattoo_checksum].last_timestamp:
            '''
            Add the Data table results to a dict in case we have duplicate
            posting over the API. We need to key off a unique time dependent
            value per datapoint to prevent different datapoints at the same
            point in time overwriting the value. This is specifically for
            removing duplicates for the _SAME_ datapoint at the same point in
            time as could possibly occur with the restart of an agent causing a
            double posting or network issues. We therefore use a tuple of
            idx_datapoint and timestamp.
            '''
            _data[(pdbr.pattoo_timestamp, idx_datapoint)] = IDXTimestampValue(
                idx_datapoint=idx_datapoint,
                polling_interval=int(pdbr.pattoo_agent_polling_interval),
                timestamp=pdbr.pattoo_timestamp,
                value=float_value)

    # Update the data table
    if bool(_data) is True:
        data.insert_rows(list(_data.values()))

    # Log message
    log_message = ('''\
Finished cache data processing for agent_id: {}'''.format(agent_id))
    log.log2debug(20113, log_message)
예제 #6
0
    def test_route_graphql(self):
        """Testing method / function add_url_rule (graphql)."""
        # Initialize key variables
        _data = []
        pattoo_checksum = data.hashstring(str(random()))
        pattoo_key = data.hashstring(str(random()))
        agent_id = data.hashstring(str(random()))
        pattoo_agent_polled_target = data.hashstring(str(random()))
        _pi = 300 * 1000
        data_type = DATA_FLOAT
        timestamp = int(time.time()) * 1000
        pattoo_value = round(uniform(1, 100), 5)

        insert = PattooDBrecord(
            pattoo_checksum=pattoo_checksum,
            pattoo_key=pattoo_key,
            pattoo_agent_id=agent_id,
            pattoo_agent_polling_interval=_pi,
            pattoo_timestamp=timestamp,
            pattoo_data_type=data_type,
            pattoo_value=pattoo_value,
            pattoo_agent_polled_target=pattoo_agent_polled_target,
            pattoo_agent_program='pattoo_agent_program',
            pattoo_agent_hostname='pattoo_agent_hostname',
            pattoo_metadata=[])

        # Create checksum entry in the DB, then update the data table
        idx_datapoint = datapoint.idx_datapoint(insert)
        _data.append(
            IDXTimestampValue(idx_datapoint=idx_datapoint,
                              polling_interval=_pi,
                              timestamp=timestamp,
                              value=pattoo_value))

        # Insert rows of new data
        lib_data.insert_rows(_data)

        # Creating required test admin
        test_admin = {
            "username": "******",
            "first_name": "Pattoo Test",
            "last_name": "Pattoo Test",
            "password": "******",
            "role": 0,
            "password_expired": 0,
            "enabled": 1
        }
        user.insert_row(DbRowUser(**test_admin))

        # Get accesss token to make test queries
        acesss_query = ('''\
mutation{
    authenticate(Input: {username: "******", password: "******"}) {
        accessToken
        refreshToken
    }
}

''')

        # Replacing username and password in access_query
        acesss_query = acesss_query.replace("USERNAME", test_admin['username'])
        acesss_query = acesss_query.replace("PASSWORD", test_admin['password'])

        access_request = _query(acesss_query, query_type="Mutation")
        acesss_token = access_request['data']['authenticate']['accessToken']

        # Test
        query = ('''\
{
allDatapoints(idxDatapoint: "IDX", token: "TOKEN") {
    edges {
      node {
        checksum
      }
    }
  }
}
''')

        # Replacing IDX and TOKEN in query
        query = query.replace("IDX", str(idx_datapoint))
        query = query.replace("TOKEN", str(acesss_token))

        # Test
        graphql_result = _query(query)
        result = graphql_result['data']['allDatapoints']['edges'][0]['node']
        self.assertEqual(result['checksum'], pattoo_checksum)