예제 #1
0
    def checkout(dbapi_connection, connection_record, connection_proxy):
        """Checkout sub-processes connection for sub-processing if needed.

            Checkout is called when a connection is retrieved from the Pool.

        Args:
            dbapi_connection: Connection object
            connection_record: Connection record object
            connection_proxy: Connection proxy object

        Returns:
            None

        """
        # Get PID of main process
        pid = os.getpid()

        # Detect if this is a sub-process
        if connection_record.info['pid'] != pid:
            # substitute log.debug() or similar here as desired
            log_message = (
                'Parent process %s forked (%s) with an open '
                'database connection, '
                'which is being discarded and recreated.'
                '') % (
                    connection_record.info['pid'], pid)
            log.log2quiet(1067, log_message)

            connection_record.connection = connection_proxy.connection = None
            raise exc.DisconnectionError(
                "Connection record belongs to pid %s, "
                "attempting to check out in pid %s" %
                (connection_record.info['pid'], pid)
            )
예제 #2
0
    def start(self):
        """Start the daemon.

        Args:
            None

        Returns:

        """
        # Check for a pidfile to see if the daemon already runs
        try:
            with open(self.pidfile, 'r') as pf_handle:
                pid = int(pf_handle.read().strip())

        except IOError:
            pid = None

        if pid:
            log_message = (
                'PID file: %s already exists. Daemon already running?'
                '') % (self.pidfile)
            log.log2die(1062, log_message)

        # Start the daemon
        self.daemonize()

        # Log success
        log_message = ('Daemon Started - PID file: %s') % (self.pidfile)
        log.log2quiet(1070, log_message)

        # Run code for daemon
        self.run()
예제 #3
0
    def purge(self):
        """Purge cache file that was read.

        Args:
            None

        Returns:
            success: "True" if successful

        """
        # Initialize key variables
        success = True

        try:
            os.remove(self.filename)
        except:
            success = False

        # Report success
        if success is True:
            log_message = (
                'Ingest cache file %s deleted') % (self.filename)
            log.log2quiet(1046, log_message)
        else:
            log_message = (
                'Failed to delete ingest cache file %s') % (self.filename)
            log.log2warn(1087, log_message)

        # Return
        return success
예제 #4
0
def _check_when_disabled(agent_name):
    """Stop agent.

    Args:
        agent_filepath: Filepath of agent to be restarted.
        agent_name: Agent name

    Returns:
        None

    """
    # Initialize key variables
    agentconfig = jm_configuration.ConfigAgent(agent_name)
    agent_filename = agentconfig.agent_filename()

    # Get agent status variables
    root_dir = jm_general.root_directory()
    agent_filepath = ('%s/%s') % (root_dir, agent_filename)
    pid = hidden.File()
    pidfile = pid.pid(agent_name)

    # Shutdown agent if running
    if os.path.isfile(pidfile) is True:
        with open(pidfile, 'r') as f_handle:
            pidvalue = int(f_handle.readline().strip())
        if psutil.pid_exists(pidvalue) is True:
            log_message = (
                'Agent "%s" is alive, but should be disabled. '
                'Attempting to stop.'
                '') % (agent_name)
            log.log2quiet(1032, log_message)
            _stop(agent_filepath, agent_name)
예제 #5
0
    def stop(self):
        """Stop the daemon.

        Args:
            None

        Returns:

        """
        # Get the pid from the pidfile
        try:
            with open(self.pidfile, 'r') as pf_handle:
                pid = int(pf_handle.read().strip())
        except IOError:
            pid = None

        if not pid:
            log_message = (
                'PID file: %s does not exist. Daemon not running?'
                '') % (self.pidfile)
            log.log2warn(1063, log_message)
            # Not an error in a restart
            return

        # Try killing the daemon process
        try:
            while 1:
                if self.lockfile is None:
                    os.kill(pid, signal.SIGTERM)
                else:
                    time.sleep(0.3)
                    if os.path.exists(self.lockfile) is True:
                        continue
                    else:
                        os.kill(pid, signal.SIGTERM)
                time.sleep(0.3)
        except OSError as err:
            error = str(err.args)
            if error.find("No such process") > 0:
                self.delpid()
                self.dellock()
            else:
                log_message = (str(err.args))
                log_message = (
                    '%s - PID file: %s') % (log_message, self.pidfile)
                log.log2die(1068, log_message)
        except:
            log_message = (
                'Unknown daemon "stop" error for PID file: %s'
                '') % (self.pidfile)
            log.log2die(1066, log_message)

        # Log success
        self.delpid()
        self.dellock()
        log_message = ('Daemon Stopped - PID file: %s') % (self.pidfile)
        log.log2quiet(1071, log_message)
예제 #6
0
def api_make(config, host, verbose=False):
    """Process 'pagemaker' CLI option.

    Args:
        config: Configuration object
        verbose: Verbose output if True

    Returns:
        None

    """
    # Initialize key variables
    threads_in_pool = 1
    device_file_found = False

    # Create directory if needed
    perm_dir = config.web_directory()
    temp_dir = tempfile.mkdtemp()

    # Delete all files in temporary directory
    jm_general.delete_files(temp_dir)

    # Skip if device file not found
    if os.path.isfile(config.snmp_device_file(host)) is False:
        log_message = (
            'No YAML device file for host %s found in %s. '
            'Run toolbox.py with the "poll" option first.'
            '') % (host, config.snmp_directory())
        log.log2quiet(1018, log_message)
    else:
        device_file_found = True

    ####################################################################
    #
    # Define variables that will be required for the database update
    # We have to initialize the dict during every loop to prevent
    # data corruption
    #
    ####################################################################
    data_dict = {}
    data_dict['host'] = host
    data_dict['config'] = config
    data_dict['verbose'] = verbose
    data_dict['temp_dir'] = temp_dir

    table = HTMLTable(config, host)

    # Create HTML output
    html = ('%s%s\n%s\n\n%s\n') % (
        _html_header(host), host, table.device(),
        table.ethernet())


    # Do the rest if device_file_found
    if device_file_found is True:
        # Wait on the queue until everything has been processed
        return html
예제 #7
0
    def post(self, save=True, data=None):
        """Post data to central server.

        Args:
            save: When True, save data to cache directory if postinf fails
            data: Data to post. If None, then uses self.data

        Returns:
            success: "True: if successful

        """
        # Initialize key variables
        success = False
        response = False
        timestamp = self.data['timestamp']
        uid = self.data['uid']

        # Create data to post
        if data is None:
            data = self.data

        # Post data save to cache if this fails
        try:
            result = requests.post(self.url, json=data)
            response = True
        except:
            if save is True:
                # Create a unique very long filename to reduce risk of
                hosthash = jm_general.hashstring(self.data['hostname'], sha=1)
                filename = ('%s/%s_%s_%s.json') % (
                    self.cache_dir, timestamp, uid, hosthash)

                # Save data
                with open(filename, 'w') as f_handle:
                    json.dump(data, f_handle)

        # Define success
        if response is True:
            if result.status_code == 200:
                success = True

        # Log message
        if success is True:
            log_message = (
                'Agent "%s" successfully contacted server %s'
                '') % (self.name(), self.url)
            log.log2quiet(1027, log_message)
        else:
            log_message = (
                'Agent "%s" failed to contact server %s'
                '') % (self.name(), self.url)
            log.log2warn(1028, log_message)

        # Return
        return success
예제 #8
0
파일: agent.py 프로젝트: Pr0x1m4/infoset
    def purge(self):
        """Purge data from cache by posting to central server.

        Args:
            None

        Returns:
            success: "True: if successful

        """
        # Initialize key variables
        uid = self.data["uid"]

        # Add files in cache directory to list
        filenames = [
            filename
            for filename in os.listdir(self.cache_dir)
            if os.path.isfile(os.path.join(self.cache_dir, filename))
        ]

        # Read cache file
        for filename in filenames:
            # Only post files for our own UID value
            if uid not in filename:
                continue

            # Get the full filepath for the cache file and post
            filepath = os.path.join(self.cache_dir, filename)
            with open(filepath, "r") as f_handle:
                try:
                    data = json.load(f_handle)
                except:
                    # Log removal
                    log_message = (
                        "Error reading previously cached agent data file %s " "for agent %s. May be corrupted." ""
                    ) % (filepath, self.name())
                    log.log2die(1064, log_message)

            # Post file
            success = self.post(save=False, data=data)

            # Delete file if successful
            if success is True:
                os.remove(filepath)

                # Log removal
                log_message = ("Purging cache file %s after successfully " "contacting server %s" "") % (
                    filepath,
                    self.url,
                )
                log.log2quiet(1029, log_message)
예제 #9
0
def create(config, host):
    """Create topology page for host.

    Args:
        config: Configuration object
        host: Hostname to create pages for

    Returns:
        None

    """
    # Initialize key variables
    device_file_found = False

    # Create directory if needed
    temp_dir = tempfile.mkdtemp()

    # Delete all files in temporary directory
    jm_general.delete_files(temp_dir)

    # Skip if device file not found
    if os.path.isfile(config.topology_device_file(host)) is False:
        log_message = (
            'No YAML device file for host %s found in %s. '
            'topoloy agent has not discovered it yet.'
            '') % (host, config.topology_directory())
        log.log2quiet(1018, log_message)
    else:
        device_file_found = True

    ####################################################################
    #
    # Define variables that will be required for the database update
    # We have to initialize the dict during every loop to prevent
    # data corruption
    #
    ####################################################################
    table = HTMLTable(config, host)

    # Create HTML output
    html = ('%s%s\n%s\n\n%s\n') % (
        _html_header(host), host, table.device(),
        table.ethernet())

    # Do the rest if device_file_found
    if device_file_found is True:
        # Wait on the queue until everything has been processed
        return html
예제 #10
0
파일: poll.py 프로젝트: Pr0x1m4/infoset
    def run(self):
        """Update the database using threads."""
        while True:
            # Get the data_dict
            data_dict = self.queue.get()
            host = data_dict['host']
            config = data_dict['config']
            verbose = data_dict['verbose']
            temp_dir = data_dict['temp_dir']

            # Show host information
            validate = snmp_manager.Validate(host, config.snmp_auth())
            snmp_params = validate.credentials()
            snmp_object = snmp_manager.Interact(snmp_params)

            # Verbose output
            if verbose is True:
                output = ('Processing on: host %s') % (host)
                print(output)

            # Skip invalid, and uncontactable hosts
            if bool(snmp_params) is False:
                if verbose is True:
                    log_message = (
                        'Uncontactable host %s or no valid SNMP '
                        'credentials found for it.') % (host)
                    log.log2quiet(1019, log_message)
                continue

            # Process if valid
            if bool(snmp_params) is True:
                # Get data
                status = snmp_info.Query(snmp_object)
                data = status.everything()
                yaml_string = jm_general.dict2yaml(data)

                # Dump data
                temp_file = ('%s/%s.yaml') % (temp_dir, host)
                with open(temp_file, 'w') as file_handle:
                    file_handle.write(yaml_string)

                # Verbose output
                if verbose is True:
                    output = ('Completed run: host %s') % (host)
                    print(output)

            # Signals to queue job is done
            self.queue.task_done()
예제 #11
0
def _stop(agent_filepath, agent_name):
    """Stop agent.

    Args:
        agent_filepath: Filepath of agent to be restarted.
        agent_name: Agent name

    Returns:
        None

    """
    # Restart
    log_message = (
        'Stopping agent "%s".'
        '') % (agent_name)
    log.log2quiet(1033, log_message)
    command2run = ('%s --stop --force') % (agent_filepath)
    _execute(command2run)
예제 #12
0
def _start(agent_filepath, agent_name):
    """Start agent.

    Args:
        agent_filepath: Filepath of agent to be started.
        agent_name: Agent name

    Returns:
        None

    """
    # Start
    log_message = (
        'Starting agent "%s".'
        '') % (agent_name)
    log.log2quiet(1077, log_message)
    command2run = ('%s --start') % (agent_filepath)
    _execute(command2run)
예제 #13
0
    def query(self):
        """Query all remote hosts for data.

        Args:
            None

        Returns:
            None

        """
        # Check SNMP supported
        if bool(self.snmp_params) is True:
            # Get datapoints
            self._create_yaml()
        else:
            log_message = (
                'Uncontactable host %s or no valid SNMP '
                'credentials found for it.') % (self.hostname)
            log.log2quiet(1021, log_message)
예제 #14
0
    def _create_yaml(self):
        """Create the master dictionary for the host.

        Args:
            None
        Returns:
            value: Index value

        """
        # Initialize key variables
        temp_dir = tempfile.mkdtemp()
        temp_file = ('%s/%s.yaml') % (temp_dir, self.hostname)
        perm_file = self.server_config.topology_device_file(self.hostname)

        # Get data
        log_message = (
            'Querying topology data from host %s.'
            '') % (self.hostname)
        log.log2quiet(1125, log_message)

        # Create YAML file by polling device
        status = snmp_info.Query(self.snmp_object)
        data = status.everything()
        yaml_string = jm_general.dict2yaml(data)

        # Dump data
        with open(temp_file, 'w') as file_handle:
            file_handle.write(yaml_string)

        # Get data
        log_message = (
            'Completed topology query from host %s.'
            '') % (self.hostname)
        log.log2quiet(1019, log_message)

        # Clean up files
        os.rename(temp_file, perm_file)
        os.rmdir(temp_dir)
예제 #15
0
    def query(self):
        """Query all remote hosts for data.

        Args:
            None

        Returns:
            None

        """
        # Initialize key variables
        if self.config.agent_port():
            data = self.config.agent_port()
            port = int(data)
        else:
            port = 5001

        # Do stuff
        log_message = (
            'Starting agent %s on localhost port %s.'
            '') % (self.agent_name, port)
        log.log2quiet(1088, log_message)
        APP.run(host='0.0.0.0', port=port)
예제 #16
0
def _check_when_enabled(agent_name):
    """Stop agent.

    Args:
        agent_filepath: Filepath of agent to be restarted.
        agent_name: Agent name

    Returns:
        None

    """
    # Initialize key variables
    agentconfig = jm_configuration.ConfigAgent(agent_name)
    agent_filename = agentconfig.agent_filename()

    # Get agent status variables
    root_dir = jm_general.root_directory()
    agent_filepath = ('%s/%s') % (root_dir, agent_filename)
    pid = hidden.File()
    pidfile = pid.pid(agent_name)
    lockfile = pid.lock(agent_name)

    # Ignore agents that cannot be found
    if os.path.isfile(agent_filepath) is False:
        log_message = (
            'Agent executable file %s listed in the '
            'configuration file '
            'of agent "%s" does not exist. Please fix.'
            '') % (agent_filepath, agent_name)
        log.log2quiet(1075, log_message)
        return

    # Check for pid file
    if os.path.isfile(pidfile) is True:
        with open(pidfile, 'r') as f_handle:
            pidvalue = int(f_handle.readline().strip())

        # Check if service died catastrophically. No PID file
        if psutil.pid_exists(pidvalue) is False:
            log_message = (
                'Agent "%s" is dead. Attempting to restart.'
                '') % (agent_name)
            log.log2quiet(1041, log_message)

            # Remove PID file and restart
            os.remove(pidfile)
            _restart(agent_filepath, agent_name)

        else:
            # Check if agent hung without updating the PID
            if agentconfig.monitor_agent_pid() is True:
                try:
                    mtime = os.path.getmtime(pidfile)
                except OSError:
                    mtime = 0
                if mtime < int(time.time()) - (60 * 10):
                    log_message = (
                        'Agent "%s" is hung. Attempting to restart.'
                        '') % (agent_name)
                    log.log2quiet(1076, log_message)
                    _restart(agent_filepath, agent_name)
    else:
        if os.path.isfile(lockfile) is True:
            _restart(agent_filepath, agent_name)
        else:
            _start(agent_filepath, agent_name)
예제 #17
0
파일: cache.py 프로젝트: jhar123/infoset
def _update_unchartable(mapping, ingest, config):
    """Update unchartable data into the database "iset_datapoint" table.

    Args:
        mapping: Map of DIDs to database row index values
        ingest: Drain object
        config: Config object

    Returns:
        None

    """
    # Initialize key variables
    data = ingest.other()
    data_list = []
    timestamp_tracker = {}

    # Update data
    for item in data:
        # Process each datapoint item found
        (_, did, tuple_value, timestamp) = item
        idx_datapoint = int(mapping[did][0])
        last_timestamp = int(mapping[did][2])
        value = ('%s') % (tuple_value)

        # Only update with data collected after
        # the most recent update. Don't do anything more
        if timestamp > last_timestamp:
            data_list.append(
                (idx_datapoint, value)
            )

            # Update DID's last updated timestamp
            if idx_datapoint in timestamp_tracker:
                timestamp_tracker[idx_datapoint] = max(
                    timestamp, timestamp_tracker[idx_datapoint])
            else:
                timestamp_tracker[idx_datapoint] = timestamp

    # Update if there is data
    if bool(data_list) is True:
        for item in data_list:
            (idx_datapoint, value) = item
            fixed_value = str(value)[0:128]

            # Prepare SQL query to read a record from the database.
            sql_modify = (
                'UPDATE iset_datapoint set uncharted_value="%s" WHERE '
                'idx=%s') % (fixed_value, idx_datapoint)

            # Do query and get results
            database = db.Database(config)
            database.modify(sql_modify, 1037)

        # Change the last updated timestamp
        for idx_datapoint, last_timestamp in timestamp_tracker.items():
            # Prepare SQL query to read a record from the database.
            sql_modify = (
                'UPDATE iset_datapoint SET last_timestamp=%s '
                'WHERE iset_datapoint.idx=%s'
                '') % (last_timestamp, idx_datapoint)
            database.modify(sql_modify, 1044)

        # Report success
        log_message = (
            'Successful cache drain (Uncharted Data) '
            'for UID %s at timestamp %s') % (
                ingest.uid(), ingest.timestamp())
        log.log2quiet(1045, log_message)
예제 #18
0
    def _update_unchartable(self, mapping):
        """Update unchartable data into the database "iset_datapoint" table.

        Args:
            mapping: Map of DIDs to database row index values

        Returns:
            None

        """
        # Initialize key variables
        data = self.agent_data['unchartable']
        data_list = []
        timestamp_tracker = {}

        # Update data
        for item in data:
            # Process each datapoint item found
            (_, did, value, timestamp) = item
            idx_datapoint = int(mapping[did][0])
            last_timestamp = int(mapping[did][2])

            # Only update with data collected after
            # the most recent update. Don't do anything more
            if timestamp > last_timestamp:
                data_list.append(
                    (idx_datapoint, value)
                )

                # Update DID's last updated timestamp
                if idx_datapoint in timestamp_tracker:
                    timestamp_tracker[idx_datapoint] = max(
                        timestamp, timestamp_tracker[idx_datapoint])
                else:
                    timestamp_tracker[idx_datapoint] = timestamp

        # Update if there is data
        if bool(data_list) is True:
            # Create a database session
            # NOTE: We only do a single commit on the session.
            # This is much faster (20x based on testing) than
            # instantiating the database, updating records, and committing
            # after every iteration of the "for loop"
            database = db.Database()
            session = database.session()

            # Update uncharted data
            for item in data_list:
                (idx_datapoint, value) = item
                data_dict = {'uncharted_value': jm_general.encode(value)}
                session.query(Datapoint).filter(
                    Datapoint.idx == idx_datapoint).update(data_dict)

            # Change the last updated timestamp
            for idx_datapoint, last_timestamp in timestamp_tracker.items():
                data_dict = {'last_timestamp': last_timestamp}
                session.query(Datapoint).filter(
                    Datapoint.idx == idx_datapoint).update(data_dict)

            # Commit data
            database.commit(session, 1128)

            # Report success
            log_message = (
                'Successful cache drain (Uncharted Data) '
                'for UID %s') % (self.agent_data['uid'])
            log.log2quiet(1045, log_message)
예제 #19
0
def make(config, verbose=False):
    """Process 'pagemaker' CLI option.

    Args:
        config: Configuration object
        verbose: Verbose output if True

    Returns:
        None

    """
    # Initialize key variables
    threads_in_pool = 10
    device_file_found = False

    # Create directory if needed
    perm_dir = config.web_directory()
    temp_dir = tempfile.mkdtemp()

    # Delete all files in temporary directory
    jm_general.delete_files(temp_dir)

    # Spawn a pool of threads, and pass them queue instance
    for _ in range(threads_in_pool):
        update_thread = PageMaker(THREAD_QUEUE)
        update_thread.daemon = True
        update_thread.start()

    # Get host data and write to file
    for host in config.hosts():
        # Skip if device file not found
        if os.path.isfile(config.snmp_device_file(host)) is False:
            log_message = (
                'No YAML device file for host %s found in %s. '
                'Run toolbox.py with the "poll" option first.'
                '') % (host, config.snmp_directory())
            log.log2quiet(1018, log_message)
            continue
        else:
            device_file_found = True

        ####################################################################
        #
        # Define variables that will be required for the database update
        # We have to initialize the dict during every loop to prevent
        # data corruption
        #
        ####################################################################
        data_dict = {}
        data_dict['host'] = host
        data_dict['config'] = config
        data_dict['verbose'] = verbose
        data_dict['temp_dir'] = temp_dir
        THREAD_QUEUE.put(data_dict)

    # Do the rest if device_file_found
    if device_file_found is True:
        # Wait on the queue until everything has been processed
        THREAD_QUEUE.join()

        # PYTHON BUG. Join can occur while threads are still shutting down.
        # This can create spurious "Exception in thread (most likely raised
        # during interpreter shutdown)" errors.
        # The "time.sleep(1)" adds a delay to make sure things really terminate
        # properly. This seems to be an issue on virtual machines in Dev only
        time.sleep(1)

        # Create index file
        write_file = ('%s/index.html') % (temp_dir)
        index_html = _index_html(config)
        with open(write_file, 'w') as file_handle:
            file_handle.write(index_html)

        # Cleanup, move temporary files to clean permanent directory.
        # Delete temporary directory
        if os.path.isdir(perm_dir):
            jm_general.delete_files(perm_dir)
        else:
            os.makedirs(perm_dir, 0o755)
        jm_general.move_files(temp_dir, perm_dir)

    # Clean up
    os.rmdir(temp_dir)
예제 #20
0
    def process(self):
        """Update the database using threads."""
        # Initialize key variables
        updated = False
        hostnames = []
        uids = []
        ingests = []
        agent_names = []
        agent_data = {
            'hostname': None,
            'uid': None,
            'sources': [],
            'chartable': [],
            'unchartable': []
        }

        # Get the data_dict
        metadata = self.metadata
        config = self.config

        # Initialize other values
        max_timestamp = 0

        # Get start time for activity
        start_ts = time.time()

        # Sort metadata by timestamp
        metadata.sort()

        # Process file for each timestamp, starting from the oldes file
        for (timestamp, filepath) in metadata:
            # Read in data
            ingest = drain.Drain(filepath)

            # Make sure file is OK
            # Move it to a directory for further analysis
            # by administrators
            if ingest.valid() is False:
                log_message = (
                    'Cache ingest file %s is invalid. Moving.'
                    '') % (filepath)
                log.log2warn(1054, log_message)
                shutil.copy(
                    filepath, config.ingest_failures_directory())
                os.remove(filepath)
                continue

            # Append data
            agent_data['chartable'].extend(ingest.chartable())
            agent_data['unchartable'].extend(ingest.other())
            agent_data['sources'].extend(ingest.sources())
            hostnames.append(ingest.hostname())
            uids.append(ingest.uid())
            ingests.append(ingest)
            agent_names.append(ingest.agent())

            # Purge source file
            # ingest.purge()

            # Get the max timestamp
            max_timestamp = max(timestamp, max_timestamp)

            # Update update flag
            updated = True

        # Verify that we have only processed data for the same hostname
        # UID and agent name
        if (jm_general.all_same(hostnames) is False) or (
                jm_general.all_same(uids) is False) or (
                    jm_general.all_same(agent_names) is False):
            log_message = (
                'Cache ingest files error for hostname %s,'
                'agent name %s, UID %s.'
                '') % (hostnames[0], agent_names[0], uids[0])
            log.log2quiet(1083, log_message)

        # Process the rest
        if updated is True:
            # Update remaining agent data
            agent_data['hostname'] = hostnames[0]
            agent_data['uid'] = uids[0]
            agent_data['agent_name'] = agent_names[0]

            # Update database
            dbase = UpdateDB(agent_data)
            dbase.update()

            # Update the last time the agent was contacted
            _update_agent_last_update(agent_data['uid'], max_timestamp)

            # Update the host / agent table timestamp if
            # hostname was processed
            _host_agent_last_update(
                agent_data['hostname'], agent_data['uid'], max_timestamp)

            # Purge source files. Only done after complete
            # success of database updates. If not we could lose data in the
            # event of an ingester crash. Ingester would re-read the files
            # and process the non-duplicates, while deleting the duplicates.
            for ingest in ingests:
                ingest.purge()

            # Log duration of activity
            duration = time.time() - start_ts
            log_message = (
                'UID %s was processed in %s seconds.'
                '') % (agent_data['uid'], duration)
            log.log2quiet(1127, log_message)
예제 #21
0
    def valid(self):
        """Master method that defines whether data is OK.

        Args:
            None

        Returns:
            all_ok:

        """
        # Initialize key variables
        validity = []
        ts_start = time.time()

        # Check primary keys
        validity.append(
            _check_primary_keys_exist(self.information))

        # Check timestamp key
        if False not in validity:
            validity.append(
                _check_timestamp_key(self.information))

        # Check validity of primary keys in file
        if False not in validity:
            validity.append(
                self._check_primary_keys_in_file())

        # Check chartable and unchartable data in the data
        if False not in validity:
            validity.append(_check_reported_data(self.information))
        if False not in validity:
            validity.append(_check_chartable_data(self.information))

        # Check if data to be validated is already in the database
        if False not in validity:
            validity.append(_check_duplicates(self.information))

        # Do final check
        if False in validity:
            # Log failure
            if self.filepath is None:
                mid_string = ''
            else:
                mid_string = ('in %s') % (self.filepath)
            log_message = ('Cache data %s is invalid') % (mid_string)
            log.log2warn(1059, log_message)
            all_ok = False
        else:
            # Log success
            ts_stop = time.time()
            duration = ts_stop - ts_start
            if self.filepath is None:
                mid_string = ''
            else:
                mid_string = ('of %s') % (self.filepath)
            log_message = (
                'Data validation %s took %s seconds.'
                '') % (mid_string, round(duration, 4))
            log.log2quiet(1126, log_message)
            all_ok = True

        # Return
        return all_ok
예제 #22
0
파일: cache.py 프로젝트: jhar123/infoset
def _update_chartable(mapping, ingest, config):
    """Insert data into the database "iset_data" table.

    Args:
        mapping: Map of DIDs to database row index values
        ingest: Drain object
        config: Config object

    Returns:
        None

    """
    # Initialize key variables
    data = ingest.chartable()
    data_list = []
    timestamp_tracker = {}

    # Update data
    for item in data:
        # Process each datapoint item found
        (_, did, tuple_value, timestamp) = item
        idx_datapoint = int(mapping[did][0])
        idx_agent = int(mapping[did][1])
        last_timestamp = int(mapping[did][2])
        value = float(tuple_value)

        # Only update with data collected after
        # the most recent update. Don't do anything more
        if timestamp > last_timestamp:
            data_list.append(
                (idx_datapoint, idx_agent, value, timestamp)
            )

            # Update DID's last updated timestamp
            if idx_datapoint in timestamp_tracker:
                timestamp_tracker[idx_datapoint] = max(
                    timestamp, timestamp_tracker[idx_datapoint])
            else:
                timestamp_tracker[idx_datapoint] = timestamp

    # Update if there is data
    if bool(data_list) is True:
        # Prepare SQL query to read a record from the database.
        sql_insert = (
            'REPLACE INTO iset_data '
            '(idx_datapoint, idx_agent, value, timestamp) VALUES '
            '(%s, %s, %s, %s)')

        # Do query and get results
        database = db.Database(config)
        database.modify(sql_insert, 1056, data_list=data_list)

        # Change the last updated timestamp
        for idx_datapoint, last_timestamp in timestamp_tracker.items():
            # Prepare SQL query to read a record from the database.
            sql_modify = (
                'UPDATE iset_datapoint SET last_timestamp=%s '
                'WHERE iset_datapoint.idx=%s'
                '') % (last_timestamp, idx_datapoint)
            database.modify(sql_modify, 1057)

        # Report success
        log_message = (
            'Successful cache drain for UID %s at timestamp %s') % (
                ingest.uid(), ingest.timestamp())
        log.log2quiet(1058, log_message)
예제 #23
0
파일: cache.py 프로젝트: Pr0x1m4/infoset
    def _update_unchartable(self, mapping):
        """Update unchartable data into the database "iset_datapoint" table.

        Args:
            mapping: Map of DIDs to database row index values

        Returns:
            None

        """
        # Initialize key variables
        data = self.ingest.other()
        data_list = []
        timestamp_tracker = {}

        # Update data
        for item in data:
            # Process each datapoint item found
            (_, did, value, timestamp) = item
            idx_datapoint = int(mapping[did][0])
            last_timestamp = int(mapping[did][2])

            # Only update with data collected after
            # the most recent update. Don't do anything more
            if timestamp > last_timestamp:
                data_list.append(
                    (idx_datapoint, value)
                )

                # Update DID's last updated timestamp
                if idx_datapoint in timestamp_tracker:
                    timestamp_tracker[idx_datapoint] = max(
                        timestamp, timestamp_tracker[idx_datapoint])
                else:
                    timestamp_tracker[idx_datapoint] = timestamp

        # Update if there is data
        if bool(data_list) is True:
            for item in data_list:
                (idx_datapoint, value) = item

                # Update database
                database = db.Database()
                session = database.session()
                record = session.query(Datapoint).filter(
                    Datapoint.idx == idx_datapoint).one()
                record.uncharted_value = jm_general.encode(value)
                database.commit(session, 1037)

            # Change the last updated timestamp
            for idx_datapoint, last_timestamp in timestamp_tracker.items():
                # Update database
                database = db.Database()
                session = database.session()
                record = session.query(Datapoint).filter(
                    Datapoint.idx == idx_datapoint).one()
                record.last_timestamp = last_timestamp
                database.commit(session, 1083)

            # Report success
            log_message = (
                'Successful cache drain (Uncharted Data) '
                'for UID %s at timestamp %s') % (
                    self.ingest.uid(), self.ingest.timestamp())
            log.log2quiet(1045, log_message)
예제 #24
0
파일: cache.py 프로젝트: Pr0x1m4/infoset
    def _update_chartable(self, mapping):
        """Insert data into the database "iset_data" table.

        Args:
            mapping: Map of DIDs to database row index values

        Returns:
            None

        """
        # Initialize key variables
        data = self.ingest.chartable()
        data_list = []
        timestamp_tracker = {}

        # Update data
        for item in data:
            # Process each datapoint item found
            (_, did, string_value, timestamp) = item
            idx_datapoint = int(mapping[did][0])
            last_timestamp = int(mapping[did][2])
            value = float(string_value)

            # Only update with data collected after
            # the most recent DID update. Don't do anything more
            if timestamp > last_timestamp:
                data_list.append(
                    Data(
                        idx_datapoint=idx_datapoint,
                        value=value,
                        timestamp=timestamp
                    )
                )

                # Update DID's last updated timestamp
                if idx_datapoint in timestamp_tracker:
                    timestamp_tracker[idx_datapoint] = max(
                        timestamp, timestamp_tracker[idx_datapoint])
                else:
                    timestamp_tracker[idx_datapoint] = timestamp

        # Update if there is data
        if bool(data_list) is True:
            # Do performance data update
            database = db.Database()
            database.add_all(data_list, 1056)

            # Change the last updated timestamp
            for idx_datapoint, last_timestamp in timestamp_tracker.items():
                # Update the database
                database = db.Database()
                session = database.session()
                record = session.query(Datapoint).filter(
                    Datapoint.idx == idx_datapoint).one()
                record.last_timestamp = last_timestamp
                database.commit(session, 1057)

            # Report success
            log_message = (
                'Successful cache drain for UID %s at timestamp %s') % (
                    self.ingest.uid(), self.ingest.timestamp())
            log.log2quiet(1058, log_message)