def _save_configuration(self, browser_params): """ Saves crawl configuration details to db and logfile""" cur = self.db.cursor() # Get git version and commit information openwpm_v, browser_v = get_version() # Record task details cur.execute(("INSERT INTO task " "(manager_params, openwpm_version, browser_version) " "VALUES (?,?,?)"), (json.dumps(self.manager_params), openwpm_v, browser_v)) self.db.commit() self.task_id = cur.lastrowid # Record browser details for each brower for i in xrange(self.num_browsers): cur.execute("INSERT INTO crawl (task_id, browser_params) VALUES (?,?)", (self.task_id, json.dumps(browser_params[i]))) self.db.commit() browser_params[i]['crawl_id'] = cur.lastrowid # Print the configuration details self.logger.info(get_configuration_string(self.manager_params, browser_params, (openwpm_v, browser_v)))
def _save_configuration(self, browser_params): """ Saves crawl configuration details to db and logfile""" cur = self.db.cursor() # Get git version and commit information openwpm_v, browser_v = get_version() # Record task details cur.execute(("INSERT INTO task " "(manager_params, openwpm_version, browser_version) " "VALUES (?,?,?)"), (json.dumps(self.manager_params), openwpm_v, browser_v)) self.db.commit() self.task_id = cur.lastrowid # Record browser details for each brower for i in xrange(self.num_browsers): cur.execute( "INSERT INTO crawl (task_id, browser_params) VALUES (?,?)", (self.task_id, json.dumps(browser_params[i]))) self.db.commit() browser_params[i]['crawl_id'] = cur.lastrowid # Print the configuration details self.logger.info( get_configuration_string(self.manager_params, browser_params, (openwpm_v, browser_v)))
def __init__(self, manager_params, browser_params, process_watchdog=False): # Make paths absolute in manager_params for path in ['data_directory', 'log_directory']: if manager_params[path] is not None: manager_params[path] = os.path.expanduser(manager_params[path]) manager_params['database_name'] = os.path.join( manager_params['data_directory'], manager_params['database_name']) manager_params['log_file'] = os.path.join( manager_params['log_directory'], manager_params['log_file']) manager_params['screenshot_path'] = os.path.join( manager_params['data_directory'], 'screenshots') manager_params['source_dump_path'] = os.path.join( manager_params['data_directory'], 'sources') self.manager_params = manager_params self.browser_params = browser_params # Create data directories if they do not exist if not os.path.exists(manager_params['screenshot_path']): os.makedirs(manager_params['screenshot_path']) if not os.path.exists(manager_params['source_dump_path']): os.makedirs(manager_params['source_dump_path']) # check size of parameter dictionary self.num_browsers = manager_params['num_browsers'] if len(browser_params) != self.num_browsers: raise Exception("Number of <browser_params> dicts is not the same " "as manager_params['num_browsers']") # Flow control self.closing = False self.failure_status = None self.threadlock = threading.Lock() self.failurecount = 0 if manager_params['failure_limit'] is not None: self.failure_limit = manager_params['failure_limit'] else: self.failure_limit = self.num_browsers * 2 + 10 self.process_watchdog = process_watchdog # sets up logging server + connect a client self.logging_status_queue = None self.loggingserver = self._launch_loggingserver() # socket location: (address, port) self.manager_params['logger_address'] = self.logging_status_queue.get() self.logger = MPLogger.loggingclient( *self.manager_params['logger_address']) # Initialize the data aggregators self._launch_aggregators() # sets up the BrowserManager(s) + associated queues self.browsers = self._initialize_browsers(browser_params) self._launch_browsers() # start the manager watchdog thread = threading.Thread(target=self._manager_watchdog, args=()) thread.daemon = True thread.start() # Save crawl config information to database openwpm_v, browser_v = get_version() self.data_aggregator.save_configuration(openwpm_v, browser_v) self.logger.info( get_configuration_string(self.manager_params, browser_params, (openwpm_v, browser_v)))