def _save_configuration(self, browser_params):
""" Saves crawl configuration details to db and logfile"""
cur = self.db.cursor()
# Get git version and commit information
openwpm_v, browser_v = get_version()
# Record task details
cur.execute(("INSERT INTO task "
"(manager_params, openwpm_version, browser_version) "
"VALUES (?,?,?)"),
(json.dumps(self.manager_params), openwpm_v, browser_v))
self.db.commit()
self.task_id = cur.lastrowid
# Record browser details for each brower
for i in xrange(self.num_browsers):
cur.execute("INSERT INTO crawl (task_id, browser_params) VALUES (?,?)",
(self.task_id, json.dumps(browser_params[i])))
self.db.commit()
browser_params[i]['crawl_id'] = cur.lastrowid
# Print the configuration details
self.logger.info(get_configuration_string(self.manager_params,
browser_params,
(openwpm_v, browser_v)))
def _save_configuration(self, browser_params):
""" Saves crawl configuration details to db and logfile"""
cur = self.db.cursor()
# Get git version and commit information
openwpm_v, browser_v = get_version()
# Record task details
cur.execute(("INSERT INTO task "
"(manager_params, openwpm_version, browser_version) "
"VALUES (?,?,?)"),
(json.dumps(self.manager_params), openwpm_v, browser_v))
self.db.commit()
self.task_id = cur.lastrowid
# Record browser details for each brower
for i in xrange(self.num_browsers):
cur.execute(
"INSERT INTO crawl (task_id, browser_params) VALUES (?,?)",
(self.task_id, json.dumps(browser_params[i])))
self.db.commit()
browser_params[i]['crawl_id'] = cur.lastrowid
# Print the configuration details
self.logger.info(
get_configuration_string(self.manager_params, browser_params,
(openwpm_v, browser_v)))
def __init__(self, manager_params, browser_params, process_watchdog=False):
# Make paths absolute in manager_params
for path in ['data_directory', 'log_directory']:
if manager_params[path] is not None:
manager_params[path] = os.path.expanduser(manager_params[path])
manager_params['database_name'] = os.path.join(
manager_params['data_directory'], manager_params['database_name'])
manager_params['log_file'] = os.path.join(
manager_params['log_directory'], manager_params['log_file'])
manager_params['screenshot_path'] = os.path.join(
manager_params['data_directory'], 'screenshots')
manager_params['source_dump_path'] = os.path.join(
manager_params['data_directory'], 'sources')
self.manager_params = manager_params
self.browser_params = browser_params
# Create data directories if they do not exist
if not os.path.exists(manager_params['screenshot_path']):
os.makedirs(manager_params['screenshot_path'])
if not os.path.exists(manager_params['source_dump_path']):
os.makedirs(manager_params['source_dump_path'])
# check size of parameter dictionary
self.num_browsers = manager_params['num_browsers']
if len(browser_params) != self.num_browsers:
raise Exception("Number of <browser_params> dicts is not the same "
"as manager_params['num_browsers']")
# Flow control
self.closing = False
self.failure_status = None
self.threadlock = threading.Lock()
self.failurecount = 0
if manager_params['failure_limit'] is not None:
self.failure_limit = manager_params['failure_limit']
else:
self.failure_limit = self.num_browsers * 2 + 10
self.process_watchdog = process_watchdog
# sets up logging server + connect a client
self.logging_status_queue = None
self.loggingserver = self._launch_loggingserver()
# socket location: (address, port)
self.manager_params['logger_address'] = self.logging_status_queue.get()
self.logger = MPLogger.loggingclient(
*self.manager_params['logger_address'])
# Initialize the data aggregators
self._launch_aggregators()
# sets up the BrowserManager(s) + associated queues
self.browsers = self._initialize_browsers(browser_params)
self._launch_browsers()
# start the manager watchdog
thread = threading.Thread(target=self._manager_watchdog, args=())
thread.daemon = True
thread.start()
# Save crawl config information to database
openwpm_v, browser_v = get_version()
self.data_aggregator.save_configuration(openwpm_v, browser_v)
self.logger.info(
get_configuration_string(self.manager_params, browser_params,
(openwpm_v, browser_v)))