def record_observations_in_db(res_list): 
	if len(res_list) == 0: 
		return
	try: 
		conn = sqlite3.connect(notary_db)
		for r in res_list: 
			notary_common.report_observation_with_conn( \
						conn, r[0], r[1]) 
		conn.commit()
		conn.close() 
	except:
		# TODO: we should probably retry here 
		print "DB Error: Failed to write res_list of length %s" % \
					len(res_list)
		traceback.print_exc(file=sys.stdout)
all_sids = [ line.rstrip() for line in f ]

for sid in all_sids:  
	try: 
		# ignore non SSL services	
		if sid.split(",")[1] == "2": 
			stats.num_started += 1
			t = ScanThread(sid,stats,timeout_sec)
			t.start()
 
		if (stats.num_started % rate) == 0: 
			time.sleep(1)
			try: 
				conn = sqlite3.connect(notary_db)
				for r in res_list: 
					notary_common.report_observation_with_conn(conn, r[0], r[1]) 
				conn.commit()
				conn.close() 
			except:
				# TODO: we should probably retry here 
				print "DB Error: Failed to write res_list of length %s" % len(res_list)
				traceback.print_exc(file=sys.stdout)
				
			res_list = [] 
			so_far = int(time.time() - start_time)
			print "%s seconds passed.  %s complete, %s failures.  %s Active threads" % \
				(so_far, stats.num_completed, 
					stats.failures, stats.active_threads)
			print "failure details: timeouts = %s, ssl-alerts = %s, no-route = %s, conn-refused = %s, conn-reset = %s, dns = %s, other = %s" % \
				(stats.failure_timeouts,
				stats.failure_ssl_alert,
all_sids = [line.rstrip() for line in f]

for sid in all_sids:
    try:
        # ignore non SSL services
        if sid.split(",")[1] == "2":
            stats.num_started += 1
            t = ScanThread(sid, stats, timeout_sec)
            t.start()

        if (stats.num_started % rate) == 0:
            time.sleep(1)
            try:
                conn = sqlite3.connect(notary_db)
                for r in res_list:
                    notary_common.report_observation_with_conn(
                        conn, r[0], r[1])
                conn.commit()
                conn.close()
            except:
                # TODO: we should probably retry here
                print "DB Error: Failed to write res_list of length %s" % len(
                    res_list)
                traceback.print_exc(file=sys.stdout)

            res_list = []
            so_far = int(time.time() - start_time)
            print "%s seconds passed.  %s complete, %s failures.  %s Active threads" % \
             (so_far, stats.num_completed,
              stats.failures, stats.active_threads)
            print "failure details: timeouts = %s, ssl-alerts = %s, no-route = %s, conn-refused = %s, conn-reset = %s, dns = %s, other = %s" % \
             (stats.failure_timeouts,