def __init__( self, setup_file_name):
setup_file_values = setup_file_reader.read_in_setup_file(setup_file_name)
assert( setup_file_values['host'] and
setup_file_values['username'] and
setup_file_values['passwd'] and
setup_file_values['db_name'] and
setup_file_values['table_columns'] ) #check for non-nullity and non-emptiness
self.host = setup_file_values['host']
self.username = setup_file_values['username']
self.passwd = setup_file_values['passwd']
self.db_name = setup_file_values['db_name']
self.table_columns = setup_file_values['table_columns'].split(',')
self.column_datatypes = setup_file_values['column_datatypes'].split(',')
self.db_connection = mysql.connector.connect( host=self.host, user=self.username, passwd=self.passwd, db=self.db_name )
self.cursor = self.db_connection.cursor()
import setup_file_reader as setup
"""
Server/Client code for the fishpaste crawler.
This handles passing out new sites for each crawler to visit.
"""
#client/server needs to run with sudo.
BUFFER_SIZE = 4096
TCP_PORT = 666
TCP_CALLBACK_PORT = 667
setup_info = setup.read_in_setup_file("./fishpaste_server.setup")
SERVER_IP = setup_info['host']
CLIENT_IP = setup_info['client']
def encode_for_transport( string ):
return bytes( string, "utf-8" )
def client_get_more_links( exclusion_url ):
if not exclusion_url:
exclusion_url = "None"
TRIES = 10
string_data = None
s = socket.socket( socket.AF_INET, socket.SOCK_STREAM )
