def __init__(self): self.total_new = 0 self.table = 'lagou_basic' self.table2 = 'lagou_company_label' self.url_base = 'http://www.lagou.com' self.url_params = '/jobs/positionAjax.json?px=new' try: # generate insert query self.model = model.dbSqlite() self.iq_1 = self.model.insertQuery(self.table) self.ip_1 = self.model.insertParam(self.table) except Exception as e: msg = time.strftime('%Y-%m-%d %H:%M:%S') + '[Error][Init] ' + str( e) func.logger('crawler', msg) exit()
def __init__(self): self.total_new = 0 self.table = 'lagou_basic' self.table2 = 'lagou_company_label' self.url_base = 'http://www.lagou.com' self.url_params = '/jobs/positionAjax.json?px=new' try: # generate insert query self.model = model.dbSqlite() self.iq_1 = self.model.insertQuery(self.table) self.ip_1 = self.model.insertParam(self.table) except Exception as e: msg = time.strftime( '%Y-%m-%d %H:%M:%S') + '[Error][Init] ' + str(e) func.logger('crawler', msg) exit()
import sys # in order to import module from parent directory sys.path.append('..') import model m = model.dbSqlite() m.createTable('lagou_basic', 'lagou_basic.sql') m.createTable('lagou_company_label', 'lagou_company_label.sql') ''' this method shouldn't be used anymore as it doesn't store data in txt file. m.importFromFileToTable('lagou_basic','data/position_lagou.txt') '''
import sys # in order to import module from parent directory sys.path.append('..') import model m = model.dbSqlite() m.createTable('lagou_basic','lagou_basic.sql') m.createTable('lagou_company_label','lagou_company_label.sql') ''' this method shouldn't be used anymore as it doesn't store data in txt file. m.importFromFileToTable('lagou_basic','data/position_lagou.txt') '''