def test_connect():
""" Connect to the PostgreSQL database server """
conn = None
try:
# read connection parameters
params = config()
# connect to the PostgreSQL server
conn = psycopg2.connect(**params)
# create a cursor
cur = conn.cursor()
# Check database version of postgresql
cur.execute('SELECT version()')
db_version = cur.fetchone()
# display the PostgreSQL database server version
print(db_version)
# close the communication with the PostgreSQL
cur.close()
conn.commit()
except (Exception, psycopg2.DatabaseError) as error:
print(params)
print(error)
return "TEST CONNECTION FAILED"
finally:
if conn is not None:
conn.close()
print('Database connection closed.')
return "Success - CONNECTION CLOSED..."
return "CONNECTION NOT CLOSED - conn is nulll :( "
def connect(self):
""" Connect to the PostgreSQL database server """
try:
# read connection parameters
params = config('../config/database.ini')
# connect to the PostgreSQL server
print('Connecting to the PostgreSQL database...')
self.conn = psycopg2.connect(**params)
print('Connection to DB successful')
print('INSERT IMAGE code to page_type table')
self.conn.cursor().execute(
"INSERT INTO crawldb.page_type (code) VALUES ('IMAGE') ON CONFLICT (code) DO UPDATE SET code = 'IMAGE'"
)
self.conn.commit()
# Get all page types from database
cur = self.conn.cursor()
cur.execute("SELECT code FROM crawldb.data_type")
self.data_types = cur.fetchall()
for idx, row in enumerate(self.data_types):
self.data_types[idx] = row[0]
return self.conn
except (Exception, psycopg2.DatabaseError) as error:
print(error)
def connect(self):
try:
# read connection parameters
params = config()
# connect to the PostgreSQL server
print('Connecting to the PostgreSQL database...')
self.conn = psycopg2.connect(**params)
# create a cursor
self.cur = self.conn.cursor()
# execute a statement
except (Exception, psycopg2.DatabaseError) as error:
print(error)
import logging
import multiprocessing
from bs4 import BeautifulSoup
import time
import pickle
from dbconfig import config
reload(sys)
sys.setdefaultencoding('utf-8')
date = time.strftime('%Y-%m-%d', time.localtime())
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(filename)s line:%(lineno)d [%(levelname)s]:%(message)s',
filename='logs/' + date + '.log',
filemode='a')
[dbadd, user, password, database] = config()
# 读取职位列表
def readweb():
print u'get jobdict.....'
configmap = {}
jobnamesfile = 'dict/jobnames.pkl'
if os.path.isfile(jobnamesfile):
configmap = pickle.load(open(jobnamesfile))
return configmap
p = Proxy()
while (True):
proxies = p.getproxies()
try:
r = requests.get(url='http://www.lagou.com/',
def __init__(self):
self.dbConf = config()
logging.basicConfig(filename=baseDir + 'Logs/' + 'eventdb.log',
format='%(name)s - %(levelname)s - %(message)s')