def put_page(self):
lock_uid = 'page'
# single_oracle.execute('update company_basic_info set PAGE_SPIDER=0')
# self.server.delete(lock_uid)
status, sec = self.get_lock(lock_uid)
if status:
pages_length = self.server.llen('pages')
if pages_length > 1000:
print('users length={}'.format(pages_length))
self.del_lock(lock_uid, sec)
return
sql = "SELECT search_name,txt_id FROM company_basic_info where parsed=0 and page_spider=0 and {} and rownum <= 10000"
try:
tablename_condition = self.get_configs_by_tablename('put_page')
results = single_oracle.oracle_find_by_param_all(
sql.format(tablename_condition[0]))
# results = single_oracle.oracle_find_by_param_all(sql)
if results:
print(len(results))
if tablename_condition[1] == 0:
update_sql = "update config set over = 2 where id = {}".format(
tablename_condition[2])
single_oracle.oracle_update(update_sql)
if len(results) > 10:
results = results[10:]
for page in results:
txt_id = page[1]
# url = page[0]
# if self.isContains(txt_id):
# single_oracle.oracle_update(
# "update company_basic_info set PAGE_SPIDER=1 where txt_id={}".format(txt_id))
# continue
# self.insert(txt_id)
self.server.lpush('pages', txt_id)
else:
single_oracle.oracle_update(
"update config set over = 1 where id = {}".format(
tablename_condition[2]))
return
sql = "SELECT search_name,txt_id FROM company_basic_info where parsed=0 and page_spider=0 "
results = single_oracle.oracle_find_by_param_all(sql)
for page in results:
txt_id = page[1]
if self.isContains(txt_id):
single_oracle.oracle_update(
"update company_basic_info set PAGE_SPIDER=1 where txt_id={}"
.format(txt_id))
continue
self.insert(txt_id)
self.server.lpush('pages', txt_id)
self.del_lock(lock_uid, sec)
except Exception as e:
self.del_lock(lock_uid, sec)
print(e)
def put_11315_increased(self):
'''
11315新增企业名单
:return:
'''
lock_uid = '11315_increased_s'
status, sec = self.get_lock(lock_uid)
if status:
incre_length = self.server.llen('11315_increased')
if incre_length > 50000:
logger.debug('parses incre_length={}'.format(incre_length))
self.del_lock(lock_uid, sec)
return
try:
incre_sql = 'SELECT max(company_number) FROM company_11315 where {}'
# select max(company_number) from company_11315 where company_name != 'NA'
tablename_condition = self.get_configs_by_tablename(
'put_11315_increased')
print(incre_sql.format(tablename_condition[0]))
results = single_oracle.oracle_find_by_param_all(
incre_sql.format(tablename_condition[0]))
print(results)
if results[0][0]:
if tablename_condition[1] == 0:
sql = 'update config set over = 2 where id = {}'.format(
tablename_condition[2])
single_oracle.oracle_update(sql)
max_num = single_oracle.oracle_find_by_param_all(
'select max(company_number) from company_11315')
if max_num[0][0] > results[0][0]:
single_oracle.execute(
'delete from company_11315 where company_number > {}'
.format(results[0][0]))
# XXX: 需要判段什么时候抓取完毕
for i in range(results[0][0] + 1, results[0][0] + 2000000):
company_number = i
# print(company_number)
self.server.lpush('11315_increased', company_number)
else:
single_oracle.oracle_update(
'update config set over = 1, over_time = SYSDATE where id = {}'
.format(tablename_condition[2]))
self.del_lock(lock_uid, sec)
except Exception as e:
self.del_lock(lock_uid, sec)
logger.exception(e)
def put_company_(self):
lock_uid = 'ent'
status, sec = self.get_lock(lock_uid)
if status:
users_length = self.server.llen('ents')
if users_length > 1000:
print('users length={}'.format(users_length))
self.del_lock(lock_uid, sec)
return
ents = {}
# 慈溪
sql = 'SELECT company_number,company_name,address_3 FROM company_11315 where mark = 11 and searched != 1'
try:
companys = single_oracle.oracle_find_by_param_all(sql)
print(len(companys))
ents['table'] = 'company_11315'
ents['type'] = 'company_11315s'
if companys:
for company in companys:
if company[1] == 'None' or company[1] == 'NA' or company[1] == None:
print('发现空的公司名称={}'.format(company))
single_oracle.oracle_update(
"update {table} set searched=1 where company_number={number}".format(
table=ents['table'], number=company[0]))
continue
ents['company_name'] = company[1].encode('utf-8')
print('not exists!')
ents['address'] = company[2] or ''
ents['number'] = company[0]
self.server.lpush('ents', str(ents))
self.del_lock(lock_uid, sec)
except Exception as e:
self.del_lock(lock_uid, sec)
logger.debug(e)
def put_11315_update(self):
lock_uid = '11315_update'
status, sec = self.get_lock(lock_uid)
if status:
parses_length = self.server.llen('11315_updates')
if parses_length > 50000:
logger.debug('parses length={}'.format(parses_length))
self.del_lock(lock_uid, sec)
return
# mongo = single_mongodb
# param = {'parse': 0}
# projection = {'_id': 1}
sql = "SELECT company_number FROM {} where company_type is null and error=0 and branch is not null and searched=1 and rownum<=100000"
try:
# max1 = single_oracle.oracle_find_by_param_all(sql.format('company_11315_zyfd')
# )
max2 = single_oracle.oracle_find_by_param_all(
sql.format('company_11315'))
# for i in max1:
# self.server.lpush('11315_types', i[0])
for i in max2:
self.server.lpush('11315_updates', i[0])
self.del_lock(lock_uid, sec)
except Exception as e:
self.del_lock(lock_uid, sec)
logger.exception(e)
def put_page(self):
lock_uid = 'page'
status, sec = self.get_lock(lock_uid)
if status:
pages_length = self.server.llen('pages')
if pages_length > 500:
print('users length={}'.format(pages_length))
self.del_lock(lock_uid, sec)
return
sql = "SELECT txt_id FROM COMPANY_BASIC_INFO_XIONG_AN where REGISTERED_CAPITAL!='-' and txt_id not in(select txt_id from tyc_qybj_jbxx)"
try:
resuls = single_oracle.oracle_find_by_param_all(sql)
if resuls:
print(len(resuls))
for page in resuls:
txt_id = page[0]
# url = page[0]
self.server.lpush('pages', txt_id)
else:
time.sleep(10 % 30)
self.del_lock(lock_uid, sec)
self.put_page()
self.del_lock(lock_uid, sec)
except Exception as e:
self.del_lock(lock_uid, sec)
print(e)
def put_user(self):
lock_uid = 'user'
status, sec = self.get_lock(lock_uid)
if status:
users_length = self.server.llen('users')
# if users_length > 100:
# self.del_lock(lock_uid, sec)
# logger.debug('users length={}'.format(users_length))
# return
# SQL 查询语句
sql = 'SELECT username FROM tyc_user '
try:
# 执行SQL语句
# cursor.execute(sql)
# 获取所有记录列表
# results = cursor.fetchall()
results = single_oracle.oracle_find_by_param_all(sql)
if results:
for row in results:
self.server.lpush('users', row[0])
self.del_lock(lock_uid, sec)
return
except Exception as e:
logger.exception(e)
self.del_lock(lock_uid, sec)
time.sleep(0.001)
def put_page(self):
lock_uid = 'page'
status, sec = self.get_lock(lock_uid)
if status:
pages_length = self.server.llen('pages')
if pages_length > 500:
print('users length={}'.format(pages_length))
self.del_lock(lock_uid, sec)
return
sql = 'SELECT url,txt_id FROM company_basic_info WHERE page_spider=0 AND parsed=0 and ROWNUM <=10000 order by txt_id'
try:
resuls = single_oracle.oracle_find_by_param_all(sql)
if resuls:
print(len(resuls))
for page in resuls:
txt_id = page[1]
# url = page[0]
self.server.lpush('pages', txt_id)
else:
time.sleep(10 * 30)
self.del_lock(lock_uid, sec)
self.put_page()
self.del_lock(lock_uid, sec)
except Exception as e:
self.del_lock(lock_uid, sec)
print(e)
def func():
sql = "select count(company_number) from {table} where instr({address},'{area}') > 0 "
for i in area_list:
for table in [company_11315, company_11315_zyfd]:
for address in ['address_2', 'address_3']:
sql_ = sql.format(table=table, address=address, area=i)
print(single_oracle.oracle_find_by_param_all(sql_))
def put_parse(self):
lock_uid = 'parse_lock'
# self.server.delete(lock_uid)
status, sec = self.get_lock(lock_uid)
print(status, sec)
if status:
# parses_length = self.server.llen('parses')
#
# if parses_length > 500:
# logger.debug('parses length={}'.format(parses_length))
# self.del_lock(lock_uid, sec)
# return
# mongo = single_mongodb
param = {'parse': 0}
projection = {'_id': 1}
try:
sql = "SELECT txt_id,search_name FROM company_basic_info WHERE {} and rownum <= 10000 and txt_id is not null"
# sql = "SELECT txt_id,search_name FROM company_basic_info WHERE parsed=0 and page_spider=1 and rownum <= 10000"
# sql='select B.txt_id,B.search_name from company_basic_info B where (select count(1) as num from tyc_qybj_jbxx A where A.txt_id = B.txt_id) = 0'
# results = single_mongodb.mongodb_find_limit('company_detail_info', param,projection, 1000)
# if results.count() == 0:
# param = {'parse': {'$exists': False}}
# results = single_mongodb.mongodb_find_limit('company_detail_info', param,projection, 100000)
tablename_condition = self.get_configs_by_tablename('put_parse')
results = single_oracle.oracle_find_by_param_all(sql.format(tablename_condition[0]))
print(len(results))
if not results:
time.sleep(60 * 30)
else:
if tablename_condition[1] == 0:
update_sql = "update config set over = 2 where id = {}".format(tablename_condition[2])
single_oracle.oracle_update(update_sql)
if len(results) > 10:
results = results[10:]
for page in results:
txt_id = page[0]
search_name = page[1]
if isinstance(txt_id, bytes):
search_name = search_name.decode()
if isinstance(txt_id, bytes):
txt_id = txt_id.decode()
value = txt_id + ',' + search_name
# if self.isContains(value):
# single_oracle.oracle_update(
# "update company_basic_info set parsed=1 where txt_id={}".format(txt_id))
# continue
# self.insert(value)
self.server.lpush('parses', value)
logger.debug('塞完待解析数据')
self.del_lock('parse_lock', sec)
logger.debug('yi shan chu lock_uid:parse_lock')
except Exception as e:
self.del_lock('parse_lock', sec)
logger.exception(e)
def put_company(self):
lock_uid = 'ent'
status, sec = self.get_lock(lock_uid)
if status:
users_length = self.server.llen('ents')
# if users_length > 2000:
# print('users length={}'.format(users_length))
# self.del_lock(lock_uid, sec)
# return
ents = {}
# sql = 'select B.* from batch_detail B where (select count(1) as num from company_basic_info A where A.search_name = B.COMPANY_NAME) = 0'
# sql='select distinct t1.company_number,t1.company_name from BATCH_DETAIL_1 t1 where t1.company_name not in (select t.search_name from company_basic_info t)'
# companys = single_oracle.oracle_find_by_param_all(sql)
resuls = []
# 分支机构名单
sql_fzjg = 'select distinct ent_name from tyc_qybj_fzjg'
# 对外投资名单
sql_dwtz = 'select distinct invest_company from tyc_qybj_dwtz'
resuls.extend(single_oracle.oracle_find_by_param_all(sql_fzjg))
resuls.extend(single_oracle.oracle_find_by_param_all(sql_dwtz))
print(len(resuls))
# logger.debug(companys)
ents['table'] = 'batch_detail'
ents['type'] = 'batch_details'
if resuls:
for company in resuls:
ents['company_name'] = company[0]
# if self.isContains(ents['company_name']): # 判断字符串是否存在
# print('exists!')
# continue
# else:
# print ('not exists!')
# self.insert(ents['company_name'])
ents['agency_num'] = 'NA'
ents['agency_name'] = 'NA'
ents['batch'] = 'NA'
# ents['company_name'] = company[2]
ents['address'] = 'NA'
ents['number'] = 'NA'
self.server.lpush('ents_fzjg_dwtz', str(ents))
# self.del_lock(lock_uid, sec)
self.del_lock(lock_uid, sec)
def execute(sql):
'''
:param sql: oracle sql
:return: 查询结果列表
'''
# cursor = db_conn()
# sql = 'select company_address from company_11315_xa'
# res = cursor.execute(sql)
res = single_oracle.oracle_find_by_param_all(sql)
return res
def get_configs_by_tablename(self, function):
sql_conf = "select c.condition, c.over, c.id from (select min(a.priority) min_p from config a where a.function = '{}' and a.over != 1) b, config c where c.priority = b.min_p and c.over != 1 and c.function = '{}'".format(
function, function)
# sql = "SELECT company_number, company_name, branch, address_3 from company_11315 where address_3 like '%通州%' and address_2 = '北京市' and searched=0"
configs = single_oracle.oracle_find_by_param_all(sql_conf)
print(configs)
if not configs or not configs[0]:
return
condition = configs[0][0]
over = configs[0][1]
id = configs[0][2]
return [condition, over, id]
def put_11315_update(self):
lock_uid = '11315:update'
status, sec = self.get_lock(lock_uid)
if status:
parses_length = self.server.llen('11315_updates')
if parses_length > 50000:
print('parses length={}'.format(parses_length))
self.del_lock(lock_uid, sec)
return
sql = "SELECT company_number,company_address FROM {} where parse!=6 and company_name!='NoCompany' and company_address is not null and (company_name like '%公司%' or company_name like '%企业%' or company_name like '%集团%') and rownum<=200000"
# mongo = single_mongodb
# param = {'parse': 0}
# projection = {'_id': 1}
try:
# max1 = single_oracle.oracle_find_by_param_all(sql.format('company_11315_zyfd')
# )
max2 = single_oracle.oracle_find_by_param_all(
sql.format('company_11315'))
if not max2:
sql = "SELECT company_number,company_address FROM {} where parse!=6 and company_name!='NoCompany' and company_address is not null and rownum<=200000"
max2 = single_oracle.oracle_find_by_param_all(
sql.format('company_11315'))
# for i in max1:
# self.server.lpush('11315_types', i[0])
print(len(max2))
for i in max2:
if i and i[1]:
self.server.lpush('11315_updates',
str(i[0]) + ';' + i[1])
else:
print(i)
# single_oracle.oracle_update(
# "update company_11315 set parse=5 where company_number={}".format(i[0]))
self.del_lock(lock_uid, sec)
except Exception as e:
self.del_lock(lock_uid, sec)
print(e)
def put_11315_omit(self):
'''
11315遗漏的企业名单
:return:
'''
lock_uid = '11315_omit_s'
status, sec = self.get_lock(lock_uid)
if status:
omit_length = self.server.llen('11315_omit')
if omit_length > 50000:
logger.debug('parses omit_length={}'.format(omit_length))
self.del_lock(lock_uid, sec)
return
try:
omit_sql = 'select company_number from company_11315 where {}'
# select company_number from company_11315 where company_numnber > 3138637 and company_number < 4000000 order by company_number asc
tablename_condition = self.get_configs_by_tablename(
'put_11315_omit')
# print(omit_sql.format(tablename_condition[0]))
results = single_oracle.oracle_find_by_param_all(
omit_sql.format(tablename_condition[0]))
if results[0][0]:
if tablename_condition[1] == 0:
# TODO
sql = 'update config set over = 2 where id = {}'.format(
tablename_condition[2])
single_oracle.oracle_update(sql)
num_1 = [x[0] for x in results]
num_2 = [x for x in range(min(num_1), max(num_1))]
omit_list = set(num_2) - set(num_1)
for company_number in omit_list:
# company_number = result[0]
self.server.lpush('11315_omit', company_number)
if len(omit_list) == 0:
sql = 'update config set over = 1,over_time = SYSDATE where id = {}'.format(
tablename_condition[2])
single_oracle.oracle_update(sql)
print('此次操作已经完成')
time.sleep(10 * 60)
# time.sleep('60'*60)
self.del_lock(lock_uid, sec)
except Exception as e:
s = self.del_lock(lock_uid, sec)
logger.exception(e)
def put_parse(self):
lock_uid = 'parse'
status, sec = self.get_lock(lock_uid)
if status:
parses_length = self.server.llen('parses')
if parses_length > 500:
logger.debug('parses length={}'.format(parses_length))
self.del_lock(lock_uid, sec)
return
# mongo = single_mongodb
param = {'parse': 0}
projection = {'_id': 1}
try:
sql = "SELECT txt_id,search_name FROM company_basic_info WHERE parsed=0 and page_spider=1 and rownum <= 10000"
# resuls = single_mongodb.mongodb_find_limit('company_detail_info', param,projection, 1000)
# if resuls.count() == 0:
# param = {'parse': {'$exists': False}}
# resuls = single_mongodb.mongodb_find_limit('company_detail_info', param,projection, 100000)
results = single_oracle.oracle_find_by_param_all(sql)
print(len(results))
if not results:
time.sleep(60 % 30)
# self.del_lock(lock_uid, sec)
# self.put_parse()
else:
for page in results:
txt_id = page[0]
search_name = page[1]
# print ('已解析')
# print(page[1])
# find_sql='select count(*) from tyc_qybj_jbxx where txt_id='{}''.format(txt_id)
# print (mysql.mysql_find_by_param(find_sql))
# mysql.mysql_update_param(
# 'update company_basic_info set parsed=1 where txt_id='{}''.format(txt_id))
# print(txt_id)
# continue
self.server.lpush('parses', txt_id + ',' + search_name)
self.del_lock(lock_uid, sec)
except Exception as e:
self.del_lock(lock_uid, sec)
logger.exception(e)
def put_phone(self):
lock_uid = 'phone'
status, sec = self.get_lock(lock_uid)
if status:
users_length = self.server.llen('phones')
if users_length > 0:
logger.debug('phones length={}'.format(users_length))
self.del_lock(lock_uid, sec)
return
companys = single_oracle.oracle_find_by_param_all('user')
print(companys)
for company in companys:
self.server.lpush('phones', company[0])
# print(company[0])
self.del_lock(lock_uid, sec)
def put_parse(self):
lock_uid = 'parse'
status, sec = self.get_lock(lock_uid)
if status:
parses_length = self.server.llen('parses')
print(parses_length)
try:
# txt_ids = single_oracle.oracle_find_by_param_all(
# "select B.txt_id from company_basic_info B where (select count(1) as num from tyc_qybj_jbxx A where A.ent_name = B.ent_name) = 0")
# sql = 'select B.* from company_basic_info B where (select count(1) as num from tyc_qybj_jbxx A where A.txt_id = B.txt_id) = 0'
# sql='select a.txt_id,b.company_name from company_basic_info a, batch_detail_1 b where a.search_name=b.company_name'
# sql='select distinct txt_id,company_name from tyc_sffx_flss'
# sql = 'select B.txt_id,B.search_name from company_basic_info B ,batch_detail_1 C where B.search_name=C.company_name '
# 分支机构，对外投资
sql = "SELECT txt_id,search_name FROM company_basic_info_xiong_an "
resuls = single_oracle.oracle_find_by_param_all(sql)
print(len(resuls))
# if resuls.count() == 0:
# param = {'parse': {'$exists': False}}
# resuls = mongo.mongodb_find_limit('company_detail_info', param,projection, 100000)
# results = single_oracle.oracle_find_by_param_all(sql)
for page in resuls:
txt_id = page[0]
company_name = page[1]
# print ('已解析')
# print(page[1])
# find_sql='select count(*) from tyc_qybj_jbxx where txt_id='{}''.format(txt_id)
# print (mysql.mysql_find_by_param(find_sql))
# mysql.mysql_update_param(
# 'update company_basic_info set parsed=1 where txt_id='{}''.format(txt_id))
# print(txt_id)
# continue
# self.server.lpush('sxr_parses',str(company_name))
self.server.lpush('parses',
str(txt_id) + ',' + str(company_name))
self.del_lock(lock_uid, sec)
except Exception as e:
self.del_lock(lock_uid, sec)
print(e)
def put_11315_increased(self):
'''
11315新增企业名单
:return:
'''
lock_uid = '11315_increased_s'
status, sec = self.get_lock(lock_uid)
if status:
incre_length = self.server.llen('11315_increased')
if incre_length > 50000:
logger.debug('parses length={}'.format(parses_length))
self.del_lock(lock_uid, sec)
return
try:
incre_sql = 'SELECT max(company_number) FROM company_11315 where {}'
# select company_number from company_11315 where 1=1
tablename_condition = self.get_configs_by_tablename(
'put_11315_increased')
results = single_oracle.oracle_find_by_param_all(
omit_sql.format(tablename_condition[0]))
if results[0][0]:
if tablename_condition[1] == 0:
sql = 'update config set over = 2 where id = {}'.format(
tablename_condition[2])
single_oracle.oracle_update(sql)
# XXX: 需要判段什么时候抓取完毕
for i in ranage(results[0][0] + 1, i + 100000):
company_number = i
self.server.lpush('11315_omit', company_number)
else:
single_oracle.oracle_update(
'update config set over = 1, over_time = SYSDATE where id = {}'
.format(tablename_condition[2]))
self.del_lock(lock_uid, sec)
except Exception as e:
self.del_lock(lock_uid, sec)
logger.exception(e)
def main(args):
print(u'启动', args)
# import sys
#
# reload(sys)
# check_first_start=True
flag = single_reids.server.get('11315_increased_flag')
if not flag or int(flag.decode()) == 0:
single_reids.put_flag()
flag = single_reids.server.get('11315_increased_flag')
while True:
NA_count = single_reids.server.get('11315_NA_count')
if NA_count and int(NA_count.decode()) >= int(flag.decode()):
time.sleep(60 * 30)
continue
# i = single_reids.server.rpop('11315_omit')
i = single_reids.server.rpop('11315_increased')
#i=12872429
print('-' * 20, i)
if not i:
# time.sleep(60*10)
# single_reids.put_11315_omit()
single_reids.put_11315_increased()
continue
Retry = 1
i = i.decode()
res = single_oracle.oracle_find_by_param_all(
'select count(*) from company_11315 where company_number = {}'.
format(i))
if res[0][0] != 0:
print('该id{}已存在'.format(i))
single_oracle.execute(
'delete from company_11315 where company_number = {}'.format(
i))
url = "http://" + str(i) + ".11315.com/"
print(url)
# 一个公司连接最多访问10次
while Retry <= 10:
logger.info(url + "------%d", Retry)
try:
con = web_spider(url)
# 对访问状态及页面内容进行初步判断
if con.status_code == 200:
html = con.text
if "系统检测到您的请求存在异常" not in html and "你可能访问的太快了" not in html and "正在 请求 请 稍候" not in html:
try:
web_parse(html, i, url)
except Exception:
logger.info("此number号已存在: %s", i)
break
else:
logger.info("IP 被封或者访问过快")
Retry += 1
continue
else:
logger.info("访问状态码异常%s", str(con.status_code))
Retry += 1
time.sleep(3)
continue
except Exception as e:
logger.info("访问超时或者连接出错")
Retry += 1
logger.exception(e)
continue
except AttributeError as w:
logger.exception(w)
logger.info("无法获取二次连接url")
Retry += 1
continue
except Exception as w:
logger.exception(w)
logger.info("未知访问错误")
logger.exception("Exception Logged")
Retry += 1
continue
def put_company(self):
lock_uid = 'ent'
status, sec = self.get_lock(lock_uid)
if status:
users_length = self.server.llen('ents')
if users_length > 0:
print('users length={}'.format(users_length))
self.del_lock(lock_uid, sec)
return
ents = {}
sql = 'SELECT * FROM batch_detail WHERE searched=0 '
companys = single_oracle.oracle_find_by_param_all(sql)
# logger.debug(companys)
ents['table'] = 'batch_detail'
ents['type'] = 'batch_details'
if companys:
for company in companys:
ents['company_name'] = company[2].encode('utf-8')
# if self.isContains(ents['company_name']): # 判断字符串是否存在
# print('exists!')
# # continue
# else:
# print ('not exists!')
# self.insert(ents['company_name'])
ents['agency_num'] = company[7] or 'NA'
ents['agency_name'] = company[8] or 'NA'
ents['batch'] = company[9] or 'NA'
# ents['company_name'] = company[2]
ents['address'] = company[3] or 'NA'
ents['number'] = company[0] or 'NA'
self.server.lpush('ents', str(ents))
# self.del_lock(lock_uid, sec)
else:
# and branch in ('蕲春县',
# '京山县',
# '谷城县',
# '枣阳市',
# '老河口市',
# '潜江经济开发区',
# '潜江区',
# '潜江市经济开发区',
# '湖北省潜江市浩口镇',
# '潜江市总口管理区',
# '潜江县',
# '监利县',
# '松滋市',
# '公安县',
# '沙洋县',
# '南漳县',
# '黄梅县',
# '武穴市',
# '大冶市',
# '宜城市',
# '宜城县',
# '曹县 ',
# '沂水县',
# '青州市',
# '临邑县',
# '单县 ',
# '曲阜市',
# '五莲县',
# '栖霞市',
# '栖霞区',
# '巨野县',
# '东明县',
# '汶上县',
# '龙口市',
# '宁海县',
# '镇海区',
# '象山区',
# '象山县',
# '北仑区',
# '全椒县',
# '来安县',
# '界首市',
# '颍上县',
# '阜南县',
# '太和县',
# '临泉县',
# '临颍县',
# '滑县 ',
# '项城市',
# '杞县 ',
# '淮阳县',
# '沈丘县',
# '石柱县',
# '重庆石柱',
# '石柱土家族自治县',
# '长寿区',
# '合川区',
# '重庆合川区',
# '市辖区合川区',
# '奉节县',
# '重庆奉节县',
# '巫山县',
# '重庆巫山县',
# '万州区',
# '市辖区万州区',
# '重庆城口县',
# '城口县',
# '涪陵区',
# '市辖区涪陵区',
# '垫江县',
# '重庆垫江县',
# '巫溪县',
# '重庆巫溪县',
# '南岸区',
# '安福县',
# '响水县',
# '岳池县',
# '四川岳池县',
# '广安岳池县',
# '蓬溪县',
# '武胜县',
# '广安武胜县',
# '邻水县',
# '广安邻水县',
# '达川区',
# '四川巴中经济开发区',
# '巴中经济开发区',
# '巴中区',
# '四川巴中通江县',
# '巴中军分区',
# '郫都区',
# '平川区',
# '泾川县',
# '平凉泾川县',
# '习水县',
# '贵州习水县',
# '赤水市',
# '桐梓县',
# '湄潭县',
# '务川 ',
# '务川县',
# '正安县',
# '道真县',
# '道真 ',
# '道真自治县',
# '凤冈县',
# '余庆县',
# '镇赉县',
# '汨罗市',
# '通州区',
# '大通 ',
# '大通县',
# '大通区',
# '西青区',
# '市辖区西青区',
# '龙岗区',
# '龙岗镇平湖区',
# '深圳市龙岗区',
# '龙岗开发区',
# '宜丰县',
# '上饶县',
# '泰和县',
# '新干县',
# '乾县 ',
# '蒲城县',
# '凤翔县',
# '兴平市',
# '兴平县',
# '汉滨区',
# '城固县',
# '汉中城固县',
# '蓝田县',
# '睢宁县',
# '吴江区',
# '繁昌县',
# '万州县',
# '桃江县',
# '丰宁县',
# '新野县',
# '江肇东县',
# '泰兴县',
# '高淳县',
# '武进县',
# '锡山县',
# '慈溪县',
# '宁海县',
# '浦东县',
# '常熟县',
# '滕州县',
# '诸城县',
# '邹城县',
# '文登县',
# '招远县',
# '安塞县',
# '苍南县',
# '武义县',
# '青田县',
# '江山县',
# '淳安县',
# '丽水县',
# '繁昌区',
# '万州区',
# '桃江区',
# '丰宁区',
# '新野区',
# '江肇东区',
# '泰兴区',
# '高淳区',
# '武进区',
# '锡山区',
# '慈溪区',
# '宁海区',
# '浦东区',
# '常熟区',
# '滕州区',
# '诸城区',
# '邹城区',
# '文登区',
# '招远区',
# '安塞区',
# '苍南区',
# '武义区',
# '青田区',
# '江山区',
# '淳安区',
# '丽水区')
ents['type'] = '11315'
ents['table'] = 'company_11315'
sql = """SELECT company_number, company_name, branch, address_3
FROM {table}
WHERE searched = 0
and error=0
and address_3 in ('繁昌县',
'万州区',
'桃江县',
'丰宁县',
'新野县',
'肇东市',
'海门市',
'泰兴市',
'高淳区',
'武进区',
'锡山区',
'慈溪市',
'宁海县',
'浦东区',
'常熟市',
'滕州市',
'诸城市',
'邹城市',
'文登区',
'招远市',
'安塞区',
'苍南县',
'武义县',
'青田县',
'江山市',
'淳安县',
'丽水市'
)
and rownum <= 40000
order by company_number
"""
companys = single_oracle.oracle_find_by_param_all(
sql.format(table=ents['table']))
logger.debug(
' find from table={} and type={} and lenth={}'
.format(ents['table'], type(companys), len(companys)))
if not companys or len(companys) == 0:
ents['table'] = 'company_11315_zyfd'
companys = single_oracle.oracle_find_by_param_all(
sql.format(table=ents['table']))
logger.debug(
' find from table={} and type={} and lenth={}'
.format(ents['table'], type(companys), len(companys)))
for company in companys:
if company[1] == 'None' or company[1] is None or company[
1] == None:
print('发现空的公司名称={}'.format(company))
single_oracle.oracle_update(
"update {table} set searched=1 where company_number={number}"
.format(table=ents['table'], number=company[0]))
continue
ents['company_name'] = company[1]
if self.isContains(ents['company_name']): # 判断字符串是否存在
print('exists!')
# continue
company_sql = "select count(*) from company_basic_info where search_name='{}'"
update_11315 = 'update {} set searched=1 where company_number={}'
check = single_oracle.oracle_find_by_param_all(
company_sql.format(ents['company_name']))
print(check)
logger.debug('check type[0] {}'.format(check))
if check[0][0] != 0:
single_oracle.oracle_update(
update_11315.format(ents['table'], company[0]))
continue
else:
print('放入过滤队列')
# self.insert(ents['company_name'])
else:
print(company)
print('not exists!')
# self.insert(ents['company_name'])
ents['branch'] = company[2] or ''
ents['address'] = company[3] or ''
# print cache['company_name']
ents['number'] = company[0]
self.server.lpush('ents', ''.join(ents))
self.del_lock(lock_uid, sec)
def put_company(self):
lock_uid = 'ent'
status, sec = self.get_lock(lock_uid)
if status:
users_length = self.server.llen('ents')
if users_length > 0:
print('users length={}'.format(users_length))
self.del_lock(lock_uid, sec)
return
ents = {}
sql = 'SELECT * FROM batch_detail WHERE searched=0 and rownum<=10000'
companys = single_oracle.oracle_find_by_param_all(sql)
# logger.debug(companys)
ents['table'] = 'batch_detail'
ents['type'] = 'batch_details'
if companys:
for company in companys:
ents['company_name'] = company[2].encode('utf-8')
if self.isContains(ents['company_name']): # 判断字符串是否存在
print('exists!')
continue
else:
print('not exists!')
self.insert(ents['company_name'])
ents['agency_num'] = company[7] or ''
ents['agency_name'] = company[8] or ''
ents['batch'] = company[9] or ''
ents['company_name'] = company[2]
ents['address'] = company[3] or ''
ents['number'] = company[0] or ''
self.server.lpush('ents', ents)
# self.del_lock(lock_uid, sec)
else:
ents['type'] = '11315'
ents['table'] = 'company_11315'
# sql = 'SELECT company_number,company_name,branch,address_3
# FROM 11315_company WHERE searched=0 and address_3 in
# ('蕲春县','京山县','谷城县','枣阳市','老河口市','潜江经济开发区','潜江区','潜江市经济开发区','湖北省潜江市浩口镇','潜江市总口管理区','潜江县','监利县','松滋市','公安县','沙洋县','南漳县','黄梅县','武穴市','大冶市','宜城市','宜城县','曹县
# ','沂水县','青州市','临邑县','单县
# ','曲阜市','五莲县','栖霞市','栖霞区','巨野县','东明县','汶上县','龙口市','宁海县','镇海区','象山区','象山县','北仑区','全椒县','来安县','界首市','颍上县','阜南县','太和县','临泉县','临颍县','滑县
# ','项城市','杞县
# ','淮阳县','沈丘县','石柱县','重庆石柱','石柱土家族自治县','长寿区','合川区','重庆合川区','市辖区合川区','奉节县','重庆奉节县','巫山县','重庆巫山县','万州区','市辖区万州区','重庆城口县','城口县','涪陵区','市辖区涪陵区','垫江县','重庆垫江县','巫溪县','重庆巫溪县','南岸区','安福县','响水县','岳池县','四川岳池县','广安岳池县','蓬溪县','武胜县','广安武胜县','邻水县','广安邻水县','达川区','四川巴中经济开发区','巴中经济开发区','巴中区','四川巴中通江县','巴中军分区','郫都区','平川区','泾川县','平凉泾川县','习水县','贵州习水县','赤水市','桐梓县','湄潭县','务川
# ','务川县','正安县','道真县','道真
# ','道真自治县','凤冈县','余庆县','镇赉县','汨罗市','通州区','大通
# ','大通县','大通区','西青区','市辖区西青区','龙岗区','龙岗镇平湖区','深圳市龙岗区','龙岗开发区','宜丰县','上饶县','泰和县','新干县','乾县
# ','蒲城县','凤翔县','兴平市','兴平县','汉滨区','城固县','汉中城固县','蓝田县','睢宁县','吴江区')
# limit 100000'
sql = "select company_number,company_name,branch,address_3 FROM company_11315 WHERE searched=0 and address_3 in ('宁海','宁海县','宁海区') "
companys = single_oracle.oracle_find_by_param_all(sql)
if not companys or len(companys) == 0:
ents['table'] = 'company_11315_zyfd'
sql = "select company_number,company_name,branch,address_3 FROM company_11315_zyfd WHERE searched=0 and address_3 in ('宁海','宁海县','宁海区') "
companys = single_oracle.oracle_find_by_param_all(sql)
if not companys or len(companys) == 0:
ents['table'] = 'company_11315'
sql = """SELECT company_number, company_name, branch, address_3
FROM company_11315
WHERE searched = 0
and address_3 in ('蕲春县',
'京山县',
'谷城县',
'枣阳市',
'老河口市',
'潜江经济开发区',
'潜江区',
'潜江市经济开发区',
'湖北省潜江市浩口镇',
'潜江市总口管理区',
'潜江县',
'监利县',
'松滋市',
'公安县',
'沙洋县',
'南漳县',
'黄梅县',
'武穴市',
'大冶市',
'宜城市',
'宜城县',
'曹县 ',
'沂水县',
'青州市',
'临邑县',
'单县 ',
'曲阜市',
'五莲县',
'栖霞市',
'栖霞区',
'巨野县',
'东明县',
'汶上县',
'龙口市',
'宁海县',
'镇海区',
'象山区',
'象山县',
'北仑区',
'全椒县',
'来安县',
'界首市',
'颍上县',
'阜南县',
'太和县',
'临泉县',
'临颍县',
'滑县 ',
'项城市',
'杞县 ',
'淮阳县',
'沈丘县',
'石柱县',
'重庆石柱',
'石柱土家族自治县',
'长寿区',
'合川区',
'重庆合川区',
'市辖区合川区',
'奉节县',
'重庆奉节县',
'巫山县',
'重庆巫山县',
'万州区',
'市辖区万州区',
'重庆城口县',
'城口县',
'涪陵区',
'市辖区涪陵区',
'垫江县',
'重庆垫江县',
'巫溪县',
'重庆巫溪县',
'南岸区',
'安福县',
'响水县',
'岳池县',
'四川岳池县',
'广安岳池县',
'蓬溪县',
'武胜县',
'广安武胜县',
'邻水县',
'广安邻水县',
'达川区',
'四川巴中经济开发区',
'巴中经济开发区',
'巴中区',
'四川巴中通江县',
'巴中军分区',
'郫都区',
'平川区',
'泾川县',
'平凉泾川县',
'习水县',
'贵州习水县',
'赤水市',
'桐梓县',
'湄潭县',
'务川 ',
'务川县',
'正安县',
'道真县',
'道真 ',
'道真自治县',
'凤冈县',
'余庆县',
'镇赉县',
'汨罗市',
'通州区',
'大通 ',
'大通县',
'大通区',
'西青区',
'市辖区西青区',
'龙岗区',
'龙岗镇平湖区',
'深圳市龙岗区',
'龙岗开发区',
'宜丰县',
'上饶县',
'泰和县',
'新干县',
'乾县 ',
'蒲城县',
'凤翔县',
'兴平市',
'兴平县',
'汉滨区',
'城固县',
'汉中城固县',
'蓝田县',
'睢宁县',
'吴江区',
'繁昌县',
'万州县',
'桃江县',
'丰宁县',
'新野县',
'江肇东县',
'泰兴县',
'高淳县',
'武进县',
'锡山县',
'慈溪县',
'宁海县',
'浦东县',
'常熟县',
'滕州县',
'诸城县',
'邹城县',
'文登县',
'招远县',
'安塞县',
'苍南县',
'武义县',
'青田县',
'江山县',
'淳安县',
'丽水县',
'繁昌区',
'万州区',
'桃江区',
'丰宁区',
'新野区',
'江肇东区',
'泰兴区',
'高淳区',
'武进区',
'锡山区',
'慈溪区',
'宁海区',
'浦东区',
'常熟区',
'滕州区',
'诸城区',
'邹城区',
'文登区',
'招远区',
'安塞区',
'苍南区',
'武义区',
'青田区',
'江山区',
'淳安区',
'丽水区')
and rownum <= 10000
order by company_number
"""
companys = single_oracle.oracle_find_by_param_all(sql)
for company in companys:
if company[1] == 'None' or company[1] is None or company[
1] is None:
print('发现空的公司名称={}'.format(company))
single_oracle.oracle_update(
"update company_11315 set searched=1 where company_number={}"
.format(company[0]))
continue
ents['company_name'] = company[1]
if self.isContains(ents['company_name']): # 判断字符串是否存在
print('exists!')
# continue
company_sql = "select count(*) from company_basic_info where search_name='{}'"
update_11315 = 'update {} set searched=1 where company_number={}'
check = single_oracle.oracle_find_by_param_all(
company_sql.format(ents['company_name']))
print(check)
logger.debug('check type[0] {}'.format(check))
if check[0][0] != 0:
single_oracle.oracle_update(
update_11315.format(ents['table'], company[0]))
continue
else:
print('放入过滤队列')
# self.insert(ents['company_name'])
else:
print(company)
print('not exists!')
self.insert(ents['company_name'])
ents['branch'] = company[2] or ''
ents['address'] = company[3] or ''
# print cache['company_name']
ents['number'] = company[0]
self.server.lpush('ents', ents)
self.del_lock(lock_uid, sec)
def put_company(self):
lock_uid = 'ent'
status, sec = self.get_lock(lock_uid)
if status:
users_length = self.server.llen('ents')
if users_length > 1000:
print('users length={}'.format(users_length))
self.del_lock(lock_uid, sec)
return
ents = {}
sql = "SELECT company_number, company_name, branch, address_3 from company_11315 where {} and rownum <= 10000"
# sql = 'SELECT * FROM batch_detail WHERE searched=0'
# sql = 'SELECT company_number,company_name,company_address FROM company_11315'
#
# sql = "SELECT company_number, company_name, branch, address_3 from company_11315 where address_3 like '%颍上%' and searched = 0"
try:
# companys = single_oracle.oracle_find_by_param_all(sql)
# print(len(companys))
# if not companys or len(companys) == 0:
tablename_condition = self.get_configs_by_tablename('put_company')
print(tablename_condition)
tablename_condition = tablename_condition if tablename_condition else ['searched=0']
sql = sql.format(tablename_condition[0])
# print(sql)
companys = single_oracle.oracle_find_by_param_all(sql)
# print(companys)
# logger.debug(companys)
ents['table'] = 'company_11315'
ents['type'] = 'company_11315s'
if companys[0][0]:
if tablename_condition[1] == 0:
update_sql = "update config set over = 2 where id = {}".format(tablename_condition[2])
single_oracle.oracle_update(update_sql)
if len(companys) > 10:
companys=companys[10:]
for company in companys:
if company[1] == 'None' or company[1] is None or company[1] == None:
print('发现空的公司名称={}'.format(company))
single_oracle.oracle_update(
"update {table} set searched=1 where company_number={number}".format(
table=ents['table'], number=company[0]))
continue
ents['company_name'] = company[1]
# if self.isContains(ents['company_name']): # 判断字符串是否存在
# print('exists!')
# single_oracle.oracle_update(
# "update {table} set searched=1 where company_number={number}".format(
# table=ents['table'], number=company[0]))
# continue
# print('not exists!')
# self.insert(ents['company_name'])
ents['agency_num'] = 'NA'
ents['agency_name'] = 'NA'
ents['batch'] = 'NA'
# ents['company_name'] = company[2]
ents['address'] = company[3] or 'NA'
ents['number'] = company[0] or 'NA'
ents['branch'] = company[2] or 'NA'
self.server.lpush('ents', str(ents))
# self.del_lock(lock_uid, sec)
# else:
# # and branch in ('蕲春县',
# # '京山县',
# # '谷城县',
# # '枣阳市',
# # '老河口市',
# # '潜江经济开发区',
# # '潜江区',
# # '潜江市经济开发区',
# # '湖北省潜江市浩口镇',
# # '潜江市总口管理区',
# # '潜江县',
# # '监利县',
# # '松滋市',
# # '公安县',
# # '沙洋县',
# # '南漳县',
# # '黄梅县',
# # '武穴市',
# # '大冶市',
# # '宜城市',
# # '宜城县',
# # '曹县 ',
# # '沂水县',
# # '青州市',
# # '临邑县',
# # '单县 ',
# # '曲阜市',
# # '五莲县',
# # '栖霞市',
# # '栖霞区',
# # '巨野县',
# # '东明县',
# # '汶上县',
# # '龙口市',
# # '宁海县',
# # '镇海区',
# # '象山区',
# # '象山县',
# # '北仑区',
# # '全椒县',
# # '来安县',
# # '界首市',
# # '颍上县',
# # '阜南县',
# # '太和县',
# # '临泉县',
# # '临颍县',
# # '滑县 ',
# # '项城市',
# # '杞县 ',
# # '淮阳县',
# # '沈丘县',
# # '石柱县',
# # '重庆石柱',
# # '石柱土家族自治县',
# # '长寿区',
# # '合川区',
# # '重庆合川区',
# # '市辖区合川区',
# # '奉节县',
# # '重庆奉节县',
# # '巫山县',
# # '重庆巫山县',
# # '万州区',
# # '市辖区万州区',
# # '重庆城口县',
# # '城口县',
# # '涪陵区',
# # '市辖区涪陵区',
# # '垫江县',
# # '重庆垫江县',
# # '巫溪县',
# # '重庆巫溪县',
# # '南岸区',
# # '安福县',
# # '响水县',
# # '岳池县',
# # '四川岳池县',
# # '广安岳池县',
# # '蓬溪县',
# # '武胜县',
# # '广安武胜县',
# # '邻水县',
# # '广安邻水县',
# # '达川区',
# # '四川巴中经济开发区',
# # '巴中经济开发区',
# # '巴中区',
# # '四川巴中通江县',
# # '巴中军分区',
# # '郫都区',
# # '平川区',
# # '泾川县',
# # '平凉泾川县',
# # '习水县',
# # '贵州习水县',
# # '赤水市',
# # '桐梓县',
# # '湄潭县',
# # '务川 ',
# # '务川县',
# # '正安县',
# # '道真县',
# # '道真 ',
# # '道真自治县',
# # '凤冈县',
# # '余庆县',
# #
# # '汨罗市',
# # '通州区',
# # '大通 ',
# # '大通县',
# # '大通区',
# # '西青区',
# # '市辖区西青区',
# # '龙岗区',
# # '龙岗镇平湖区',
# # '深圳市龙岗区',
# # '龙岗开发区',
# # '宜丰县',
# # '上饶县',
# # '泰和县',
# # '新干县',
# # '乾县 ',
# # '蒲城县',
# # '凤翔县',
# # '兴平市',
# # '兴平县',
# # '汉滨区',
# # '城固县',
# # '汉中城固县',
# # '蓝田县',
# # '睢宁县',
# # '吴江区',
# # '繁昌县',
# # '万州县',
# # '桃江县',
# # '丰宁县',
# # '新野县',
# # '江肇东县',
# # '泰兴县',
# # '高淳县',
# # '武进县',
# # '锡山县',
# # '慈溪县',
# # '宁海县',
# # '浦东县',
# # '常熟县',
# # '滕州县',
# # '诸城县',
# # '邹城县',
# # '文登县',
# # '招远县',
# # '安塞县',
# # '苍南县',
# # '武义县',
# # '青田县',
# # '江山县',
# # '淳安县',
# # '丽水县',
# # '繁昌区',
# # '万州区',
# # '桃江区',
# # '丰宁区',
# # '新野区',
# # '江肇东区',
# # '泰兴区',
# # '高淳区',
# # '武进区',
# # '锡山区',
# # '慈溪区',
# # '宁海区',
# # '浦东区',
# # '常熟区',
# # '滕州区',
# # '诸城区',
# # '邹城区',
# # '文登区',
# # '招远区',
# # '安塞区',
# # '苍南区',
# # '武义区',
# # '青田区',
# # '江山区',
# # '淳安区',
# # '丽水区')
# ents['type'] = '11315'
# ents['table'] = 'company_11315'
#
#
# # logger.debug(
# # 'find from table={} and type={} and lenth={}'.format(ents['table'], type(companys), len(companys)))
# # if not companys or len(companys) == 0:
# # ents['table'] = 'company_11315_zyfd'
# # companys = single_oracle.oracle_find_by_param_all(sql.format(table=ents['table']))
# # logger.debug('find from table={} and type={} and lenth={}'.format(ents['table'], type(companys),
# # len(companys)))
#
# for company in companys:
# if company[1] == 'None' or company[1] is None or company[1] == None:
# print('发现空的公司名称={}'.format(company))
# single_oracle.oracle_update(
# "update {table} set searched=1 where company_number={number}".format(table=ents['table'], number=company[0]))
# continue
# ents['company_name'] = company[1]
# if self.isContains(ents['company_name']): # 判断字符串是否存在
# print('exists!')
#
# company_sql = "select count(*) from company_basic_info where search_name='{}'"
# update_11315 = 'update {} set searched=1 where company_number={}'
# check = single_oracle.oracle_find_by_param_all(company_sql.format(ents['company_name']))
# print(check)
# logger.debug('check type[0] {}'.format(check))
# if check[0][0] != 0:
# single_oracle.oracle_update(update_11315.format(ents['table'], company[0]))
#
# else:
# print('放入过滤队列')
# self.insert(ents['company_name'])
# continue
# else:
# print(company)
# print('not exists!')
# self.insert(ents['company_name'])
# ents['branch'] = company[2] or ''
# ents['address'] = company[3] or ''
# # print cache['company_name']
# ents['number'] = company[0]
# self.server.lpush('ents', ''.join(ents))
else:
single_oracle.oracle_update("update config set over = 1, over_time = sysdate where id = {}".format(tablename_condition[2]))
self.del_lock(lock_uid, sec)
except Exception as e:
self.del_lock(lock_uid, sec)
logger.debug(e)
