def get_topic_priority(platform):
select_priority_sql = sql.get_priority(platform)
try:
n = db.handle_db(select_priority_sql)['data']
print 'there is :', n
return n
except:
traceback.print_exc()
return
def get_topic_url(platform):
iscrawled_list = get_topic_priority(platform)
all_urls = []
for iscrawled in iscrawled_list:
topic_urls = sql.get_topic_bycrawed(platform, '0', iscrawled[0], settings.DOWN_NUM)
urls = db.handle_db(topic_urls)['data']
all_urls.extend(urls)
if len(all_urls) < settings.DOWN_NUM:
continue
else:
break
all_urls = all_urls[:10]
return all_urls
def new_spidername(spidername):
get_platform_sql = sql.get_platform(spidername)
n = db.handle_db(get_platform_sql)['data'][0]
spidername = n[3]
return spidername