def word_repost_relationship(batch_num, temp_dir, searchList, breakpos=None):
# 据进程名生成日志
name = f'getRepost_batchNum{str(batch_num)}'
log = Logger(name)
logger = log.getLogger()
# 断点处理
if not breakpos:
# 每个进程维护一个sublist转发关系的层级目录（临时）
level_dir = temp_dir + f'temp_{name}/'
if not os.path.exists(level_dir):
os.mkdir(level_dir)
# 生成写文件
repost_file = temp_dir + name + '.csv'
repost_writer = csvWriter(repost_file, repost=True)
else:
level_dir = temp_dir + breakpos['level_dir']
repost_file = temp_dir + breakpos['repost_file']
repost_writer = csvWriter(repost_file, repost=True, breakpos=True)
# 先爬取完断点id，再对余下id按常规爬取
get_repost_relationship(breakpos['center_bw_id'], repost_writer,
level_dir, logger, breakpos)
searchList = searchList[1:]
# 常规爬取
logger.info('Strat getting repost...')
for id in searchList:
get_repost_relationship(id, repost_writer, level_dir, logger)
logger.info('Finish!')
# 爬取完后，删除日志
# 日志主要用于处理断点，若爬取完成则不再需要
log.remove()
def word_repost_relationship(temp_dir, searchList, breakpos=None):
# 据进程名生成日志
name = 'getRepost_' + str(os.getpid())
log = Logger(name)
logger = log.getLogger()
# 每个进程维护一个sublist转发关系的层级目录（临时）
level_dir = temp_dir + f'temp_{name}/'
os.mkdir(level_dir)
# 断点处理
if not breakpos:
# 生成写文件
repost_file = temp_dir + name + '.csv'
repost_writer = csvWriter(repost_file, repost=True)
else:
repost_file = temp_dir + breakpos['repost_file']
repost_writer = csvWriter(repost_file, repost=True, breakpos=True)
# 先爬取完断点id，再对余下id按常规爬取
get_repost_relationship(breakpos['center_bw_id'], repost_writer,
level_dir, logger, breakpos)
searchList = searchList[1:] # 有问题！！断点再看，常常爬取不成
# 常规爬取
logger.info('Strat getting repost...')
for id in searchList:
get_repost_relationship(id, repost_writer, level_dir, logger)
logger.info('Finish!')
# 爬取完后，删除日志
# 日志主要用于处理断点，若爬取完成则不再需要
log.remove()
def word_repost_relationship(temp_dir, searchList, breakpos=None):
# 据进程名生成日志
name = 'getRepost_' + str(os.getpid())
logger = getLogger(name)
# 每个进程维护一个sublist转发关系的层级目录（临时）
level_dir = temp_dir + 'temp/'
os.mkdir(level_dir)
# 断点处理
if not breakpos:
# 生成写文件
repost_file = temp_dir + name + '.csv'
repost_writer = csvWriter(repost_file, repost=True)
else:
repost_file = temp_dir + breakpos['repost_file']
repost_writer = csvWriter(repost_file, repost=True, breakpos=True)
# 先爬取完断点id，再对余下id按常规爬取
get_repost_relationship(breakpos['center_bw_id'], repost_writer,
level_dir, logger, breakpos)
searchList = searchList[1:]
# 常规爬取
logger.info('Strat getting repost...')
for id in searchList:
get_repost_relationship(id, repost_writer, level_dir, logger)
logger.info('Finish!')
# 删除临时目录
shutil.rmtree(level_dir)
def word_spider(searchlist):
# 加载设置文件，获取数据输出路径和检索词
config = load_config()
hot_dir = config['hot_dir']
topic_dir = config['topic_dir']
repost_dir = config['repost_dir']
# 根据规定日志目录创建目录实例
name = multiprocessing.current_process().name
logger = getLogger(name)
topic_dir += name + '_'
# 记录载入检索词列表的次数
epoch = 1
while True:
# 对每一个词爬取相关微博和各微博的转发关系
for wd in searchlist:
logger.info(f'EPOCH: {epoch}. Keyword: {wd}. Start crawling ...')
search_file = hot_dir + 'search_result_' + str(wd) + '.csv'
repost_file = repost_dir + 'repost_Relationship_' + str(wd) + '.csv'
# 创建两个写的对象,同时创建文件
search_writer = csvWriter(search_file, search=True)
repost_writer = csvWriter(repost_file, repost=True)
# 获取该检索词的所有相关微博，至多能获取1000条
get_query_info(wd, search_writer, logger)
# 获取相关微博id组成的列表
idList = search_writer.get_idList()
# 获取各相关微博的转发关系
for bw_id in idList:
get_repost_relationship(bw_id, repost_writer, logger)
repost_writer.drop_duplicates()
# 获取该词相关所有话题作为之后的检索词
get_more_topic(wd, epoch, topic_dir, logger)
# 结束一轮检索爬取
# 获取新检索词列表
filename = topic_dir + 'Topics_' + str(epoch) + '.csv'
with open(filename, 'r', encoding='utf-8-sig') as f:
rows = csv.reader(f)
searchlist = [row[0].strip() for row in rows]
# 删除中间文件
os.remove(filename)
epoch += 1
def get_repost_relationship(bw_id,
repost_writer,
level_dir,
logger,
breakpos=None):
# center_bw_id记录最原始的bw_id
center_bw_id = bw_id
# 类层次遍历处理转发关系
# 为了节省内存，将每一层的层级关系写入level_dir
# 断点处理
if not breakpos:
# 初始化层数为0，仍可以获取转发关系
level = 1
idList = [bw_id]
else:
level = breakpos['level']
break_file = level_dir + f'Level_{level}_{center_bw_id}.csv'
temp_writer = csvWriter(break_file, temp=True, breakpos=True)
idList = temp_writer.get_idList(breakpos.get('break_id'))
# 爬取转发
while len(idList) > 0:
# 创建下一层的原博文件，即该层的转发微博的微博id
temp_file = level_dir + f'Level_{level+1}_{center_bw_id}.csv'
if level == breakpos and breakpos.get('break_id'):
temp_writer = csvWriter(
temp_file, temp=True,
breakpos=True) # 断点为本层的中间，所以其下一层文件早已创建，直接往后添加
else:
temp_writer = csvWriter(temp_file, temp=True) # 非断点，则照常创建新文件
# 获得该层所有bw_id的直接转发关系
for bw_id in idList:
get_repost_info(center_bw_id, bw_id, level, repost_writer, logger,
temp_writer)
# 获取下一level的原博id
idList = temp_writer.get_idList()
# 删除存储本层idList的文件
if not level == 1:
os.remove(level_dir + f'Level_{level}_{center_bw_id}.csv')
level += 1
# 爬取结束后，删除最后一次的temp_file
os.remove(temp_file)
def one_word_spider():
# 加载设置文件，获取数据输出路径和检索词
config = load_config()
hot_dir = config['hot_dir']
repost_dir = config['repost_dir']
one_repost_dir = config['one_repost_dir']
searchlist = config['searchlist']
if type(searchlist) is str or len(searchlist) == 1:
wd = searchlist
else:
raise ValueError('one_word_spider() can only accept one search word!')
# 写文件的文件名
search_file = hot_dir + 'search_result_' + str(wd) + '.csv'
# 创建写的对象,同时创建文件
search_writer = csvWriter(search_file, search=True)
# 获取该检索词的所有相关微博，至多能获取1000条
# 为了日志正常输出（python的logging非进程安全），需要用到进程池
print(f'[{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}] Keyword: {wd}. Start crawling ...')
p = Pool(1)
p.apply_async(one_word_get_query_info, args=(wd, search_writer))
p.close()
p.join()
# 获取相关微博id组成的列表
# 将其分为10个列表，用10个进程进行转发关系爬取
raw_searchlist = search_writer.get_idList()
searchList = split_searchList(raw_searchlist, 10)
# 生成10进程的进程池
print(f'[{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}] Generating Process Pool Containing 10 Process...')
p = Pool(10)
for group in searchList:
p.apply_async(one_word_repost_relationship, args=(group,))
p.close()
p.join()
# 将10个进程的csv文件进行合并
print(f'[{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}] Start Merging csv Files...')
merge_csv(wd, one_repost_dir, repost_dir)
def one_word_continue():
# 加载设置文件，获取数据输出路径和检索词
config = load_config()
repost_dir = config['repost_dir']
one_repost_dir = config['one_repost_dir']
searchlist = config['searchlist']
if type(searchlist) is str or len(searchlist) == 1:
wd = searchlist
else:
raise ValueError('one_word_spider() can only accept one search word!')
# 读取已爬取好的检索词相关微博
filename = config['hot_dir'] + 'search_result_' + str(wd) + '.csv'
reader = csvWriter(filename, search=True, breakpos=True)
raw_searchlist = reader.get_idList()
# 根据每个进程此前中断的center_bw_id重新进行爬取
breakpos = config['breakpos']
allList = getBreakList(raw_searchlist, breakpos)
# 启动进程池
pool_size = len(allList)
p = Pool(pool_size)
print(f'[{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}] Generating Process Pool Containing {str(pool_size)} Process...')
print(f'[{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}] Start crawling repost relationship...')
for sublist in allList:
if sublist.get('breakpos'):
p.apply_async(one_word_repost_relationship, args=(sublist['thisList'], sublist['breakpos']))
else:
p.apply_async(one_word_repost_relationship, args=(sublist['thisList'],))
p.close()
p.join()
print(f'[{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}] Finish crawling repost relationship!')
# 将所有csv文件进行合并
merge_csv(wd, one_repost_dir, repost_dir)
def word_spider():
# 加载设置文件，获取数据输出路径和检索词
config = load_config()
hot_dir = config['hot_dir']
repost_dir = config['repost_dir']
process_num = config['process_num']
searchlist = config['searchlist']
expand_topic = config['expand_topic']
if expand_topic:
topic_dir = config['topic_dir']
# 记录载入检索词列表的次数
epoch = 1
while True:
# 对每一个词爬取相关微博和各微博的转发关系
for wd in searchlist:
if wd == config.get('breakList'):
config['breakPoint'] = True
else:
config['breakPoint'] = False
print(
f'[{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}] EPOCH: {epoch}. Keyword: {wd}.'
)
search_file = hot_dir + 'search_result_' + str(wd) + '.csv'
repost_file = repost_dir + 'repost_Relationship_' + str(
wd) + '.csv'
# 创建两个写的对象,同时创建文件
# 如果是断点，则不创建新文件，只指定各文件的字段
search_writer = csvWriter(search_file,
search=True,
breakpos=config['breakPoint'])
repost_writer = csvWriter(repost_file,
repost=True,
breakpos=config['breakPoint'])
# 需要临时目录来存储多个进程的文件
temp = repost_dir + wd + '/'
if not config['breakPoint']:
# 创建临时目录
os.mkdir(temp)
# 获取该检索词的所有相关微博，至多能获取1000条
# 因为Python不兼容多进程的日志，而之后repost需要多进程
# 所以凡日志输出，都需要新开进程处理
print(
f'[{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}] Start searching keyword: {wd}.'
)
p = Pool(1)
p.apply_async(word_get_query_info, args=(wd, search_writer))
p.close()
p.join()
print(
f'[{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}] Finished searching keyword: {wd}.'
)
# 获取相关微博id组成的列表
raw_idList = search_writer.get_idList()
# 分割待爬取转发关系的id，以便开启进程
idList = splitList(raw_idList,
process_num,
breakpos=config['breakPoint'])
# 多进程爬取转发关系
print(
f'[{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}] Start crawling repost relationship...'
)
# 正常爬取时，遵循用户设置的进程数
# 断点处理时，遵循每个进程处理不多于10个center_bw_id的原则
p = Pool(len(idList))
for num, item in enumerate(idList):
if item.get('breakpos'):
p.apply_async(word_repost_relationship,
args=(num, temp, item['sublist'],
item['breakpos']))
else:
p.apply_async(word_repost_relationship,
args=(num, temp, item['sublist']))
p.close()
p.join()
# 整合中间文件成完整文件并去重
repost_writer.merge_csv(temp)
# 获取该词相关所有话题作为之后的检索词
if expand_topic:
get_more_topic(wd, epoch, topic_dir)
if expand_topic:
# 结束一轮检索爬取
# 获取新检索词列表
filename = topic_dir + 'Topics_' + str(epoch) + '.csv'
with open(filename, 'r', encoding='utf-8-sig') as f:
rows = csv.reader(f)
searchlist = list(set([row[0].strip() for row in rows]))
os.remove(filename)
epoch += 1
else:
break
def word_spider():
# 加载设置文件，获取数据输出路径和检索词
config = load_config()
hot_dir = config['hot_dir']
topic_dir = config['topic_dir']
repost_dir = config['repost_dir']
process_num = config['process_num']
one_word = config['one_word']
searchlist = config['searchlist']
# 记录载入检索词列表的次数
epoch = 1
while True:
# 对每一个词爬取相关微博和各微博的转发关系
for wd in searchlist:
print(
f'[{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}] EPOCH: {epoch}. Keyword: {wd}.'
)
search_file = hot_dir + 'search_result_' + str(wd) + '.csv'
repost_file = repost_dir + 'repost_Relationship_' + str(
wd) + '.csv'
# 创建两个写的对象,同时创建文件
search_writer = csvWriter(search_file, search=True)
repost_writer = csvWriter(repost_file, repost=True)
# 获取该检索词的所有相关微博，至多能获取1000条
# 因为Python不兼容多进程的日志，而之后repost需要多进程
# 所以凡日志输出，都需要新开进程处理
print(
f'[{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}] Start searching keyword: {wd}.'
)
p = Pool(1)
p.apply_async(word_get_query_info, args=(wd, search_writer))
p.close()
p.join()
print(
f'[{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}] Finished searching keyword: {wd}.'
)
# 获取相关微博id组成的列表
raw_idList = search_writer.get_idList()
# 分割待爬取转发关系的id，以便开启进程
idList = splitList(raw_idList, process_num)
# 需要临时目录来存储多个进程的文件
temp = repost_dir + wd + '/'
os.mkdir(temp)
# 多进程爬取转发关系
print(
f'[{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}] Start crawling repost relationship...'
)
p = Pool(process_num)
for sublist in idList:
p.apply_async(word_repost_relationship, args=(temp, sublist))
p.close()
p.join()
# 整合中间文件成完整文件并去重
repost_writer.merge_csv(temp)
# 获取该词相关所有话题作为之后的检索词
if one_word:
break
else:
get_more_topic(wd, epoch, topic_dir)
# 结束一轮检索爬取
# 获取新检索词列表
filename = topic_dir + 'Topics_' + str(epoch) + '.csv'
with open(filename, 'r', encoding='utf-8-sig') as f:
rows = csv.reader(f)
searchlist = list(set([row[0].strip() for row in rows]))
os.remove(filename)
epoch += 1