def __waitforImgs(self, urlImgList, newsId):
"""
对一组图片进行下载，只要有一个失败，就报异常
:param urlImgList:
:param newsId:
:return:
"""
urlImgListAfterFilter = []
for img in urlImgList:
if img[0:2] == '//':
img = 'http:' + img
urlImgListAfterFilter.append(img)
missionBean = MissionBean(img, 7000, [])
missionBean.isFileTag = True
missionBean.downloadCallback = 'set'
redisDownloadSaveDb3(
str(self.DEEP_DOWNLOAD) + '_' + self.FILE_KEY_DOWNLOAD,
missionBean.getRedisDict())
for url in urlImgListAfterFilter:
timeStart = datetime.datetime.now().timestamp()
while True:
# 下载超时30秒
if datetime.datetime.now().timestamp() - timeStart > 30:
raise Exception('图片下载超时')
msgStr = redisGet(
3, self.FILE_KEY_DOWNLOAD + '_callback_' +
str(self.TYPE_DOWNLOAD) + ':' + url)
if msgStr is None:
sleep(1)
continue
else:
dictMsg = json.loads(msgStr)
print('图片下载成功,正在存储')
dictInsert = {}
fileUrl = dictMsg['fileUrl']
fileSome = fileUrl.split('/')
urlSome = img.split('/')
lstTag = ['.jpg', '.jpeg', '.gif', '.bmp', '.png']
for tag in lstTag:
if tag in fileUrl:
dictInsert['fileUrl'] = 'testFileName' + tag
break
dictInsert['fileUrl'] = 'testFileName.jpg'
dictInsert['Uploaded size'] = dictMsg['info']['file'][
'size']
dictInsert['Storage IP'] = fileSome[2]
dictInsert['Remote file_id'] = fileUrl.split(fileSome[2] +
'/')[1]
dictInsert['imgUrl'] = dictMsg['url']
dictInsert['imgName'] = urlSome[len(urlSome) - 1]
dictInsert['Group name'] = fileSome[3]
dictInsert['articleId'] = newsId
self.new_db['d_news_images'].save(dictInsert)
break
from zywa_database_core.dao.mongo.mongoClientMyself import MongoClientMyself
from zywa_extract_helper.model.missionBean import MissionBean
if __name__ == '__main__':
__mongoClient = MongoClientMyself(host="172.10.3.219",
port=20000,
db="xiaociwei",
user="******",
password="******")
items = __mongoClient.selectAll(tableName='iqiyi_video')
i = 0
for item in items:
missionBean = MissionBean('', 0, [])
missionBean.__dict__ = item
print(i)
print(missionBean.title)
redisLPush(4, 'data_clear_' + str(missionBean.type),
missionBean.getRedisDict())
i += 1