def get_aicar_img_entrance(limt_num):
car_entrances = []
with session_scope(DBSession,
'CarEntrance get_car_img_entrance') as session:
car_entrances = session.query(AiCarEntrance).filter(
AiCarEntrance.is_crawled == 0).limit(limt_num)
return car_entrances
def process_item(self, item, spider):
if isinstance(item, CarImEntranceItem):
"""保存汽车之家各个车型图片入口url"""
car_entrance = AiCarEntrance(
entran_im_url = item['entran_im_url'],
entran_brand = item['entran_brand'],
entran_series=item['entran_series'],
entran_series_id = item['entran_series_id'],
is_crawled = 0
)
with session_scope(self.Session, 'insert_car_entrance') as session:
session.add(car_entrance)
return item
elif isinstance(item, CarPicItem):
"""按车型保存汽车之家图片"""
try:
car_folder = item['car_model']
image_fold = img_path+'homeOfcars/'+car_folder+'/'
image_name = image_fold + item['pic_id'] + '.jpg'
print('****************'+' '+image_name)
if not os.path.exists(image_fold):
# os.mkdir(img_path+img_folder+'/')
os.makedirs(image_fold)
with open(image_name, 'wb') as f:
f.write(item['pic_content'])
except Exception as e:
utils.send_mail(str(e), 'ImagespiderPipeline', 'jason', 'CarPicItem_exception')
return item
def aicar_crawled_to0():
with session_scope(DBSession, 'CarEntrance update_to0') as sess:
sess.query(AiCarEntrance).update({AiCarEntrance.is_crawled: 0})
# Base.metadata.create_all(bind=engine)
def aicar_setcrawled_base_series_id(series_id, is_crawled=1):
with session_scope(DBSession,
'CarEntrance hoc_setcrawled_base_series_id') as sess:
sess.query(AiCarEntrance).filter(
AiCarEntrance.entran_series_id == series_id).update(
{AiCarEntrance.is_crawled: is_crawled})
def aicar_set_crawled(entrance_id, is_crawled=1):
with session_scope(DBSession, 'CarEntrance hoc_set_crawled') as session:
session.query(AiCarEntrance).filter(
AiCarEntrance.id == entrance_id).update(
{AiCarEntrance.is_crawled: is_crawled})
def update_is_crawl(car_number, code):
with session_scope(DBSession, 'update yipei is_crawl') as session:
session.query(YiPei).filter(YiPei.car_number == car_number).update(
{YiPei.is_crawled: code})
def get_number(limt_num):
ins_user = []
with session_scope(DBSession, 'get_yipei_number') as session:
ins_user = session.query(YiPei).filter(
YiPei.is_crawled == 0).limit(limt_num)
return ins_user