def fix_item2(kwargs): #signal.signal(signal.SIGINT, signal.SIG_IGN) item = kwargs['item'] crawl_path = kwargs['crawl_path'] server_path = kwargs['server_path'] org_server_path = kwargs['org_server_path'] is_remove = kwargs['is_remove'] #thumbs = ((350,350),(710,10000)) thumbs = ((710,10000),) org_path = "%s%s" % (org_server_path, item[4]) if not os.path.exists(org_path): logger.error("Org image not found :" + item[3] + " " + item[4]) for width, height in thumbs: thumb_path = "%s%sx%s_%s" % (server_path, width, height, item[4]) if not os.path.exists(thumb_path): logger.error("Thumb image not found :" + item[3] + " " + item[4]) # fix it try: image = Image.open(cStringIO.StringIO(open(org_path).read())) if image.mode not in ('L', 'RGB'): image = image.convert('RGB') if width != 710: image.thumbnail((width, height), Image.ANTIALIAS) elif item[1] < 710: image.thumbnail((710, 710*item[2]/item[1]), Image.ANTIALIAS) make_dirs_for_file(thumb_path) thumbfile = open(thumb_path, "w") image.save(thumbfile, "JPEG") thumbfile.close() logger.info("fix suc!") except: logger.info("generate thumb failed %s %s %sx%s error : %s", item[0], thumb_path, width, height, traceback.format_exc())
def download_image(pic_url, shop_id, item_id, local_pic_url, fdfs_client): big_path = "%s/%s/big/%s" % (FLAGS.path, shop_id, local_pic_url) mid2_path = "%s/%s/mid2/%s" % (FLAGS.path, shop_id, local_pic_url) mid_path = "%s/%s/mid/%s" % (FLAGS.path, shop_id, local_pic_url) sma_path = "%s/%s/sma/%s" % (FLAGS.path, shop_id, local_pic_url) sma2_path = "%s/%s/small2/%s" % (FLAGS.path, shop_id, local_pic_url) sma3_path = "%s/%s/small3/%s" % (FLAGS.path, shop_id, local_pic_url) headers = { 'Referer': "http://www.j.cn/product/%s.htm" % item_id, 'User-Agent': "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)" } # pic_url 包含http走下载,不包含即fastdfs get if "http://" in pic_url: data = download(pic_url, headers) save_image(big_path, data) else: try: if not os.path.exists(os.path.dirname(big_path)): make_dirs_for_file(big_path) fdfs_client.download_to_file(big_path, pic_url) except (ConnectionError, ResponseError, DataError), e: fastdfs_filename = "http://image2.guang.j.cn/images/%s/big/%s" % (shop_id, local_pic_url) data = download(fastdfs_filename, headers) save_image(big_path, data) logger.info("%s:%s fdfs get failed: %s, usage http download", item_id, pic_url, e)
def download_image(pic_url, shop_id, item_id, local_pic_url, fdfs_client): big_path = "%s/%s/big/%s" % (FLAGS.path, shop_id, local_pic_url) mid2_path = "%s/%s/mid2/%s" % (FLAGS.path, shop_id, local_pic_url) mid_path = "%s/%s/mid/%s" % (FLAGS.path, shop_id, local_pic_url) sma_path = "%s/%s/sma/%s" % (FLAGS.path, shop_id, local_pic_url) sma2_path = "%s/%s/small2/%s" % (FLAGS.path, shop_id, local_pic_url) sma3_path = "%s/%s/small3/%s" % (FLAGS.path, shop_id, local_pic_url) headers = { 'Referer': "http://www.j.cn/product/%s.htm" % item_id, 'User-Agent': "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)" } # pic_url 包含http走下载,不包含即fastdfs get if "http://" in pic_url: data = download(pic_url, headers) save_image(big_path, data) else: try: if not os.path.exists(os.path.dirname(big_path)): make_dirs_for_file(big_path) fdfs_client.download_to_file(big_path, pic_url) except (ConnectionError, ResponseError, DataError), e: fastdfs_filename = "http://image2.guang.j.cn/images/%s/big/%s" % ( shop_id, local_pic_url) data = download(fastdfs_filename, headers) save_image(big_path, data) logger.info("%s:%s fdfs get failed: %s, usage http download", item_id, pic_url, e)
def imagemagick_resize(width, height, image_filename, thumb_filename): # validate dir if not os.path.exists(os.path.dirname(thumb_filename)): make_dirs_for_file(thumb_filename) #validate file if not os.path.isfile(thumb_filename): cmd = "convert -resize %sx%s -strip -density 72x72 %s %s" % (width, height, image_filename, thumb_filename) os.system(cmd)
def save_image(image_filename, data): if not os.path.exists(os.path.dirname(image_filename)) and not FLAGS.dryrun: make_dirs_for_file(image_filename) if os.path.exists(image_filename) and not FLAGS.dryrun: os.remove(image_filename) if not FLAGS.dryrun: f = file(image_filename, "wb") f.write(data) f.close()
def imagemagick_resize(self, width, height, image_filename, thumb_filename): if not os.path.exists(os.path.dirname(thumb_filename)): make_dirs_for_file(thumb_filename) x = 0.3 y = 0.4 cmd = "convert +profile \"*\" -interlace Line -quality 95%% -resize %sx%s -sharpen %s,%s %s %s" % ( width, height, x, y, image_filename, thumb_filename) os.system(cmd)
def imagemagick_resize(width, height, image_filename, thumb_filename): # validate dir if not os.path.exists(os.path.dirname(thumb_filename)): make_dirs_for_file(thumb_filename) #validate file if not os.path.isfile(thumb_filename): cmd = "convert -resize %sx%s -strip -density 72x72 %s %s" % ( width, height, image_filename, thumb_filename) os.system(cmd)
def save_image(image_filename, data): if not os.path.exists( os.path.dirname(image_filename)) and not FLAGS.dryrun: make_dirs_for_file(image_filename) if os.path.exists(image_filename) and not FLAGS.dryrun: os.remove(image_filename) if not FLAGS.dryrun: f = file(image_filename, "wb") f.write(data) f.close()
def imagemagick_resize(width, height, image_filename, thumb_filename): if not os.path.exists(os.path.dirname(thumb_filename)) and not FLAGS.dryrun: make_dirs_for_file(thumb_filename) if os.path.exists(thumb_filename) and not FLAGS.dryrun: os.remove(thumb_filename) x = 0.3 y = 0.4 cmd = "convert +profile \"*\" -interlace Line -quality 95%% -resize %sx%s -sharpen %s,%s %s %s" % (width,height,x,y,image_filename, thumb_filename) logger.info("running %s", cmd) if not FLAGS.dryrun: os.system(cmd)
def imagemagick_resize(width, height, image_filename, thumb_filename): if not os.path.exists( os.path.dirname(thumb_filename)) and not FLAGS.dryrun: make_dirs_for_file(thumb_filename) if os.path.exists(thumb_filename) and not FLAGS.dryrun: os.remove(thumb_filename) x = 0.3 y = 0.4 cmd = "convert +profile \"*\" -interlace Line -quality 95%% -resize %sx%s -sharpen %s,%s %s %s" % ( width, height, x, y, image_filename, thumb_filename) logger.info("running %s", cmd) if not FLAGS.dryrun: os.system(cmd)
def save_image(image_filename, data): if not os.path.exists(os.path.dirname(image_filename)): make_dirs_for_file(image_filename) f = file(image_filename, "wb") f.write(data) f.close()
def save_image(self, image_filename, data): if not os.path.exists(os.path.dirname(image_filename)): make_dirs_for_file(image_filename) f = file(image_filename, "wb") f.write(data) f.close()