def fix_item2(kwargs):
#signal.signal(signal.SIGINT, signal.SIG_IGN)
item = kwargs['item']
crawl_path = kwargs['crawl_path']
server_path = kwargs['server_path']
org_server_path = kwargs['org_server_path']
is_remove = kwargs['is_remove']
#thumbs = ((350,350),(710,10000))
thumbs = ((710,10000),)
org_path = "%s%s" % (org_server_path, item[4])
if not os.path.exists(org_path):
logger.error("Org image not found :" + item[3] + " " + item[4])
for width, height in thumbs:
thumb_path = "%s%sx%s_%s" % (server_path, width, height, item[4])
if not os.path.exists(thumb_path):
logger.error("Thumb image not found :" + item[3] + " " + item[4])
# fix it
try:
image = Image.open(cStringIO.StringIO(open(org_path).read()))
if image.mode not in ('L', 'RGB'):
image = image.convert('RGB')
if width != 710:
image.thumbnail((width, height), Image.ANTIALIAS)
elif item[1] < 710:
image.thumbnail((710, 710*item[2]/item[1]), Image.ANTIALIAS)
make_dirs_for_file(thumb_path)
thumbfile = open(thumb_path, "w")
image.save(thumbfile, "JPEG")
thumbfile.close()
logger.info("fix suc!")
except:
logger.info("generate thumb failed %s %s %sx%s error : %s", item[0], thumb_path, width, height, traceback.format_exc())
def download_image(pic_url, shop_id, item_id, local_pic_url, fdfs_client):
big_path = "%s/%s/big/%s" % (FLAGS.path, shop_id, local_pic_url)
mid2_path = "%s/%s/mid2/%s" % (FLAGS.path, shop_id, local_pic_url)
mid_path = "%s/%s/mid/%s" % (FLAGS.path, shop_id, local_pic_url)
sma_path = "%s/%s/sma/%s" % (FLAGS.path, shop_id, local_pic_url)
sma2_path = "%s/%s/small2/%s" % (FLAGS.path, shop_id, local_pic_url)
sma3_path = "%s/%s/small3/%s" % (FLAGS.path, shop_id, local_pic_url)
headers = {
'Referer': "http://www.j.cn/product/%s.htm" % item_id,
'User-Agent': "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)"
}
# pic_url 包含http走下载，不包含即fastdfs get
if "http://" in pic_url:
data = download(pic_url, headers)
save_image(big_path, data)
else:
try:
if not os.path.exists(os.path.dirname(big_path)):
make_dirs_for_file(big_path)
fdfs_client.download_to_file(big_path, pic_url)
except (ConnectionError, ResponseError, DataError), e:
fastdfs_filename = "http://image2.guang.j.cn/images/%s/big/%s" % (shop_id, local_pic_url)
data = download(fastdfs_filename, headers)
save_image(big_path, data)
logger.info("%s:%s fdfs get failed: %s, usage http download", item_id, pic_url, e)
def download_image(pic_url, shop_id, item_id, local_pic_url, fdfs_client):
big_path = "%s/%s/big/%s" % (FLAGS.path, shop_id, local_pic_url)
mid2_path = "%s/%s/mid2/%s" % (FLAGS.path, shop_id, local_pic_url)
mid_path = "%s/%s/mid/%s" % (FLAGS.path, shop_id, local_pic_url)
sma_path = "%s/%s/sma/%s" % (FLAGS.path, shop_id, local_pic_url)
sma2_path = "%s/%s/small2/%s" % (FLAGS.path, shop_id, local_pic_url)
sma3_path = "%s/%s/small3/%s" % (FLAGS.path, shop_id, local_pic_url)
headers = {
'Referer':
"http://www.j.cn/product/%s.htm" % item_id,
'User-Agent':
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)"
}
# pic_url 包含http走下载，不包含即fastdfs get
if "http://" in pic_url:
data = download(pic_url, headers)
save_image(big_path, data)
else:
try:
if not os.path.exists(os.path.dirname(big_path)):
make_dirs_for_file(big_path)
fdfs_client.download_to_file(big_path, pic_url)
except (ConnectionError, ResponseError, DataError), e:
fastdfs_filename = "http://image2.guang.j.cn/images/%s/big/%s" % (
shop_id, local_pic_url)
data = download(fastdfs_filename, headers)
save_image(big_path, data)
logger.info("%s:%s fdfs get failed: %s, usage http download",
item_id, pic_url, e)
def imagemagick_resize(width, height, image_filename, thumb_filename):
# validate dir
if not os.path.exists(os.path.dirname(thumb_filename)):
make_dirs_for_file(thumb_filename)
#validate file
if not os.path.isfile(thumb_filename):
cmd = "convert -resize %sx%s -strip -density 72x72 %s %s" % (width, height, image_filename, thumb_filename)
os.system(cmd)
def save_image(image_filename, data):
if not os.path.exists(os.path.dirname(image_filename)) and not FLAGS.dryrun:
make_dirs_for_file(image_filename)
if os.path.exists(image_filename) and not FLAGS.dryrun:
os.remove(image_filename)
if not FLAGS.dryrun:
f = file(image_filename, "wb")
f.write(data)
f.close()
def imagemagick_resize(self, width, height, image_filename, thumb_filename):
if not os.path.exists(os.path.dirname(thumb_filename)):
make_dirs_for_file(thumb_filename)
x = 0.3
y = 0.4
cmd = "convert +profile \"*\" -interlace Line -quality 95%% -resize %sx%s -sharpen %s,%s %s %s" % (
width, height, x, y, image_filename, thumb_filename)
os.system(cmd)
def imagemagick_resize(width, height, image_filename, thumb_filename):
# validate dir
if not os.path.exists(os.path.dirname(thumb_filename)):
make_dirs_for_file(thumb_filename)
#validate file
if not os.path.isfile(thumb_filename):
cmd = "convert -resize %sx%s -strip -density 72x72 %s %s" % (
width, height, image_filename, thumb_filename)
os.system(cmd)
def imagemagick_resize(self, width, height, image_filename,
thumb_filename):
if not os.path.exists(os.path.dirname(thumb_filename)):
make_dirs_for_file(thumb_filename)
x = 0.3
y = 0.4
cmd = "convert +profile \"*\" -interlace Line -quality 95%% -resize %sx%s -sharpen %s,%s %s %s" % (
width, height, x, y, image_filename, thumb_filename)
os.system(cmd)
def save_image(image_filename, data):
if not os.path.exists(
os.path.dirname(image_filename)) and not FLAGS.dryrun:
make_dirs_for_file(image_filename)
if os.path.exists(image_filename) and not FLAGS.dryrun:
os.remove(image_filename)
if not FLAGS.dryrun:
f = file(image_filename, "wb")
f.write(data)
f.close()
def imagemagick_resize(width, height, image_filename, thumb_filename):
if not os.path.exists(os.path.dirname(thumb_filename)) and not FLAGS.dryrun:
make_dirs_for_file(thumb_filename)
if os.path.exists(thumb_filename) and not FLAGS.dryrun:
os.remove(thumb_filename)
x = 0.3
y = 0.4
cmd = "convert +profile \"*\" -interlace Line -quality 95%% -resize %sx%s -sharpen %s,%s %s %s" % (width,height,x,y,image_filename, thumb_filename)
logger.info("running %s", cmd)
if not FLAGS.dryrun:
os.system(cmd)
def imagemagick_resize(width, height, image_filename, thumb_filename):
if not os.path.exists(
os.path.dirname(thumb_filename)) and not FLAGS.dryrun:
make_dirs_for_file(thumb_filename)
if os.path.exists(thumb_filename) and not FLAGS.dryrun:
os.remove(thumb_filename)
x = 0.3
y = 0.4
cmd = "convert +profile \"*\" -interlace Line -quality 95%% -resize %sx%s -sharpen %s,%s %s %s" % (
width, height, x, y, image_filename, thumb_filename)
logger.info("running %s", cmd)
if not FLAGS.dryrun:
os.system(cmd)
def save_image(image_filename, data):
if not os.path.exists(os.path.dirname(image_filename)):
make_dirs_for_file(image_filename)
f = file(image_filename, "wb")
f.write(data)
f.close()
def save_image(self, image_filename, data):
if not os.path.exists(os.path.dirname(image_filename)):
make_dirs_for_file(image_filename)
f = file(image_filename, "wb")
f.write(data)
f.close()
