def __init__(self, **kwargs):
self.lr = LRequest(string_proxy=kwargs.get('string_proxy', ''))
self.captcha = GsaCaptcha(ip=kwargs.get('gsa_ip', '192.168.1.188'), port=kwargs.get('gsa_port', '8000'))
self.CACHE_ROOT = kwargs.get('cache_root', 'I:\\cache_amazon')
self.CACHE_PAGES_ROOT = kwargs.get('cache_page', os.path.join(self.CACHE_ROOT, 'pages'))
self.CACHE_IMAGES_ROOT = kwargs.get('cache_image', os.path.join(self.CACHE_ROOT, 'images'))
if not os.path.exists(self.CACHE_ROOT): os.makedirs(self.CACHE_ROOT)
if not os.path.exists(self.CACHE_PAGES_ROOT): os.makedirs(self.CACHE_PAGES_ROOT)
if not os.path.exists(self.CACHE_IMAGES_ROOT): os.makedirs(self.CACHE_IMAGES_ROOT)
self.domain = kwargs.get('domain', 'amazon.com')
self.CACHE_EXPIRED_DAYS = kwargs.get('cache_expired_days', 15)
def __init__(self, **kwargs):
Amazon.CACHE_ROOT = kwargs.get('CACHE_ROOT', 'I:\\cache')
Amazon.CACHE_EXPIRED_DAYS = kwargs.get('CACHE_EXPIRED_DAYS', 15)
Amazon.max_workers = kwargs.get('max_workers', 1)
Amazon.string_proxies = kwargs.get('string_proxies', [])
if len(Amazon.string_proxies) > 0:
Amazon.lr = LRequest(string_proxy=Amazon.string_proxies[0])
Amazon.captcha = GsaCaptcha(ip='192.168.1.188', port='8000')
self.executor = LThreadPoolExecutor(max_workers=Amazon.max_workers)
from lutils.lrequest import LRequest
from lutils.captcha.gsa_captcha import GsaCaptcha
logging.config.fileConfig('logging.conf')
logger = logging.getLogger('verbose')
string_proxies = [
'socks4://192.168.1.188:1080',
'socks4://192.168.1.188:1081',
'socks4://192.168.1.188:1082',
'socks4://192.168.1.188:1083',
# 'socks4://192.168.1.188:1084',
'socks4://192.168.1.188:1085',
]
captcha = GsaCaptcha(ip='192.168.1.188', port=8000)
def check_captcha(lr):
if captcha is not None:
captcha_img_ele = lr.xpath(
'//form[contains(@action, "Captcha")]//img[contains(@src, "captcha")]'
)
if captcha_img_ele is not None:
while 1:
try:
if captcha_img_ele is not None:
logger.info('Need Captcha')
form = lr.get_forms()[0]
lr.load(captcha_img_ele.attrib['src'])
class AmazonBase(object):
CACHE_ROOT = ''
CACHE_PAGES_ROOT = ''
CACHE_IMAGES_ROOT = ''
CACHE_EXPIRED_DAYS = 15
captcha = None
def __init__(self, **kwargs):
self.lr = LRequest(string_proxy=kwargs.get('string_proxy', ''))
self.captcha = GsaCaptcha(ip=kwargs.get('gsa_ip', '192.168.1.188'),
port=kwargs.get('gsa_port', '8000'))
self.CACHE_ROOT = config.AMAZON_CACHE_ROOT
self.CACHE_PAGES_ROOT = kwargs.get(
'cache_page', os.path.join(self.CACHE_ROOT, 'pages'))
self.CACHE_IMAGES_ROOT = kwargs.get(
'cache_image', os.path.join(self.CACHE_ROOT, 'images'))
if not os.path.exists(self.CACHE_ROOT): os.makedirs(self.CACHE_ROOT)
if not os.path.exists(self.CACHE_PAGES_ROOT):
os.makedirs(self.CACHE_PAGES_ROOT)
if not os.path.exists(self.CACHE_IMAGES_ROOT):
os.makedirs(self.CACHE_IMAGES_ROOT)
self.domain = kwargs.get('domain', 'amazon.com')
self.CACHE_EXPIRED_DAYS = kwargs.get('cache_expired_days', 15)
def load(self, url, is_xpath=True, is_decode=True):
# logger.info('Load Url: %s' % url)
url = urllib.parse.quote(url, safe='https:/')
self.lr.load(url, is_xpath=is_xpath, is_decode=is_decode)
if self.check_captcha():
self.lr.load(url, is_xpath=is_xpath, is_decode=is_decode)
def check_captcha(self):
if self.captcha is not None:
captcha_img_ele = self.lr.xpath(
'//form[contains(@action, "Captcha")]//img[contains(@src, "captcha")]'
)
if captcha_img_ele is not None:
while 1:
logger.info('Need Captcha')
try:
if captcha_img_ele is not None:
print('##### %s ' % captcha_img_ele.attrib['src'])
form = self.lr.get_forms()[0]
self.lr.load(captcha_img_ele.attrib['src'])
cap = self.captcha.decode_stream(self.lr.body)
logger.info('Captcha: %s' % cap)
form['field-keywords'] = cap
self.lr.load(form.click())
else:
return True
captcha_img_ele = self.lr.xpath(
'//form[contains(@action, "Captcha")]//img[contains(@src, "captcha")]'
)
except KeyboardInterrupt:
raise
except IndexError:
self.lr.load(self.lr.current_url)
captcha_img_ele = self.lr.xpath(
'//form[contains(@action, "Captcha")]//img[contains(@src, "captcha")]'
)
if captcha_img_ele is None:
return True
except:
# open(os.path.join('I:\\captcha_error_page', '%s.html' % time.time()), 'w').write(self.lr.body)
logger.error(traceback.format_exc())
return False
else:
raise RuntimeError('Not Captcha Server...')
def exists_cache(self, cache_name):
cache_path = os.path.join(self.CACHE_PAGES_ROOT, cache_name[0],
cache_name[1], cache_name)
return os.path.exists(cache_path)
def remove_cache(self, cache_name):
cache_path = os.path.join(self.CACHE_PAGES_ROOT, cache_name[0],
cache_name[1], cache_name)
if os.path.exists(cache_path):
try:
os.remove(cache_path)
except:
pass
def load_cache(self, cache_name):
cache_path = os.path.join(self.CACHE_PAGES_ROOT, cache_name[0],
cache_name[1], cache_name)
if os.path.exists(cache_path):
try:
return pickle.loads(gzip.GzipFile(cache_path, 'rb').read())
except:
return {}
return {}
def save_cache(self, cache_name, data):
_p = os.path.join(self.CACHE_PAGES_ROOT, cache_name[0], cache_name[1])
if not os.path.exists(_p): os.makedirs(_p)
cache_path = os.path.join(self.CACHE_PAGES_ROOT, cache_name[0],
cache_name[1], cache_name)
gzip_file = gzip.open(cache_path, 'wb')
gzip_file.write(pickle.dumps(data))
gzip_file.close()
def exists_image(self, name):
image_path = os.path.join(self.CACHE_IMAGES_ROOT, name[0], name[1],
name)
return os.path.exists(image_path)
def save_image(self, name, data):
_p = os.path.join(self.CACHE_IMAGES_ROOT, name[0], name[1])
if not os.path.exists(_p): os.makedirs(_p)
image_path = os.path.join(self.CACHE_IMAGES_ROOT, name[0], name[1],
name)
open(image_path, 'wb').write(data)
@staticmethod
def wrapped_url(url):
return url.split('/ref', 1)[0]
@cache()
@load_html
@name
@price
@brand
@merchant
@sold_by
@reviews
@star
@ranks_str
@other_seller
@weight_ounces
def product_detail(self, asin, is_cache=True, **kwargs):
return kwargs.get('product_info', {})
@cache()
@load_html
@image_urls
@image_data
def product(self, asin, is_cache=True, **kwargs):
return kwargs.get('product_info', {})