def __init__(self):
verbose('INIT MYSQL', {
'host': settings.mysql_host,
'port': settings.mysql_port,
'db': settings.mysql_database
},
level=2)
pass
def worker2(product, store):
verbose(product, store, level=1, label='INFO')
ad = spider.get_ad_page(product, store)
verbose(ad, level=4, label='INFO')
product['id'] = ad['id']
product['prices'] = ad['prices']
pool.insert_ad(ad['id'], store, product)
def __init__(self):
verbose('INIT REDIS', {
'host': settings.redis_host,
'port': settings.redis_port,
'db': settings.redis_db,
'password': settings.redis_pass
},
level=2)
self.redis = redis.StrictRedis(host=settings.redis_host,
port=settings.redis_port,
db=settings.redis_db,
password=settings.redis_pass)
def get_stores(self):
conn = self.database()
cursor = conn.cursor()
sql = 'select slug from mvp_b2w_store where is_active=1'
verbose(sql, level=3)
res = cursor.execute(sql)
result = []
for row in cursor.fetchall():
result.append(row[0])
return result
def insert_mlb(self, MLB, price, sold_quantity, available_quantity, store):
conn = self.database()
cursor = conn.cursor()
sql = f'''INSERT INTO mvp_ml_prices (MLB, price, sold_quantity,
available_quantity, store, datetime) VALUES ('{MLB}', '{price}',
'{sold_quantity}', '{available_quantity}', '{store}', NOW())
ON DUPLICATE KEY UPDATE price='{price}',
sold_quantity='{sold_quantity}',
available_quantity='{available_quantity}',
store='{store}', datetime=NOW();'''
verbose(sql, level=3)
cursor.execute(sql)
cursor.close()
conn.commit()
def insert_ad(self, id, store, data):
try:
key = f'{id}_{store}'
self.redis.set(name=key,
value=json.dumps(data),
ex=settings.redis_ttl)
verbose(key, level=2, label='INFO')
except:
verbose('Error trying to set Redis key',
key, {
'host': settings.redis_host,
'port': settings.redis_port,
'db': settings.redis_db,
'password': settings.redis_pass
},
label='ERROR')
def print_ip():
sources = [
"http://whatismyip.akamai.com",
"http://ifconfig.me",
"http://ipecho.net/plain",
"http://ipinfo.io/ip",
"http://ifconfig.co",
]
for source in sources:
try:
ip = requests.get(source,
headers=settings.requests_headers,
proxies=settings.requests_proxy).text
verbose(ip, level=2)
break
except:
verbose("Não foi possivel buscar o IP", source, label='WARN')
def worker(store):
verbose("Requesting Ads", level=1, label='INFO')
products = spider.get_products(store)
verbose("get_products", products, level=4, label='INFO')
all_ads = []
pular = 1
for prod in products:
if prod['url'].find('lead?oid') != -1:
all_ads.append(prod)
continue
if pular > 0:
pular = pular - 1
continue
# serial mode
worker2(prod, store)
exit()
#
# if stop == True:
# verbose('STOP', stop)
# return
#
# stores = database.get_stores()
# verbose(stores, level=2)
stores = ['celular/smartphone']
# serial mode
worker(stores[0])
exit()
with concurrent.futures.ThreadPoolExecutor(
max_workers=settings.max_workers) as executor:
future_spider = {executor.submit(worker, job): job for job in stores}
spider = Spider()
database = Mysql()
pool = Redis()
while True:
try:
print_ip()
main()
except Exception as exc:
verbose('%r generated an exception: %s' % (url, exc), label='ERROR')
# break
def bla(v=1):
verbose('running bla method', 'Process:', v, label='INFO')
def runsleep():
verbose('So tired. I\'ll get some sleep', label='INFO')
time.sleep(1)
verbose('Wake up. What\'s time is it?', label='WARN')
def blablabla():
try:
unexistent()
except:
verbose('Unexistent function called', label='ERROR')
def blabla():
if 1 == 2:
pass
else:
verbose('Something strange here, but its ok', label='WARN')
if 1 == 2:
pass
else:
verbose('Something strange here, but its ok', label='WARN')
def blablabla():
try:
unexistent()
except:
verbose('Unexistent function called', label='ERROR')
def runsleep():
verbose('So tired. I\'ll get some sleep', label='INFO')
time.sleep(1)
verbose('Wake up. What\'s time is it?', label='WARN')
bla()
blabla()
verbose('I\'m a -vv INFO text', level=2, label='INFO')
verbose('I\'m a -vvv WARN text', level=3, label='WARN')
blablabla()
runsleep()
jobs = [1, 2, 3]
with concurrent.futures.ProcessPoolExecutor(max_workers=3) as executor:
future_mbl = {executor.submit(bla, job): job for job in jobs}
