def __init__(self): verbose('INIT MYSQL', { 'host': settings.mysql_host, 'port': settings.mysql_port, 'db': settings.mysql_database }, level=2) pass
def worker2(product, store): verbose(product, store, level=1, label='INFO') ad = spider.get_ad_page(product, store) verbose(ad, level=4, label='INFO') product['id'] = ad['id'] product['prices'] = ad['prices'] pool.insert_ad(ad['id'], store, product)
def __init__(self): verbose('INIT REDIS', { 'host': settings.redis_host, 'port': settings.redis_port, 'db': settings.redis_db, 'password': settings.redis_pass }, level=2) self.redis = redis.StrictRedis(host=settings.redis_host, port=settings.redis_port, db=settings.redis_db, password=settings.redis_pass)
def get_stores(self): conn = self.database() cursor = conn.cursor() sql = 'select slug from mvp_b2w_store where is_active=1' verbose(sql, level=3) res = cursor.execute(sql) result = [] for row in cursor.fetchall(): result.append(row[0]) return result
def insert_mlb(self, MLB, price, sold_quantity, available_quantity, store): conn = self.database() cursor = conn.cursor() sql = f'''INSERT INTO mvp_ml_prices (MLB, price, sold_quantity, available_quantity, store, datetime) VALUES ('{MLB}', '{price}', '{sold_quantity}', '{available_quantity}', '{store}', NOW()) ON DUPLICATE KEY UPDATE price='{price}', sold_quantity='{sold_quantity}', available_quantity='{available_quantity}', store='{store}', datetime=NOW();''' verbose(sql, level=3) cursor.execute(sql) cursor.close() conn.commit()
def insert_ad(self, id, store, data): try: key = f'{id}_{store}' self.redis.set(name=key, value=json.dumps(data), ex=settings.redis_ttl) verbose(key, level=2, label='INFO') except: verbose('Error trying to set Redis key', key, { 'host': settings.redis_host, 'port': settings.redis_port, 'db': settings.redis_db, 'password': settings.redis_pass }, label='ERROR')
def print_ip(): sources = [ "http://whatismyip.akamai.com", "http://ifconfig.me", "http://ipecho.net/plain", "http://ipinfo.io/ip", "http://ifconfig.co", ] for source in sources: try: ip = requests.get(source, headers=settings.requests_headers, proxies=settings.requests_proxy).text verbose(ip, level=2) break except: verbose("Não foi possivel buscar o IP", source, label='WARN')
def worker(store): verbose("Requesting Ads", level=1, label='INFO') products = spider.get_products(store) verbose("get_products", products, level=4, label='INFO') all_ads = [] pular = 1 for prod in products: if prod['url'].find('lead?oid') != -1: all_ads.append(prod) continue if pular > 0: pular = pular - 1 continue # serial mode worker2(prod, store) exit()
# # if stop == True: # verbose('STOP', stop) # return # # stores = database.get_stores() # verbose(stores, level=2) stores = ['celular/smartphone'] # serial mode worker(stores[0]) exit() with concurrent.futures.ThreadPoolExecutor( max_workers=settings.max_workers) as executor: future_spider = {executor.submit(worker, job): job for job in stores} spider = Spider() database = Mysql() pool = Redis() while True: try: print_ip() main() except Exception as exc: verbose('%r generated an exception: %s' % (url, exc), label='ERROR') # break
def bla(v=1): verbose('running bla method', 'Process:', v, label='INFO')
def runsleep(): verbose('So tired. I\'ll get some sleep', label='INFO') time.sleep(1) verbose('Wake up. What\'s time is it?', label='WARN')
def blablabla(): try: unexistent() except: verbose('Unexistent function called', label='ERROR')
def blabla(): if 1 == 2: pass else: verbose('Something strange here, but its ok', label='WARN')
if 1 == 2: pass else: verbose('Something strange here, but its ok', label='WARN') def blablabla(): try: unexistent() except: verbose('Unexistent function called', label='ERROR') def runsleep(): verbose('So tired. I\'ll get some sleep', label='INFO') time.sleep(1) verbose('Wake up. What\'s time is it?', label='WARN') bla() blabla() verbose('I\'m a -vv INFO text', level=2, label='INFO') verbose('I\'m a -vvv WARN text', level=3, label='WARN') blablabla() runsleep() jobs = [1, 2, 3] with concurrent.futures.ProcessPoolExecutor(max_workers=3) as executor: future_mbl = {executor.submit(bla, job): job for job in jobs}