Beispiel #1
0
 def __init__(self):
     verbose('INIT MYSQL', {
         'host': settings.mysql_host,
         'port': settings.mysql_port,
         'db': settings.mysql_database
     },
             level=2)
     pass
Beispiel #2
0
def worker2(product, store):
    verbose(product, store, level=1, label='INFO')

    ad = spider.get_ad_page(product, store)
    verbose(ad, level=4, label='INFO')

    product['id'] = ad['id']
    product['prices'] = ad['prices']

    pool.insert_ad(ad['id'], store, product)
Beispiel #3
0
 def __init__(self):
     verbose('INIT REDIS', {
         'host': settings.redis_host,
         'port': settings.redis_port,
         'db': settings.redis_db,
         'password': settings.redis_pass
     },
             level=2)
     self.redis = redis.StrictRedis(host=settings.redis_host,
                                    port=settings.redis_port,
                                    db=settings.redis_db,
                                    password=settings.redis_pass)
Beispiel #4
0
    def get_stores(self):

        conn = self.database()
        cursor = conn.cursor()
        sql = 'select slug from mvp_b2w_store where is_active=1'
        verbose(sql, level=3)
        res = cursor.execute(sql)

        result = []
        for row in cursor.fetchall():
            result.append(row[0])

        return result
Beispiel #5
0
 def insert_mlb(self, MLB, price, sold_quantity, available_quantity, store):
     conn = self.database()
     cursor = conn.cursor()
     sql = f'''INSERT INTO mvp_ml_prices (MLB, price, sold_quantity,
         available_quantity, store, datetime) VALUES ('{MLB}', '{price}',
         '{sold_quantity}', '{available_quantity}', '{store}', NOW())
         ON DUPLICATE KEY UPDATE price='{price}',
         sold_quantity='{sold_quantity}',
         available_quantity='{available_quantity}',
         store='{store}', datetime=NOW();'''
     verbose(sql, level=3)
     cursor.execute(sql)
     cursor.close()
     conn.commit()
Beispiel #6
0
 def insert_ad(self, id, store, data):
     try:
         key = f'{id}_{store}'
         self.redis.set(name=key,
                        value=json.dumps(data),
                        ex=settings.redis_ttl)
         verbose(key, level=2, label='INFO')
     except:
         verbose('Error trying to set Redis key',
                 key, {
                     'host': settings.redis_host,
                     'port': settings.redis_port,
                     'db': settings.redis_db,
                     'password': settings.redis_pass
                 },
                 label='ERROR')
Beispiel #7
0
def print_ip():
    sources = [
        "http://whatismyip.akamai.com",
        "http://ifconfig.me",
        "http://ipecho.net/plain",
        "http://ipinfo.io/ip",
        "http://ifconfig.co",
    ]
    for source in sources:
        try:
            ip = requests.get(source,
                              headers=settings.requests_headers,
                              proxies=settings.requests_proxy).text
            verbose(ip, level=2)
            break
        except:
            verbose("Não foi possivel buscar o IP", source, label='WARN')
Beispiel #8
0
def worker(store):
    verbose("Requesting Ads", level=1, label='INFO')
    products = spider.get_products(store)
    verbose("get_products", products, level=4, label='INFO')

    all_ads = []
    pular = 1
    for prod in products:
        if prod['url'].find('lead?oid') != -1:
            all_ads.append(prod)
            continue

        if pular > 0:
            pular = pular - 1
            continue
        # serial mode
        worker2(prod, store)
        exit()
Beispiel #9
0
    #
    # if stop == True:
    #     verbose('STOP', stop)
    #     return
    #
    # stores = database.get_stores()
    # verbose(stores, level=2)

    stores = ['celular/smartphone']

    # serial mode
    worker(stores[0])

    exit()
    with concurrent.futures.ThreadPoolExecutor(
            max_workers=settings.max_workers) as executor:
        future_spider = {executor.submit(worker, job): job for job in stores}


spider = Spider()
database = Mysql()
pool = Redis()

while True:
    try:
        print_ip()
        main()
    except Exception as exc:
        verbose('%r generated an exception: %s' % (url, exc), label='ERROR')
    # break
def bla(v=1):
    verbose('running bla method', 'Process:', v, label='INFO')
def runsleep():
    verbose('So tired. I\'ll get some sleep', label='INFO')
    time.sleep(1)
    verbose('Wake up. What\'s time is it?', label='WARN')
def blablabla():
    try:
        unexistent()
    except:
        verbose('Unexistent  function called', label='ERROR')
def blabla():
    if 1 == 2:
        pass
    else:
        verbose('Something strange here, but its ok', label='WARN')
    if 1 == 2:
        pass
    else:
        verbose('Something strange here, but its ok', label='WARN')


def blablabla():
    try:
        unexistent()
    except:
        verbose('Unexistent  function called', label='ERROR')


def runsleep():
    verbose('So tired. I\'ll get some sleep', label='INFO')
    time.sleep(1)
    verbose('Wake up. What\'s time is it?', label='WARN')


bla()
blabla()
verbose('I\'m a -vv INFO text', level=2, label='INFO')
verbose('I\'m a -vvv WARN text', level=3, label='WARN')
blablabla()
runsleep()

jobs = [1, 2, 3]

with concurrent.futures.ProcessPoolExecutor(max_workers=3) as executor:
    future_mbl = {executor.submit(bla, job): job for job in jobs}