def main(): company = 'Champion' c_hour = str(datetime.now().strftime("%H")) date = datetime.now().strftime("%d-%m-%Y %H") date = str(date) + 'hrs' temp = create_xls(date, company) output = build_table(temp, 'blue_light') time.sleep(5) a = send_email(c_hour, date, company, output) if a == 1: log.insert_log('OK', 'Correo enviado') else: log.insert_log('ERROR', 'Correo no enviado')
status = 'Agotado' #print(product_name) else: status = 'Posible Quiebre' #print(status) info = { 'store_id': rs, 'category': rc, 'product_id': product_id, 'product_name': product_name, 'product_code': product_code, 'brand': brand, 'product_url': product_url, 'status': status, 'formato': formato, 'price': price, 'price_wo_offer': price_wo_offer, 'create_date': create_date, 'offer': offer } result.append(info) except: pass insert_db(result) print('HORA FIN: ' + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) log.insert_log('End', 'Jumbo VM ' + str(id_vm), id_log, datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
import json import pandas as pd import io import log from bs4 import BeautifulSoup import requests import threading import time import log from random import randrange from datetime import datetime from sqlalchemy import create_engine, Column, MetaData, Table, DateTime, String, Integer, ForeignKey, BIGINT, TEXT, FLOAT, inspect, event result = [] id_vm = 6 log.insert_log('Start', 'Cornershop VM ' + str(id_vm)) def stores_aisles(inicio, fin, vm): #se determina en la variable local, que locales va a scapear local = [] sublocal = [] for i in range(inicio, fin): #print(result_stores[i]) local.append(result_stores[i]) #balance de carga en las VM num_vm = 6 num_local = len(local) #temporal para pruebas
import time import log import pandas as pd import config import io import threading import json from datetime import datetime from bs4 import BeautifulSoup from sqlalchemy import create_engine, Column, MetaData, Table, DateTime, String, Integer, ForeignKey, BIGINT, TEXT, FLOAT, inspect, event id_vm = 6 #encabezado log print('Log VM-' + str(id_vm)) print('HORA INICIO: ' + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) log.insert_log('Start', 'Lider VM ' + str(id_vm)) def insert_db(informacion): df = pd.DataFrame(informacion) df = df.reindex(columns=[ 'store_id', 'category_url', 'product_url', 'name', 'sku', 'brand', 'price', 'format', 'status', 'create_date', 'product_number', 'normal_price', 'label_price', 'array_position', 'offer' ]) #conexion a la db engine = create_engine( 'postgresql://' + config.DATABASE_CONFIG['user'] + ':' + config.DATABASE_CONFIG['password'] + '@' + config.DATABASE_CONFIG['host'] + ':' + config.DATABASE_CONFIG['port'] +
for k in range(vm1, vm2): sublocal.append(result_stores[k]) elif id_vm == 3: for k in range(vm2, vm3): sublocal.append(result_stores[k]) else: for k in range(vm3, vm4): sublocal.append(result_stores[k]) return sublocal #main #encabezado log print('Log Rappi Convenience-'+str(id_vm)) print('HORA INICIO: '+str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) log.insert_log('Start', 'Rappi - Convenience VM-'+str(id_vm)) engine = create_engine('postgresql://'+config.DATABASE_CONFIG['user']+':'+config.DATABASE_CONFIG['password']+'@'+config.DATABASE_CONFIG['host']+':'+config.DATABASE_CONFIG['port']+'/'+config.DATABASE_CONFIG['dbname'] , connect_args={'options': '-csearch_path={}'.format(config.DATABASE_CONFIG['schema'])}) connection = engine.connect() result_stores = connection.execute(config.RAPPI_CONVENIENCE_PRODUCT['query_stores_prod']).fetchall() connection.close() locales = balance_cargas() #for i in range(0, stores_len): # #print(result_stores[i]['category_id']) # get_product(int(result_stores[i]['store_id']), str(result_stores[i]['category_id']), 0 ) for rc in locales:
'170.0.203.9:1080', '84.22.59.202:5678', '104.248.48.233:30588', '134.249.198.7:7777', '203.128.72.62:4145', '150.107.103.64:4145', '69.163.164.116:18309', '103.20.191.242:4145', ] #encabezado log print('Log VM-' + str(id_vm) + ' Cornershop') print('HORA INICIO: ' + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) id_log = str(datetime.now().strftime( "%Y-%m-%d %H:%M:%S")) + ' - Cornershop VM ' + str(id_vm) log.insert_log('Start', 'Cornershop VM ' + str(id_vm), id_log, datetime.now().strftime("%Y-%m-%d %H:%M:%S")) #obtenemos las tiendas y pasillos a scrapear result_stores = connection.execute( config.CORNERSHOP_STORES['query_stores_lopez']).fetchall() result_aisles = connection.execute( config.CORNERSHOP_AISLES['query_aisles_lopez']).fetchall() connection.close() len_stores = len(result_stores) n_thread = 4 i0 = 0 i1 = len_stores // n_thread i2 = (len_stores // n_thread) * 2 i3 = (len_stores // n_thread) * 3 i4 = len(result_stores)