def main():
company = 'Champion'
c_hour = str(datetime.now().strftime("%H"))
date = datetime.now().strftime("%d-%m-%Y %H")
date = str(date) + 'hrs'
temp = create_xls(date, company)
output = build_table(temp, 'blue_light')
time.sleep(5)
a = send_email(c_hour, date, company, output)
if a == 1:
log.insert_log('OK', 'Correo enviado')
else:
log.insert_log('ERROR', 'Correo no enviado')
status = 'Agotado'
#print(product_name)
else:
status = 'Posible Quiebre'
#print(status)
info = {
'store_id': rs,
'category': rc,
'product_id': product_id,
'product_name': product_name,
'product_code': product_code,
'brand': brand,
'product_url': product_url,
'status': status,
'formato': formato,
'price': price,
'price_wo_offer': price_wo_offer,
'create_date': create_date,
'offer': offer
}
result.append(info)
except:
pass
insert_db(result)
print('HORA FIN: ' + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
log.insert_log('End', 'Jumbo VM ' + str(id_vm), id_log,
datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
import json
import pandas as pd
import io
import log
from bs4 import BeautifulSoup
import requests
import threading
import time
import log
from random import randrange
from datetime import datetime
from sqlalchemy import create_engine, Column, MetaData, Table, DateTime, String, Integer, ForeignKey, BIGINT, TEXT, FLOAT, inspect, event
result = []
id_vm = 6
log.insert_log('Start', 'Cornershop VM ' + str(id_vm))
def stores_aisles(inicio, fin, vm):
#se determina en la variable local, que locales va a scapear
local = []
sublocal = []
for i in range(inicio, fin):
#print(result_stores[i])
local.append(result_stores[i])
#balance de carga en las VM
num_vm = 6
num_local = len(local)
#temporal para pruebas
import time
import log
import pandas as pd
import config
import io
import threading
import json
from datetime import datetime
from bs4 import BeautifulSoup
from sqlalchemy import create_engine, Column, MetaData, Table, DateTime, String, Integer, ForeignKey, BIGINT, TEXT, FLOAT, inspect, event
id_vm = 6
#encabezado log
print('Log VM-' + str(id_vm))
print('HORA INICIO: ' + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
log.insert_log('Start', 'Lider VM ' + str(id_vm))
def insert_db(informacion):
df = pd.DataFrame(informacion)
df = df.reindex(columns=[
'store_id', 'category_url', 'product_url', 'name', 'sku', 'brand',
'price', 'format', 'status', 'create_date', 'product_number',
'normal_price', 'label_price', 'array_position', 'offer'
])
#conexion a la db
engine = create_engine(
'postgresql://' + config.DATABASE_CONFIG['user'] + ':' +
config.DATABASE_CONFIG['password'] + '@' +
config.DATABASE_CONFIG['host'] + ':' + config.DATABASE_CONFIG['port'] +
for k in range(vm1, vm2):
sublocal.append(result_stores[k])
elif id_vm == 3:
for k in range(vm2, vm3):
sublocal.append(result_stores[k])
else:
for k in range(vm3, vm4):
sublocal.append(result_stores[k])
return sublocal
#main
#encabezado log
print('Log Rappi Convenience-'+str(id_vm))
print('HORA INICIO: '+str(datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
log.insert_log('Start', 'Rappi - Convenience VM-'+str(id_vm))
engine = create_engine('postgresql://'+config.DATABASE_CONFIG['user']+':'+config.DATABASE_CONFIG['password']+'@'+config.DATABASE_CONFIG['host']+':'+config.DATABASE_CONFIG['port']+'/'+config.DATABASE_CONFIG['dbname']
, connect_args={'options': '-csearch_path={}'.format(config.DATABASE_CONFIG['schema'])})
connection = engine.connect()
result_stores = connection.execute(config.RAPPI_CONVENIENCE_PRODUCT['query_stores_prod']).fetchall()
connection.close()
locales = balance_cargas()
#for i in range(0, stores_len):
# #print(result_stores[i]['category_id'])
# get_product(int(result_stores[i]['store_id']), str(result_stores[i]['category_id']), 0 )
for rc in locales:
'170.0.203.9:1080',
'84.22.59.202:5678',
'104.248.48.233:30588',
'134.249.198.7:7777',
'203.128.72.62:4145',
'150.107.103.64:4145',
'69.163.164.116:18309',
'103.20.191.242:4145',
]
#encabezado log
print('Log VM-' + str(id_vm) + ' Cornershop')
print('HORA INICIO: ' + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
id_log = str(datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")) + ' - Cornershop VM ' + str(id_vm)
log.insert_log('Start', 'Cornershop VM ' + str(id_vm), id_log,
datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
#obtenemos las tiendas y pasillos a scrapear
result_stores = connection.execute(
config.CORNERSHOP_STORES['query_stores_lopez']).fetchall()
result_aisles = connection.execute(
config.CORNERSHOP_AISLES['query_aisles_lopez']).fetchall()
connection.close()
len_stores = len(result_stores)
n_thread = 4
i0 = 0
i1 = len_stores // n_thread
i2 = (len_stores // n_thread) * 2
i3 = (len_stores // n_thread) * 3
i4 = len(result_stores)