def __init__(self, initial_url):
self.browser = webdriver.Firefox(
) #todo: enum to choose which browser to use
self.browser.get(initial_url)
self.browser.maximize_window()
self.logger = get_logger(self)
self.config = get_config()
self.timeout = self.config.getint('DriverServices', 'timeout')
def _save_models(self, models, directory):
with open('{}/best_{}.pkl'.format(directory, self.name),
'wb') as handle:
pickle.dump(models, handle, protocol=pickle.HIGHEST_PROTOCOL)
if __name__ == "__main__":
# Import directories
paths = get_paths(station=args.s)
# get some data
data_dir, pkl_dir = paths['data_dir'], paths['pkl_dir']
lgbm_cont_dir, log_dir = paths['lgbm_cont_search'], paths['logs']
# Get new logger
logger = get_logger('LGBMContRandSearch', log_dir)
logger.info("########## Started new {} session ##########".format(
config.CORE_SETTINGS['name']))
logger.info("loading data ...")
X = pandas.read_csv(pkl_dir + "/train_imp_na_df.csv")
y = X["TARGET"].values
X = X.drop("TARGET", axis=1)
meta = pandas.read_pickle(pkl_dir + '/meta_df.pkl')
for x in set(meta.columns):
if x in set(X.columns):
X = X.drop(x, axis=1)
del meta
X = X.values
import asyncio
from aiohttp import ClientSession
from models import get_session, get_bancos, add_data
from bancos_data import load_data
from scraper import parse_cotizacion
from logger_factory import get_logger
logger = get_logger("runner")
sem = asyncio.Semaphore(5)
db_session = get_session()
load_data(db_session)
bancos = get_bancos(db_session)
async def get(url):
async with ClientSession() as session:
async with session.get(url) as response:
return await response.text()
@asyncio.coroutine
def save_cotizacion(banco):
with (yield from sem):
body = yield from get(banco.url)
cotizacion = parse_cotizacion(banco, body)
if cotizacion and cotizacion.check_unique():
logger.info("Intentando guardar cotización para Banco %s",
banco.name)
try:
add_data(cotizacion, db_session)
logger.info("Guardada cotización %s", cotizacion)
# For terminal execution
parser = ArgumentParser()
parser.add_argument('-s', help='work station registered in kaggleProjects.directory_table.py')
parser.add_argument('-n', help='name of session')
parser.add_argument('-i', help='iterations of swarm optimisation')
parser.add_argument('-p', help='number of particles to generate')
args = parser.parse_args()
# Import paths
paths = get_paths(station=args.s)
pkl_dir = paths['pkl_dir']
pso_dir, log_dir = paths['temp'], paths['logs']
# Get new logger
logger = get_logger('XgbPso', log_dir)
logger.info('# started session {} #'.format(args.n))
# Import data
logger.info('importing data ...')
X = pandas.read_csv(pkl_dir + "/train_imp_na_df.csv")
y = X["TARGET"].values
X = X.drop("TARGET", axis=1)
meta = pandas.read_pickle(pkl_dir + '/meta_df.pkl')
for x in set(meta.columns):
if x in set(X.columns):
X = X.drop(x, axis=1)
del meta
X = X.values
# Init swarm optimisation
def __init__(self, element):
self.element = element
self.logger = get_logger(self)
self.config = get_config()
self.timeout = self.config.getint('DriverServices', 'timeout')
def __init__(self, browser, page):
self.logger = get_logger(self)
self.logger.debug('entering page')
self.browser = browser
self.source_page = page
self.take_page_over()
# For terminal execution
parser = ArgumentParser()
parser.add_argument('-s', help='work station registered in kaggleProjects.directory_table.py')
parser.add_argument('-i', help='iterations of swarm optimisation')
parser.add_argument('-p', help='number of particles to generate')
parser.add_argument('-d', help='cpu or gpu')
parser.add_argument('-n', help='name of session')
args = parser.parse_args()
# Import paths
paths = get_paths(station=args.s)
pkl_dir = paths['pkl_dir']
pso_dir, log_dir = paths['pso'], paths['logs']
# Get new logger
logger = get_logger('LgbmPso', log_dir)
logger.info('# started session {} #'.format(args.n))
# Import data
logger.info('importing data ...')
X = pandas.read_csv(pkl_dir + "/train_imp_na_df.csv")
y = X["TARGET"].values
X = X.drop("TARGET", axis=1)
meta = pandas.read_pickle(pkl_dir + '/meta_df.pkl')
for x in set(meta.columns):
if x in set(X.columns):
X = X.drop(x, axis=1)
del meta
X = X.values
# Init swarm optimisation
import datetime
from os import environ
from os.path import join, dirname
import telegram
from dotenv import load_dotenv
from models import get_session, get_last_cotizaciones
from utils import format_cotizaciones_for_telegram
from logger_factory import get_logger
logger = get_logger("channel")
dotenv_path = join(dirname(__file__), '../.env')
load_dotenv(dotenv_path)
TOKEN = environ.get("TOKEN", None)
CHANNEL = environ.get("CHANNEL", None)
OWNER = environ.get("OWNER", None)
def send_cotizaciones(bot):
now = datetime.datetime.now()
session = get_session()
cotizaciones = get_last_cotizaciones(now, session)
text = format_cotizaciones_for_telegram(cotizaciones)
if text:
bot.sendMessage(chat_id=CHANNEL,
text=text,
parse_mode=telegram.ParseMode.MARKDOWN)
else:
logger.warning("No hubo cotizaciones para el día de hoy")