def main(): """Main loop started from command line""" no_broadcast, current_datetime = parse_args() if current_datetime is None: current_datetime = pd.datetime.utcnow() else: current_datetime = pd.to_datetime(current_datetime) data_directory = os.path.join(config.PROJECT_DIRECTORY, 'scraped_data') model_directoy = os.path.join(config.PROJECT_DIRECTORY, 'trained_models') log_directory = os.path.join(config.PROJECT_DIRECTORY, 'logs') configure_logging(log_directory, current_datetime) logger.info('STARTING main script at {}'.format(current_datetime)) if no_broadcast: logger.info('Run without broadcasting.') else: logger.info('ATTENTION I WILL BROADCAST TO STEEMIT!!!') time.sleep(2) steem = MPSteem(nodes=config.NODES, no_broadcast=no_broadcast) # To post stuff account = config.ACCOUNT poster = Poster(account=account, steem=steem) prediction_frame = tpgd.scrape_hour_data(steem=steem, current_datetime=current_datetime, ncores=32, offset_hours=2) prediction_frame = tppp.preprocess(prediction_frame, ncores=8) permalink = 'daily-truffle-picks-2018-03-27' overview_permalink = 'weekly-truffle-updates-2018-12' logger.info('Computing the top trending without bidbots') logger.info('Searching for bid bots and bought votes') min_datetime = prediction_frame.created.min() max_datetime = prediction_frame.created.max() + pd.Timedelta(days=1) upvote_payments, bots = tpad.get_upvote_payments_to_bots(steem=steem, min_datetime=min_datetime, max_datetime=max_datetime) logger.info('Adjusting votes and reward') sorted_frame = tppp.compute_bidbot_correction(post_frame=prediction_frame, upvote_payments=upvote_payments) tt0b.create_trending_post(sorted_frame, upvote_payments=upvote_payments, poster=poster, topN_permalink=permalink, overview_permalink=overview_permalink, current_datetime=current_datetime, bots=bots) logger.info('DONE at {}'.format(current_datetime))
def test_logging_config(temp_dir): assert len(os.listdir(temp_dir)) == 0 tfut.configure_logging(temp_dir, pd.datetime.utcnow()) logging.getLogger().info('foobar') assert len(os.listdir(temp_dir)) == 1
def large_mp_preprocess(log_directory, current_datetime, steem, data_directory, days, offset_days): """Helper function to spawn in child process""" configure_logging(log_directory, current_datetime) post_frame = tpgd.load_or_scrape_training_data(steem, data_directory, current_datetime=current_datetime, days=days, offset_days=offset_days, ncores=32) return tppp.preprocess(post_frame, ncores=8)
def main(): """Main loop started from command line""" no_broadcast, current_datetime = parse_args() if current_datetime is None: current_datetime = pd.datetime.utcnow() else: current_datetime = pd.to_datetime(current_datetime) data_directory = os.path.join(config.PROJECT_DIRECTORY, 'scraped_data') model_directoy = os.path.join(config.PROJECT_DIRECTORY, 'trained_models') log_directory = os.path.join(config.PROJECT_DIRECTORY, 'logs') configure_logging(log_directory, current_datetime) logger.info('STARTING main script at {}'.format(current_datetime)) if no_broadcast: logger.info('Run without broadcasting.') else: logger.info('ATTENTION I WILL BROADCAST TO STEEMIT!!!') time.sleep(2) steem = MPSteem(nodes=config.NODES, no_broadcast=no_broadcast) # To post stuff account = config.ACCOUNT poster = Poster(account=account, steem=steem) tppd.create_wallet(steem, config.PASSWORD, posting_key=config.POSTING_KEY, active_key=config.ACTIVE_KEY) pipeline = tpmo.load_or_train_pipeline( None, model_directoy, current_datetime, regressor_kwargs={}, topic_kwargs={}, targets=['adjusted_reward', 'adjusted_votes']) overview_permalink = tppw.PERMALINK_TEMPLATE.format( date=current_datetime.strftime('%Y-%V')) permalink = tppd.PERMALINK_TEMPLATE.format( date=current_datetime.strftime('%Y-%m-%d')) logger.info('Done with normal duty, answering manual calls!') tfod.call_a_pig(poster=poster, pipeline=pipeline, topN_permalink=permalink, current_datetime=current_datetime, offset_hours=2, hours=24, overview_permalink=overview_permalink)
def main(): """Main loop started from command line""" no_broadcast, current_datetime = parse_args() if current_datetime is None: current_datetime = pd.datetime.utcnow() else: current_datetime = pd.to_datetime(current_datetime) data_directory = os.path.join(config.PROJECT_DIRECTORY, 'scraped_data') model_directoy = os.path.join(config.PROJECT_DIRECTORY, 'trained_models') log_directory = os.path.join(config.PROJECT_DIRECTORY, 'logs') configure_logging(log_directory, current_datetime) logger.info('STARTING main script at {}'.format(current_datetime)) if no_broadcast: logger.info('Run without broadcasting.') else: logger.info('ATTENTION I WILL BROADCAST TO STEEMIT!!!') time.sleep(2) steem = MPSteem(nodes=config.NODES, no_broadcast=no_broadcast) # hack to allow for payments, because of https://github.com/steemit/steem-python/issues/191 noapisteem = MPSteem(nodes=config.NODES[1:], no_broadcast=no_broadcast) # To post stuff account = config.ACCOUNT poster = Poster(account=account, steem=noapisteem) tppd.create_wallet(steem, config.PASSWORD, posting_key=config.POSTING_KEY, active_key=config.ACTIVE_KEY) logger.info('Paying out investors') tpde.pay_delegates( account=account, steem=noapisteem, # use a steem instance without api.steem! current_datetime=current_datetime) if not tpmo.model_exists(current_datetime, model_directoy): post_frame = load_and_preprocess_2_frames( log_directory=log_directory, current_datetime=current_datetime, steem=steem, noapisteem=noapisteem, data_directory=data_directory) logger.info('Garbage collecting') gc.collect() else: post_frame = None regressor_kwargs = dict(n_estimators=256, max_leaf_nodes=5000, max_features=0.2, n_jobs=-1, verbose=1, random_state=42) topic_kwargs = dict(num_topics=128, no_below=7, no_above=0.1, ngrams=(1, 2), keep_n=333000) if post_frame is not None and len(post_frame) > MAX_DOCUMENTS: logger.info('Frame has {} Documents, too many, ' 'reducing to {}'.format(len(post_frame), MAX_DOCUMENTS)) post_frame.sort_values('created', inplace=True, ascending=False) train_frame = post_frame.iloc[:MAX_DOCUMENTS, :] else: train_frame = post_frame pipeline = tpmo.load_or_train_pipeline( train_frame, model_directoy, current_datetime, regressor_kwargs=regressor_kwargs, topic_kwargs=topic_kwargs, targets=['adjusted_reward', 'adjusted_votes']) tpmo.log_pipeline_info(pipeline=pipeline) overview_permalink = tppw.return_overview_permalink_if_exists( account=account, current_datetime=current_datetime, steem=steem) if not overview_permalink: if post_frame is None: logger.info('Need to reaload data for weekly overview') post_frame = load_and_preprocess_2_frames( log_directory=log_directory, current_datetime=current_datetime, steem=steem, noapisteem=noapisteem, data_directory=data_directory) logger.info('I want to post my weekly overview') overview_permalink = tppw.post_weakly_update( pipeline=pipeline, post_frame=post_frame, poster=poster, current_datetime=current_datetime) logger.info('Garbage collecting') del post_frame gc.collect() prediction_frame = tpgd.scrape_hour_data(steem=steem, current_datetime=current_datetime, ncores=32, offset_hours=2) prediction_frame = tppp.preprocess(prediction_frame, ncores=8) sorted_frame = tpmo.find_truffles(prediction_frame, pipeline, account=account) permalink = tppd.post_topN_list(sorted_frame, poster=poster, current_datetime=current_datetime, overview_permalink=overview_permalink) tppd.comment_on_own_top_list(sorted_frame, poster=poster, topN_permalink=permalink) tppd.vote_and_comment_on_topK(sorted_frame, poster=poster, topN_permalink=permalink, overview_permalink=overview_permalink) logger.info('Computing the top trending without bidbots') logger.info('Searching for bid bots and bought votes') min_datetime = sorted_frame.created.min() max_datetime = sorted_frame.created.max() + pd.Timedelta(days=1) upvote_payments, bots = tpad.get_upvote_payments_to_bots( steem=noapisteem, min_datetime=min_datetime, max_datetime=max_datetime) logger.info('Adjusting votes and reward') sorted_frame = tppp.compute_bidbot_correction( post_frame=sorted_frame, upvote_payments=upvote_payments) tt0b.create_trending_post(sorted_frame, upvote_payments=upvote_payments, poster=poster, topN_permalink=permalink, overview_permalink=overview_permalink, current_datetime=current_datetime, bots=bots) logger.info('Done with normal duty, answering manual calls!') tfod.call_a_pig(poster=poster, pipeline=pipeline, topN_permalink=permalink, current_datetime=current_datetime, offset_hours=2, hours=24, overview_permalink=overview_permalink) logger.info('Cleaning up after myself') tfut.clean_up_directory(model_directoy, keep_last=3) tfut.clean_up_directory(data_directory, keep_last=25) tfut.clean_up_directory(log_directory, keep_last=14) logger.info('Preloading -8 days for later training') tpgd.load_or_scrape_training_data(steem, data_directory, current_datetime=current_datetime, days=1, offset_days=8, ncores=32) logger.info('DONE at {}'.format(current_datetime))