def download(_): conf.log = setup_log(logging.DEBUG) if conf.args['debug'] else setup_log() tempdir = tempfile.mkdtemp() cdn_url = conf.checkpoints_cdn.format(conf.checkpoints_version) temp_zip = os.path.join(tempdir, "{}.zip".format(conf.checkpoints_version)) try: conf.log.info("Downloading {}".format(cdn_url)) dl_file(conf.checkpoints_cdn.format(conf.checkpoints_version), temp_zip) conf.log.info("Extracting {}".format(temp_zip)) unzip(temp_zip, conf.args['checkpoints']) conf.log.info("Moving Checkpoints To Final Location") for c in ("cm.lib", "mm.lib", "mn.lib"): if os.path.isfile(os.path.join(conf.args['checkpoints'], c)): os.remove(os.path.join(conf.args['checkpoints'], c)) shutil.move( os.path.join(conf.args['checkpoints'], 'checkpoints', c), conf.args['checkpoints']) shutil.rmtree(os.path.join(conf.args['checkpoints'], 'checkpoints')) except Exception as e: conf.log.error(e) conf.log.error( "Something Gone Bad Download Downloading The Checkpoints") shutil.rmtree(tempdir) sys.exit(1) shutil.rmtree(tempdir) conf.log.info("Checkpoints Downloaded Successfully")
def main(args): utils.setup_log(LOGGER, verbosity=args.verbosity, logfile=args.logfile) LOGGER.debug( u'Loaded the LOOT API v{} using wrapper version {}'.format( loot.Version.string(), loot.WrapperVersion.string() ) ) game_data = [ (u'Oblivion', u'Oblivion.esm', u'oblivion', loot.GameType.tes4), (u'Skyrim', u'Skyrim.esm', u'skyrim', loot.GameType.tes5), (u'SkyrimSE', u'Skyrim.esm', u'skyrimse', loot.GameType.tes5se), (u'Fallout3', u'Fallout3.esm', u'fallout3', loot.GameType.fo3), (u'FalloutNV', u'FalloutNV.esm', u'falloutnv', loot.GameType.fonv), (u'Fallout4', u'Fallout4.esm', u'fallout4', loot.GameType.fo4), ] for game_name, master_name, repository, game_type in game_data: game_install_path = mock_game_install(master_name) masterlist_path = os.path.join(game_install_path, u"masterlist.yaml") game_dir = os.path.join(MOPY_PATH, u"Bash Patches", game_name) taglist_path = os.path.join(game_dir, u"taglist.yaml") if not os.path.exists(game_dir): LOGGER.error( u"Skipping taglist for {} as its output " u"directory does not exist".format(game_name) ) continue download_masterlist(repository, args.masterlist_version, masterlist_path) loot_game = loot.create_game_handle(game_type, game_install_path) loot_db = loot_game.get_database() loot_db.load_lists(masterlist_path) loot_db.write_minimal_list(taglist_path, True) LOGGER.info(u"{} masterlist converted.".format(game_name)) shutil.rmtree(game_install_path)
def main(verbosity=logging.INFO, logfile=LOGFILE, masterlist_version=MASTERLIST_VERSION): utils.setup_log(LOGGER, verbosity=verbosity, logfile=logfile) for game_name, repository in GAME_DATA.iteritems(): game_dir = os.path.join(MOPY_PATH, u'taglists', game_name) taglist_path = os.path.join(game_dir, u'taglist.yaml') if not os.path.exists(game_dir): os.makedirs(game_dir) download_masterlist(repository, masterlist_version, taglist_path) LOGGER.info(u'{} masterlist downloaded.'.format(game_name))
def main(args): utils.setup_log(LOGGER, verbosity=args.verbosity, logfile=args.logfile) download_dir = tempfile.mkdtemp() # if url is given in command line, always dl and install if not is_msvc_redist_installed(14, 15, 26429) or args.loot_msvc is not None: install_msvc_redist(download_dir, args.loot_msvc) if is_loot_api_installed(args.loot_version, args.loot_revision): LOGGER.info("Found LOOT API wrapper version {}.{}".format( args.loot_version, args.loot_revision)) else: install_loot_api(args.loot_version, args.loot_revision, download_dir, MOPY_PATH) os.rmdir(download_dir)
def main(_): conf.log = setup_log(logging.DEBUG) if conf.args['debug'] else setup_log() if sum([ 1 for x in ["cm.lib", "mm.lib", "mn.lib"] if os.path.isfile(os.path.join(conf.args['checkpoints'], x)) ]): conf.log.info("Checkpoints Found In {}".format( conf.args['checkpoints'])) else: conf.log.warn("Checkpoints Not Found In {}".format( conf.args['checkpoints'])) conf.log.info( "You Can Download Them Using : {} checkpoints download".format( sys.argv[0]))
def download(_): """ Start checkpoints download logic. :param _: None :return: None """ Conf.log = setup_log(logging.DEBUG) if Conf.args['debug'] else setup_log() tempdir = tempfile.mkdtemp() cdn_url = Conf.checkpoints_cdn.format(Conf.checkpoints_version) temp_zip = os.path.join(tempdir, "{}.zip".format(Conf.checkpoints_version)) try: Conf.log.info("Downloading {}".format(cdn_url)) dl_file(Conf.checkpoints_cdn.format(Conf.checkpoints_version), temp_zip) if not os.path.exists(Conf.args['checkpoints']['checkpoints_path']): os.mkdir(Conf.args['checkpoints']['checkpoints_path']) Conf.log.info("Extracting {}".format(temp_zip)) unzip(temp_zip, Conf.args['checkpoints']['checkpoints_path']) Conf.log.info("Moving Checkpoints To Final Location") for c in ("cm.lib", "mm.lib", "mn.lib"): if os.path.isfile( os.path.join(Conf.args['checkpoints']['checkpoints_path'], c)): os.remove( os.path.join(Conf.args['checkpoints']['checkpoints_path'], c)) shutil.move( os.path.join(Conf.args['checkpoints']['checkpoints_path'], 'checkpoints', c), Conf.args['checkpoints']['checkpoints_path']) shutil.rmtree( os.path.join(Conf.args['checkpoints']['checkpoints_path'], 'checkpoints')) except Exception as e: Conf.log.error(e) Conf.log.error( "Something Gone Bad Download Downloading The Checkpoints") shutil.rmtree(tempdir) sys.exit(1) shutil.rmtree(tempdir) Conf.log.info("Checkpoints Downloaded Successfully")
def train(parameters): model_folder = setup_log(parameters, 'train') set_seed(parameters['seed']) ################################### # Data Loading ################################### print('Loading training data ...') train_loader = DataLoader(parameters['train_data'], parameters) train_loader(embeds=parameters['embeds']) train_data = DocRelationDataset(train_loader, 'train', parameters, train_loader).__call__() print('\nLoading testing data ...') test_loader = DataLoader(parameters['test_data'], parameters) test_loader() test_data = DocRelationDataset(test_loader, 'test', parameters, train_loader).__call__() ################################### # Training ################################### trainer = Trainer(train_loader, parameters, {'train': train_data, 'test': test_data}, model_folder) trainer.run() if parameters['plot']: plot_learning_curve(trainer, model_folder) if parameters['save_model']: save_model(model_folder, trainer, train_loader)
def run(): Parser.parser = init_parser() if len(sys.argv) == 1: Parser.parser.print_usage() Parser.parser.exit() args = Parser.parser.parse_args() Conf.log = setup_log(logging.DEBUG) if args.debug else setup_log() args = config_args(Parser.parser, args) Conf.log.spam("Args : {}".format(args)) Conf.args = vars(args) args.func(args)
def __init__(self) -> None: """Initialization""" self.return_code = -1 self.state = self.State(self.State.START.value + 1) self.new_user = dict( username="******", password="******", full_name="John Doe", role=AuthenticationMessages.Role.OPERATOR.value, updated_full_name="John J. Doe", updated_password="******", updated_role=AuthenticationMessages.Role.ADMIN.value, ) self.settings = get_settings() self.logger = setup_log("AuthenticationExample", self.settings.log_level) self.client = Connections( hostname=self.settings.hostname, logger=self.logger, authentication_on_open=self.authentication_on_open, authentication_on_message=self.authentication_on_message, authentication_on_error=self.authentication_on_error, authentication_on_close=self.authentication_on_close, ) self.authentication = AuthenticationMessages( self.logger, self.settings.protocol_version)
def __init__(self) -> None: """Initialization""" self.return_code = -1 self.state = self.State(self.State.START.value + 1) self.authentication_thread = None self.metadata_thread = None self.realtime_situation_thread = None self.total_node_count = 0 self.loaded_node_count = 0 self.settings = get_settings() self.logger = setup_log("ComponentsInformationExample", self.settings.log_level) self.client = Connections( hostname=self.settings.hostname, logger=self.logger, authentication_on_open=self.authentication_on_open, authentication_on_message=self.authentication_on_message, authentication_on_error=self.authentication_on_error, authentication_on_close=self.authentication_on_close, metadata_on_open=self.metadata_on_open, metadata_on_message=self.metadata_on_message, metadata_on_error=self.metadata_on_error, metadata_on_close=self.metadata_on_close, realtime_situation_on_open=self.realtime_situation_on_open, realtime_situation_on_message=self.realtime_situation_on_message, realtime_situation_on_error=self.realtime_situation_on_error, realtime_situation_on_close=self.realtime_situation_on_close, ) self.messages = Messages(self.logger, self.settings.protocol_version)
def __init__(self) -> None: """Initialization""" self.return_code = -1 self.state = self.State(self.State.START.value + 1) self.authentication_thread = None self.metadata_thread = None self.settings = get_settings() self.logger = setup_log("BuildingExample", self.settings.log_level) self.client = Connections( hostname=self.settings.hostname, logger=self.logger, authentication_on_open=self.authentication_on_open, authentication_on_message=self.authentication_on_message, authentication_on_error=self.authentication_on_error, authentication_on_close=self.authentication_on_close, metadata_on_open=self.metadata_on_open, metadata_on_message=self.metadata_on_message, metadata_on_error=self.metadata_on_error, metadata_on_close=self.metadata_on_close, ) self.messages = BuildingMessages(self.logger, self.settings.protocol_version)
def train(parameters): model_folder = setup_log(parameters, 'train') set_seed(0) ################################### # Data Loading ################################### print('\nLoading training data ...') train_loader = DataLoader(parameters['train_data'], parameters) train_loader(embeds=parameters['embeds']) train_data = RelationDataset(train_loader, 'train', parameters['unk_w_prob'], train_loader).__call__() print('\nLoading testing data ...') test_loader = DataLoader(parameters['test_data'], parameters) test_loader() test_data = RelationDataset(test_loader, 'test', parameters['unk_w_prob'], train_loader).__call__() ################################### # TRAINING ################################### trainer = Trainer({ 'train': train_data, 'test': test_data }, parameters, train_loader, model_folder) trainer.run() trainer.eval_epoch(final=True, save_predictions=True) if parameters['plot']: plot_learning_curve(trainer, model_folder)
def main(): parser = argparse.ArgumentParser(description='Code Template') # model parser.add_argument('-nn', '--n_neurons', type=int, nargs='+', default=[32, 64], help='number of neurons (default: [32, 64])') parser.add_argument('-sc', '--use_skip_connection', action='store_true', default=False, help='use skip connection (default: False)') # hyperparameter parser.add_argument('-b', '--batch_size', type=int, default=32, help='batch size (default: 32)') parser.add_argument('-dr', '--drop_rate', type=float, default=0.8, help='drop rate (default: 0.8)') parser.add_argument('-lr', '--learning_rate', type=float, default=0.001, help='learning rate (default: 0.001)') # else parser.add_argument('--log_dir', default='logs', help='log directory to save') parser.add_argument('--log_level', default='debug', choices=['debug', 'info', 'warning', 'error'], help='default: debug') flags = parser.parse_args() setup_log(flags) model.train(flags) model.test(flags)
def __init__(self) -> None: """Initialization""" self.return_code = -1 self.state = self.State(self.State.START.value + 1) # Network id needs to point to a valid network with at least one sink # online self.network_id = 777555 self.sink_ids = None # Uncomment the line below for setting appconfig for specific sinks only # self.sink_ids = [1, 101] # When running more than once for the same network, the diagnostics interval # or application data needs to changed as WNT server will not try to set the # application configuration if it is already the same. self.diagnostics_interval = 60 self.application_data = "00112233445566778899AABBCCDDEEFF" self.is_override_on = False self.authentication_thread = None self.metadata_thread = None self.realtime_situation_thread = None self.total_node_count = 0 self.loaded_node_count = 0 self.settings = get_settings() self.logger = setup_log( "ApplicationConfigurationExample", self.settings.log_level ) self.client = Connections( hostname=self.settings.hostname, logger=self.logger, authentication_on_open=self.authentication_on_open, authentication_on_message=self.authentication_on_message, authentication_on_error=self.authentication_on_error, authentication_on_close=self.authentication_on_close, metadata_on_open=self.metadata_on_open, metadata_on_message=self.metadata_on_message, metadata_on_error=self.metadata_on_error, metadata_on_close=self.metadata_on_close, realtime_situation_on_open=self.realtime_situation_on_open, realtime_situation_on_message=self.realtime_situation_on_message, realtime_situation_on_error=self.realtime_situation_on_error, realtime_situation_on_close=self.realtime_situation_on_close, ) self.messages = Messages(self.logger, self.settings.protocol_version)
def main(_): """ Main logic entry point """ conf.log = setup_log(logging.DEBUG) if conf.args['debug'] else setup_log() conf.log.debug("Args : {}".format(conf.args)) conf.log.info("Welcome to DreamPower") if conf.args['gpu_ids']: conf.log.info("GAN Processing Will Use GPU IDs: {}".format( conf.args['gpu_ids'])) else: conf.log.info("GAN Processing Will Use CPU") # Processing start = time.time() select_processing().run() conf.log.info("Done! We have taken {} seconds".format( round(time.time() - start, 2))) # Exit sys.exit()
def main(args): utils.setup_log(LOGGER, verbosity=args.verbosity, logfile=LOGFILE) # check nightly timestamp is different than previous if not check_timestamp(args.version): raise OSError("Aborting build due to equal nightly timestamps.") with handle_apps_folder(), update_file_version(args.version, args.commit): # Get repository files version_info = get_version_info(args.version) # create distributable directory utils.mkdir(DIST_PATH, exists_ok=True) if args.manual: LOGGER.info("Creating python source distributable...") pack_manual(args.version) if not args.standalone and not args.installer: return with build_executable(args.version, version_info): if args.standalone: LOGGER.info("Creating standalone distributable...") pack_standalone(args.version) if args.installer: LOGGER.info("Creating installer distributable...") pack_installer(args.nsis, args.version, version_info)
def test(parameters): model_folder = setup_log(parameters, 'test') print('\nLoading mappings ...') train_loader = load_mappings(model_folder) print('\nLoading testing data ...') test_loader = DataLoader(parameters['test_data'], parameters) test_loader() test_data = DocRelationDataset(test_loader, 'test', parameters, train_loader).__call__() m = Trainer(train_loader, parameters, {'train': [], 'test': test_data}, model_folder) trainer = load_model(model_folder, m) trainer.eval_epoch(final=True, save_predictions=True)
def __init__(self) -> None: """Initialization""" self.floor_plan_image_id = None self.floor_plan_image_thumbnail_id = None self.return_code = -1 self.state = self.State(self.State.START.value + 1) self.authentication_thread = None self.metadata_thread = None self.settings = get_settings() self.logger = setup_log("FloorPlanAreaExample", self.settings.log_level) self.client = Connections( hostname=self.settings.hostname, logger=self.logger, authentication_on_open=self.authentication_on_open, authentication_on_message=self.authentication_on_message, authentication_on_error=self.authentication_on_error, authentication_on_close=self.authentication_on_close, metadata_on_open=self.metadata_on_open, metadata_on_message=self.metadata_on_message, metadata_on_error=self.metadata_on_error, metadata_on_close=self.metadata_on_close, ) self.messages = Messages(self.logger, self.settings.protocol_version) script_path = os.path.dirname(os.path.realpath(__file__)) self.floor_plan_image_file_path = os.path.join( script_path, "assets/floor_plan.png" ) self.floor_plan_image_thumbnail_file_path = os.path.join( script_path, "assets/floor_plan_thumbnail.png" ) self.floor_plan_image_width = 8989 self.floor_plan_image_height = 4432 self.temp_floor_plan_image_file_path = ( self.floor_plan_image_file_path + ".tmp.png" ) self.temp_floor_plan_image_thumbnail_file_path = ( self.floor_plan_image_thumbnail_file_path + ".tmp.png" )
def main(args): utils.setup_log(LOGGER, verbosity=args.verbosity, logfile=LOGFILE) # check nightly timestamp is different than previous if not check_timestamp(args.version): raise OSError(u'Aborting build due to equal nightly timestamps.') with handle_apps_folder(), update_file_version(args.version, args.commit): # Get repository files version_info = get_version_info(args.version) # create distributable directory utils.mkdir(DIST_PATH, exists_ok=True) # Copy the license so it's included in the built releases license_real = os.path.join(ROOT_PATH, u'LICENSE.md') license_temp = os.path.join(MOPY_PATH, u'LICENSE.md') try: cpy(license_real, license_temp) # Check if we need to update the LOOT taglists if args.force_tl_update or taglists_need_update(): update_taglist.main() # Remember the last LOOT version we generated taglists for with open(TAGINFO, u'w') as out: out.write(update_taglist.MASTERLIST_VERSION) if args.manual: LOGGER.info(u'Creating python source distributable...') pack_manual(args.version) if not args.standalone and not args.installer: return with build_executable(args.version, version_info): if args.standalone: LOGGER.info(u'Creating standalone distributable...') pack_standalone(args.version) if args.installer: LOGGER.info(u'Creating installer distributable...') pack_installer(args.nsis, args.version, version_info) finally: # Clean up the temp copy of the license rm(license_temp)
def __init__(self): config = ConfigLoader() self.parameters = config.load_config() self.model_folder = setup_log(self.parameters, 'train') set_seed(0) ################################### # Data Loading ################################### print('\nLoading training data ...') self.train_loader = DataLoader(parameters['train_data'], self.parameters) train_loader(embeds=self.parameters['embeds']) self.train_data = RelationDataset(train_loader, 'train', self.parameters['unk_w_prob'], train_loader).__call__() print('\nLoading testing data ...') test_loader = DataLoader(self.parameters['test_data'], parameters) test_loader() self.test_data = RelationDataset(test_loader, 'test', self.parameters['unk_w_prob'], train_loader).__call__()
def test(parameters): print('*** Testing Model ***') model_folder = setup_log(parameters, 'test') print('Loading mappings ...') with open(os.path.join(model_folder, 'mappings.pkl'), 'rb') as f: loader = pkl.load(f) print('Loading testing data ...') test_loader = DataLoader(parameters['test_data'], parameters) test_loader.__call__() test_data = RelationDataset(test_loader, 'test', parameters['unk_w_prob'], loader).__call__() m = Trainer({ 'train': [], 'test': test_data }, parameters, loader, model_folder) trainer = load_model(model_folder, m) trainer.eval_epoch(final=True, save_predictions=True)
import feedparser import ftfy import googlemaps import newspaper import requests from PIL import Image from bs4 import BeautifulSoup from unidecode import unidecode from utils import setup_log, log, u8, u16, u32, u32_littleendian, s16 import importlib with open("./Channels/News_Channel/config.json", "rb") as f: config = json.load(f) if config["production"]: setup_log(config["sentry_url"], True) """Define information about news sources""" sources = { # urls string argument is category key # reference parse_feed "ap_english": { "name": "AP", "url": "https://afs-prod.appspot.com/api/v2/feed/tag?tags=%s", "lang": "en", "cat": collections.OrderedDict([("apf-usnews", "national"), ("apf-intlnews", "world"),
print(traceback.format_exc(), file=sys.stderr) _thread.interrupt_main() def main(): if len(sys.argv) != 3: raise ValueError('Missing argument') _, address, role = sys.argv server_address = (address, 10000) role = net.PLAYER if role == 'player' else net.SPECTATOR hw_interface = cd.HardwareInterface() display_args_glob = [None] command_queue = queue.Queue() render_thread = threading.Thread(target=display_updater, args=(hw_interface, display_args_glob, command_queue), daemon=True) render_thread.start() with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as base_socket: base_socket.connect(server_address) base_socket.setblocking(False) s = net.PacketSocket(base_socket) client = client_core.Client(s, hw_interface, display_args_glob, role, command_queue) client.run() if __name__ == '__main__': utils.setup_log(logfile='client.log') main()
import sys import time import utils from datetime import timedelta, datetime, date import rsa from . import newsdownload from datadog import statsd from utils import setup_log, log, mkdir_p, u8, u16, u32, u32_littleendian with open("./Channels/News_Channel/config.json", "rb") as f: config = json.load(f) # load config if config["production"]: setup_log(config["sentry_url"], True) # error logging sources = { "ap_english": { "topics_news": { "National News": "national", "International News": "world", "Sports": "sports", "Arts/Entertainment": "entertainment", "Business": "business", "Science/Health": "science", "Technology": "technology", "Oddities": "oddities", }, "languages": [1, 3, 4], "language_code": 1,
import textwrap import time import mysql.connector import requests import rsa from mysql.connector import errorcode from config import * from utils import setup_log, log, u8, u16, u32 from voteslists import * with open("./Channels/Everybody_Votes_Channel/config.json", "rb") as f: config = json.load(f) if config["production"]: setup_log(config["sentry_url"], False) print "Everybody Votes Channel File Generator \n" print "By John Pansera / Larsen Vallecillo / www.rc24.xyz \n" worldwide = 0 national = 0 national_results = 0 worldwide_results = 0 question_data = collections.OrderedDict() results = collections.OrderedDict() country_code = 49 country_count = 0 language_code = 1 languages = {} num = 0
import subprocess import sys import time import utils from datetime import timedelta, datetime, date import rsa from . import newsdownload from datadog import statsd from utils import setup_log, log, mkdir_p, u8, u16, u32, u32_littleendian with open("./Channels/News_Channel/config.json", "rb") as f: config = json.load(f) # load config if config["production"]: setup_log(config["sentry_url"], True) # error logging sources = { "ap_english": { "topics_news": collections.OrderedDict([ ("National News", "national"), ("International News", "world"), ("Sports", "sports"), ("Arts/Entertainment", "entertainment"), ("Business", "business"), ("Science/Health", "science"), ("Technology", "technology"), ("Oddities", "oddities") ]), "languages": [1, 3, 4], "language_code": 1,
import googlemaps import newspaper import requests import zlib from PIL import Image from bs4 import BeautifulSoup from unidecode import unidecode from utils import setup_log, log, u8, u16, u32, u32_littleendian, s16 import importlib with open("./Channels/News_Channel/config.json", "rb") as f: config = json.load(f) if config["production"]: setup_log(config["sentry_url"], True) # define information about news sources sources = { # urls string argument is category key # reference parse_feed "ap_english": { "name": "AP", "url": "https://storage.googleapis.com/afs-prod/feeds/%s.json.gz", "lang": "en", "cat": { "us-news": "national", "world-news": "world", "sports": "sports", "entertainment": "entertainment",
""" logging.info('starting median_unique') if not args.output_file: logging.error('no output file defined') sys.exit(-1) # init vars mf = MedianFinder(args.output_file) # read file bringing the word list tfile = TweetFile(args.input_file) for words in tfile.get_words(): # make an unique set of words and get its length mf.process_tweet(words) # save to file the current median mf.write_results() # log the program is finished logging.info('program finished') if __name__ == '__main__': args = parse_args() # run logging any error try: setup_log() main(args) except: logging.error(get_error()) logging.info('Exiting. Bye')
import poi from utils import Filename from utils import setup_log from utils import save_model if __name__ == "__main__": mdname = "bpr" fn = Filename("foursquare") setup_log(fn.log(mdname)) train_cks = poi.load_checkins(open(fn.train)) test_cks = poi.load_checkins(open(fn.test)) eva = poi.Evaluation(test_cks, full=False) def hook(model): eva.assess(model) save_model(model, "./model/model_%s_%i.pkl" % (mdname, model.current)) mf = poi.BPR(train_cks, learn_rate = 0.1, reg_user=0.08, reg_item=0.08, ) mf.train(after=hook)
twitter_stream = Stream(auth, Listener(cfg, stats)) logging.debug('Authenticated') # start monitoring tags logging.info('Monitoring tags: %s' % cfg['monitored_tags']) try: twitter_stream.filter(track=cfg['monitored_tags']) except KeyboardInterrupt: # user finished logging.info('Program ended by user') finally: # save the stats stats.save_info() if __name__ == '__main__': # single param, config file. parser = argparse.ArgumentParser(description='') parser.add_argument('config_file', nargs='?', help='a yaml file with the configuration for this process', default='./src/config.yaml') args = parser.parse_args() # run logging any error try: # read the file, set the logging level and call the main program config = yaml.load(file(args.config_file)) setup_log(config.get('logging_level')) main(config) except: logging.error(get_error())
# start monitoring tags logging.info('Monitoring tags: %s' % cfg['monitored_tags']) try: twitter_stream.filter(track=cfg['monitored_tags']) except KeyboardInterrupt: # user finished logging.info('Program ended by user') finally: # save the stats stats.save_info() if __name__ == '__main__': # single param, config file. parser = argparse.ArgumentParser(description='') parser.add_argument( 'config_file', nargs='?', help='a yaml file with the configuration for this process', default='./src/config.yaml') args = parser.parse_args() # run logging any error try: # read the file, set the logging level and call the main program config = yaml.load(file(args.config_file)) setup_log(config.get('logging_level')) main(config) except: logging.error(get_error())
from torch import nn from torch import optim from sklearn.preprocessing import StandardScaler from sklearn.externals import joblib import matplotlib.pyplot as plt import pandas as pd import numpy as np import utils from modules import Encoder, Decoder from custom_types import DaRnnNet, TrainData, TrainConfig from utils import numpy_to_tvar from constants import device logger = utils.setup_log() logger.info(f"Using computation device: {device}") TRAIN_SIZE = 5 * 10**3 VALI_SIZE = 5 * 10**3 def preprocess_data(dat, col_names) -> Tuple[TrainData, StandardScaler]: scale = StandardScaler().fit(dat) proc_dat = scale.transform(dat) mask = np.ones(proc_dat.shape[1], dtype=bool) dat_cols = list(dat.columns) for col_name in col_names: mask[dat_cols.index(col_name)] = False
logging.error('no output file defined') sys.exit(-1) # create the counting structure wc = WordsCounter(args.output_file) # read file bringing the word list tfile = TweetFile(args.input_file) for words in tfile.get_words(): # add each word in the dictionary wc.process_tweet(words) # save the info to output wc.write_results() logging.debug('Finished writing to %s' % args.output_file) # log the program is finished logging.info('program finished') if __name__ == '__main__': args = parse_args() # run logging any error try: setup_log() main(args) except: logging.error(get_error()) logging.info('Exiting. Bye')