def download(_):
conf.log = setup_log(logging.DEBUG) if conf.args['debug'] else setup_log()
tempdir = tempfile.mkdtemp()
cdn_url = conf.checkpoints_cdn.format(conf.checkpoints_version)
temp_zip = os.path.join(tempdir, "{}.zip".format(conf.checkpoints_version))
try:
conf.log.info("Downloading {}".format(cdn_url))
dl_file(conf.checkpoints_cdn.format(conf.checkpoints_version),
temp_zip)
conf.log.info("Extracting {}".format(temp_zip))
unzip(temp_zip, conf.args['checkpoints'])
conf.log.info("Moving Checkpoints To Final Location")
for c in ("cm.lib", "mm.lib", "mn.lib"):
if os.path.isfile(os.path.join(conf.args['checkpoints'], c)):
os.remove(os.path.join(conf.args['checkpoints'], c))
shutil.move(
os.path.join(conf.args['checkpoints'], 'checkpoints', c),
conf.args['checkpoints'])
shutil.rmtree(os.path.join(conf.args['checkpoints'], 'checkpoints'))
except Exception as e:
conf.log.error(e)
conf.log.error(
"Something Gone Bad Download Downloading The Checkpoints")
shutil.rmtree(tempdir)
sys.exit(1)
shutil.rmtree(tempdir)
conf.log.info("Checkpoints Downloaded Successfully")
def main(args):
utils.setup_log(LOGGER, verbosity=args.verbosity, logfile=args.logfile)
LOGGER.debug(
u'Loaded the LOOT API v{} using wrapper version {}'.format(
loot.Version.string(), loot.WrapperVersion.string()
)
)
game_data = [
(u'Oblivion', u'Oblivion.esm', u'oblivion', loot.GameType.tes4),
(u'Skyrim', u'Skyrim.esm', u'skyrim', loot.GameType.tes5),
(u'SkyrimSE', u'Skyrim.esm', u'skyrimse', loot.GameType.tes5se),
(u'Fallout3', u'Fallout3.esm', u'fallout3', loot.GameType.fo3),
(u'FalloutNV', u'FalloutNV.esm', u'falloutnv', loot.GameType.fonv),
(u'Fallout4', u'Fallout4.esm', u'fallout4', loot.GameType.fo4),
]
for game_name, master_name, repository, game_type in game_data:
game_install_path = mock_game_install(master_name)
masterlist_path = os.path.join(game_install_path, u"masterlist.yaml")
game_dir = os.path.join(MOPY_PATH, u"Bash Patches", game_name)
taglist_path = os.path.join(game_dir, u"taglist.yaml")
if not os.path.exists(game_dir):
LOGGER.error(
u"Skipping taglist for {} as its output "
u"directory does not exist".format(game_name)
)
continue
download_masterlist(repository, args.masterlist_version, masterlist_path)
loot_game = loot.create_game_handle(game_type, game_install_path)
loot_db = loot_game.get_database()
loot_db.load_lists(masterlist_path)
loot_db.write_minimal_list(taglist_path, True)
LOGGER.info(u"{} masterlist converted.".format(game_name))
shutil.rmtree(game_install_path)
def main(verbosity=logging.INFO,
logfile=LOGFILE,
masterlist_version=MASTERLIST_VERSION):
utils.setup_log(LOGGER, verbosity=verbosity, logfile=logfile)
for game_name, repository in GAME_DATA.iteritems():
game_dir = os.path.join(MOPY_PATH, u'taglists', game_name)
taglist_path = os.path.join(game_dir, u'taglist.yaml')
if not os.path.exists(game_dir):
os.makedirs(game_dir)
download_masterlist(repository, masterlist_version, taglist_path)
LOGGER.info(u'{} masterlist downloaded.'.format(game_name))
def main(args):
utils.setup_log(LOGGER, verbosity=args.verbosity, logfile=args.logfile)
download_dir = tempfile.mkdtemp()
# if url is given in command line, always dl and install
if not is_msvc_redist_installed(14, 15,
26429) or args.loot_msvc is not None:
install_msvc_redist(download_dir, args.loot_msvc)
if is_loot_api_installed(args.loot_version, args.loot_revision):
LOGGER.info("Found LOOT API wrapper version {}.{}".format(
args.loot_version, args.loot_revision))
else:
install_loot_api(args.loot_version, args.loot_revision, download_dir,
MOPY_PATH)
os.rmdir(download_dir)
def main(_):
conf.log = setup_log(logging.DEBUG) if conf.args['debug'] else setup_log()
if sum([
1 for x in ["cm.lib", "mm.lib", "mn.lib"]
if os.path.isfile(os.path.join(conf.args['checkpoints'], x))
]):
conf.log.info("Checkpoints Found In {}".format(
conf.args['checkpoints']))
else:
conf.log.warn("Checkpoints Not Found In {}".format(
conf.args['checkpoints']))
conf.log.info(
"You Can Download Them Using : {} checkpoints download".format(
sys.argv[0]))
def download(_):
"""
Start checkpoints download logic.
:param _: None
:return: None
"""
Conf.log = setup_log(logging.DEBUG) if Conf.args['debug'] else setup_log()
tempdir = tempfile.mkdtemp()
cdn_url = Conf.checkpoints_cdn.format(Conf.checkpoints_version)
temp_zip = os.path.join(tempdir, "{}.zip".format(Conf.checkpoints_version))
try:
Conf.log.info("Downloading {}".format(cdn_url))
dl_file(Conf.checkpoints_cdn.format(Conf.checkpoints_version),
temp_zip)
if not os.path.exists(Conf.args['checkpoints']['checkpoints_path']):
os.mkdir(Conf.args['checkpoints']['checkpoints_path'])
Conf.log.info("Extracting {}".format(temp_zip))
unzip(temp_zip, Conf.args['checkpoints']['checkpoints_path'])
Conf.log.info("Moving Checkpoints To Final Location")
for c in ("cm.lib", "mm.lib", "mn.lib"):
if os.path.isfile(
os.path.join(Conf.args['checkpoints']['checkpoints_path'],
c)):
os.remove(
os.path.join(Conf.args['checkpoints']['checkpoints_path'],
c))
shutil.move(
os.path.join(Conf.args['checkpoints']['checkpoints_path'],
'checkpoints', c),
Conf.args['checkpoints']['checkpoints_path'])
shutil.rmtree(
os.path.join(Conf.args['checkpoints']['checkpoints_path'],
'checkpoints'))
except Exception as e:
Conf.log.error(e)
Conf.log.error(
"Something Gone Bad Download Downloading The Checkpoints")
shutil.rmtree(tempdir)
sys.exit(1)
shutil.rmtree(tempdir)
Conf.log.info("Checkpoints Downloaded Successfully")
def train(parameters):
model_folder = setup_log(parameters, 'train')
set_seed(parameters['seed'])
###################################
# Data Loading
###################################
print('Loading training data ...')
train_loader = DataLoader(parameters['train_data'], parameters)
train_loader(embeds=parameters['embeds'])
train_data = DocRelationDataset(train_loader, 'train', parameters, train_loader).__call__()
print('\nLoading testing data ...')
test_loader = DataLoader(parameters['test_data'], parameters)
test_loader()
test_data = DocRelationDataset(test_loader, 'test', parameters, train_loader).__call__()
###################################
# Training
###################################
trainer = Trainer(train_loader, parameters, {'train': train_data, 'test': test_data}, model_folder)
trainer.run()
if parameters['plot']:
plot_learning_curve(trainer, model_folder)
if parameters['save_model']:
save_model(model_folder, trainer, train_loader)
def run():
Parser.parser = init_parser()
if len(sys.argv) == 1:
Parser.parser.print_usage()
Parser.parser.exit()
args = Parser.parser.parse_args()
Conf.log = setup_log(logging.DEBUG) if args.debug else setup_log()
args = config_args(Parser.parser, args)
Conf.log.spam("Args : {}".format(args))
Conf.args = vars(args)
args.func(args)
def __init__(self) -> None:
"""Initialization"""
self.return_code = -1
self.state = self.State(self.State.START.value + 1)
self.new_user = dict(
username="******",
password="******",
full_name="John Doe",
role=AuthenticationMessages.Role.OPERATOR.value,
updated_full_name="John J. Doe",
updated_password="******",
updated_role=AuthenticationMessages.Role.ADMIN.value,
)
self.settings = get_settings()
self.logger = setup_log("AuthenticationExample",
self.settings.log_level)
self.client = Connections(
hostname=self.settings.hostname,
logger=self.logger,
authentication_on_open=self.authentication_on_open,
authentication_on_message=self.authentication_on_message,
authentication_on_error=self.authentication_on_error,
authentication_on_close=self.authentication_on_close,
)
self.authentication = AuthenticationMessages(
self.logger, self.settings.protocol_version)
def __init__(self) -> None:
"""Initialization"""
self.return_code = -1
self.state = self.State(self.State.START.value + 1)
self.authentication_thread = None
self.metadata_thread = None
self.realtime_situation_thread = None
self.total_node_count = 0
self.loaded_node_count = 0
self.settings = get_settings()
self.logger = setup_log("ComponentsInformationExample",
self.settings.log_level)
self.client = Connections(
hostname=self.settings.hostname,
logger=self.logger,
authentication_on_open=self.authentication_on_open,
authentication_on_message=self.authentication_on_message,
authentication_on_error=self.authentication_on_error,
authentication_on_close=self.authentication_on_close,
metadata_on_open=self.metadata_on_open,
metadata_on_message=self.metadata_on_message,
metadata_on_error=self.metadata_on_error,
metadata_on_close=self.metadata_on_close,
realtime_situation_on_open=self.realtime_situation_on_open,
realtime_situation_on_message=self.realtime_situation_on_message,
realtime_situation_on_error=self.realtime_situation_on_error,
realtime_situation_on_close=self.realtime_situation_on_close,
)
self.messages = Messages(self.logger, self.settings.protocol_version)
def __init__(self) -> None:
"""Initialization"""
self.return_code = -1
self.state = self.State(self.State.START.value + 1)
self.authentication_thread = None
self.metadata_thread = None
self.settings = get_settings()
self.logger = setup_log("BuildingExample", self.settings.log_level)
self.client = Connections(
hostname=self.settings.hostname,
logger=self.logger,
authentication_on_open=self.authentication_on_open,
authentication_on_message=self.authentication_on_message,
authentication_on_error=self.authentication_on_error,
authentication_on_close=self.authentication_on_close,
metadata_on_open=self.metadata_on_open,
metadata_on_message=self.metadata_on_message,
metadata_on_error=self.metadata_on_error,
metadata_on_close=self.metadata_on_close,
)
self.messages = BuildingMessages(self.logger,
self.settings.protocol_version)
def train(parameters):
model_folder = setup_log(parameters, 'train')
set_seed(0)
###################################
# Data Loading
###################################
print('\nLoading training data ...')
train_loader = DataLoader(parameters['train_data'], parameters)
train_loader(embeds=parameters['embeds'])
train_data = RelationDataset(train_loader, 'train',
parameters['unk_w_prob'],
train_loader).__call__()
print('\nLoading testing data ...')
test_loader = DataLoader(parameters['test_data'], parameters)
test_loader()
test_data = RelationDataset(test_loader, 'test', parameters['unk_w_prob'],
train_loader).__call__()
###################################
# TRAINING
###################################
trainer = Trainer({
'train': train_data,
'test': test_data
}, parameters, train_loader, model_folder)
trainer.run()
trainer.eval_epoch(final=True, save_predictions=True)
if parameters['plot']:
plot_learning_curve(trainer, model_folder)
def main():
parser = argparse.ArgumentParser(description='Code Template')
# model
parser.add_argument('-nn',
'--n_neurons',
type=int,
nargs='+',
default=[32, 64],
help='number of neurons (default: [32, 64])')
parser.add_argument('-sc',
'--use_skip_connection',
action='store_true',
default=False,
help='use skip connection (default: False)')
# hyperparameter
parser.add_argument('-b',
'--batch_size',
type=int,
default=32,
help='batch size (default: 32)')
parser.add_argument('-dr',
'--drop_rate',
type=float,
default=0.8,
help='drop rate (default: 0.8)')
parser.add_argument('-lr',
'--learning_rate',
type=float,
default=0.001,
help='learning rate (default: 0.001)')
# else
parser.add_argument('--log_dir',
default='logs',
help='log directory to save')
parser.add_argument('--log_level',
default='debug',
choices=['debug', 'info', 'warning', 'error'],
help='default: debug')
flags = parser.parse_args()
setup_log(flags)
model.train(flags)
model.test(flags)
def __init__(self) -> None:
"""Initialization"""
self.return_code = -1
self.state = self.State(self.State.START.value + 1)
# Network id needs to point to a valid network with at least one sink
# online
self.network_id = 777555
self.sink_ids = None
# Uncomment the line below for setting appconfig for specific sinks only
# self.sink_ids = [1, 101]
# When running more than once for the same network, the diagnostics interval
# or application data needs to changed as WNT server will not try to set the
# application configuration if it is already the same.
self.diagnostics_interval = 60
self.application_data = "00112233445566778899AABBCCDDEEFF"
self.is_override_on = False
self.authentication_thread = None
self.metadata_thread = None
self.realtime_situation_thread = None
self.total_node_count = 0
self.loaded_node_count = 0
self.settings = get_settings()
self.logger = setup_log(
"ApplicationConfigurationExample", self.settings.log_level
)
self.client = Connections(
hostname=self.settings.hostname,
logger=self.logger,
authentication_on_open=self.authentication_on_open,
authentication_on_message=self.authentication_on_message,
authentication_on_error=self.authentication_on_error,
authentication_on_close=self.authentication_on_close,
metadata_on_open=self.metadata_on_open,
metadata_on_message=self.metadata_on_message,
metadata_on_error=self.metadata_on_error,
metadata_on_close=self.metadata_on_close,
realtime_situation_on_open=self.realtime_situation_on_open,
realtime_situation_on_message=self.realtime_situation_on_message,
realtime_situation_on_error=self.realtime_situation_on_error,
realtime_situation_on_close=self.realtime_situation_on_close,
)
self.messages = Messages(self.logger, self.settings.protocol_version)
def main(_):
"""
Main logic entry point
"""
conf.log = setup_log(logging.DEBUG) if conf.args['debug'] else setup_log()
conf.log.debug("Args : {}".format(conf.args))
conf.log.info("Welcome to DreamPower")
if conf.args['gpu_ids']:
conf.log.info("GAN Processing Will Use GPU IDs: {}".format(
conf.args['gpu_ids']))
else:
conf.log.info("GAN Processing Will Use CPU")
# Processing
start = time.time()
select_processing().run()
conf.log.info("Done! We have taken {} seconds".format(
round(time.time() - start, 2)))
# Exit
sys.exit()
def main(args):
utils.setup_log(LOGGER, verbosity=args.verbosity, logfile=LOGFILE)
# check nightly timestamp is different than previous
if not check_timestamp(args.version):
raise OSError("Aborting build due to equal nightly timestamps.")
with handle_apps_folder(), update_file_version(args.version, args.commit):
# Get repository files
version_info = get_version_info(args.version)
# create distributable directory
utils.mkdir(DIST_PATH, exists_ok=True)
if args.manual:
LOGGER.info("Creating python source distributable...")
pack_manual(args.version)
if not args.standalone and not args.installer:
return
with build_executable(args.version, version_info):
if args.standalone:
LOGGER.info("Creating standalone distributable...")
pack_standalone(args.version)
if args.installer:
LOGGER.info("Creating installer distributable...")
pack_installer(args.nsis, args.version, version_info)
def test(parameters):
model_folder = setup_log(parameters, 'test')
print('\nLoading mappings ...')
train_loader = load_mappings(model_folder)
print('\nLoading testing data ...')
test_loader = DataLoader(parameters['test_data'], parameters)
test_loader()
test_data = DocRelationDataset(test_loader, 'test', parameters, train_loader).__call__()
m = Trainer(train_loader, parameters, {'train': [], 'test': test_data}, model_folder)
trainer = load_model(model_folder, m)
trainer.eval_epoch(final=True, save_predictions=True)
def __init__(self) -> None:
"""Initialization"""
self.floor_plan_image_id = None
self.floor_plan_image_thumbnail_id = None
self.return_code = -1
self.state = self.State(self.State.START.value + 1)
self.authentication_thread = None
self.metadata_thread = None
self.settings = get_settings()
self.logger = setup_log("FloorPlanAreaExample", self.settings.log_level)
self.client = Connections(
hostname=self.settings.hostname,
logger=self.logger,
authentication_on_open=self.authentication_on_open,
authentication_on_message=self.authentication_on_message,
authentication_on_error=self.authentication_on_error,
authentication_on_close=self.authentication_on_close,
metadata_on_open=self.metadata_on_open,
metadata_on_message=self.metadata_on_message,
metadata_on_error=self.metadata_on_error,
metadata_on_close=self.metadata_on_close,
)
self.messages = Messages(self.logger, self.settings.protocol_version)
script_path = os.path.dirname(os.path.realpath(__file__))
self.floor_plan_image_file_path = os.path.join(
script_path, "assets/floor_plan.png"
)
self.floor_plan_image_thumbnail_file_path = os.path.join(
script_path, "assets/floor_plan_thumbnail.png"
)
self.floor_plan_image_width = 8989
self.floor_plan_image_height = 4432
self.temp_floor_plan_image_file_path = (
self.floor_plan_image_file_path + ".tmp.png"
)
self.temp_floor_plan_image_thumbnail_file_path = (
self.floor_plan_image_thumbnail_file_path + ".tmp.png"
)
def main(args):
utils.setup_log(LOGGER, verbosity=args.verbosity, logfile=LOGFILE)
# check nightly timestamp is different than previous
if not check_timestamp(args.version):
raise OSError(u'Aborting build due to equal nightly timestamps.')
with handle_apps_folder(), update_file_version(args.version, args.commit):
# Get repository files
version_info = get_version_info(args.version)
# create distributable directory
utils.mkdir(DIST_PATH, exists_ok=True)
# Copy the license so it's included in the built releases
license_real = os.path.join(ROOT_PATH, u'LICENSE.md')
license_temp = os.path.join(MOPY_PATH, u'LICENSE.md')
try:
cpy(license_real, license_temp)
# Check if we need to update the LOOT taglists
if args.force_tl_update or taglists_need_update():
update_taglist.main()
# Remember the last LOOT version we generated taglists for
with open(TAGINFO, u'w') as out:
out.write(update_taglist.MASTERLIST_VERSION)
if args.manual:
LOGGER.info(u'Creating python source distributable...')
pack_manual(args.version)
if not args.standalone and not args.installer:
return
with build_executable(args.version, version_info):
if args.standalone:
LOGGER.info(u'Creating standalone distributable...')
pack_standalone(args.version)
if args.installer:
LOGGER.info(u'Creating installer distributable...')
pack_installer(args.nsis, args.version, version_info)
finally:
# Clean up the temp copy of the license
rm(license_temp)
def __init__(self):
config = ConfigLoader()
self.parameters = config.load_config()
self.model_folder = setup_log(self.parameters, 'train')
set_seed(0)
###################################
# Data Loading
###################################
print('\nLoading training data ...')
self.train_loader = DataLoader(parameters['train_data'], self.parameters)
train_loader(embeds=self.parameters['embeds'])
self.train_data = RelationDataset(train_loader, 'train', self.parameters['unk_w_prob'], train_loader).__call__()
print('\nLoading testing data ...')
test_loader = DataLoader(self.parameters['test_data'], parameters)
test_loader()
self.test_data = RelationDataset(test_loader, 'test', self.parameters['unk_w_prob'], train_loader).__call__()
def test(parameters):
print('*** Testing Model ***')
model_folder = setup_log(parameters, 'test')
print('Loading mappings ...')
with open(os.path.join(model_folder, 'mappings.pkl'), 'rb') as f:
loader = pkl.load(f)
print('Loading testing data ...')
test_loader = DataLoader(parameters['test_data'], parameters)
test_loader.__call__()
test_data = RelationDataset(test_loader, 'test', parameters['unk_w_prob'],
loader).__call__()
m = Trainer({
'train': [],
'test': test_data
}, parameters, loader, model_folder)
trainer = load_model(model_folder, m)
trainer.eval_epoch(final=True, save_predictions=True)
import feedparser
import ftfy
import googlemaps
import newspaper
import requests
from PIL import Image
from bs4 import BeautifulSoup
from unidecode import unidecode
from utils import setup_log, log, u8, u16, u32, u32_littleendian, s16
import importlib
with open("./Channels/News_Channel/config.json", "rb") as f:
config = json.load(f)
if config["production"]: setup_log(config["sentry_url"], True)
"""Define information about news sources"""
sources = {
# urls string argument is category key
# reference parse_feed
"ap_english": {
"name":
"AP",
"url":
"https://afs-prod.appspot.com/api/v2/feed/tag?tags=%s",
"lang":
"en",
"cat":
collections.OrderedDict([("apf-usnews", "national"),
("apf-intlnews", "world"),
print(traceback.format_exc(), file=sys.stderr)
_thread.interrupt_main()
def main():
if len(sys.argv) != 3:
raise ValueError('Missing argument')
_, address, role = sys.argv
server_address = (address, 10000)
role = net.PLAYER if role == 'player' else net.SPECTATOR
hw_interface = cd.HardwareInterface()
display_args_glob = [None]
command_queue = queue.Queue()
render_thread = threading.Thread(target=display_updater,
args=(hw_interface, display_args_glob,
command_queue),
daemon=True)
render_thread.start()
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as base_socket:
base_socket.connect(server_address)
base_socket.setblocking(False)
s = net.PacketSocket(base_socket)
client = client_core.Client(s, hw_interface, display_args_glob, role,
command_queue)
client.run()
if __name__ == '__main__':
utils.setup_log(logfile='client.log')
main()
import sys
import time
import utils
from datetime import timedelta, datetime, date
import rsa
from . import newsdownload
from datadog import statsd
from utils import setup_log, log, mkdir_p, u8, u16, u32, u32_littleendian
with open("./Channels/News_Channel/config.json", "rb") as f:
config = json.load(f) # load config
if config["production"]:
setup_log(config["sentry_url"], True) # error logging
sources = {
"ap_english": {
"topics_news": {
"National News": "national",
"International News": "world",
"Sports": "sports",
"Arts/Entertainment": "entertainment",
"Business": "business",
"Science/Health": "science",
"Technology": "technology",
"Oddities": "oddities",
},
"languages": [1, 3, 4],
"language_code": 1,
import textwrap
import time
import mysql.connector
import requests
import rsa
from mysql.connector import errorcode
from config import *
from utils import setup_log, log, u8, u16, u32
from voteslists import *
with open("./Channels/Everybody_Votes_Channel/config.json", "rb") as f:
config = json.load(f)
if config["production"]:
setup_log(config["sentry_url"], False)
print "Everybody Votes Channel File Generator \n"
print "By John Pansera / Larsen Vallecillo / www.rc24.xyz \n"
worldwide = 0
national = 0
national_results = 0
worldwide_results = 0
question_data = collections.OrderedDict()
results = collections.OrderedDict()
country_code = 49
country_count = 0
language_code = 1
languages = {}
num = 0
import subprocess
import sys
import time
import utils
from datetime import timedelta, datetime, date
import rsa
from . import newsdownload
from datadog import statsd
from utils import setup_log, log, mkdir_p, u8, u16, u32, u32_littleendian
with open("./Channels/News_Channel/config.json", "rb") as f:
config = json.load(f) # load config
if config["production"]: setup_log(config["sentry_url"], True) # error logging
sources = {
"ap_english": {
"topics_news": collections.OrderedDict([
("National News", "national"),
("International News", "world"),
("Sports", "sports"),
("Arts/Entertainment", "entertainment"),
("Business", "business"),
("Science/Health", "science"),
("Technology", "technology"),
("Oddities", "oddities")
]),
"languages": [1, 3, 4],
"language_code": 1,
import googlemaps
import newspaper
import requests
import zlib
from PIL import Image
from bs4 import BeautifulSoup
from unidecode import unidecode
from utils import setup_log, log, u8, u16, u32, u32_littleendian, s16
import importlib
with open("./Channels/News_Channel/config.json", "rb") as f:
config = json.load(f)
if config["production"]:
setup_log(config["sentry_url"], True)
# define information about news sources
sources = {
# urls string argument is category key
# reference parse_feed
"ap_english": {
"name": "AP",
"url": "https://storage.googleapis.com/afs-prod/feeds/%s.json.gz",
"lang": "en",
"cat": {
"us-news": "national",
"world-news": "world",
"sports": "sports",
"entertainment": "entertainment",
"""
logging.info('starting median_unique')
if not args.output_file:
logging.error('no output file defined')
sys.exit(-1)
# init vars
mf = MedianFinder(args.output_file)
# read file bringing the word list
tfile = TweetFile(args.input_file)
for words in tfile.get_words():
# make an unique set of words and get its length
mf.process_tweet(words)
# save to file the current median
mf.write_results()
# log the program is finished
logging.info('program finished')
if __name__ == '__main__':
args = parse_args()
# run logging any error
try:
setup_log()
main(args)
except:
logging.error(get_error())
logging.info('Exiting. Bye')
import poi
from utils import Filename
from utils import setup_log
from utils import save_model
if __name__ == "__main__":
mdname = "bpr"
fn = Filename("foursquare")
setup_log(fn.log(mdname))
train_cks = poi.load_checkins(open(fn.train))
test_cks = poi.load_checkins(open(fn.test))
eva = poi.Evaluation(test_cks, full=False)
def hook(model):
eva.assess(model)
save_model(model, "./model/model_%s_%i.pkl" % (mdname, model.current))
mf = poi.BPR(train_cks,
learn_rate = 0.1,
reg_user=0.08,
reg_item=0.08,
)
mf.train(after=hook)
twitter_stream = Stream(auth, Listener(cfg, stats))
logging.debug('Authenticated')
# start monitoring tags
logging.info('Monitoring tags: %s' % cfg['monitored_tags'])
try:
twitter_stream.filter(track=cfg['monitored_tags'])
except KeyboardInterrupt:
# user finished
logging.info('Program ended by user')
finally:
# save the stats
stats.save_info()
if __name__ == '__main__':
# single param, config file.
parser = argparse.ArgumentParser(description='')
parser.add_argument('config_file', nargs='?',
help='a yaml file with the configuration for this process',
default='./src/config.yaml')
args = parser.parse_args()
# run logging any error
try:
# read the file, set the logging level and call the main program
config = yaml.load(file(args.config_file))
setup_log(config.get('logging_level'))
main(config)
except:
logging.error(get_error())
# start monitoring tags
logging.info('Monitoring tags: %s' % cfg['monitored_tags'])
try:
twitter_stream.filter(track=cfg['monitored_tags'])
except KeyboardInterrupt:
# user finished
logging.info('Program ended by user')
finally:
# save the stats
stats.save_info()
if __name__ == '__main__':
# single param, config file.
parser = argparse.ArgumentParser(description='')
parser.add_argument(
'config_file',
nargs='?',
help='a yaml file with the configuration for this process',
default='./src/config.yaml')
args = parser.parse_args()
# run logging any error
try:
# read the file, set the logging level and call the main program
config = yaml.load(file(args.config_file))
setup_log(config.get('logging_level'))
main(config)
except:
logging.error(get_error())
from torch import nn
from torch import optim
from sklearn.preprocessing import StandardScaler
from sklearn.externals import joblib
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import utils
from modules import Encoder, Decoder
from custom_types import DaRnnNet, TrainData, TrainConfig
from utils import numpy_to_tvar
from constants import device
logger = utils.setup_log()
logger.info(f"Using computation device: {device}")
TRAIN_SIZE = 5 * 10**3
VALI_SIZE = 5 * 10**3
def preprocess_data(dat, col_names) -> Tuple[TrainData, StandardScaler]:
scale = StandardScaler().fit(dat)
proc_dat = scale.transform(dat)
mask = np.ones(proc_dat.shape[1], dtype=bool)
dat_cols = list(dat.columns)
for col_name in col_names:
mask[dat_cols.index(col_name)] = False
logging.error('no output file defined')
sys.exit(-1)
# create the counting structure
wc = WordsCounter(args.output_file)
# read file bringing the word list
tfile = TweetFile(args.input_file)
for words in tfile.get_words():
# add each word in the dictionary
wc.process_tweet(words)
# save the info to output
wc.write_results()
logging.debug('Finished writing to %s' % args.output_file)
# log the program is finished
logging.info('program finished')
if __name__ == '__main__':
args = parse_args()
# run logging any error
try:
setup_log()
main(args)
except:
logging.error(get_error())
logging.info('Exiting. Bye')
