def run():
root = Tk()
# Try making window a dialog if the system allows it.
try:
root.attributes('-type', 'dialog')
except TclError:
pass
args = parser.parse_args()
if args.manual:
player = UserPlayer(root)
else:
player = SelectedPlayer()
adversary = RandomAdversary(DEFAULT_SEED)
board = Board(BOARD_WIDTH, BOARD_HEIGHT)
def runner():
for move in board.run(player, adversary):
# When not playing manually, allow some time to see the move.
if not args.manual:
sleep(0.001)
Visual(board)
background = Thread(target=runner)
background.daemon = True
background.start()
root.mainloop()
raise SystemExit
def run():
board = Board(BOARD_WIDTH, BOARD_HEIGHT)
adversary = RandomAdversary(DEFAULT_SEED)
args = parser.parse_args()
if args.manual:
player = UserPlayer()
else:
player = SelectedPlayer()
pygame.init()
screen = pygame.display.set_mode([(BOARD_WIDTH + 6) * CELL_WIDTH,
BOARD_HEIGHT * CELL_HEIGHT])
clock = pygame.time.Clock()
# Set timer to force block down when no input is given.
pygame.time.set_timer(EVENT_FORCE_DOWN, INTERVAL)
for move in board.run(player, adversary):
render(screen, board)
pygame.display.flip()
# If we are not playing manually, clear the events.
if not args.manual:
check_stop()
clock.tick(FRAMES_PER_SECOND)
while True:
check_stop()
def main():
args = parser.parse_args()
modify_arguments(args)
# setting random seeds
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
with open(args.config_file, 'r') as stream:
config = yaml.load(stream)
args.config = Munch(modify_config(args, config))
logger.info(args)
if args.mode == 'train':
train.train(args, device)
elif args.mode == 'test':
pass
elif args.mode == 'analysis':
analysis.analyze(args, device)
elif args.mode == 'generate':
pass
elif args.mode == 'classify':
analysis.classify(args, device)
elif args.mode == 'classify_coqa':
analysis.classify_coqa(args, device)
elif args.mode == 'classify_final':
analysis.classify_final(args, device)
def run(window):
board = Board(BOARD_WIDTH, BOARD_HEIGHT)
adversary = RandomAdversary(DEFAULT_SEED)
args = parser.parse_args()
if args.manual:
window.timeout(INTERVAL)
player = UserPlayer(window)
else:
window.timeout(0)
player = SelectedPlayer()
for move in board.run(player, adversary):
render(window, board)
if not args.manual:
while True:
key = window.getch()
if key == -1:
break
elif key == curses.ascii.ESC:
raise SystemExit
sleep(0.1)
window.timeout(-1)
window.getch()
def run(window, seed):
global score
board = Board(BOARD_WIDTH, BOARD_HEIGHT)
adversary = RandomAdversary(seed, BLOCK_LIMIT)
args = parser.parse_args()
if args.manual:
window.timeout(INTERVAL)
player = UserPlayer(window)
else:
window.timeout(0)
player = SelectedPlayer()
try:
for move in board.run(player, adversary):
render(window, board)
if not args.manual:
while True:
key = window.getch()
if key == -1:
break
elif key == curses.ascii.ESC:
raise SystemExit
#sleep(0.1)
finally:
score = board.score
def main():
# Parse arguments
global args
args = parser.parse_args()
# Extract parameters
filename_left = args.CONFIG_LEFT[0]
filename_right = args.CONFIG_RIGHT[0]
# Start parsing
comparer = KComparer(filename_left, filename_right)
comparer.load_files()
comparer.parse_files()
# Compare
compare_result = comparer.start_compare()
# Apply filter and regex
filtered_compare_result = comparer.filter_results(
results=compare_result,
filter=args.filter[0],
regex_conf_name=args.regex[0]
)
# Output result
table_exporter = TableExporter(filtered_compare_result, filename_left, filename_right)
table_exporter.print_table()
# Print warning for duplicated items
comparer.warn_duplicated_items()
# At the end of program, close stderr stream to mute down BrokenPipeError info
# This is a disadvantage of Python...
sys.stderr.close()
def main():
"""The main method of script."""
args = parser.parse_args()
with open(args.config_file, 'r') as stream:
args.config = munchify(yaml.load(stream))
args.save_dir = os.path.join(args.save_dir, args.job_id)
args.best_dir = os.path.join(args.best_dir, args.job_id)
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
if not os.path.exists(args.best_dir):
os.makedirs(args.best_dir)
logger.info(args)
if args.mode == 'train':
train(args)
elif args.mode == 'test' or args.mode == 'valid':
test(args)
elif args.mode == 'generate':
generate(args)
def run(window):
board = Board(BOARD_WIDTH, BOARD_HEIGHT)
adversary = RandomAdversary(DEFAULT_SEED)
args = parser.parse_args()
if args.manual:
window.timeout(INTERVAL)
player = UserPlayer(window)
else:
window.timeout(0)
player = SelectedPlayer()
for move in board.run(player, adversary):
render(window, board)
if not args.manual:
while True:
key = window.getch()
if key == -1:
break
elif key == curses.ascii.ESC:
raise SystemExit
sleep(0.1)
# window.timeout(-1)
window.getch()
try:
fin = open('scores.txt', 'r')
content = fin.readlines()
fin.close()
content.append(str(board.score) + '\n')
except FileNotFoundError:
content = [f'{board.score}\n']
fin = open('scores.txt', 'w')
fin.writelines(content)
fin.close()
def main():
arguments = parser.parse_args()
command = parse_command(arguments)
command.validate()
command.execute()
def train():
args = parser.parse_args()
# logging is set to INFO (resp. WARN) for main (resp. auxiliary) process. logger.info => log main process only, logger.warning => log all processes
logging.basicConfig(level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
logger.warning("Running process %d", args.local_rank) # This is a logger.warning: it will be printed by all distributed processes
logger.info("Arguments: %s", pformat(args))
# Initialize distributed training if needed
args.distributed = (args.local_rank != -1)
if args.distributed:
torch.cuda.set_device(args.local_rank)
args.device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend='nccl', init_method='env://')
logger.info("Prepare tokenizer, pretrained model and optimizer - add special tokens for fine-tuning")
tokenizer_class = GPT2Tokenizer if "gpt2" in args.model_checkpoint else OpenAIGPTTokenizer
tokenizer = tokenizer_class.from_pretrained(args.model_checkpoint)
model_class = GPT2LMHeadModel if "gpt2" in args.model_checkpoint else OpenAIGPTLMHeadModel
model = model_class.from_pretrained(args.model_checkpoint)
tokenizer.set_special_tokens(SPECIAL_TOKENS)
model.set_num_special_tokens(len(SPECIAL_TOKENS))
model.to(args.device)
optimizer = OpenAIAdam(model.parameters(), lr=args.lr)
# Prepare model for FP16 and distributed training if needed (order is important, distributed should be the last)
if args.fp16:
from apex import amp # Apex is only required if we use fp16 training
model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16)
if args.distributed:
model = DistributedDataParallel(model, device_ids=[args.local_rank], output_device=args.local_rank)
logger.info("Prepare datasets")
train_loader, val_loader, train_sampler, valid_sampler = get_data_loaders(args, tokenizer)
# Training function and trainer
def update(engine, batch):
model.train()
batch = tuple(input_tensor.to(args.device) for input_tensor in batch)
lm_loss = model(*batch)
loss = lm_loss / args.gradient_accumulation_steps
if args.fp16:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_norm)
else:
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_norm)
if engine.state.iteration % args.gradient_accumulation_steps == 0:
optimizer.step()
optimizer.zero_grad()
return loss.item()
trainer = Engine(update)
# Evaluation function and evaluator (evaluator output is the input of the metrics)
def inference(engine, batch):
model.eval()
with torch.no_grad():
batch = tuple(input_tensor.to(args.device) for input_tensor in batch)
input_ids, lm_labels, token_type_ids = batch
# logger.info(tokenizer.decode(input_ids[0, :].tolist()))
model_outputs = model(input_ids, token_type_ids=token_type_ids)
lm_logits = model_outputs[0]
lm_logits_flat_shifted = lm_logits[..., :-1, :].contiguous().view(-1, lm_logits.size(-1))
lm_labels_flat_shifted = lm_labels[..., 1:].contiguous().view(-1)
return lm_logits_flat_shifted, lm_labels_flat_shifted
evaluator = Engine(inference)
# Attach evaluation to trainer: we evaluate when we start the training and at the end of each epoch
trainer.add_event_handler(Events.EPOCH_COMPLETED, lambda _: evaluator.run(val_loader))
if args.n_epochs < 1:
trainer.add_event_handler(Events.COMPLETED, lambda _: evaluator.run(val_loader))
if args.eval_before_start:
trainer.add_event_handler(Events.STARTED, lambda _: evaluator.run(val_loader))
# Make sure distributed data samplers split the dataset nicely between the distributed processes
if args.distributed:
trainer.add_event_handler(Events.EPOCH_STARTED, lambda engine: train_sampler.set_epoch(engine.state.epoch))
evaluator.add_event_handler(Events.EPOCH_STARTED, lambda engine: valid_sampler.set_epoch(engine.state.epoch))
# Linearly decrease the learning rate from lr to zero
scheduler = PiecewiseLinear(optimizer, "lr", [(0, args.lr), (args.n_epochs * len(train_loader), 0.0)])
trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)
# Prepare metrics - note how we compute distributed metrics
RunningAverage(output_transform=lambda x: x).attach(trainer, "loss")
metrics = {
"nll": Loss(torch.nn.CrossEntropyLoss(ignore_index=-1))
}
metrics.update({
"average_nll": MetricsLambda(average_distributed_scalar, metrics["nll"], args)
})
metrics["average_ppl"] = MetricsLambda(math.exp, metrics["average_nll"])
for name, metric in metrics.items():
metric.attach(evaluator, name)
# On the main process: add progress bar, tensorboard, checkpoints and save model, configuration and tokenizer before we start to train
if args.local_rank in [-1, 0]:
pbar = ProgressBar(persist=True)
pbar.attach(trainer, metric_names=["loss"])
evaluator.add_event_handler(Events.COMPLETED, lambda _: pbar.log_message("Validation: %s" % pformat(evaluator.state.metrics)))
tb_logger = TensorboardLogger(log_dir=args.output_dir)
tb_logger.attach(trainer, log_handler=OutputHandler(tag="training", metric_names=["loss"]), event_name=Events.ITERATION_COMPLETED)
tb_logger.attach(trainer, log_handler=OptimizerParamsHandler(optimizer), event_name=Events.ITERATION_STARTED)
tb_logger.attach(evaluator, log_handler=OutputHandler(tag="validation", metric_names=list(metrics.keys()), another_engine=trainer), event_name=Events.EPOCH_COMPLETED)
checkpoint_handler = ModelCheckpoint(tb_logger.writer.log_dir, 'checkpoint', save_interval=1, n_saved=3)
trainer.add_event_handler(Events.EPOCH_COMPLETED, checkpoint_handler, {'mymodel': getattr(model, 'module', model)}) # "getattr" take care of distributed encapsulation
torch.save(args, tb_logger.writer.log_dir + '/model_training_args.bin')
getattr(model, 'module', model).config.to_json_file(os.path.join(tb_logger.writer.log_dir, CONFIG_NAME))
tokenizer.save_vocabulary(tb_logger.writer.log_dir)
# Run the training
trainer.run(train_loader, max_epochs=args.n_epochs)
# On the main process: close tensorboard logger and rename the last checkpoint (for easy re-loading with OpenAIGPTModel.from_pretrained method)
if args.local_rank in [-1, 0] and args.n_epochs > 0:
os.rename(checkpoint_handler._saved[-1][1][-1], os.path.join(tb_logger.writer.log_dir, WEIGHTS_NAME)) # TODO: PR in ignite to have better access to saved file paths (cleaner)
tb_logger.close()
if not historic
]
if not dry:
calendar = get_calendar_service()
create_remote_appointments(appointments_to_be_created, calendar)
delete_remote_appointments(appointments_to_be_deleted, calendar)
appointments_to_be_cached = [
appointment for appointment in schedule_appointments
if not appointment.is_historic and appointment.remote_event_id
]
save_remote_appointments_to_cache(appointments_to_be_cached)
save_appointment_meta(appointment_meta)
else:
logging.info(
f"Would create {len(appointments_to_be_created)} event(s) in total."
)
logging.info(
f"Would delete {len(appointments_to_be_deleted)} event(s) in total."
)
if __name__ == '__main__':
options = parser.parse_args()
logging.basicConfig(level=logging.getLevelName(options.log_level))
main(options.dry)
import csv
import logging
import sys
import asyncio
import download_media
from arguments import parser
import twint
import os
args = parser.parse_args()
csv_file_path = f"{args.destination_directory.rstrip(os.sep)}{os.sep}{args.user}{os.sep}data.csv"
if os.path.isfile(csv_file_path):
logging.info(f"{csv_file_path} already exists; exiting.")
sys.exit(-1)
destination_directory = f"{args.destination_directory.rstrip(os.sep)}{os.sep}{args.user}"
os.makedirs(destination_directory, exist_ok=True)
twint_config = twint.Config()
twint_config.Username = args.user
twint_config.Store_csv = True
twint_config.Output = csv_file_path
twint_config.Profile_full = True
async def main():
with open(csv_file_path) as csv_vile:
for row in csv.DictReader(csv_vile, delimiter=","):
import threading, datetime
from pprint import pprint
sys.path.append('.')
import scons
import SCons.Script
import workers
from arguments import parser
from qt_meta import QTFile
from context_managers import filedescriptor, restore_cwd, eyecandy
from utils import Eyecandy
import conf
# C:/Users/kosi/AppData/Local/Amazon/Kindle/application/Kindle.exe
args = parser.parse_args([r'C:\Users\kosii\AppData\Local\Amazon\Kindle\application\Kindle.exe'])
conf.debug = bool(args.debug)
if args.do_not_regenerate:
with filedescriptor(args.input_file, os.O_RDONLY) as fd:
with contextlib.closing(mmap.mmap(fd, length=0, access=mmap.ACCESS_READ)) as mmapped_file:
with open('injector/injected.cpp', 'w') as injected_dll_source:
print 'Identifying Qt classes in {input_file} ...'.format(input_file=args.input_file)
with eyecandy():
injected_dll_source.write(QTFile(mmapped_file, n=args.n).render())
if args.do_not_recompile:
with restore_cwd():
os.environ['SCONSFLAGS'] = "-C injector -Q -s"
try:
print "Compiling injected dll ..."
