def init():
global api
api = config.create_twitter_api()
logger.info('Bot Started: %s' % datetime.utcnow())
config.load_cfg()
imgpicker.init_img_list()
users.init_user_list()
def main():
dirname = sys.argv[1]
image_number = int(sys.argv[2])
(model_dir, img_dir, width, height, num_images, ensemble_size, max_levels, \
sampled, assignments, average_img) = config.load_cfg(dirname)
init(model_dir, img_dir)
print("image_number = %s, num_images = %s"\
% (str(image_number), str(num_images)))
test_data, network_data = kmeans2d.assignment_data_to_test_data(\
assignments, image_number, num_images, average_img)
input_map = np.zeros((height, width, 3), dtype = np.float64)
image_out = np.zeros((height, width, 3), dtype = np.float64)
thread_data = [(data, test_data, model_dir) for data in network_data]
start_time = time.clock()
pool = ThreadPool(8)
results = pool.map(predict_thread, thread_data)
pool.close()
pool.join()
for result in results:
# predictions
(level, network_id, start, count, predictions, test_out) = result
kmeans2d.predictions_to_image(image_out, test_out, predictions)
# debug data
prediction_out = np.zeros((height, width, 3), dtype = np.float64)
update_input_map(level, network_id, width, height, test_data,\
[level, network_id, start, count], input_map)
kmeans2d.predictions_to_image(prediction_out, test_out, predictions)
prediction_file_name = "render_images/prediction_%04d_%04d.png" %\
(level, network_id)
misc.imsave(prediction_file_name, prediction_out);
image_out = np.divide(image_out, float(ensemble_size))
image_out = np.clip(255.0 * image_out, 0.0, 255.0)
end_time = time.clock()
image_file_name = "render_images/" + str(sys.argv[2]) + '.png'
misc.imsave("render_images/render_input_map.png", input_map)
misc.imsave(image_file_name, image_out)
print("time = %5.5f" % (end_time - start_time))
print("saved %s" % (image_file_name))
def generate(load_model_path, max_sentences):
config.cfg = config.load_cfg(load_model_path)
config.cfg["load_model_path"] = load_model_path
p_reader = PartialsReader(max_sentences=max_sentences)
p_reader.load_dict(config.cfg["dictionary_name"])
p_reader.read_sentences(config.cfg["path"]["continuation"])
import model
m = model.Model(cfg=config.cfg)
m.build_forward_prop()
m.load_model(config.cfg["load_model_path"])
#Revert dictionary for generation
reverted_dict = dict([(y, x)
for x, y in list(p_reader.vocab_dict.items())])
m.generate(p_reader.id_sequences, reverted_dict)
def test(args):
cfg = load_cfg(args.cfg)
weight_path = args.wts
img_path = args.im_path
segnet = SegNet().float().cuda()
segnet.load_state_dict(torch.load(weight_path))
segnet.eval()
im = cv2.imread(img_path).transpose(2, 0, 1)
im = torch.tensor(im[np.newaxis, :], dtype=torch.float).cuda()
out = segnet(im)
out = out.detach().cpu().numpy().transpose(0, 2, 3, 1)
out = np.argmax(out, axis=3).astype(np.uint8)[0]
out = out[:, :, np.newaxis]
out = out * 20
cv2.imshow('f**k', out)
cv2.waitKey(0)
#!/usr/bin/env python3
import config
import sys
from mbot import bot
def usage_hint():
s = "\n" +\
"usage:\n" +\
" $ ./run.py <config>\n" +\
"\n" +\
"example:\n" +\
" $ ./run.py cfg/mb-bot.json"
return s
if __name__ == "__main__":
if len(sys.argv) == 1:
print(usage_hint())
sys.exit(0)
cfg = config.load_cfg(sys.argv[1])
bot.run(cfg["TOKEN"])
def train(cfg_file):
## config
cfg = load_cfg(cfg_file)
## logging
FORMAT = '%(levelname)s %(filename)s: %(message)s'
logging.basicConfig(level=logging.INFO, format=FORMAT, stream=sys.stdout)
logger = logging.getLogger(__name__)
## data loader
trainset = get_datasets(cfg.train.datasets, 'train')
valset = get_datasets(cfg.val.datasets, 'val')
trainloader = DataLoader(trainset,
batch_size = cfg.train.batch_size,
shuffle = True,
num_workers = 4,
drop_last = True)
valloader = DataLoader(valset,
batch_size = 4,
shuffle = True,
num_workers = 4,
drop_last = True)
## network and checkpoint
model = get_model(cfg).float().cuda()
print(model)
if cfg.model.class_weight is not None:
weight = torch.Tensor(cfg.model.class_weight) # ignore some labels or set weight
Loss = nn.CrossEntropyLoss(weight = weight).cuda()
## optimizer
optimizer = Optimizer(model.parameters(), cfg)
## checkpoint
save_path = cfg.out_path
if not os.path.exists(save_path): os.makedirs(save_path)
save_name = os.path.join(save_path, 'model.pytorch')
if os.path.exists(save_name): return
model_ckpts = os.listdir(save_path)
its = [int(os.path.splitext(el)[0].split('_')[2]) for el in model_ckpts if el[:5] == 'model']
ckpt_max_it = 0
if len(its) > 0:
ckpt_max_it = max(its)
logger.info('resume from checkpoint iter: {}'.format(ckpt_max_it))
model_ckpt = os.path.join(save_path, ''.join(['model_iter_', str(ckpt_max_it), '.pytorch']))
model.load_state_dict(torch.load(model_ckpt))
optimizer.load_checkpoint(save_path, ckpt_max_it)
start_it = ckpt_max_it + 1
## multi-gpu
model = nn.DataParallel(model, device_ids = None)
## train
result = AttrDict({
'train_loss': [],
'val_loss': [],
'cfg': cfg
})
trainiter = iter(trainloader)
for it in range(start_it, cfg.train.max_iter):
model.train()
optimizer.zero_grad()
try:
im, label = next(trainiter)
if not im.shape[0] == cfg.train.batch_size: continue
except StopIteration:
trainiter = iter(trainloader)
im, label = next(trainiter)
im = im.cuda().float()
num_class = cfg.model.num_class
logits = model(im)
label = resize_label(label, logits.shape[2:])
label = label.cuda().long().contiguous().view(-1, )
logits = logits.permute(0, 2, 3, 1).contiguous().view(-1, num_class)
loss = Loss(logits, label)
loss_value = loss.detach().cpu().numpy()
result.train_loss.append(loss_value)
loss.backward()
optimizer.step()
if it == 0: continue
if it % 20 == 0:
logger.info('iter: {}/{}, loss: {}'.format(it, cfg.train.max_iter, loss_value))
if it % cfg.val.valid_iter == 0:
valid_loss, acc_clss, acc_all = val_one_epoch(model, Loss, valloader, cfg)
result.val_loss.append(valid_loss)
if it % cfg.train.snapshot_iter == 0:
save_model_name = os.path.join(save_path, ''.join(['model_iter_', str(it), '.pytorch']))
logger.info('saving snapshot to: {}'.format(save_path))
torch.save(model.module.state_dict(), save_model_name)
optimizer.save_checkpoint(save_path, it)
logger.info('training done')
save_name = os.path.join(save_path, 'model.pytorch')
logger.info('saving model to: {}'.format(save_name))
model.cpu()
torch.save(model.module.state_dict(), save_name)
with open(save_path + '/result.pkl', 'wb') as fw:
pickle.dump(result, fw)
while True:
try:
im, label = next(trainiter)
except StopIteration:
break
print('everything done')
help='Specify the path to the config.yml file')
args = parser.parse_args()
writedb = args.writedb
diffs = args.diffs
refresh = args.refresh
wStatus = args.wStatus
updateConfig = args.updateConfig
outputErrorDF = args.outputErrorDF
debug_flag = "DEBUG" if args.debug_flag else "CRITICAL"
cmd_log_level = args.logger_level
config_path = Path(args.config_path.replace('"',"").strip()) / "config.yml"
import config
config.load_cfg(config_path)
cfg = config.cfg
wStatus_suffix = "_wStatus" if wStatus else ""
global export_path
global archive_path
global error_path
global log_path
export_path = cfg['informer']['export_path' + wStatus_suffix]
archive_path = cfg['ccdw']['archive_path' + wStatus_suffix]
error_path = cfg['ccdw']['error_path']
log_path = cfg['ccdw']['log_path']
prefix = cfg['informer']['prefix']
from config import load_cfg
from slowfast.models import SlowFast
cfg = load_cfg('./test_config.yaml')
model = SlowFast(cfg)
global writedb
global diffs
global refresh
global log
global cfg
run_datetime = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S%f")
def timestamp():
return (datetime.datetime.now().isoformat())
import config
config.load_cfg()
cfg = config.cfg
#with open("//helix/divisions/IERG/Data/config.yml","r") as ymlfile:
# cfg = yaml.load(ymlfile)
parser = argparse.ArgumentParser(description='Import CCDW data')
parser.add_argument(
'--nodb',
dest='writedb',
action='store_false',
default=True,
help='Do not write to database (default: write to database)')
parser.add_argument(
'--diffs',
dest='diffs',
def main():
cfg = config.cfg
timestamp = time.strftime('%Y-%m-%d--%H_%M_%S')
sys.stdout = Logger(timestamp)
# region Parameter arguments
parser = argparse.ArgumentParser()
parser.add_argument('--mode',
choices={'train', 'chat', 'test'},
default='train',
help="mode. if not specified, it's in the train mode")
parser.add_argument('--cornell',
action='store_true',
help="use the cornell movie dialogue corpus")
parser.add_argument(
'--conversations',
help="limit the number of conversations used in the dataset")
parser.add_argument(
'--model',
help=
'specify name (timestamp) of a previously used model. If none is provided then the '
'newest model available will be used.')
parser.add_argument(
'--load_iter',
help=
'if you do not want to use an intermediate version of the trained network, specify the '
'iteration number here')
parser.add_argument('test_file', type=str, nargs='?')
parser.add_argument(
'--test_conversations',
help="limit the number of test conversations used in the dataset")
parser.add_argument(
'--softmax',
action='store_true',
help='use standard softmax loss instead of sampled softmax loss')
parser.add_argument(
'--clear',
action='store_true',
help="delete all existing models to free up disk space")
parser.add_argument(
'--keep_prev',
action='store_true',
help='keep only the most recent version of the trained network')
parser.add_argument(
'--epochs',
help=
'how many times the network should pass over the entire dataset. Note: Due to random '
'bucketing, this is an approximation.')
parser.add_argument('--save_end',
action='store_true',
help='save the model ONLY at the end of training')
parser.add_argument(
'--processed_path',
help='Specify if you want to use exisiting preprocessed our_data')
parser.add_argument(
'--no_save_at_exit',
action='store_true',
help='deactivate automatic model saving at keyboard interrupt')
parser.add_argument('--combined_data',
action='store_true',
help='Both cornell data and our data will be used')
args = parser.parse_args()
if args.clear:
shutil.rmtree(cfg['MODELS_PATH'])
if not os.path.exists(cfg['MODELS_PATH']):
os.makedirs(cfg['MODELS_PATH'])
########### load old config file if necessary ############
if args.model:
# load the corresponding config file
cfg = config.load_cfg(args.model)
else:
if args.mode == 'chat':
# default mode: load the config of the last used model
saved_models = [
name for name in os.listdir(cfg['MODELS_PATH'])
if os.path.isdir(os.path.join(cfg['MODELS_PATH'], name))
]
saved_models = sorted(saved_models) # sorted in ascending order
last_model = saved_models[-1]
cfg = config.load_cfg(last_model)
# pp = pprint.PrettyPrinter(indent=4)
# pp.pprint(cfg)
else:
# create a new model
config.adapt_to_dataset(args.cornell, args.combined_data)
cfg['MODEL_NAME'] = timestamp
directory = os.path.join(cfg['MODELS_PATH'], cfg['MODEL_NAME'])
if not os.path.exists(directory):
os.makedirs(directory)
config.adapt_paths_to_model()
####### process other commmand line arguments ##############
if args.conversations:
cfg['MAX_TURNS'] = int(args.conversations)
if args.test_conversations:
cfg['TESTSET_SIZE'] = int(args.test_conversations)
if args.softmax:
cfg['STANDARD_SOFTMAX'] = args.softmax
if args.keep_prev:
cfg['KEEP_PREV'] = True
if args.load_iter:
args.load_iter = int(args.load_iter)
if args.epochs:
cfg['EPOCHS'] = int(args.epochs)
if args.processed_path:
cfg['PROCESSED_PATH'] = args.processed_path
vocab_sizes_dict = pickle.load(
open(os.path.join(cfg['PROCESSED_PATH'], "vocab_sizes"), "rb"))
cfg.update(vocab_sizes_dict)
if args.no_save_at_exit:
cfg['SAVE_AT_EXIT'] = False
# endregion
############## read our_data #################
if args.cornell:
import cornell_data as data
elif args.combined_data:
import data_reader as data
else:
import our_data as data
reader = data.Reader(cfg)
# reader = data.Reader(cfg)
if not os.path.isdir(
cfg['PROCESSED_PATH']
): # Not be done if we're in test mode, chat mode or continue training
reader.prepare_raw_data()
reader.process_data()
print('Data ready!')
########### start using the actual model##################
# create checkpoints folder if there isn't one already
reader.make_dir(cfg['CPT_PATH'])
bot = Chatbot(config=cfg, reader=reader)
if args.mode == 'train':
bot.train(args.save_end)
elif args.mode == 'chat':
bot.chat(args.load_iter)
elif args.mode == 'test':
bot.test_performance(args.test_file, args.load_iter)
