def main():
args = process_args()
os.environ["CUDA_VISIBLE_DEVICES"] = args.cuda
if args.seed == -1:
RANDOMSEED = None
else:
RANDOMSEED = args.seed
torch.manual_seed(RANDOMSEED)
torch.cuda.manual_seed(RANDOMSEED)
embedding = 'random'
torch.backends.cudnn.deterministic = True
config = Config(args, embedding, 'POR')
start_time = time.time()
print("Loading data...")
vocab, train_data, test_data = build_adaptive_dataset(config, args.word)
train_iter2 = build_iterator(train_data, config, doubly_flag=False)
test_iter = build_test_iterator(test_data, config)
print("Time used: ", get_time_dif(start_time))
config.n_vocab = len(vocab)
model = POR(config).cuda()
init_model(model)
print("start training...")
train_por(config, model, train_iter, train_iter2, test_iter)
def main():
args = process_args()
os.environ["CUDA_VISIBLE_DEVICES"] = args.cuda
if args.seed == -1:
RANDOMSEED = None
else:
RANDOMSEED = args.seed
torch.manual_seed(RANDOMSEED)
torch.cuda.manual_seed(RANDOMSEED)
IMP_WEIGHT = args.imp_weight
if not IMP_WEIGHT:
imp = torch.ones(args.class_num - 1, dtype=torch.float)
elif IMP_WEIGHT == 1:
pass
else:
raise ValueError('Incorrect importance weight parameter.')
imp = imp.cuda()
embedding = 'random'
torch.backends.cudnn.deterministic = True
config = Config(args, embedding, 'OR')
start_time = time.time()
print("Loading data...")
vocab, train_data, test_data = build_dataset(config, args.word)
train_iter = build_iterator(train_data, config, doubly_flag=False)
# train_iter = buil_random_iterator(train_data,config)
test_iter = build_test_iterator(test_data, config)
print("Time used: ", get_time_dif(start_time))
config.n_vocab = len(vocab)
model = OR(config).cuda()
init_model(model)
print("start training...")
train_or(config, model, train_iter, test_iter, imp)
def __init__(self, inifile):
c = ConfigParser.ConfigParser()
c.read(inifile)
uri = c.get("db", "uri")
model.init_model(model.sa.create_engine(uri))
self.networks = c.get("scan", "networks").split(";")
self.nmap_sudo = asbool(c.get("nmap", "use_sudo"))
def __init__(self, inifile):
c = ConfigParser.ConfigParser()
c.read(inifile)
uri = c.get("db","uri")
model.init_model(model.sa.create_engine(uri))
self.networks = c.get("scan", "networks").split(";")
self.nmap_sudo = asbool(c.get("nmap", "use_sudo"))
def __init__(self):
self.engine = create_engine(config.engine_url, **config.engine_params)
init_model(self.engine)
self.log = logging.getLogger('batch.%s' % (self.__class__.__name__,))
self.opt_parser = OptionParser()
self.opt_parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False)
self.arguments()
def compute_features(keyframes, path_out, layer='conv5_1', max_dim=340):
"""
store all local features from conv5_1 of vgg16 at 340 mac res and
store then in path_data/[mode]/layer/max_dim
"""
# create folder if it does not exist
if not os.path.exists(path_out):
os.makedirs(path_out)
# init model
model = init_model(layer)
# message
desc_text = "Feature extraction --> Layer: {}, Max_dim: {}, total_images={}".format(
layer, max_dim, keyframes.shape[0])
# process keyframes
for k, keyframe in tqdm(enumerate(keyframes),
ascii=True,
desc=desc_text):
feats = model.predict(preprocess_image(
keyframe, max_dim=max_dim)).squeeze(axis=0)
np.save(os.path.join(path_out, "{}".format(k)), feats)
# resume computation
else:
computed = np.array(
[int(k.split('.')[0]) for k in os.listdir(path_out)])
# if features has been computed...
if computed.shape[0] == keyframes.shape[0]:
return path_out
# start from the last computed...
elif computed.shape[0] == 0:
last = 0
else:
last = np.sort(computed)[::-1][0]
# init model
model = init_model(layer)
desc_text = "Feature extraction --> Layer: {}, Max_dim: {}, total_images={}".format(
layer, max_dim, keyframes.shape[0] - last)
for k, keyframe in tqdm(enumerate(keyframes[last:]),
ascii=True,
desc=desc_text):
feats = model.predict(preprocess_image(
keyframe, max_dim=max_dim)).squeeze(axis=0)
k += last
np.save(os.path.join(path_out, "{}".format(k)), feats)
return path_out
def init_classifier():
global model
global t
model = init_model()
model.load_weights(model_file_name)
with open('tokenizer.pickle', 'rb') as handle:
t = pickle.load(handle)
def main():
print("loading expert actions")
expert_behavior = 'data/cloning/experts/{}.p'.format(args.envname)
expert_policy = args.expert_policy_file
observations, actions = parse_expert.actions(expert_behavior)
obs_shape, action_shape = parse_expert.model_shapes(expert_behavior)
actions = actions.reshape(-1, action_shape)
print("standardize training data")
obs_mean, obs_sdev = parse_expert.standardizers(expert_policy)
obs_std = normalize(observations, obs_mean, obs_sdev)
model = init_model(obs_shape, action_shape)
results = defaultdict(list)
for i in range(11):
result = render_and_eval(model, obs_shape, action_shape, obs_mean,
obs_sdev)
results['clone_epoch'].append(result)
model.fit(obs_std, actions, epochs=5, batch_size=256)
results['expert'] = parse_expert.returns(expert_behavior)
results['epochs_per_result'] = 5
pickle.dump(results, open("results/cloning/{}.p".format(args.envname),
"wb"))
def train(cfg_name, resume):
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print(f'running on {device}')
cfg = load_cfg(cfg_name)
log = Logger(device=device)
envs = make_vec_envs(**cfg['env'])
model, n_start = init_model(cfg, envs, device, resume)
runner = EnvRunner(rollout_size=cfg['train']['rollout_size'],
envs=envs,
model=model,
device=device)
optim = ParamOptim(**cfg['optimizer'], params=model.parameters())
agent = Agent(model=model, optim=optim, **cfg['agent'])
cp_iter = cfg['train']['checkpoint_every']
log_iter = cfg['train']['log_every']
n_end = cfg['train']['steps']
cp_name = cfg['train']['checkpoint_name']
for n_iter, rollout in zip(trange(n_start, n_end), runner):
agent_log = agent.update(rollout)
if n_iter % log_iter == 0:
log.output({**agent_log, **runner.get_logs()}, n_iter)
if n_iter > n_start and n_iter % cp_iter == 0:
f = cp_name.format(n_iter=n_iter // cp_iter)
torch.save(model.state_dict(), f)
def train(img_dir, coords_file, model_file):
"""
Trains model on images from IMG folder with coordinates from COORDS
csv file and saves trained model in hdf5 file MODEL
"""
coords = read_coords(coords_file)
X = []
y = []
for filename in tqdm(coords.keys(), desc='Reading data'):
img = imread(join(img_dir, filename), as_grey=True)
points = coords[filename]
height, width = img.shape
img = resize(img, (100, 100))
points[::2] *= 100 / width
points[1::2] *= 100 / height
X.append(img)
y.append(points)
X = np.array(X).reshape((-1, 100, 100, 1))
y = np.array(y)
X = center_circle(X)
model = init_model()
model.fit(X, y, batch_size=128, epochs=1)
model.save(model_file)
def load():
encoder = tfds.features.text.SubwordTextEncoder.load_from_file(
"encoder.tf")
model = init_model(encoder.vocab_size)
model.load_weights("weights/weights")
print(encoder)
print(model)
return encoder, model
def render(cfg_name, steps):
cfg = load_cfg(cfg_name)
cfg['env']['num'] = 1
env = make_vec_envs(**cfg['env'])
model, n_start = init_model(cfg, env, 'cpu', resume=True)
assert n_start > 0
model.eval()
print(f'running {n_start}')
obs = env.reset()
for n_iter in trange(steps):
with torch.no_grad():
a = model(obs)[0].sample().unsqueeze(1)
obs = env.step(a)[0]
env.render()
time.sleep(1 / 30)
def main():
torch.manual_seed(1)
os.environ['CUDA_VISIBLE_DEVICES'] = gpu_devices
use_gpu = torch.cuda.is_available()
if use_cpu: use_gpu = False
if use_gpu:
print("Currently using GPU {}".format(gpu_devices))
cudnn.benchmark = True
torch.cuda.manual_seed_all(1)
else:
print("Currently using CPU (GPU is highly recommended)")
print("Initializing model: {}".format(arch))
model = init_model(name=arch, num_classes=576, loss_type=loss_type)
print("Model size: {:.3f} M".format(count_num_param(model)))
if use_gpu:
model = nn.DataParallel(model).cuda()
optimizer = init_optim(optim, model.parameters(), lr, weight_decay)
scheduler = lr_scheduler.MultiStepLR(optimizer,
milestones=stepsize,
gamma=gamma)
model.train()
cnt = 0
for epoch in range(start_epoch, max_epoch, 2):
for step in range(2):
x = torch.randn(1, 3, 200, 200)
y = torch.randint(low=0, high=576, size=(1, ), dtype=torch.int64)
if use_gpu:
x = x.cuda()
y = y.cuda()
scheduler.step()
cnt += 1
print(cnt, scheduler.get_lr())
output = model(x)
# loss = nn.CrossEntropyLoss()(output[0], y)
loss = torch.tensor(0.0, dtype=torch.float32).cuda()
# loss = torch.tensor(0.0, dtype=torch.float32)
loss.requires_grad = True
optimizer.zero_grad()
loss.backward()
print(loss)
print(loss._grad)
optimizer.step()
print('Done.')
def train():
# TODO: Move this to config
test_ratio = 1 / 4
# Receiving clean data arrays here
print("Parsing data...")
train_text, train_lbls, test_text, test_lbls, len1, len2 = parser2.get_clean_data(
)
# Shuffle data for more plain distribution
print(test_text.shape)
print(test_lbls.shape)
print(train_text.shape)
print(train_lbls.shape)
# TODO: Move this to config
print("Compiling model...")
# Init optimizer, loss function and metrics
optimizer = Adam(lr=0.1)
loss = 'sparse_categorical_crossentropy'
metrics = ['accuracy'] # ['categorical_accuracy']
m = model.init_model(len1, len2)
m.compile(optimizer=optimizer, loss=loss, metrics=metrics)
# TODO: Move this to config
print("Establishing checkpoints...")
checkpoint_path = "training/cp.ckpt"
# Train model with new callback
print("Training model...")
fit = m.fit(train_text,
train_lbls,
epochs=5,
validation_data=(test_text,
test_lbls)) # Pass callback to training
# valuate = m.evaluate(test_text, test_lbls)
m.save(f'training/model.h5')
print('Training complete:')
print(fit.history)
print("Accuracy: {}".format(fit.history['val_accuracy'][-1]))
print("Loss: {}".format(fit.history['val_loss'][-1]))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('expert_policy_file', type=str)
parser.add_argument('envname', type=str)
args = parser.parse_args()
expert_policy = args.expert_policy_file
expert_behavior = 'data/cloning/experts/{}.p'.format(args.envname)
print('loading and building expert policy')
expert_model = load_policy.load_policy(expert_policy)
print('loaded and built')
obs_shape, action_shape = parse_expert.model_shapes(expert_policy)
learning_model = init_model(obs_shape, action_shape)
obs_mean, obs_sdev = parse_expert.standardizers(expert_policy)
results = defaultdict(list)
with tf.Session():
tf_util.initialize()
for i in range(50):
print("iteration {}".format(i))
returns, observations, expert_actions = gather_observations(
learning_model, expert_model, args.envname, obs_mean, obs_sdev)
print('')
print("n resets: {}".format(len(returns)))
print("score: {} *-* {} *-* {}".format(
np.mean(returns) - np.std(returns), np.mean(returns),
np.mean(returns) + np.std(returns)))
results['dagger_epoch'].append(returns)
obs_std = normalize(observations, obs_mean, obs_sdev)
expert_actions = expert_actions.reshape(-1, action_shape)
learning_model.fit(obs_std,
expert_actions,
epochs=10,
batch_size=16)
results['expert'] = parse_expert.returns(expert_behavior)
pickle.dump(results,
open("results/dagger/{}.p".format(args.envname), "wb"))
def eval(args):
model = init_model()
state_dict = torch.load(args.weights, map_location=args.device)
model.load_state_dict(state_dict)
model.to(args.device)
model.eval()
transform_test = Compose([
ToTensor(),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
test_dataset = Imagenette(DATAPATH,
mode='val',
size=IMG_SIZE,
transform=transform_test)
test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=BATCH_SIZE,
shuffle=False,
num_workers=NUM_WORKERS,
pin_memory=True)
all_output = []
all_gt = []
with torch.no_grad():
for img, labels in tqdm(test_loader):
img = img.to(args.device).float()
labels = labels.to(args.device)
output = model(img)
all_gt.extend(labels.max(1)[1].cpu().numpy().tolist())
all_output.extend(output.max(1)[1].cpu().numpy().tolist())
# report = classification_report(all_gt, all_output)
accuracy = accuracy_score(all_gt, all_output)
# print(report)
print("\nTest Accuracy {:.4f}".format(accuracy))
#!/usr/bin/env python
import site
site.addsitedir('/srv/www/domains/mhjones.org')
site.addsitedir('/srv/www/domains/mhjones.org/virtualenv/lib/python2.6/site-packages')
from tornado.options import logging
import wsgiref
from sqlalchemy import create_engine
import tornado.wsgi
import config
from model import init_model
import handlers
log = logging.getLogger('app.wsgi')
application = tornado.wsgi.WSGIApplication(handlers.app_urls.urls, **config.http_params)
read_conn = create_engine(config.engine_url, **config.engine_params)
init_model(read_conn)
if __name__ == "__main__":
wsgiref.handlers.CGIHandler().run(application)
def test_users_model_api_database():
sys.path.insert(0, os.getcwd()+'/examples/user/database-model')
try:
from authkit.users.sqlalchemy_driver import UsersFromDatabase
except ImportError:
raise Exception("Could not run the SQLAlchemy tests, not installed")
if os.path.exists("test.db"):
os.remove("test.db")
import model as test_model
# Setup SQLAlchemy database engine
from sqlalchemy import engine_from_config
engine = engine_from_config({'sqlalchemy.url':'sqlite:///test.db'}, 'sqlalchemy.')
test_model.init_model(engine)
test_model.engine = engine
d = UsersFromDatabase(test_model)
test_model.meta.metadata.create_all(test_model.engine)
d.role_create("wiki")
d.role_create("adMin")
d.role_create("editor")
d.group_create("pyLOns")
d.group_create("dJAngo")
d.user_create("jaMEs", "passWOrd1", "pyLoNs")
d.user_create("ben", "password2")
d.user_create("Simon", "password3")
d.user_create("ian", "paSsword4")
assertEqual(d.list_roles(),["admin", "editor", "wiki"])
assertEqual(d.list_groups(),["django", "pylons"])
assertEqual(d.list_users(),['ben', 'ian', 'james', 'simon'])
assertEqual(d.user_has_password("james", "passWOrd1"), True)
assertEqual(d.user_has_password("james", "password1"), False)
d.role_create("test_role")
d.group_create("test_group")
d.user_create("test_user", "password")
assertEqual(d.list_roles(),["admin", "editor", "test_role", "wiki"])
assertEqual(d.list_groups(),["django", "pylons", "test_group"])
assertEqual(d.list_users(),['ben', 'ian', 'james', 'simon', "test_user"])
d.role_delete("test_role")
d.group_delete("test_group")
d.user_delete("test_user")
assertEqual(d.list_roles(),["admin", "editor", "wiki"])
assertEqual(d.list_groups(),["django", "pylons"])
assertEqual(d.list_users(),['ben', 'ian', 'james', 'simon'])
assertEqual(d.user_has_role("james", "admin"), False)
d.user_add_role("james", "admin")
assertEqual(d.user_has_role("james", "admin"), True)
d.user_remove_role("james", "admin")
assertEqual(d.user_has_role("james", "admin"), False)
d.user_add_role("james", "wiki")
d.user_add_role("simon", "wiki")
d.user_add_role("james", "admin")
#d.user_add_role("james", "editor")
d.user_add_role("ben", "editor")
assertEqual(d.user_has_group("james", "pylons"), True)
assertEqual(d.user_has_group("simon", None), True)
assertEqual(d.user_has_group("simon", "django"), False)
d.user_set_group("simon", "dJangO")
assertEqual(d.user_has_group("simon", None), False)
d.user_set_group("bEn", "PyLONS")
assertEqual(d.user_has_group("simon", "django"), True)
assertEqual(d.user_has_group("bEn", "pYlons"), True)
d.user_remove_group("bEn")
assertEqual(d.user_has_group("bEn", "pYlons"), False)
d.user_set_group("bEn", "PyLONS")
assertEqual(d.user_has_group("bEn", "pYlons"), True)
assertEqual(d.list_users(),['ben', 'ian', 'james', 'simon'])
d.user_set_username("james", "jim")
assertEqual(d.list_users(),['ben', 'ian', 'jim', 'simon'])
d.user_set_username("jim", "james")
from authkit.users import UsersFromFile, UsersFromString, AuthKitNoSuchUserError, AuthKitNoSuchGroupError,AuthKitNoSuchRoleError
string_data = """jaMEs:passWOrd1:pyLOns wiki adMin
ben:password2:pylons admin editor
simon:password3:dJAngo
ian:paSsword4 wiki
"""
filename = 'test/user_file_data.txt'
s = UsersFromString(string_data)
f = UsersFromFile(filename)
# Test Parsing
assertAllEqual(
s.passwords,
f.passwords,
{
'james':'passWOrd1',
'ben':'password2',
'simon':'password3',
'ian':'paSsword4',
},
)
assertAllEqual(
s.roles,
f.roles,
{
'james':['admin', 'wiki'],
'ben':['admin','editor'],
'ian':['wiki'],
'simon':[],
},
)
assertAllEqual(
s.groups,
f.groups,
{
'james':'pylons',
'ben':'pylons',
'ian': None,
'simon':'django',
},
)
assertAllEqual(
s.usernames,
f.usernames,
['ben', 'ian', 'james', 'simon'],
)
# Test list functions
assertAllEqual(
s.list_users(),
f.list_users(),
d.list_users(),
['ben', 'ian', 'james', 'simon'],
)
assertAllEqual(
s.list_roles(),
f.list_roles(),
d.list_roles(),
['admin', 'editor', 'wiki'],
)
assertAllEqual(
s.list_groups(),
f.list_groups(),
d.list_groups(),
['django','pylons'],
)
# Test user has functions
assertAllEqual(
s.user_has_role('jAMes','WiKi'),
f.user_has_role('jAMes','WiKi'),
d.user_has_role('jAMes','WiKi'),
True
)
assertAllEqual(
s.user_has_role('jAMes','editOr'),
f.user_has_role('jAMes','editOr'),
d.user_has_role('jAMes','editOr'),
False
)
assertAllEqual(
s.user_has_group('jAMeS','PyLons'),
f.user_has_group('jAMes','pylOns'),
d.user_has_group('jAMes','pylOns'),
True
)
assertAllEqual(
s.user_has_group('jameS','djaNgo'),
f.user_has_group('JAMes','djAngo'),
d.user_has_group('JAMes','djAngo'),
False
)
assertAllEqual(
s.user_has_password('jAMeS','passWOrd1'),
f.user_has_password('jAMes','passWOrd1'),
d.user_has_password('jAMes','passWOrd1'),
True
)
assertAllEqual(
s.user_has_password('jameS','PASSWORD1'),
f.user_has_password('JAMes','PASSWORD1'),
d.user_has_password('JAMes','PASSWORD1'),
False
)
# Existence Methods
assertAllEqual(
s.user_exists('jAMeS'),
f.user_exists('jAMes'),
d.user_exists('jAMes'),
True
)
assertAllEqual(
s.user_exists('nobody'),
f.user_exists('nobody'),
d.user_exists('nobody'),
False
)
# Existence Methods
assertAllEqual(
s.role_exists('wiKi'),
f.role_exists('Wiki'),
d.role_exists('Wiki'),
True
)
assertAllEqual(
s.role_exists('norole'),
f.role_exists('norole'),
d.role_exists('norole'),
False
)
assertAllEqual(
s.group_exists('pyLons'),
f.group_exists('PYlons'),
d.group_exists('PYlons'),
True
)
assertAllEqual(
s.group_exists('nogroup'),
f.group_exists('nogroup'),
d.group_exists('nogroup'),
False
)
# User Methods
assertAllEqual(
s.user('James'),
f.user('James'),
d.user('James'),
{
'username': '******',
'group': 'pylons',
'password': '******',
'roles': ['admin','wiki'],
}
)
# Test all user methods raise:
for plugin in [s,f,d]:
for func in [
'user',
'user_roles',
'user_group',
'user_password',
]:
try:
getattr(plugin, func)('nouser')
except AuthKitNoSuchUserError, e:
pass
else:
raise AssertionError("Failed to throw a no user error")
logging.basicConfig(
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
level=logging.INFO)
logger = logging.getLogger(__name__)
app = FastAPI()
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
class_dict, counters, label_to_name = init_model()
def predict(label_name):
try:
class_id = class_dict[label_name]
except KeyError:
return {}
sorted_classes = sorted(enumerate(counters[class_id]),
reverse=False,
key=lambda x: x[1])
sorted_classes = [x for x in sorted_classes if x[1] > 0]
return [{
"prediction": label_to_name[label],
"confidence": probability
} for label, probability in sorted_classes]
#parser.add_argument('model', type=str,
#help='Path to model definition json. Model weights should be on the same path.')
#args = parser.parse_args()
#with open(args.model, 'r') as jfile:
# NOTE: if you saved the file by calling json.dump(model.to_json(), ...)
# then you will have to call:
#
# model = model_from_json(json.loads(jfile.read()))\
#
# instead.
#model = model_from_json(jfile.read())
#model.compile("adam", "mse")
#weights_file = args.model.replace('json', 'h5')
#model.load_weights(weights_file)
guided = True
with tf.device('/cpu:0'):
concept_model, discriminator, visual_generator, action_generator, generator_discriminator, discriminator_copy = model.init_model(
(40, 80, 1))
images = deque([])
actions = deque([])
# wrap Flask application with engineio's middleware
app = socketio.Middleware(sio, app)
# deploy as an eventlet WSGI server
eventlet.wsgi.server(eventlet.listen(('', 4567)), app)
ReplyKeyboardMarkup, KeyboardButton, \
InlineKeyboardMarkup, InlineKeyboardButton
from aiofiles import os as aio_os
from torchvision import transforms
from model import init_model, predict_image
N_BEST = 5
best_recipes = dict()
TYPE = 'image'
# bot = Bot(token=os.environ.get('TOKEN', None),
TOKEN = 'TOKEN'
bot = Bot(token=TOKEN)
dp = Dispatcher(bot, storage=MemoryStorage())
init_model()
logging.info("Model was init")
logging.basicConfig(
filename='log.txt',
filemode='a',
format='%(asctime)s, %(msecs) d %(name)s %(levelname) s %(message) s',
datefmt='%H:%M:%S',
level=logging.INFO)
inline_keyboard_markup = types.InlineKeyboardMarkup()
inline_keyboard_markup.add(
types.InlineKeyboardButton('Применить стиль Сезанна',
callback_data='sezanne'))
class TestStates(Helper):
import model
from authkit.users.sqlalchemy_driver import UsersFromDatabase
# Setup SQLAlchemy database engine
from sqlalchemy import engine_from_config
engine = engine_from_config({"sqlalchemy.url": "sqlite:///test.db"}, "sqlalchemy.")
model.init_model(engine)
model.engine = engine
users = UsersFromDatabase(model)
model.meta.metadata.create_all(model.engine)
users.group_create("pylons")
users.role_create("admin")
users.user_create("james", password="******", group="pylons")
users.user_create("ben", password="******")
users.user_add_role("ben", role="admin")
# Commit the changes
model.meta.Session.commit()
model.meta.Session.remove()
import numpy as np
import model
Wxh, Whh, Why, bh, by = model.init_model() #model parameters
def Loss(inputs, targets, hprev):
x, h, y, p = {}, {}, {}, {} #Empty dicts
loss = 0.0
h[-1] = np.copy(hprev)
#forward pass
for t in range(len(inputs)):
x[t] = np.zeros(
(1, data.vocab_size)) # for one-hot-row vector representation
x[t][0][inputs[
t]] = 1 # placing the t-th input in one-hot-row vector representation
h[t] = np.tanh(np.dot(x[t], Wxh) + np.dot(h[t - 1], Whh) +
bh) # hidden state
y[t] = np.dot(h[t], Why) + by #output
p[t] = np.exp(y[t]) / np.sum(np.exp(
y[t])) # probabilities for output chars
loss += -np.log(p[t][0][targets[t]]) # softmax loss
#bakward pass
dWxh, dWhh, dWy = np.zeros_like(Wxh), np.zeros_like(Whh), np.zeros_like(
Why)
dbh, dby = np.zeros_like(bh), np.zeros_like(by)
dhprev = np.zeros_like(hs[0])
for t in reversed(range(len(inputs))):
def main():
parser = create_parser()
args = parser.parse_args()
if args.setup:
create_directories()
if args.debug:
dataset = DATASETS['debug']
args.dataset = "debug"
features, _, labels, _ = preprocess_data(args.patch_size,
args.distribution,
dataset=dataset)
#print(features, 'debug')
#print("length of features: ",type(features), len(features),'element.shape: ',features[0][0])
features_train, features_test = features[:100], features[100:120]
labels_train, labels_test = labels[:100], labels[100:120]
elif args.train_model or args.evaluate_model or args.preprocess_data:
dataset = DATASETS[args.dataset]
#print(dataset.values())
load_from_cache = not args.preprocess_data
try:
features_train, features_test, labels_train, labels_test = preprocess_data(
args.patch_size,
args.distribution,
dataset=dataset,
only_cache=load_from_cache)
#print(features_train, 'train_model or evaluate_model or preprocess_data')
print("Length of features_train: ", len(features_train))
except IOError:
print("Cache file does not exist. Please run again with -p flag.")
sys.exit(1)
if args.visualise:
visualise_labels(labels_train, args.patch_size, LABELS_DIR)
visualise_labels(labels_test, args.patch_size, LABELS_DIR)
if not args.model_id:
timestamp = time.strftime("%d_%m_%Y_%H%M")
model_id = "{}_{}_{}".format(timestamp, args.dataset,
args.architecture)
else:
model_id = args.model_id
if args.init_model or args.train_model or args.evaluate_model:
model_dir = os.path.join(OUTPUT_DIR, model_id)
save_makedirs(model_dir)
# Hyperparameters for the model. Since there are so many of them it is
# more convenient to set them in the source code as opposed to passing
# them as arguments to the Command Line Interface. We use a list of tuples instead of a
# dict since we want to print the hyperparameters and for that purpose
# keep them in the predefined order.
hyperparameters = [
("architecture", args.architecture),
# Hyperparameters for the first convolutional layer.
("nb_filters_1", 64),
("filter_size_1", 9),
("stride_1", (2, 2)),
# Hyperparameter for the first pooling layer.
("pool_size_1", (2, 2)),
# Hyperparameter for the second convolutional layer (when
# two layer architecture is used).
("nb_filters_2", 128),
("filter_size_2", 5),
("stride_2", (1, 1)),
# Hyperparameters for Stochastic Gradient Descent.
("learning_rate", 0.05),
("momentum", 0.9),
("decay", 0.0)
]
hyperparameters_mnih = [
("architecture", args.architecture),
# Hyperparameters for the first convolutional layer.
("nb_filters_1", 64),
("filter_size_1", 16),
("stride_1", (4, 4)),
# Hyperparameter for the first pooling layer.
("pool_size_1", (2, 2)),
("pool_stride", 1),
# Hyperparameter for the second convolutional layer).
("nb_filters_2", 112),
("filter_size_2", 4),
("stride_2", (1, 1)),
# Hyperparameter for the third convolutional layer).
("nb_filters_3", 80),
("filter_size_3", 3),
("stride_3", (1, 1)),
# Hyperparameters for Stochastic Gradient Descent.
("learning_rate", 0.05),
("momentum", 0.9),
("decay", 0.0)
]
if args.init_model:
model = init_model(args.patch_size, model_id,
**dict(hyperparameters_mnih))
save_model_summary(hyperparameters_mnih, model, model_dir)
elif args.train_model or args.evaluate_model:
hyperparameters = dict(hyperparameters_mnih)
model = load_model(model_id)
model = compile_model(model, hyperparameters["learning_rate"],
hyperparameters['momentum'],
hyperparameters["decay"])
if args.train_model:
model = train_model(model,
features_train,
labels_train,
args.patch_size,
model_id,
model_dir,
nb_epoch=args.epochs,
checkpoints=args.checkpoints,
tensorboard=args.tensorboard,
earlystop=args.earlystop)
if args.evaluate_model:
evaluate_model(model,
features_test,
labels_test,
args.patch_size,
model_dir,
out_format=args.out_format)
def updated_training_loop():
gen_iterations = 0
if opt.oldmodelD:
D = net_d()
init_model(D, 'normal')
else:
D = SRGAN_D(6, 1)
init_model(D, 'normal')
if opt.CGAND:
D = CGAN_D(opt)
init_model(D, 'normal')
if opt.wasserstein:
D = DCGAN_D(256, 6, 32, 1)
if opt.netg_dropout:
G = net_g_dropout()
else:
G = net_g()
#init_model(G, 'normal')
#
D.train()
G.train()
losses = []
start = time.time()
EPS = opt.eps
G.cuda()
D.cuda()
beta_1 = 0.5
if opt.loss == 'mse':
criterionGAN = nn.MSELoss() #GANLoss()
elif opt.loss == 'ganloss':
criterionGAN = GANLoss()
criterionL1 = torch.nn.L1Loss()
if opt.wasserstein:
optimizer_D = torch.optim.RMSprop(D.parameters(), lr=opt.lrD)
optimizer_G = torch.optim.RMSprop(G.parameters(), lr=opt.lrG)
else:
optimizer_G = torch.optim.Adam(G.parameters(),
lr=opt.lrG,
betas=(opt.beta_1, 0.999))
optimizer_D = torch.optim.Adam(D.parameters(),
lr=opt.lrD,
betas=(opt.beta_1, 0.999))
total_epochs = opt.total_epochs
for epoch in range(total_epochs):
#print(epoch)
experiment.log_current_epoch(epoch)
data_iter = iter(dataloader)
i = 0
while i < len(dataloader):
############################
# (1) Update D network
###########################
for p in D.parameters(): # reset requires_grad
p.requires_grad = True # they are set to False below in netG update
# train the discriminator Diters times
if epoch < opt.D_pretrain or (epoch + 1) % 50 == 0:
Diters = 100
else:
Diters = opt.Diters
j = 0
#Diters = 5
while j < Diters and i < len(dataloader):
j += 1
lr, hr = data_iter.next()
i += 1
# Drop the last batch if it's not the same size as the batchsize
if lr.size(0) != opt.batch_size:
break
lr = lr.cuda()
hr = hr.cuda()
# train with real
D.zero_grad()
padding = torch.zeros(opt.batch_size, 3, 192, 64).cuda()
padding2 = torch.zeros(opt.batch_size, 3, 256, 192).cuda()
real_lr = torch.cat((torch.cat((lr, padding), 2), padding2), 3)
inputy = Variable(torch.cat((real_lr, hr), 1))
#inputy = Variable(hr)
errD_real = D(inputy, True) # can modify to feed inputv too
padding = torch.zeros(opt.batch_size, 3, 192, 64).cuda()
padding2 = torch.zeros(opt.batch_size, 3, 256, 192).cuda()
real_lr = torch.cat((torch.cat((lr, padding), 2), padding2), 3)
# completely freeze netG while we train the discriminator
inputg = Variable(lr, volatile=True)
#fake = Variable(netG(inputg).data)
fake = Variable(torch.cat((real_lr, G(inputg).data), 1))
errD_fake = D(fake, True)
# calculate discriminator loss and backprop
# errD = 0.5 * (torch.mean((errD_real - 1)**2) + torch.mean(errD_fake**2))
if opt.wasserstein:
errD = -torch.mean(errD_real) + torch.mean(errD_fake)
else:
errD = -torch.mean(
torch.log(errD_real + EPS) +
torch.log(1 - errD_fake + EPS))
errD.backward()
optimizer_D.step()
if opt.wasserstein:
for p in D.parameters():
p.data.clamp_(-opt.clamp, opt.clamp)
# print("here")
############################
# (2) Update G network
###########################
for p in D.parameters():
p.requires_grad = False # to avoid computation
G.zero_grad()
lr = lr.cuda()
hr = hr.cuda()
if lr.size(0) != opt.batch_size:
break
input_lr = Variable(lr)
input_hr = Variable(hr)
padding = torch.zeros(opt.batch_size, 3, 192, 64).cuda()
padding2 = torch.zeros(opt.batch_size, 3, 256, 192).cuda()
# print(lr.size())
# print(padding.size())
# print(padding2.size())
real_lr = torch.cat((torch.cat((lr, padding), 2), padding2), 3)
fake = G(input_lr)
#errG_1 = D(Variable(torch.cat((real_lr, fake.data), 1),requires_grad=True))
errG_1 = D(torch.cat((Variable(real_lr), fake), 1))
# maximize log(D) instead of minimize log(1 - D)
if opt.wasserstein:
errG = -torch.mean(errG_1)
else:
errG = -torch.mean(torch.log(errG_1 + EPS))
# generator accumulates loss from discriminator + MSE with true image
# loss_MSE = MSE(fake, input_hr)
# loss_SSIM = ssim_loss(fake, input_hr)
# loss_G = 0.25*errG + loss_MSE - loss_SSIM
loss_G = errG
#loss_G = criterionL1(fake, input_hr)
loss_G.backward()
losses.append(loss_G.data)
optimizer_G.step()
gen_iterations += 1
if epoch > opt.D_pretrain and (epoch + 1) % 1 == 0:
print_loss_avg = np.mean(losses).cpu().numpy()[0]
experiment.log_metric("average_g_loss", 100 * print_loss_avg)
experiment.log_metric("actual_g_loss",
losses[-1].cpu().numpy()[0] * 100)
print('%s (%d %d%%) %.4f' %
(timeSince(start, (epoch + 1) / total_epochs), epoch,
epoch / total_epochs * 100, print_loss_avg))
if (epoch + 1) % 100 == 0:
print('saving model at epoch', epoch)
torch.save(
G.state_dict(),
"checkpoints/G_{0}_{1}.pth".format(opt.output_path, epoch))
torch.save(
D.state_dict(),
"checkpoints/D_{0}_{1}.pth".format(opt.output_path, epoch))
def train(opt_path):
## initialization
log_saveroot = '../logs'
opt = utils.read_option(opt_path, log_saveroot)
if opt['use_gpu']:
device = 'cuda'
os.environ['CUDA_VISIBLE_DEVICES'] = opt['gpu_id']
else:
device = 'cpu'
if opt['fixed_random']:
utils.set_random_seed()
opt_train = opt['train']
opt_data = opt['dataset']
if opt_data['type'] == 1:
out_nc = 20
elif opt_data['type'] == 2:
out_nc = 100
pprint.pprint(opt)
## define dataset
train_dataset = dataset.Q1Dataset_pic(opt_data=opt_data,
phase='train',
root=None)
val_dataset = dataset.Q1Dataset_pic(opt_data=opt_data,
phase='val',
root=None)
train_dataloader = DataLoader(train_dataset,
batch_size=opt_train['batch_size'],
num_workers=opt_train['num_workers'],
shuffle=True)
val_dataloader = DataLoader(val_dataset,
batch_size=opt_train['batch_size'],
num_workers=opt_train['num_workers'],
shuffle=False)
## define network, optimizer, scheduler, criterion
net_m = model.init_model(opt_train, out_nc).to(device)
net_m2 = model.init_model(opt_train, out_nc).to(device)
net_m.model = nn.DataParallel(net_m.model)
net_m2.model = nn.DataParallel(net_m2.model)
optimizer = optim.init_optim(opt_train, net_m)
optimizer2 = optim.init_optim(opt_train, net_m2)
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
opt_train['milestones'],
gamma=0.2)
scheduler2 = torch.optim.lr_scheduler.MultiStepLR(optimizer2,
opt_train['milestones'],
gamma=0.2)
if opt['loss'] == "CEloss":
criterion = nn.CrossEntropyLoss()
else:
criterion = nn.CrossEntropyLoss()
## define tensorboard
path_log = opt['save_root']
all_scalars = {}
writer = utils.create_writer(path_log)
## train
with tqdm(total=opt_train['epoch'] * len(train_dataloader)) as pbar:
for epoch in range(opt_train['epoch']):
train_loss = 0.0
train_accu = 0.0
total = 0
total_right = 0
net_m.train()
net_m2.train()
scheduler.step()
scheduler2.step()
for img, label in train_dataloader:
optimizer.zero_grad()
optimizer2.zero_grad()
img = img.to(device)
label = label.to(device)
total += label.shape[0]
if opt_train['model'] == 'ResNext':
att_outputs, out, _ = net_m(img)
att_loss = criterion(att_outputs, label)
per_loss = criterion(out, label)
# loss = att_loss + per_loss
att_outputs2, out2, _ = net_m2(img)
att_loss2 = criterion(att_outputs2, label)
per_loss2 = criterion(out2, label)
loss = att_loss + per_loss + att_loss2 + per_loss2
else:
out = net_m(img)
out2 = net_m2(img)
loss = criterion(out, label) + criterion(out2, label)
loss.backward()
if opt_train['optim'] == 'SAM':
optimizer.first_step(zero_grad=True)
criterion(net_m(img), label).backward()
optimizer.second_step(zero_grad=True)
optimizer2.first_step(zero_grad=True)
criterion(net_m2(img), label).backward()
optimizer2.second_step(zero_grad=True)
else:
optimizer.step()
optimizer2.step()
train_loss += loss.item()
_, pred = (out + out2).max(1)
total_right += (pred == label).sum().item()
pbar.update(1)
train_accu = total_right / total
writer, all_scalars = utils.add_scalar(path_log, all_scalars,
writer, 'train/loss',
train_loss, epoch)
writer, all_scalars = utils.add_scalar(path_log, all_scalars,
writer, 'train/accu',
train_accu, epoch)
tqdm.write("Epoch: %d, loss: %.6f, accu: %.6f" %
((epoch + 1), train_loss, train_accu))
## val
if ((epoch + 1) % opt_train['val_interval'] == 0):
net_m.eval()
net_m2.eval()
val_total = 0
val_total_right = 0
for img, label in val_dataloader:
img = img.to(device)
label = label.to(device)
val_total += label.shape[0]
with torch.no_grad():
if opt_train['model'] == 'ResNext':
_, out, _ = net_m(img)
_, out2, _ = net_m2(img)
else:
out = net_m(img)
out2 = net_m2(img)
_, pred = (out + out2).max(1)
val_total_right += (pred == label).sum().item()
val_accu = val_total_right / val_total
writer, all_scalars = utils.add_scalar(path_log, all_scalars,
writer, 'val/accu',
val_accu, epoch)
tqdm.write("VAL Epoch: %d, VAL accu: %.6f" %
((epoch + 1), val_accu))
## save model
if ((epoch + 1) % opt_train['save_interval'] == 0):
save_model_path = os.path.join(
opt['save_root'] + '/checkpoints',
'model_%05d.pth' % (epoch + 1))
net_m.save(save_model_path)
save_model_path = os.path.join(
opt['save_root'] + '/checkpoints',
'model2_%05d.pth' % (epoch + 1))
net_m2.save(save_model_path)
''' write for using webservice '''
from Control.main import externalWebservice, OfficialWebservice
''' All server Initialization'''
from model import init_model
from model.__cmdbinit__ import cmdbinit_model
from model.initData import initdata
from model.cminitData import cminitdata
from BaseClass.logger import LoggerRecord
logger = LoggerRecord().initlog()
if __name__ == '__main__':
init_model() # init all tables
cmdbinit_model() # init cmdb all tables
initdata() # init all data
cminitdata() # init cmdb all data
# starting tcpserver
factory = Factory()
# starting webservice
soap_app=soaplib.core.Application([externalWebservice, OfficialWebservice], 'tns')
wsgi_app=wsgi.Application(soap_app)
# outside Statement
logger.debug('* Webservice started on port:7789... ')
logger.debug('* wsdl AT: http://192.168.82.89:7789/SOAP/?wsdl')
logger.debug('* wsdl AT: http://222.73.33.131:7789/SOAP/?wsdl')
# model_idx = options.model
batch_size = options.batch_size
num_epoch = options.epochs
data_root = options.data_root
input_size = options.input_size
train_csv_path = data_root + "train.csv"
test_csv_path = data_root + "test.csv"
images_dir = data_root + "images/"
submission_df_path = data_root + "sample_submission.csv"
num_classes = 4
num_cv_folds = 5
device = get_device()
model, _ = init_model(num_classes, use_pretrained=options.pre_train)
feature_center = torch.zeros(4, 32 * model.num_features).to(device)
criterion = get_loss_fn()
optimizer = torch.optim.SGD(model.parameters(),
lr=0.001,
momentum=0.9,
weight_decay=1e-5)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=2,
gamma=0.95)
tr_df_all = pd.read_csv(train_csv_path)
tr_df, val_df = train_test_split(tr_df_all, test_size=0.2)
val_df = val_df.reset_index(drop=True)
img = (img + 1) / 2.
if filename:
cv2.imwrite(filename, img*255)
cv2.imshow(window, img)
key = cv2.waitKey(1)
BATCH_SIZE = 1
FRAMES_PER_CONCEPT = 4
BATCH_COUNT = 100000
with tf.device('/gpu:0'):
(generator, discriminator,
generator_discriminator) = model.init_model((20, 80, 1), FRAMES_PER_CONCEPT)
generator = sensorimotor_generator(generator, scale_factor=.25,
batch_size=BATCH_SIZE,
frames_per_concept=FRAMES_PER_CONCEPT)
for i in range(BATCH_COUNT):
(real_images, real_actions,
fake_images, fake_actions) = next(generator)
d_loss = discriminator.train_on_batch([real_images, real_actions], np.array([1,] * BATCH_SIZE))
print("Batch", i, "d_loss_real_real=", d_loss)
d_loss = discriminator.train_on_batch([fake_images, fake_actions], np.array([0,] * BATCH_SIZE))
print("Batch", i, "d_loss_fake_fake=", d_loss)
import datetime
import sys
import traceback
import logging
e = create_engine(
'sqlite+pysqlite:///proverbaro.db',
module=sqlite,
encoding="utf-8",
)
logger = logging.getLogger(__name__)
Base = declarative_base()
repository = init_model(Base)
class TwitterPublisher(object):
def __init__(
self, consumer_key, consumer_secret, access_token,
access_token_key, hashtag):
self.consumer_key = consumer_key
self.consumer_secret = consumer_secret
self.access_token = access_token
self.access_token_key = access_token_key
self.hashtag = hashtag
def post_tweet(self, proverb):
client = UserClient(self.consumer_key, self.consumer_secret,
self.access_token, self.access_token_key)
def test_users_model_api_database():
sys.path.insert(0, os.getcwd() + "/examples/user/database-model")
try:
from authkit.users.sqlalchemy_driver import UsersFromDatabase
except ImportError:
raise Exception("Could not run the SQLAlchemy tests, not installed")
if os.path.exists("test.db"):
os.remove("test.db")
import model as test_model
# Setup SQLAlchemy database engine
from sqlalchemy import engine_from_config
engine = engine_from_config({"sqlalchemy.url": "sqlite:///test.db"}, "sqlalchemy.")
test_model.init_model(engine)
test_model.engine = engine
d = UsersFromDatabase(test_model)
test_model.meta.metadata.create_all(test_model.engine)
d.role_create("wiki")
d.role_create("adMin")
d.role_create("editor")
d.group_create("pyLOns")
d.group_create("dJAngo")
d.user_create("jaMEs", "passWOrd1", "pyLoNs")
d.user_create("ben", "password2")
d.user_create("Simon", "password3")
d.user_create("ian", "paSsword4")
assertEqual(d.list_roles(), ["admin", "editor", "wiki"])
assertEqual(d.list_groups(), ["django", "pylons"])
assertEqual(d.list_users(), ["ben", "ian", "james", "simon"])
assertEqual(d.user_has_password("james", "passWOrd1"), True)
assertEqual(d.user_has_password("james", "password1"), False)
d.role_create("test_role")
d.group_create("test_group")
d.user_create("test_user", "password")
assertEqual(d.list_roles(), ["admin", "editor", "test_role", "wiki"])
assertEqual(d.list_groups(), ["django", "pylons", "test_group"])
assertEqual(d.list_users(), ["ben", "ian", "james", "simon", "test_user"])
d.role_delete("test_role")
d.group_delete("test_group")
d.user_delete("test_user")
assertEqual(d.list_roles(), ["admin", "editor", "wiki"])
assertEqual(d.list_groups(), ["django", "pylons"])
assertEqual(d.list_users(), ["ben", "ian", "james", "simon"])
assertEqual(d.user_has_role("james", "admin"), False)
d.user_add_role("james", "admin")
assertEqual(d.user_has_role("james", "admin"), True)
d.user_remove_role("james", "admin")
assertEqual(d.user_has_role("james", "admin"), False)
d.user_add_role("james", "wiki")
d.user_add_role("simon", "wiki")
d.user_add_role("james", "admin")
# d.user_add_role("james", "editor")
d.user_add_role("ben", "editor")
assertEqual(d.user_has_group("james", "pylons"), True)
assertEqual(d.user_has_group("simon", None), True)
assertEqual(d.user_has_group("simon", "django"), False)
d.user_set_group("simon", "dJangO")
assertEqual(d.user_has_group("simon", None), False)
d.user_set_group("bEn", "PyLONS")
assertEqual(d.user_has_group("simon", "django"), True)
assertEqual(d.user_has_group("bEn", "pYlons"), True)
d.user_remove_group("bEn")
assertEqual(d.user_has_group("bEn", "pYlons"), False)
d.user_set_group("bEn", "PyLONS")
assertEqual(d.user_has_group("bEn", "pYlons"), True)
assertEqual(d.list_users(), ["ben", "ian", "james", "simon"])
d.user_set_username("james", "jim")
assertEqual(d.list_users(), ["ben", "ian", "jim", "simon"])
d.user_set_username("jim", "james")
from authkit.users import (
UsersFromFile,
UsersFromString,
AuthKitNoSuchUserError,
AuthKitNoSuchGroupError,
AuthKitNoSuchRoleError,
)
string_data = """jaMEs:passWOrd1:pyLOns wiki adMin
ben:password2:pylons admin editor
simon:password3:dJAngo
ian:paSsword4 wiki
"""
filename = "test/user_file_data.txt"
s = UsersFromString(string_data)
f = UsersFromFile(filename)
# Test Parsing
assertAllEqual(
s.passwords, f.passwords, {"james": "passWOrd1", "ben": "password2", "simon": "password3", "ian": "paSsword4"}
)
assertAllEqual(
s.roles, f.roles, {"james": ["admin", "wiki"], "ben": ["admin", "editor"], "ian": ["wiki"], "simon": []}
)
assertAllEqual(s.groups, f.groups, {"james": "pylons", "ben": "pylons", "ian": None, "simon": "django"})
assertAllEqual(s.usernames, f.usernames, ["ben", "ian", "james", "simon"])
# Test list functions
assertAllEqual(s.list_users(), f.list_users(), d.list_users(), ["ben", "ian", "james", "simon"])
assertAllEqual(s.list_roles(), f.list_roles(), d.list_roles(), ["admin", "editor", "wiki"])
assertAllEqual(s.list_groups(), f.list_groups(), d.list_groups(), ["django", "pylons"])
# Test user has functions
assertAllEqual(
s.user_has_role("jAMes", "WiKi"), f.user_has_role("jAMes", "WiKi"), d.user_has_role("jAMes", "WiKi"), True
)
assertAllEqual(
s.user_has_role("jAMes", "editOr"),
f.user_has_role("jAMes", "editOr"),
d.user_has_role("jAMes", "editOr"),
False,
)
assertAllEqual(
s.user_has_group("jAMeS", "PyLons"),
f.user_has_group("jAMes", "pylOns"),
d.user_has_group("jAMes", "pylOns"),
True,
)
assertAllEqual(
s.user_has_group("jameS", "djaNgo"),
f.user_has_group("JAMes", "djAngo"),
d.user_has_group("JAMes", "djAngo"),
False,
)
assertAllEqual(
s.user_has_password("jAMeS", "passWOrd1"),
f.user_has_password("jAMes", "passWOrd1"),
d.user_has_password("jAMes", "passWOrd1"),
True,
)
assertAllEqual(
s.user_has_password("jameS", "PASSWORD1"),
f.user_has_password("JAMes", "PASSWORD1"),
d.user_has_password("JAMes", "PASSWORD1"),
False,
)
# Existence Methods
assertAllEqual(s.user_exists("jAMeS"), f.user_exists("jAMes"), d.user_exists("jAMes"), True)
assertAllEqual(s.user_exists("nobody"), f.user_exists("nobody"), d.user_exists("nobody"), False)
# Existence Methods
assertAllEqual(s.role_exists("wiKi"), f.role_exists("Wiki"), d.role_exists("Wiki"), True)
assertAllEqual(s.role_exists("norole"), f.role_exists("norole"), d.role_exists("norole"), False)
assertAllEqual(s.group_exists("pyLons"), f.group_exists("PYlons"), d.group_exists("PYlons"), True)
assertAllEqual(s.group_exists("nogroup"), f.group_exists("nogroup"), d.group_exists("nogroup"), False)
# User Methods
assertAllEqual(
s.user("James"),
f.user("James"),
d.user("James"),
{"username": "******", "group": "pylons", "password": "******", "roles": ["admin", "wiki"]},
)
# Test all user methods raise:
for plugin in [s, f, d]:
for func in ["user", "user_roles", "user_group", "user_password"]:
try:
getattr(plugin, func)("nouser")
except AuthKitNoSuchUserError, e:
pass
else:
raise AssertionError("Failed to throw a no user error")
def create_db(self):
init_model(self.engine)
import model
import img
import sys
import os
c_model = model.init_model()
if len(sys.argv) > 1:
path = sys.argv[1]
else:
print('Usage: python3 test.py folder')
sys.exit()
for filename in os.listdir(path):
captcha = img.load_captcha_file(f'{path}/{filename}')
char_array = img.cut_captcha(captcha)
text = model.get_captcha_from_array(c_model, char_array)
print(text)
os.rename(f'{path}/{filename}', f'{path}/{text}.png')
def setup_database():
init_model(engine)
teardownDatabase()
elixir.setup_all(True)
# Creating permissions
see_site = Permission()
see_site.permission_name = u'see-site'
see_site.description = u'see-site permission description'
DBSession.save(see_site)
edit_site = Permission()
edit_site.permission_name = u'edit-site'
edit_site.description = u'edit-site permission description'
DBSession.save(edit_site)
commit = Permission()
commit.permission_name = u'commit'
commit.description = u'commit permission description'
DBSession.save(commit)
# Creating groups
admins = Group()
admins.group_name = u'admins'
admins.display_name = u'Admins Group'
admins.permissions.append(edit_site)
DBSession.save(admins)
developers = Group(group_name=u'developers',
display_name=u'Developers Group')
developers.permissions = [commit, edit_site]
DBSession.save(developers)
trolls = Group(group_name=u'trolls', display_name=u'Trolls Group')
trolls.permissions.append(see_site)
DBSession.save(trolls)
# Plus a couple of groups with no permissions
php = Group(group_name=u'php', display_name=u'PHP Group')
DBSession.save(php)
python = Group(group_name=u'python', display_name=u'Python Group')
DBSession.save(python)
# Creating users
user = User()
user.user_name = u'rms'
user.password = u'freedom'
user.email_address = u'*****@*****.**'
user.groups.append(admins)
user.groups.append(developers)
DBSession.save(user)
user = User()
user.user_name = u'linus'
user.password = u'linux'
user.email_address = u'*****@*****.**'
user.groups.append(developers)
DBSession.save(user)
user = User()
user.user_name = u'sballmer'
user.password = u'developers'
user.email_address = u'*****@*****.**'
user.groups.append(trolls)
DBSession.save(user)
# Plus a couple of users without groups
user = User()
user.user_name = u'guido'
user.password = u'phytonic'
user.email_address = u'*****@*****.**'
DBSession.save(user)
user = User()
user.user_name = u'rasmus'
user.password = u'php'
user.email_address = u'*****@*****.**'
DBSession.save(user)
DBSession.commit()
def __init__(self):
self.model = init_model()
if filename:
cv2.imwrite(filename, img * 255)
cv2.imshow(window, img)
key = cv2.waitKey(1)
BATCH_SIZE = 1
FRAMES_PER_CONCEPT = 5
BATCH_COUNT = 100000
with tf.device('/gpu:0'):
(concept_model, discriminator, visual_generator, action_generator,
generator_discriminator) = model.init_model((20, 80, 1), BATCH_SIZE)
generator = sensorimotor_generator(concept_model,
visual_generator,
action_generator,
scale_factor=.25,
batch_size=BATCH_SIZE,
frames_per_concept=FRAMES_PER_CONCEPT)
for i in range(BATCH_COUNT):
(real_images, real_actions, fake_images,
fake_actions) = next(generator)
d_loss = discriminator.train_on_batch([real_images, real_actions],
np.array([
def train(best_score, start_epoch, max_epochs, learning_rate, batch_size,
img_w, img_h, sgd, context):
print("Loading dataset...", flush=True)
dataset = load_dataset("data")
split = int(len(dataset) * 0.9)
training_set = dataset[:split]
print("Training set: ", len(training_set))
validation_set = dataset[split:]
print("Validation set: ", len(validation_set))
if os.path.isfile("model/global-wheat-yolo3-darknet53.params"):
model = load_model("model/global-wheat-yolo3-darknet53.params",
ctx=context)
else:
model = init_model(ctx=context)
metrics = [
gcv.utils.metrics.VOCMApMetric(iou_thresh=iou)
for iou in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]
]
print("Learning rate: ", learning_rate)
if sgd:
print("Optimizer: SGD")
trainer = mx.gluon.Trainer(model.collect_params(), "SGD", {
"learning_rate": learning_rate,
"momentum": 0.5
})
else:
print("Optimizer: Nadam")
trainer = mx.gluon.Trainer(model.collect_params(), "Nadam",
{"learning_rate": learning_rate})
if os.path.isfile("model/global-wheat-yolo3-darknet53.state"):
trainer.load_states("model/global-wheat-yolo3-darknet53.state")
print("Traning...", flush=True)
for epoch in range(start_epoch, max_epochs):
ts = time.time()
random.shuffle(training_set)
training_total_L = 0.0
training_batches = 0
for x, objectness, center_targets, scale_targets, weights, class_targets, gt_bboxes in get_batches(
training_set,
batch_size,
width=img_w,
height=img_h,
net=model,
ctx=context):
training_batches += 1
with mx.autograd.record():
obj_loss, center_loss, scale_loss, cls_loss = model(
x, gt_bboxes, objectness, center_targets, scale_targets,
weights, class_targets)
L = obj_loss + center_loss + scale_loss + cls_loss
L.backward()
trainer.step(x.shape[0])
training_batch_L = mx.nd.mean(L).asscalar()
if training_batch_L != training_batch_L:
raise ValueError()
training_total_L += training_batch_L
print(
"[Epoch %d Batch %d] batch_loss %.10f average_loss %.10f elapsed %.2fs"
% (epoch, training_batches, training_batch_L,
training_total_L / training_batches, time.time() - ts),
flush=True)
training_avg_L = training_total_L / training_batches
for metric in metrics:
metric.reset()
for x, label in get_batches(validation_set,
batch_size,
width=img_w,
height=img_h,
ctx=context):
classes, scores, bboxes = model(x)
for metric in metrics:
metric.update(bboxes, classes.reshape((0, -1)),
scores.reshape((0, -1)), label[:, :, :4],
label[:, :, 4:5].reshape((0, -1)))
score = mx.nd.array([metric.get()[1] for metric in metrics],
ctx=context).mean()
print(
"[Epoch %d] training_loss %.10f validation_score %.10f best_score %.10f duration %.2fs"
% (epoch + 1, training_avg_L, score.asscalar(), best_score,
time.time() - ts),
flush=True)
if score.asscalar() > best_score:
best_score = score.asscalar()
model.save_parameters(
"model/global-wheat-yolo3-darknet53_best.params")
model.save_parameters("model/global-wheat-yolo3-darknet53.params")
trainer.save_states("model/global-wheat-yolo3-darknet53.state")
FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(filename='proverbaro.log', format=FORMAT)
console = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(name)s: %(message)s')
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
logger = logging.getLogger(__name__)
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///proverbaro.db'
db = SQLAlchemy(app)
model = init_model(db.Model)
@app.route('/<date>/<int:publish_id>')
def show_proverb(date, publish_id):
proverb = model.proverb_to_show(date, publish_id)
definitions = []
if proverb is not None:
definitions = [
find_definition(split_proverb_into_words(proverb.text)[0]),
find_definition(split_proverb_into_words(proverb.text)[2])
]
print definitions
return render_template(
'proverb.html',
proverb=proverb.text if proverb is not None else None,
def setup_database():
init_model(engine)
teardownDatabase()
elixir.setup_all(True)
# Creating permissions
see_site = Permission()
see_site.permission_name = u'see-site'
see_site.description = u'see-site permission description'
DBSession.save(see_site)
edit_site = Permission()
edit_site.permission_name = u'edit-site'
edit_site.description = u'edit-site permission description'
DBSession.save(edit_site)
commit = Permission()
commit.permission_name = u'commit'
commit.description = u'commit permission description'
DBSession.save(commit)
# Creating groups
admins = Group()
admins.group_name = u'admins'
admins.display_name = u'Admins Group'
admins.permissions.append(edit_site)
DBSession.save(admins)
developers = Group(group_name=u'developers',
display_name=u'Developers Group')
developers.permissions = [commit, edit_site]
DBSession.save(developers)
trolls = Group(group_name=u'trolls', display_name=u'Trolls Group')
trolls.permissions.append(see_site)
DBSession.save(trolls)
# Plus a couple of groups with no permissions
php = Group(group_name=u'php', display_name=u'PHP Group')
DBSession.save(php)
python = Group(group_name=u'python', display_name=u'Python Group')
DBSession.save(python)
# Creating users
user = User()
user.user_name = u'rms'
user.password = u'freedom'
user.email_address = u'*****@*****.**'
user.groups.append(admins)
user.groups.append(developers)
DBSession.save(user)
user = User()
user.user_name = u'linus'
user.password = u'linux'
user.email_address = u'*****@*****.**'
user.groups.append(developers)
DBSession.save(user)
user = User()
user.user_name = u'sballmer'
user.password = u'developers'
user.email_address = u'*****@*****.**'
user.groups.append(trolls)
DBSession.save(user)
# Plus a couple of users without groups
user = User()
user.user_name = u'guido'
user.password = u'phytonic'
user.email_address = u'*****@*****.**'
DBSession.save(user)
user = User()
user.user_name = u'rasmus'
user.password = u'php'
user.email_address = u'*****@*****.**'
DBSession.save(user)
DBSession.commit()
import model
from authkit.users.sqlalchemy_driver import UsersFromDatabase
# Setup SQLAlchemy database engine
from sqlalchemy import engine_from_config
engine = engine_from_config({'sqlalchemy.url': 'sqlite:///test.db'},
'sqlalchemy.')
model.init_model(engine)
model.engine = engine
users = UsersFromDatabase(model)
model.meta.metadata.create_all(model.engine)
users.group_create("pylons")
users.role_create("admin")
users.user_create("james", password="******", group="pylons")
users.user_create("ben", password="******")
users.user_add_role("ben", role="admin")
# Commit the changes
model.meta.Session.commit()
model.meta.Session.remove()
np.hstack([bboxes1, np.full((bboxes1.shape[0], 1), r)]),
np.hstack([bboxes2,
np.full((bboxes2.shape[0], 1), 1.0 - r)])
])
return mix_raw.astype("uint8"), mix_bboxes
def reconstruct_color(img):
mean = mx.nd.array([0.485, 0.456, 0.406])
std = mx.nd.array([0.229, 0.224, 0.225])
return ((img * std + mean).clip(0.0, 1.0) * 255).astype("uint8")
if __name__ == "__main__":
from model import init_model
net = init_model()
data = load_dataset("data")
print("dataset preview: ", data[:3])
print("max count of bboxes: ", max([len(bboxes) for _, _, bboxes in data]))
print("training batch preview: ", next(get_batches(data, 4, net=net)))
print("validation batch preview: ", next(get_batches(data, 4)))
import matplotlib.pyplot as plt
print("data visual preview: ")
sampler = Sampler(data, 512, 512, net)
for i, x in enumerate(data):
print(x[1])
y = sampler(i)
gcv.utils.viz.plot_bbox(reconstruct_color(y[0].transpose((1, 2, 0))),
y[6])
plt.show()
