def init(self): self.y_d = self.parms.y_d.copy() # Load Models self.models = list() for nm in xrange(int(self.parms.num_models)): self.models.append(mdl.model(self.parms)) # Load multimodel setup self.multimodels = mmdl.multimodel(float(self.parms.NinSoftmax),float(self.parms.Nout),self.parms.num_models,self.parms.eta_s, self.parms.eta_g) self.sdpos = np.zeros((self.parms.Nout,)) # Reading and setting al the initial parameters t1 = self.parms.t1 dt = self.parms.dt delta = self.parms.delta self.sysInp = np.zeros((self.parms.sNin, t1 / dt + delta + 1)) self.sysInp2 = np.zeros((self.parms.Nout, t1 / dt + delta + 1)) self.sysOutp = np.zeros((self.parms.Nout, t1 / dt + delta + 1)) self.eeVal = self.sysInp self.inited = 1 self.model_e = np.zeros((self.parms.Nout, self.parms.num_models)) self.model_outputs = np.zeros((self.parms.Nout, self.parms.num_models,t1 / dt + delta + 1)) self.h = np.zeros((self.parms.num_models,t1 / dt + delta + 1)) self.model_doutputs = np.zeros((self.parms.Nout, self.parms.num_models)) self.i = delta self.t = 0
def __raxml_to_model(self, model_name, param_names,args,s_size):
raxmlmodelstr=self.__model_to_raxmodelstr(model_name,param_names)
##think about changing this so it catches crashes and continues
rax_cmd=Popen('{raxpath} -f e -m {modelstr} -s {phy_file} -t {starting_tree} -n {output_names} -w {out_path}/{model_name}'.format(
raxpath=self.raxml_path,
modelstr=raxmlmodelstr,
output_names=param_names,
out_path=self.output,
model_name=model_name,
phy_file=self.ifile_name,
starting_tree=self.stree),
shell=True,
stdout=PIPE,
stderr=PIPE)
rax_out = rax_cmd.communicate()
full_name=model_name+param_names
self.raxstdout+='='*80+'\n'+full_name.center(80)+'\n'+'='*80+'\n'
self.raxstdout+=rax_out[0]
self.raxstderr+='='*80+'\n'+full_name.center(80)+'\n'+'='*80+'\n'
self.raxstderr+=rax_out[0]
new_model=model(model_name=model_name,
param_names=param_names,
rax_name=raxmlmodelstr,
args=self.args,
s_size=s_size
)
return new_model
def main(weights_path, base_path, base_file, style_path, style_file,
combo_path, img_width, img_height, iterations):
result_prefix = base_file[:-4] + '_' + style_file[:-4]
base_img_path = base_path + base_file
style_img_path = style_path + style_file
# get tensor representations of images
base_img = K.variable(preprocess_image(base_img_path,
img_width,
img_height))
style_img = K.variable(preprocess_image(style_img_path,
img_width,
img_height))
combo_img = K.placeholder((1, 3, img_width, img_height))
# combine the 3 images into a single Keras tensor
input_tensor = K.concatenate([base_img, style_img, combo_img],
axis=0)
print('Creating painting of {} in the style of {}'.format(base_file[:-4],
style_file[:-4]))
print('Loading model with VGG16 network weights...')
nn = model(weights_path, input_tensor, img_width, img_height)
loss, grads = calc_loss_grad(nn, combo_img, img_width, img_height)
evaluate = Evaluator(loss, grads, combo_img, img_width, img_height)
return optimizer(evaluate, img_width, img_height, combo_path,
result_prefix, iterations=iterations)
def lnlikeHFM(pars, samples, obs, u, extra=False):
'''
Generic likelihood function for importance sampling with any number of
dimensions.
Now with added jitter parameter (hierarchical)
obs should be a 2d array of observations. shape = (ndims, nobs)
u should be a 2d array of uncertainties. shape = (ndims, nobs)
samples is a 3d array of samples. shape = (ndims, nobs, nsamp)
if extra == True, the sigma has both a slope and intercept
Now with a mixture model!
'''
ndims, nobs, nsamp = samples.shape
ypred = model(pars, samples)
yobs = obs[1, :]
xobs = obs[0, :]
yerr = u[1, :]
ll1 = np.zeros((nobs, nsamp*nobs))
ll2 = np.zeros((nobs, nsamp*nobs))
Y, V, P = pars[3], pars[4], pars[5]
for i in range(nobs):
if extra:
inv_sigma2 = 1.0/(yerr[i]**2 + \
(pars[2] + pars[3] * model1(pars, xobs[i]))**2 + V)
else:
inv_sigma2 = 1.0/(yerr[i]**2 + \
(pars[2]*model1(pars, xobs[i]))**2 + V)
ll1[i, :] = -.5*((yobs[i] - ypred)**2*inv_sigma2) + np.log(inv_sigma2)
ll2[i, :] = -.5*((yobs[i] - Y)**2*inv_sigma2) + np.log(inv_sigma2)
lnlike1 = np.logaddexp.reduce(ll1, axis=1)
lnlike2 = np.logaddexp.reduce(ll2, axis=1)
loglike = np.sum(np.logaddexp(np.log(1-P) + lnlike1, np.log(P) + lnlike2))
if np.isfinite(loglike):
return loglike
return -np.inf
def keyring_auth_list(**kwargs):
"""
List all cephx authorization keys
CLI Example:
salt '*' sesceph.auth_list \\
'cluster_name'='ceph' \\
'cluster_uuid'='cluster_uuid'
Notes:
cluster_name
Set the cluster name. Defaults to "ceph".
cluster_uuid
Set the cluster UUID. Defaults to value found in ceph config file.
"""
m = model.model(**kwargs)
u = mdl_updater.model_updater(m)
u.hostname_refresh()
try:
u.defaults_refresh()
except:
return {}
u.load_confg(m.cluster_name)
u.mon_members_refresh()
u.auth_list()
p = presenter.mdl_presentor(m)
return p.auth_list()
def pool_del(pool_name, **kwargs):
"""
List all cephx authorization keys
CLI Example:
salt '*' sesceph.pool_del pool_name \
'cluster_name'='ceph' \
'cluster_uuid'='cluster_uuid' \
Notes:
cluster_name
Set the cluster name. Defaults to "ceph".
cluster_uuid
Set the cluster UUID. Defaults to value found in ceph config file.
"""
m = model.model(**kwargs)
u = mdl_updater.model_updater(m)
u.hostname_refresh()
try:
u.defaults_refresh()
except:
return {}
u.load_confg(m.cluster_name)
u.mon_members_refresh()
u.pool_list()
u.pool_del(pool_name)
return True
def keyring_present_type(**kwargs):
"""
Check if keyring exists on disk
CLI Example:
salt '*' sesceph.keyring_admin_save \\
'[mon.]\n\tkey = AQA/vZ9WyDwsKRAAxQ6wjGJH6WV8fDJeyzxHrg==\n\tcaps mon = \"allow *\"\n' \\
'cluster_name'='ceph' \\
'cluster_uuid'='cluster_uuid'
Notes:
cluster_uuid
Set the cluster UUID. Defaults to value found in ceph config file.
cluster_name
Set the cluster name. Defaults to "ceph".
keyring_type
Set the keyring type
"""
keyring_type = kwargs.get("keyring_type")
if (keyring_type is None):
raise Error("keyring_type is None")
m = model.model(**kwargs)
u = mdl_updater.model_updater(m)
u.hostname_refresh()
try:
u.defaults_refresh()
except:
pass
keyobj = keyring.keyring_facard(m)
keyobj.key_type = keyring_type
return keyobj.present()
def build_agg_snapshots(self, db='./database'):
conn = sqlite3.connect(db)
c = conn.cursor()
# force model rebuild if created within last n days.
created_within = 7
# select models which require rebuilding.
cmd = c.execute('select mid, uid, version, created_on, last_updated_on, agg_fx, granularity \
from model \
where julianday(current_date) - julianday(last_updated_on) >= expires_in or \
rebuild = 1 and julianday(current_date) - julianday(last_updated_on) >= earliest_rebuild or \
julianday(current_date) - julianday(created_on) <= ?', (created_within,))
r = cmd.fetchall()
for m in r:
mdl = model(m[0], m[2], m[6])
# update model # && force deletion existing aggregrate snapshot.
c.execute('delete from agg_location_snapshot where mid = ?', (mdl.id(), ))
mdl.inc_version()
c.execute('update model \
set version = ?, last_updated_on = current_date, \
nr_matches = 0, nr_successful = 0, streak = 0, last10 = 0 \
where mid = ?', (mdl.version(), mdl.id()))
# rebuid aggregrate snapshot.
self.__build_agg_snapshot(mdl, c)
c.close()
conn.commit()
conn.close()
def cluster_quorum(**kwargs):
"""
Get the cluster status
CLI Example:
salt '*' sesceph.cluster_status \\
'cluster_name'='ceph' \\
'cluster_uuid'='cluster_uuid'
Notes:
Get the cluster quorum status.
Scope:
Cluster wide
Arguments:
cluster_uuid
Set the cluster UUID. Defaults to value found in ceph config file.
cluster_name
Set the cluster name. Defaults to "ceph".
"""
m = model.model(**kwargs)
u = mdl_updater.model_updater(m)
u.hostname_refresh()
u.defaults_refresh()
u.load_confg(m.cluster_name)
u.mon_members_refresh()
mur = mdl_updater_remote.model_updater_remote(m)
can_connect = mur.connect()
if not can_connect:
return False
q = mdl_query.mdl_query(m)
return q.cluster_quorum()
def pool_del(pool_name, **kwargs):
"""
List all cephx authorization keys
CLI Example:
salt '*' sesceph.pool_del pool_name \\
'cluster_name'='ceph' \\
'cluster_uuid'='cluster_uuid'
Notes:
cluster_name
Set the cluster name. Defaults to "ceph".
cluster_uuid
Set the cluster UUID. Defaults to value found in ceph config file.
"""
m = model.model(**kwargs)
u = mdl_updater.model_updater(m)
u.hostname_refresh()
u.defaults_refresh()
u.load_confg(m.cluster_name)
u.mon_members_refresh()
mur = mdl_updater_remote.model_updater_remote(m)
can_connect = mur.connect()
if not can_connect:
raise Error("Cant connect to cluster.")
mur.pool_list()
return mur.pool_del(pool_name)
def setup():
import model
models =[]
tmpM = model.model()
# funcs: (nextCond, prevCond, handleTag, handleData)
tmpM.push(
lambda *parg: parg[0] == "div" and ('class', 'threadpost') in parg[1], None, None, None)
tmpM.push(
lambda *parg: parg[0] == "img" and 1 or ( (parg[0]=="div" and ('class','quote') in parg[1]) and 2),
lambda *parg: parg[0] == "hr",
lambda *parg: [v for k,v in parg[1] if k == "id" and ( ('class', 'threadpost') in parg[1] or ('class', 'reply') in parg[1])],
None)
tmpM.push(
None,
lambda *parg: parg[0] == "img",
lambda *parg: apply(MyParser.getPicIdFromTURL, [v for k, v in parg[1] if k=="src"]),
None)
tmpM.push(
None,
lambda *parg: parg[0] == "div" and 2,
None,
lambda *parg: parg)
models.append(tmpM)
return models
def cluster_status(**kwargs):
"""
Get the cluster status
CLI Example:
salt '*' sesceph.cluster_status \\
'cluster_name'='ceph' \\
'cluster_uuid'='cluster_uuid'
Notes:
Get the cluster status including health if in quorum.
Scope:
Cluster wide
Arguments:
cluster_uuid
Set the cluster UUID. Defaults to value found in ceph config file.
cluster_name
Set the cluster name. Defaults to "ceph".
"""
m = model.model(**kwargs)
u = mdl_updater.model_updater(m)
u.hostname_refresh()
u.defaults_refresh()
u.load_confg(m.cluster_name)
u.mon_members_refresh()
mur = mdl_updater_remote.model_updater_remote(m)
can_connect = mur.connect()
if not can_connect:
raise Error("Cant connect to cluster.")
p = presenter.mdl_presentor(m)
return p.cluster_status()
def lnlikeHF(pars, samples, obs, u, extra=False):
'''
Generic likelihood function for importance sampling with any number of
dimensions.
Now with added jitter parameter (hierarchical)
obs should be a 2d array of observations. shape = (ndims, nobs)
u should be a 2d array of uncertainties. shape = (ndims, nobs)
samples is a 3d array of samples. shape = (ndims, nobs, nsamp)
if extra == True, the sigma has both a slope and intercept
'''
ndims, nobs, nsamp = samples.shape
ll = np.zeros((nobs, nsamp*nobs))
for i in range(nobs):
if extra:
inv_sigma2 = 1.0/(u[1, :][i]**2 + \
pars[2]**2 + pars[3] * obs[0, :][i])
else:
inv_sigma2 = 1.0/(u[1, :][i]**2 + \
(pars[2]*model1(pars, obs[0, :][i]))**2)
ll[i, :] = -.5*((obs[1, :][i] - model(pars, samples))**2*inv_sigma2) \
+ np.log(inv_sigma2)
loglike = np.sum(np.logaddexp.reduce(ll, axis=1))
if np.isfinite(loglike):
return loglike, loglike
return -np.inf, None
def evaluate(image_path, label_path):
""" Loads network, reads image and returns IOU."""
# Load image and label
image = load_image(image_path)
label = scipy.misc.imread(label_path)
# Define the model
prediction = model.model(image, drop=False)
# Get a saver
saver = tf.train.Saver()
# Launch the graph
with tf.Session() as sess:
# Restore variables
checkpoint_path = tf.train.latest_checkpoint(checkpoint_dir)
saver.restore(sess, checkpoint_path)
model.log("Variables restored from:", checkpoint_path)
logits = prediction.eval()
segmentation = post(logits, label, threshold = -1)
iou = IOU(segmentation, label)
print("iou =", iou)
return iou
def main():
sample_size = int(sys.argv[1])
train = data_io.read_train()
print("Data Size:")
print(train.shape)
feature_eng(train)
## originally sample size = 100000
train_sample = train[:sample_size]
## Train the booking model
for i in range(0,2):
if i==0:
model_name = "Booking"
response_name = "booking_bool"
isBook = True
else:
model_name = "Click"
response_name = "click_bool"
isBook = False
print("Training the "+model_name+" Classifier...")
tstart = datetime.now()
feature_names = get_features(train_sample, isBook)
print("Using "+str(len(feature_names))+" features...")
features = train_sample[feature_names].values
target = train_sample[response_name].values
classifier = model.model()
classifier.fit(features, target)
# print the time interval
print("Time used,")
print datetime.now() - tstart
print("Saving the classifier...")
tstart = datetime.now()
data_io.save_model(classifier, isBook)
print("Time used,")
print datetime.now() - tstart
def __init__(self, **kwargs):
self.model = model.model(**kwargs)
self._clear_implementation()
u = mdl_updater.model_updater(self.model)
u.ceph_version_refresh()
q = mdl_query.mdl_query(self.model)
self.ceph_daemon_user = q.ceph_daemon_user()
def __init__(self, config):
self.config = config
self.model = model(config)
self.criterion = nn.PairwiseDistance()
self.optimizer = optim.Adam(self.model.parameters(), lr=self.config['lr'])
self.generate_dataset()
def __init__(self):
self.M = model()
self.V = view()
self.M.setPaintCallback(self.Paint)
self.M.setMessageCallback(self.systemMessage)
self.mkLoad()
self.mkSelect()
self.mkFunction()
self.mkDatabus()
self.mkAddressbus()
self.mkMemory()
self.mkPGM()
self.mkProgramControls()
self.mkHALT()
self.Paint()
m = "\n\n WELCOME TO VIRTUAL MACHINE\n"
m = m + " Memory: " + str(len(self.M.MEMORY.values())) + " bytes\n"
m = m + " ==========================================================\n"
m = m + "Press README on left...."
self.systemMessage(m)
self.V.mainloop()
def main(hps):
# Initialize Horovod.
hvd.init()
# Create tensorflow session
sess = tensorflow_session()
# Download and load dataset.
tf.set_random_seed(hvd.rank() + hvd.size() * hps.seed)
np.random.seed(hvd.rank() + hvd.size() * hps.seed)
# Get data and set train_its and valid_its
train_iterator, test_iterator, data_init = get_data(hps, sess)
hps.train_its, hps.test_its, hps.full_test_its = get_its(hps)
# Create log dir
logdir = os.path.abspath(hps.logdir) + "/"
if not os.path.exists(logdir):
os.mkdir(logdir)
# Create model
import model
model = model.model(sess, hps, train_iterator, test_iterator, data_init)
# Initialize visualization functions
visualise = init_visualizations(hps, model, logdir)
if not hps.inference:
# Perform training
train(sess, model, hps, logdir, visualise)
else:
infer(sess, model, hps, test_iterator)
def __init__(self, tf_ckpt_path):
#init the tensorflow model
self.x, _ = model.place_holders()
self.y_conv = model.model(self.x)
self.sess = tf.InteractiveSession()
self.sess.run(tf.initialize_all_variables())
saver = tf.train.Saver(tf.all_variables())
saver.restore(self.sess, tf_ckpt_path)
def __init__(self, **kwargs):
# The path to the service binary
self.path_service_bin = None
# what we are starting
self.ceph_client_id = kwargs.get("name")
self.service_name = None
self.model = model.model(**kwargs)
self.model.init = "systemd"
def get_model_func(self):
modellist=[]
ppcross=crss.crossSection(self.ppPath)
counter=0
for p in self.processes:
model=mod.model(self.parentspec[counter],p,self.data,self.source,ppcross)
modellist.append(model.model_val)
counter+=1
return modellist
def test_model():
import model
m = model.model()
pos = m.__dict__["pos"]
assert m.__dict__["pos"] == 0
m.next();
assert m.__dict__["pos"] == 0
m.prev()
assert m.__dict__["pos"] == 0
m.prev()
assert m.__dict__["pos"] == 0
assert m.getHandleData() == None
assert m.getHandleTag() == None
stack.append("\tmodel init")
f1, f2 = lambda: 1,lambda: 2
f3, f4 = lambda: 3,lambda: 4
m.push(f1)
assert m.__dict__["stack"][0] == (f1,)
m.push(f2, f1)
assert m.__dict__["stack"][1] == (f2, f1)
m.push(f1, f2, f3)
assert m.__dict__["stack"][2] == (f1, f2, f3)
m.push(f1, f2, f3, f4)
assert m.__dict__["stack"][3] == (f1, f2, f3, f4)
stack.append("\tmodel push works")
m.next()
assert m.__dict__["pos"] == 1
assert m.getHandleData() == None
assert m.getHandleTag() == None
stack.append("\tmodel get handle test 1")
m.next()
assert m.__dict__["pos"] == 3
stack.append("\tmodel next test 1")
m.prev()
assert m.__dict__["pos"] == 1
stack.append("\tmodel prev test 1")
m.next(); m.next()
assert m.__dict__["pos"] == 4
stack.append("\tmodel next test 2")
m.prev(); m.prev()
assert m.__dict__["pos"] == 4
m.push(f1, f1, f1, f1)
assert len(m.__dict__["stack"]) == 5
m.prev(); m.prev()
assert m.__dict__["pos"] == 1
stack.append("\tmodel prev test 2")
m.next()
assert m.__dict__["pos"] == 3
assert m.getHandleTag() == f3
assert m.getHandleData() == f4
stack.append("\tmodel get handle test 2")
def ceph_version():
"""
Get the version of ceph installed
"""
m = model.model()
u = mdl_updater.model_updater(m)
u.ceph_version_refresh()
p = presenter.mdl_presentor(m)
return p.ceph_version()
def step(hparams, tokens, past=None):
lm_output = model.model(hparams=hparams, X=tokens, past=past, reuse=tf.AUTO_REUSE)
logits = lm_output['logits'][:, :, :hparams.n_vocab]
presents = lm_output['present']
presents.set_shape(model.past_shape(hparams=hparams, batch_size=batch_size))
return {
'logits': logits,
'presents': presents,
}
def purge(**kwargs):
"""
purge ceph configuration on the node
CLI Example:
salt '*' sesceph.purge
"""
m = model.model(**kwargs)
purger.purge(m, **kwargs)
def inject(theta, kid):
# load data
x, y = np.genfromtxt("%s/%s_rvs.txt" % (DIR, kid)).T
if kid == "HD185":
x, y = np.genfromtxt("%s/%s_rvs_100.txt" % (DIR, kid)).T
yerr = np.ones_like(y)*2. # make up uncertainties
M1, M1_err = np.genfromtxt("%s/params/%s_mass.txt"
% (DIR, kid)).T
ecc = 0.
rv = model(theta, x, yerr, M1, ecc) # compute planet rvs
noisy_rv = white_noise(y+rv, yerr[0]) # add planet to lc with noise
return x, noisy_rv, yerr
def get_users(self, s_day, model=model.OrderModel, sql=""):
arg = [s_day]
result = self.get_all(sql, arg)
# logging.DEBUG(str(sql))
users_list = []
if not result:
return False
for row in result:
o_user_model = model()
o_user_model.setValue(row[0], int(row[1]), int(row[2]), int(row[3]))
users_list.append(o_user_model)
return users_list
def fit(self,conffile=False,initial=[]):
newpara=[]
parerror=[]
chi2s=[]
speclist=[]
modelvallist=[]
counter=0
for p in self.proc:
print "Running process No.%i"%(counter)
print "starting setting initial parameters for %s spectrum"%(p)
self.modellist.append(mod.model(self.spec[counter],p,self.data,self.source,self.ppcrs))
self.model=self.modellist[counter].model_val
if conffile:
setup=self.conf['%s_%s_%i'%(p,self.spec[counter].funcName,counter)]
npar=len(self.ipar[counter])
counter+=1
#m=minuit.Minuit(self.chi2(npar),**setup)
#m.tol=1.e2
else:
if initial==[]:
initial=self.ipar[counter]
npar=len(initial)
print 'number of intial parameters %i'%npar
setup={}
counter=0
parname=['a','b','c','d','e','f','g','h']
for i in initial:
setup[parname[counter]]=i
setup['error_%s'%parname[counter]]=0.01
limits=ut.make_para_limits(i)
setup['limit_%s'%parname[counter]]=limits
counter+=1
m=minuit.Minuit(self.chi2(npar),**setup)
m.tol=1.e2
''' start the fitting process'''
print "initial parameters set starting migrad"
m.migrad()
# m.hesse()
newpara.append(self.conv_mval2list(m.values))
parerror.append(self.conv_mval2list(m.errors))
chi2s.append(m.fval)
modelvallist.append(self.model(self.conv_mval2list(m.values)))
speclist.append(self.spec)
print 'fitting of process %s done chi^2: %f'%(p,chi2s[len(chi2s)-1])
print 'all %i processes fitted \n '%(counter)
print '\n\n'
return newpara,parerror,chi2s,modelvallist,speclist
def main(_):
if not os.path.exists(args.ckpt_dir):
os.makedirs(args.ckpt_dir)
print(args.ckpt_dir)
weightedModel = model(ckpt_dir=args.ckpt_dir, batch_size=args.batch_size, margin=args.margin, weightFile=args.weightFile,
sketch_train_list=args.sketch_train_list, sketch_test_list=args.sketch_test_list, shape_list=args.shape_list,
num_views_shape=args.num_views_shape, learning_rate=args.learning_rate, momentum=args.momentum,
logdir=args.logdir, phase=args.phase, inputFeaSize=args.inputFeaSize, outputFeaSize=args.outputFeaSize, maxiter=args.maxiter)
if args.phase == 'train':
weightedModel.train()
elif args.phase == 'evaluation':
print("evaluating using weighted feature")
weightedModel.evaluation()
from model import model
import time
settings = {
"filepath": "../data/amazon.csv",
"BATCH_SIZE": 64,
"embedding_dim": 256,
"units": 512,
"EPOCHS": 15,
"data_size": 1000,
"checkpoint_dir": "../checkpoints"
}
# data_size limits the amount of lines that are used. None = full data
time_start = time.time()
model().trainModel(settings)
time_end = time.time()
duration = round((time_end - time_start) / 60, 2)
print("Elapsed time in minutes: {}".format(duration))
def load(self):
my_model = model(self.args).get_model()
trainable = filter(lambda x: x.requires_grad, my_model.parameters())
if self.args.optimizer == 'SGD':
optimizer_function = optim.SGD
kwargs = {'momentum': self.args.momentum}
elif self.args.optimizer == 'ADAM':
optimizer_function = optim.Adam
kwargs = {
'betas': (self.args.beta1, self.args.beta2),
'eps': self.args.epsilon}
elif self.args.optimizer == 'RMSprop':
optimizer_function = optim.RMSprop
kwargs = {'eps': self.args.epsilon}
kwargs['lr'] = self.args.lr
kwargs['weight_decay'] = 0
my_optimizer = optimizer_function(trainable, **kwargs)
if self.args.decay_type == 'step':
my_scheduler = lrs.StepLR(
my_optimizer,
step_size=self.args.lr_decay,
gamma=self.args.gamma)
elif self.args.decay_type.find('step') >= 0:
milestones = self.args.decay_type.split('_')
milestones.pop(0)
milestones = list(map(lambda x: int(x), milestones))
my_scheduler = lrs.MultiStepLR(
my_optimizer,
milestones=milestones,
gamma=self.args.gamma)
self.log_training = torch.Tensor()
self.log_test = torch.Tensor()
if self.args.load == '.':
my_loss = loss(self.args).get_loss()
else:
if not self.args.test_only:
self.log_training = torch.load(self.dir + '/log_training.pt')
self.log_test = torch.load(self.dir + '/log_test.pt')
resume = self.args.resume
if resume == -1:
my_model.load_state_dict(
torch.load(self.dir + '/model/model_latest.pt'))
resume = len(self.log_test)
else:
my_model.load_state_dict(
torch.load(self.dir + '/model/model_{}.pt'.format(resume)))
my_loss = torch.load(self.dir + '/loss.pt')
my_optimizer.load_state_dict(
torch.load(self.dir + '/optimizer.pt'))
print('Load loss function from checkpoint...')
print('Continue from epoch {}...'.format(resume))
return my_model, my_loss, my_optimizer, my_scheduler
# 'out':tf.Variable(tf.zeros([num_classes]))
'out': tf.Variable(init([num_classes]))
}
#全连接神经网络
# weights = {
# 'h1':tf.Variable(tf.random_normal([num_input,num_hidden_1])),
# 'h2':tf.Variable(tf.random_normal([num_hidden_1,num_hidden_2])),
# 'out':tf.Variable(tf.random_normal([num_hidden_2,num_classes]))
# }
# biases = {
# 'b1':tf.Variable(tf.random_normal([num_hidden_1])),
# 'b2':tf.Variable(tf.random_normal([num_hidden_2])),
# 'out':tf.Variable(tf.random_normal([num_classes]))
# }
model_ = model(X, weights, biases, timesteps, num_hidden)
# logits = model_.MulTi_LSTM()
# logits = model_.MulTi_LSTM()
# logits = model_.BiRNN()
logits = model_.Basic_Rnn()
prediction = tf.nn.softmax(logits)
# # 定义损失和优化器
# 损失函数优先考虑二次代价函数又称均方误差
# loss_op = tf.reduce_mean(tf.square(Y-logits))/2
loss_op = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=Y))
tf.summary.scalar('loss', loss_op)
# loss_op = tf.reduce_mean(tf.square(logits-Y)+tf.contrib.layers.l1_regularizer(0.1))
def custom_train(dataset_path='example_train_datasets/universe',
checkpoint_dir='models',
iterations=10,
model_name='117M',
combine=50000,
encoding='utf-8',
batch_size=1,
learning_rate=0.00002,
accumulate_gradients=1,
memory_saving_gradients=False,
only_train_transformer_layers=False,
optimizer='adam',
noise=0.0,
top_k=40,
top_p=0.0,
restore_from='latest',
run_name='last',
sample_every=100,
sample_length=1023,
sample_num=1,
save_every=1000,
val_dataset=None,
val_batch_size=2,
val_batch_count=40,
val_every=0):
common_files = ["/encoder.json", "/hparams.json", "/vocab.bpe"]
dataset_name = dataset_path.split('/')
dataset_name = dataset_name[len(dataset_name) - 1]
# set the last run name to this dataset name
# run_name = dataset_name
# pre-process dataset
parse_dataset(dataset_path)
# generate npz file
generate_npz_file(dataset_path)
enc = encoder.get_encoder(model_name)
hparams = model.default_hparams()
with open(os.path.join('models', model_name, 'hparams.json')) as f:
hparams.override_from_dict(json.load(f))
if sample_length > hparams.n_ctx:
raise ValueError("Can't get samples longer than window size: %s" %
hparams.n_ctx)
if model_name == '345M':
memory_saving_gradients = True
if optimizer == 'adam':
only_train_transformer_layers = True
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.graph_options.rewrite_options.layout_optimizer = rewriter_config_pb2.RewriterConfig.OFF
with tf.Session(config=config) as sess:
context = tf.placeholder(tf.int32, [batch_size, None])
context_in = randomize(context, hparams, noise)
output = model.model(hparams=hparams, X=context_in)
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=context[:, 1:], logits=output['logits'][:, :-1]))
if val_every > 0:
val_context = tf.placeholder(tf.int32, [val_batch_size, None])
val_output = model.model(hparams=hparams, X=val_context)
val_loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=val_context[:, 1:],
logits=val_output['logits'][:, :-1]))
val_loss_summary = tf.summary.scalar('val_loss', val_loss)
tf_sample = sample.sample_sequence(hparams=hparams,
length=sample_length,
context=context,
batch_size=batch_size,
temperature=1.0,
top_k=top_k,
top_p=top_p)
all_vars = [v for v in tf.trainable_variables() if 'model' in v.name]
train_vars = [v for v in all_vars if '/h' in v.name
] if only_train_transformer_layers else all_vars
if optimizer == 'adam':
opt = tf.train.AdamOptimizer(learning_rate=learning_rate)
elif optimizer == 'sgd':
opt = tf.train.GradientDescentOptimizer(
learning_rate=learning_rate)
else:
exit('Bad optimizer:', optimizer)
if accumulate_gradients > 1:
if memory_saving_gradients:
exit(
"Memory saving gradients are not implemented for gradient accumulation yet."
)
opt = AccumulatingOptimizer(opt=opt, var_list=train_vars)
opt_reset = opt.reset()
opt_compute = opt.compute_gradients(loss)
opt_apply = opt.apply_gradients()
summary_loss = tf.summary.scalar('loss', opt_apply)
else:
if memory_saving_gradients:
opt_grads = memory_saving_gradients.gradients(loss, train_vars)
else:
opt_grads = tf.gradients(loss, train_vars)
opt_grads = list(zip(opt_grads, train_vars))
opt_apply = opt.apply_gradients(opt_grads)
summary_loss = tf.summary.scalar('loss', loss)
summary_lr = tf.summary.scalar('learning_rate', learning_rate)
summaries = tf.summary.merge([summary_lr, summary_loss])
summary_log = tf.summary.FileWriter(
os.path.join(checkpoint_dir, run_name))
saver = tf.train.Saver(var_list=all_vars,
max_to_keep=5,
keep_checkpoint_every_n_hours=2)
sess.run(tf.global_variables_initializer())
if restore_from == 'latest':
ckpt = tf.train.latest_checkpoint(
os.path.join(checkpoint_dir, run_name))
if ckpt is None:
# Get fresh GPT weights if new run.
ckpt = tf.train.latest_checkpoint(
os.path.join('models', model_name))
elif restore_from == 'fresh':
ckpt = tf.train.latest_checkpoint(
os.path.join('models', model_name))
else:
ckpt = tf.train.latest_checkpoint(restore_from)
print('Loading checkpoint', ckpt)
saver.restore(sess, ckpt)
print('Loading dataset...')
chunks = load_dataset(enc,
dataset_path + ".npz",
combine,
encoding=encoding)
data_sampler = Sampler(chunks)
if val_every > 0:
if val_dataset:
val_chunks = load_dataset(enc,
val_dataset,
combine,
encoding=encoding)
else:
val_chunks = chunks
print('dataset has', data_sampler.total_size, 'tokens')
print('Training...')
if val_every > 0:
# Sample from validation set once with fixed seed to make
# it deterministic during training as well as across runs.
val_data_sampler = Sampler(val_chunks, seed=1)
val_batches = [[
val_data_sampler.sample(1024) for _ in range(val_batch_size)
] for _ in range(val_batch_count)]
counter = 1
counter_path = os.path.join(checkpoint_dir, run_name, 'counter')
if os.path.exists(counter_path):
# Load the step number if we're resuming a run
# Add 1 so we don't immediately try to save again
with open(counter_path, 'r') as fp:
counter = int(fp.read()) + 1
def save():
maketree(os.path.join(checkpoint_dir, run_name))
print(
'Saving',
os.path.join(checkpoint_dir, run_name,
'model-{}').format(counter))
saver.save(sess,
os.path.join(checkpoint_dir, run_name, 'model'),
global_step=counter)
with open(counter_path, 'w') as fp:
fp.write(str(counter) + '\n')
def generate_samples():
print('Generating samples...')
context_tokens = data_sampler.sample(1)
all_text = []
index = 0
while index < sample_num:
out = sess.run(
tf_sample,
feed_dict={context: batch_size * [context_tokens]})
for i in range(min(sample_num - index, batch_size)):
text = enc.decode(out[i])
text = '======== SAMPLE {} ========\n{}\n'.format(
index + 1, text)
all_text.append(text)
index += 1
print(text)
maketree(os.path.join(SAMPLE_DIR, run_name))
with open(os.path.join(SAMPLE_DIR, run_name,
'samples-{}').format(counter),
'w',
encoding=encoding) as fp:
fp.write('\n'.join(all_text))
def validation():
print('Calculating validation loss...')
losses = []
for batch in tqdm.tqdm(val_batches):
losses.append(
sess.run(val_loss, feed_dict={val_context: batch}))
v_val_loss = np.mean(losses)
v_summary = sess.run(val_loss_summary,
feed_dict={val_loss: v_val_loss})
summary_log.add_summary(v_summary, counter)
summary_log.flush()
print('[{counter} | {time:2.2f}] validation loss = {loss:2.2f}'.
format(counter=counter,
time=time.time() - start_time,
loss=v_val_loss))
def sample_batch():
return [data_sampler.sample(1024) for _ in range(batch_size)]
avg_loss = (0.0, 0.0)
start_time = time.time()
try:
for i in range(iterations):
if counter % save_every == 0:
save()
if counter % sample_every == 0:
generate_samples()
if val_every > 0 and (counter % val_every == 0
or counter == 1):
validation()
if accumulate_gradients > 1:
sess.run(opt_reset)
for _ in range(accumulate_gradients):
sess.run(opt_compute,
feed_dict={context: sample_batch()})
(v_loss, v_summary) = sess.run((opt_apply, summaries))
else:
(_, v_loss, v_summary) = sess.run(
(opt_apply, loss, summaries),
feed_dict={context: sample_batch()})
summary_log.add_summary(v_summary, counter)
avg_loss = (avg_loss[0] * 0.99 + v_loss,
avg_loss[1] * 0.99 + 1.0)
print(
'[{counter} | {time:2.2f}] loss={loss:2.2f} avg={avg:2.2f}'
.format(counter=counter,
time=time.time() - start_time,
loss=v_loss,
avg=avg_loss[0] / avg_loss[1]))
counter += 1
# save results
save()
# copy base model files to the trained model folder
for f in common_files:
dest_full_path = checkpoint_dir + "/" + model_name + "/" + f
destination = checkpoint_dir + "/" + run_name + "/" + f
shutil.copyfile(dest_full_path, destination)
except KeyboardInterrupt:
print('error/exception')
# We do not add the time left over form previous dataset: time_budget += time_left_over
vprint(verbose,
"[+] Time budget for this task %5.2f sec" % time_budget)
time_spent = time.time() - start
vprint(
verbose, "[+] Remaining time after reading data %5.2f sec" %
(time_budget - time_spent))
if time_spent >= time_budget:
vprint(verbose,
"[-] Sorry, time budget exceeded, skipping this task")
execution_success = False
continue
# ========= Creating a model
vprint(verbose, "======== Creating model ==========")
M = model()
# ========= Reload trained model if it exists
vprint(verbose,
"**********************************************************")
vprint(verbose,
"****** Attempting to reload model to avoid training ******")
vprint(verbose,
"**********************************************************")
you_must_train = 1
modelname = os.path.join(submission_dir, basename)
if os.path.isfile(modelname + '_model.pickle'):
M = M.load(modelname)
you_must_train = 0
vprint(verbose, "[+] Model reloaded, no need to train!")
def main(argv=None):
import os
os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpu_list
try:
os.makedirs(FLAGS.output_dir)
except OSError as e:
if e.errno != 17:
raise
with tf.get_default_graph().as_default():
input_images = tf.placeholder(tf.float32,
shape=[None, None, None, 3],
name='input_images')
global_step = tf.get_variable('global_step', [],
initializer=tf.constant_initializer(0),
trainable=False)
f_score, f_geometry = model.model(input_images, is_training=False)
variable_averages = tf.train.ExponentialMovingAverage(
0.997, global_step)
saver = tf.train.Saver(variable_averages.variables_to_restore())
with tf.Session(config=tf.ConfigProto(
allow_soft_placement=True)) as sess:
ckpt_state = tf.train.get_checkpoint_state(FLAGS.checkpoint_path)
model_path = os.path.join(
FLAGS.checkpoint_path,
os.path.basename(ckpt_state.model_checkpoint_path))
print('Restore from {}'.format(model_path))
saver.restore(sess, model_path)
im_fn_list = get_images()
for im_fn in im_fn_list:
im = cv2.imread(im_fn)[:, :, ::-1]
start_time = time.time()
im_resized, (ratio_h, ratio_w) = resize_image(im)
timer = {'net': 0, 'restore': 0, 'nms': 0}
start = time.time()
score, geometry = sess.run(
[f_score, f_geometry],
feed_dict={input_images: [im_resized]})
timer['net'] = time.time() - start
boxes, timer = detect(score_map=score,
geo_map=geometry,
timer=timer)
print(
'{} : net {:.0f}ms, restore {:.0f}ms, nms {:.0f}ms'.format(
im_fn, timer['net'] * 1000, timer['restore'] * 1000,
timer['nms'] * 1000))
if boxes is not None:
boxes = boxes[:, :8].reshape((-1, 4, 2))
boxes[:, :, 0] /= ratio_w
boxes[:, :, 1] /= ratio_h
duration = time.time() - start_time
print('[timing] {}'.format(duration))
# save to file
if boxes is not None:
res_file = os.path.join(
FLAGS.output_dir,
'{}.txt'.format(os.path.basename(im_fn).split('.')[0]))
with open(res_file, 'w') as f:
for box in boxes:
# to avoid submitting errors
box = sort_poly(box.astype(np.int32))
if np.linalg.norm(box[0] -
box[1]) < 5 or np.linalg.norm(
box[3] - box[0]) < 5:
continue
f.write('{},{},{},{},{},{},{},{}\r\n'.format(
box[0, 0],
box[0, 1],
box[1, 0],
box[1, 1],
box[2, 0],
box[2, 1],
box[3, 0],
box[3, 1],
))
cv2.polylines(
im[:, :, ::-1],
[box.astype(np.int32).reshape((-1, 1, 2))],
True,
color=(255, 255, 0),
thickness=1)
if not FLAGS.no_write_images:
img_path = os.path.join(FLAGS.output_dir,
os.path.basename(im_fn))
cv2.imwrite(img_path, im[:, :, ::-1])
def getRes(self):
# 获取格网线图和格网点图,都是像素级别的
img_erode, joint = detectTable(self.img).run()
cv2.imshow("src", img_erode)
image, contours, hierarchy = cv2.findContours(img_erode, cv2.RETR_LIST,
cv2.CHAIN_APPROX_SIMPLE)
area_coord_roi = []
for i in range(len(contours)):
cnt = contours[i]
area = cv2.contourArea(cnt)
if area > self.areaRange[0] and area < self.areaRange[1]:
x, y, w, h = cv2.boundingRect(cnt)
roi = self.img[(y + self.border):((y + h) - self.border),
(x + self.border):((x + w) - self.border)]
area_coord_roi.append((area, (x, y, w, h), roi))
#最大面积
max_area = max([info[0] for info in area_coord_roi])
for info in area_coord_roi:
if info[0] == max_area:
max_rect = info[1]
max_info = info
bScript = cutImage.SkipMaxArea(self, max_info, area_coord_roi)
maxW = 0
totalH = 0
#计算拼接的图片大小
for each in area_coord_roi:
x, y, w, h = each[1]
#if 1 == 0: #x>max_rect[0] and y>max_rect[1] and (x+w)<(max_rect[0]+max_rect[2]) and (y+h) <(max_rect[1]+max_rect[3]):
if bScript and each[0] == max_area: #跳过最大面积的表格
pass
else:
if maxW < w:
maxW = w
totalH += h
to_image = Image.new('RGB', (maxW + 1, totalH + 1),
color='white') # 创建一个新图 size必须是元组
print(maxW + 1)
print(totalH + 1)
sumH = 0
area_coord_map = [] #原图中的位置与拼接图的位置映射
for each in area_coord_roi:
#if 1 == 0: #x>max_rect[0] and y>max_rect[1] and (x+w)<(max_rect[0]+max_rect[2]) and (y+h) <(max_rect[1]+max_rect[3]):
if bScript and each[0] == max_area: #跳过最大面积的表格
pass
else:
x, y, w, h = each[1]
dst_dir = 'clip'
if not os.path.exists(dst_dir):
os.mkdir(dst_dir)
name = '%s/%d_%d_%d_%d.png' % (dst_dir, x, y, x + w, y + h)
cv2.imwrite(
name, each[2]
) #png 0-9 ,[int(cv2.CV_IMWRITE_PNG_COMPRESSION),9]
imgtemp = Image.open(name).convert("RGB") #图片临时保存再来
to_image.paste(
imgtemp,
(0, sumH)) #((x - 1) * IMAGE_SIZE, (y - 1) * IMAGE_SIZE)
area_coord_map.append(
((x, y, w, h), (0, sumH, 0 + w, sumH + h)))
sumH += h
MeargeImageName = 'clip/final.png'
to_image.save(MeargeImageName) #95最佳 默认75 增采样
imgtempV2 = Image.open(MeargeImageName).convert("RGB") # 图片临时保存再来
W, H = imgtempV2.size
timeTake = time.time()
_, result, angle = model.model(
imgtempV2,
detectAngle=DETECTANGLE, ##是否进行文字方向检测
config=dict(
MAX_HORIZONTAL_GAP=10, ##字符之间的最大间隔，用于文本行的合并
MIN_V_OVERLAPS=0.7,
MIN_SIZE_SIM=0.7,
TEXT_PROPOSALS_MIN_SCORE=0.1,
TEXT_PROPOSALS_NMS_THRESH=0.3,
TEXT_LINE_NMS_THRESH=0.99, ##文本行之间测iou值
MIN_RATIO=1.0,
LINE_MIN_SCORE=0.2,
TEXT_PROPOSALS_WIDTH=0,
MIN_NUM_PROPOSALS=0,
),
leftAdjust=True, ##对检测的文本行进行向左延伸
rightAdjust=True, ##对检测的文本行进行向右延伸
alph=0.2, ##对检测的文本行进行向右、左延伸的倍数
ifadjustDegree=False ##是否先小角度调整文字倾斜角度
)
print(result)
boxes = []
for line in result:
cx = line['cx']
cy = line['cy']
w = line['w']
h = line['h']
text = line['text']
print(text)
return
import matplotlib.pyplot as plt
from model import model
from misc import plot_decision_boundary, load_dataset, predict, predict_dec
# %matplotlib inline
plt.rcParams['figure.figsize'] = (7.0, 4.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'
train_X, train_Y = load_dataset()
# train 3-layer model
layers_dims = [train_X.shape[0], 5, 2, 1]
parameters = model(train_X, train_Y, layers_dims, optimizer="gd")
# Predict
predictions = predict(train_X, train_Y, parameters)
# Plot decision boundary
plt.title("Model with Gradient Descent optimization")
axes = plt.gca()
axes.set_xlim([-1.5, 2.5])
axes.set_ylim([-1, 1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
train_Y)
# train 3-layer model
layers_dims = [train_X.shape[0], 5, 2, 1]
parameters = model(train_X,
train_Y,
def __init__(self,dic):
self.word_dic = dic
self.komoran = Komoran()
self.Model = model.model(len(self.word_dic))
transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
])),
batch_size=batch,
shuffle=True)
return {'train': train_loader, 'test': test_loader}
if __name__ == '__main__':
epoch = 100
loader = load_cifar10()
classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse',
'ship', 'truck')
net: model = model()
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
net.to(device)
print(device)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(params=net.parameters(),
lr=0.001,
momentum=0.9)
train_loss = []
train_acc = []
test_acc = []
for e in range(epoch):
def main():
enc = encoder.get_encoder(args.model_name)
hparams = model.default_hparams()
hparams.batch_size=args.batch_size
hparams.seq_len=args.seq_len
##data_path
args.train_data_path=args.data_dir+args.dataset+'/train.txt'
args.eval_data_path=args.data_dir+args.dataset+'/dev.txt'
args.test_data_path=args.data_dir+args.dataset+'/test.txt'
args.eval_data_path=args.test_data_path ###Test mode only!
args.gpt_save_path=args.gpt_save_dir+args.dataset+'/'
args.dis_save_path=args.dis_save_dir+args.dataset+'/'
args.gpt_sample_dir2=args.gpt_sample_dir+args.dataset+'/'
args.dis_sample_dir2=args.dis_sample_dir+args.dataset+'/'
args.log_path=args.log_dir+args.dataset+'/'
maketree(args.gpt_save_dir)
maketree(args.dis_save_dir)
maketree(args.gpt_save_path)
maketree(args.dis_save_path)
args.dis_save_path=args.dis_save_path+'/'
maketree(args.dis_save_path)
maketree(args.gpt_sample_dir2)
maketree(args.dis_sample_dir2)
args.dis_sample_dir2=args.dis_sample_dir2+'/'
maketree(args.dis_sample_dir2)
maketree(args.log_path)
with open(os.path.join('models', args.model_name, 'hparams.json')) as f:
hparams.override_from_dict(json.load(f))
if args.sample_length > hparams.n_ctx:
raise ValueError(
"Can't get samples longer than window size: %s" % hparams.n_ctx)
if args.model_name == '345M':
args.memory_saving_gradients = True
if args.optimizer == 'adam':
args.only_train_transformer_layers = True
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.graph_options.rewrite_options.layout_optimizer = rewriter_config_pb2.RewriterConfig.OFF
with tf.Session(config=config) as sess:
scope_discri='distri'
def get_dis_logit_and_prob(context, scope):
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
context=tf.reshape(context, [args.batch_size, args.seq_len])
emb=tf.get_variable(name='emb', initializer=tf.random.normal([hparams.n_vocab, 32], 0, 0.02))
context_emb=tf.nn.embedding_lookup(emb, context)
logit=dis(context_emb, scope=scope_discri)
prob=tf.nn.sigmoid(logit)
return logit, prob
##Build discriminator
def build_dis_layer(scope):
context_pos_discri = tf.placeholder(tf.int32, [args.batch_size, args.seq_len])
context_neg_discri = tf.placeholder(tf.int32, [args.batch_size, args.seq_len])
label_pos_discri=tf.ones([args.batch_size], dtype=tf.float32)
label_neg_discri=tf.zeros([args.batch_size], dtype=tf.float32)
logit_pos_discri, prob_pos_discri=get_dis_logit_and_prob(context_pos_discri, scope=scope)
logit_neg_discri, _=get_dis_logit_and_prob(context_neg_discri, scope=scope)
loss_pre_pos_discri=tf.nn.sigmoid_cross_entropy_with_logits(labels=label_pos_discri, logits=logit_pos_discri)
loss_pos_discri=tf.reduce_mean(loss_pre_pos_discri)
loss_pre_neg_discri=tf.nn.sigmoid_cross_entropy_with_logits(labels=label_neg_discri, logits=logit_neg_discri)
loss_neg_discri=tf.reduce_mean(loss_pre_neg_discri)
loss_discri=(loss_pos_discri*args.pos_loss_weight+loss_neg_discri)/(1+args.pos_loss_weight)
train_var_list_discri=[x for x in tf.global_variables() if scope in x.name]
train_op_discri=tf.train.AdamOptimizer().minimize(loss_discri, var_list=train_var_list_discri)
var_list_discri=[x for x in tf.global_variables() if scope in x.name]
initializer_discri=tf.variables_initializer(var_list_discri)
saver_discri=tf.train.Saver(var_list=var_list_discri, max_to_keep=1)
print('discri: {} build succeed!'.format(scope))
return context_pos_discri, context_neg_discri, loss_pos_discri, loss_neg_discri, loss_discri, train_op_discri, initializer_discri, saver_discri, prob_pos_discri
class dis_class:
def __init__(self, layer_num=1, scope=scope_discri):
self.model=[]
print(layer_num)
for i in range(layer_num):
layer={'scope': scope+str(i)}
layer['context_pos_discri'], layer['context_neg_discri'], layer['loss_pos_discri'], layer['loss_neg_discri'], layer['loss_discri'], layer['train_op_discri'], layer['initializer_discri'], layer['saver_discri'], layer['prob_pos_discri'] = build_dis_layer(scope+str(i))
self.model.append(layer)
def prob(self, context, layer=-1):
if layer==-1:
layer=len(self.model)
prob_final=tf.ones(tf.shape(context)[0], dtype=tf.float32)
for i in range(layer):
item=self.model[i]
scope=item['scope']
_, prob=get_dis_logit_and_prob(context, scope=scope)
prob_final*=prob
return prob_final
Dis=dis_class(layer_num=1)
context = tf.placeholder(tf.int32, [args.batch_size, None])
context_len=tf.placeholder(tf.int32, [args.batch_size])
context_mask=tf.sequence_mask(context_len-1, args.seq_len-1, dtype=tf.float32)
context_in = randomize(context, hparams, args.noise)
output = model.model(hparams=hparams, X=context_in)
loss_tensor = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=context[:, 1:], logits=output['logits'][:, :-1])*context_mask
context_prob_cut=Dis.prob(context)
loss=tf.reduce_sum(loss_tensor, axis=1)/(tf.reduce_sum(context_mask, axis=1)+1e-7)
loss_sen=tf.reduce_sum(loss)
loss=tf.reduce_mean(loss)
if args.val_every > 0:
def transform_np(x, lift=args.exponential_param):
x=x-0.5
x=x+np.abs(x)
return lift*x**2
def transform(x, lift=args.exponential_param):
x=x-0.5
x=x+tf.abs(x)
return lift*x**2
val_context = tf.placeholder(tf.int32, [args.val_batch_size, args.seq_len])
val_context_len=tf.placeholder(tf.int32, [args.batch_size])
NLL_bias=tf.placeholder(tf.float32, [])
val_context_mask=tf.sequence_mask(val_context_len-1, args.seq_len-1, dtype=tf.float32)
val_output = model.model(hparams=hparams, X=val_context)
val_loss_tensor =tf.nn.sparse_softmax_cross_entropy_with_logits(labels=val_context[:, 1:], logits=val_output['logits'][:, :-1])*val_context_mask
val_context_prob_cut=Dis.prob(val_context)
val_NLL_cut=tf.log(val_context_prob_cut+1e-7)
val_loss=tf.reduce_sum(val_loss_tensor, axis=1)/(tf.reduce_sum(val_context_mask, axis=1)+1e-7)
val_loss_cut=(tf.reduce_sum(val_loss_tensor, axis=1)+NLL_bias)/(tf.reduce_sum(val_context_mask, axis=1)+1e-7)-val_NLL_cut/tf.cast(val_context_len, tf.float32)
val_loss_sum=tf.reduce_sum(val_loss_tensor, axis=1)
val_loss_cut_sum=(tf.reduce_sum(val_loss_tensor, axis=1)+NLL_bias)-val_NLL_cut
val_loss_mean=tf.reduce_mean(val_loss)
val_loss_cut_mean=tf.reduce_mean(val_loss_cut)
val_loss_summary = tf.summary.scalar('val_loss', val_loss_mean)
tf_sample = sample.sample_sequence(
hparams=hparams,
length=args.seq_len,
context=context,
batch_size=args.batch_size,
temperature=1.0,
top_k=args.top_k,
top_p=args.top_p,
start_token=enc.encoder['<|endoftext|>'])
start_token=enc.encoder['<|endoftext|>']
all_vars = [v for v in tf.trainable_variables() if 'model' in v.name]
train_vars = [v for v in all_vars if '/h' in v.name] if args.only_train_transformer_layers else all_vars
if args.optimizer == 'adam':
opt = tf.train.AdamOptimizer(learning_rate=args.learning_rate)
elif args.optimizer == 'sgd':
opt = tf.train.GradientDescentOptimizer(learning_rate=args.learning_rate)
else:
exit('Bad optimizer:', args.optimizer)
if args.accumulate_gradients > 1:
if args.memory_saving_gradients:
exit("Memory saving gradients are not implemented for gradient accumulation yet.")
opt = AccumulatingOptimizer(
opt=opt,
var_list=train_vars)
opt_reset = opt.reset()
opt_compute = opt.compute_gradients(loss)
opt_apply = opt.apply_gradients()
summary_loss = tf.summary.scalar('loss', opt_apply)
else:
if args.memory_saving_gradients:
opt_grads = memory_saving_gradients.gradients(loss, train_vars)
else:
opt_grads = tf.gradients(loss, train_vars)
opt_grads = list(zip(opt_grads, train_vars))
opt_apply = opt.apply_gradients(opt_grads)
summary_loss = tf.summary.scalar('loss', loss)
summary_lr = tf.summary.scalar('learning_rate', args.learning_rate)
summaries = tf.summary.merge([summary_lr, summary_loss])
summary_log = tf.summary.FileWriter(
os.path.join(CHECKPOINT_DIR, args.run_name))
saver = tf.train.Saver(var_list=all_vars, max_to_keep=1)
sess.run(tf.global_variables_initializer())
if args.restore_from == 'latest':
ckpt = tf.train.latest_checkpoint(
os.path.join(CHECKPOINT_DIR, args.run_name))
if ckpt is None:
# Get fresh GPT weights if new run.
ckpt = tf.train.latest_checkpoint(
os.path.join('models', args.model_name))
elif args.restore_from == 'fresh':
ckpt = tf.train.latest_checkpoint(
os.path.join('models', args.model_name))
else:
ckpt = tf.train.latest_checkpoint(args.restore_from)
print('Loading checkpoint', ckpt)
saver.restore(sess, ckpt)
print('Loading dataset...')
data_list, data_len = load_dataset(enc, args.train_data_path, args.seq_len)
data_sampler = Sampler(data_list, data_len )
if args.val_every > 0:
val_data_list, val_data_len = load_dataset(enc, args.eval_data_path, args.seq_len)
print('dataset has', data_sampler.total_size, 'tokens')
print('Training...')
if args.val_every > 0:
# Sample from validation set once with fixed seed to make
# it deterministic during training as well as across runs.
val_data_sampler = Sampler(val_data_list, val_data_len, seed=1)
val_batches = [val_data_sampler.sample(args.batch_size) for _ in range(args.val_batch_count)]
counter = 0
counter_path = os.path.join(CHECKPOINT_DIR, args.run_name, 'counter')
if os.path.exists(counter_path):
# Load the step number if we're resuming a run
# Add 1 so we don't immediately try to save again
with open(counter_path, 'r') as fp:
counter = int(fp.read()) + 1
def save():
maketree(os.path.join(CHECKPOINT_DIR, args.run_name))
print(
'Saving',
os.path.join(CHECKPOINT_DIR, args.run_name,
'model-{}').format(counter))
saver.save(
sess,
os.path.join(CHECKPOINT_DIR, args.run_name, 'model'),
global_step=counter)
with open(counter_path, 'w') as fp:
fp.write(str(counter) + '\n')
def mask_dis(sample1, sample2):
for i in range(len(sample1)):
for j in range(len(sample1[i])):
if sample1[i][j]==start_token:
l1=j
break
for j in range(len(sample2[i])):
if sample2[i][j]==start_token:
l2=j
break
l=min(l1,l2)
if l>=2:
l_mask=np.random.randint(0, l, [])
for j in range(l_mask, len(sample1[i])):
sample1[i][j]=start_token
for j in range(l_mask, len(sample2[i])):
sample2[i][j]=start_token
return sample1, sample2
def train_step_discri(layer_id=0, mask_train_epoch=0):
pos_samples, _=data_sampler.sample(args.batch_size)
neg_samples=generate_negative_sample_and_diversify(layer_id=layer_id)[:,:args.seq_len]
_, loss=sess.run([Dis.model[layer_id]['train_op_discri'], Dis.model[layer_id]['loss_discri']], feed_dict={Dis.model[layer_id]['context_pos_discri']: pos_samples, Dis.model[layer_id]['context_neg_discri']: neg_samples})
for epoch in range(mask_train_epoch):
pos_samples, _=data_sampler.sample(args.batch_size)
neg_samples_copy=copy(neg_samples)
pos_samples, neg_samples_copy=mask_dis(pos_samples, neg_samples_copy)
_, loss_mask=sess.run([Dis.model[layer_id]['train_op_discri'], Dis.model[layer_id]['loss_discri']], feed_dict={Dis.model[layer_id]['context_pos_discri']: pos_samples, Dis.model[layer_id]['context_neg_discri']: neg_samples_copy})
return loss
def coverage_rate(list1, list2):
overlap1=0
for item in list1:
if item in list2:
overlap1+=1
overlap2=0
for item in list2:
if item in list1:
overlap2+=1
return max((overlap1+0.0)/(len(list1)+0.0), (overlap2+0.0)/(len(list2)+0.0))
def diversify(samples, samples_mem):
#return list(range(len(samples))) ##########tem
sample_split=[]
for item in samples:
sample_split.append(item.split())
sample_selected=[]
for item in sample_split:
flag=0
for item_ref in sample_selected+samples_mem:
if coverage_rate(item, item_ref)>0.5: ##temporary strategy
flag=1
break
if flag==0:
sample_selected.append(item)
id_list=[]
for item in sample_selected:
id_list.append(sample_split.index(item))
return id_list
def generate_negative_sample_and_diversify(layer_id, generate_num=args.batch_size):
result_list=[]
generate_num_now=0
samples_mem=[]
while generate_num_now<generate_num:
t=time.time()
sample_id=generate_negative_sample(layer_id=layer_id)[:,:args.seq_len]
samples=[]
t1=time.time()
selected_id_list=np.arange(len(sample_id))
t2=time.time()
result_list.append(sample_id[selected_id_list])
generate_num_now+=len(selected_id_list)
print(generate_num_now, t1-t, t2-t1)
return np.concatenate(result_list, axis=0)[:generate_num]
def generate_discri_sample2(layer_id=-1, sample_size=10000, save_path=args.dis_sample_dir2+'sample.txt'):
samples=[]
sample_id=generate_negative_sample_and_diversify(layer_id, sample_size)
for i in range(len(sample_id)):
samples.append(enc.decode(sample_id[i]).split('<|endoftext|>')[1].split('\n')[0])
print(len(samples))
with open(save_path, 'w') as g:
g.write('\n'.join(samples))
def generate_discri_sample3(layer_id=-1, sample_size=10000, save_path='/mnt/cephfs_new_wj/mlnlp/miaoning/Experiment/gpt-2-sep/samples/discri/sample2.txt'):
samples=[]
while len(samples)<sample_size:
sample_id=generate_negative_sample(layer_id)
for i in range(len(sample_id)):
sample_tem=enc.decode(sample_id[i]).split('<|endoftext|>')[1].split('\n')[0]
if len(sample_tem.split())>=3:
samples.append(sample_tem)
print(len(samples))
with open(save_path, 'w') as g:
g.write('\n'.join(samples))
def generate_sample2(save_path=args.gpt_sample_dir2+'sample.txt'):
generate_discri_sample3(layer_id=0, sample_size=3000, save_path=save_path)
def eval_discri_NLL(layer_id=0):
losses_pos=[]
losses_neg=[]
for batch in tqdm.tqdm(val_batches):
pos_samples, pos_len=batch
neg_samples=generate_negative_sample(layer_id=layer_id)[:,:args.seq_len]
loss_pos=sess.run(Dis.model[layer_id]['loss_pos_discri'], feed_dict={Dis.model[layer_id]['context_pos_discri']: pos_samples})
loss_neg=sess.run(Dis.model[layer_id]['loss_neg_discri'], feed_dict={Dis.model[layer_id]['context_neg_discri']: neg_samples})
losses_pos.append(loss_pos)
losses_neg.append(loss_neg)
return np.mean(losses_pos), np.mean(losses_neg)
def train_discri(train_step, eval_every):
#sess.run(initializer_discri)
train_losses=[]
for layer_id in range(len(Dis.model)):
for epoch in range(train_step):
if epoch % eval_every==0:
train_losses=np.mean(train_losses)
train_losses=[]
eval_NLL_pos, eval_NLL_neg=eval_discri_NLL(layer_id)
eval_loss=(eval_NLL_pos*args.pos_loss_weight+eval_NLL_neg)/(args.pos_loss_weight+1)
print('layer_id:{} discri eval loss:{}'.format(layer_id, eval_loss))
print('layer_id:{} discri NLL pos: {}, discri NLL neg: {}'.format(layer_id, eval_NLL_pos, eval_NLL_neg))
print(epoch)
if epoch==0:
eval_loss_old=eval_loss
else:
print(eval_loss, eval_loss_old)
if eval_loss<=eval_loss_old+0.01:
if eval_loss<eval_loss_old:
eval_loss_old=eval_loss
save_path=args.dis_save_path+str(layer_id)+'/'
if not os.path.isdir(save_path):
os.mkdir(save_path)
Dis.model[layer_id]['saver_discri'].save(sess, save_path+'a')
print('model discri saved!')
else:
pass
break
train_loss=train_step_discri(layer_id)
print('layer_id:{} discri train loss:{}'.format(layer_id, train_loss))
train_losses.append(train_loss)
return eval_loss_old
tf_sample_0 = sample_link.sample_sequence(
hparams=hparams,
length=args.seq_len,
context=context,
batch_size=args.batch_size,
temperature=1.0,
top_k=args.top_k,
top_p=args.top_p,
start_token=enc.encoder['<|endoftext|>'])
tf_sample_dict={}
def generate_negative_sample(layer_id=0):
##output the filtered result of layer layer_id-1
if layer_id==0:
tf_sample=tf_sample_0
sample = data_sampler.sample(args.batch_size)[0][:,0:1]
out = sess.run(
tf_sample,
feed_dict={context: sample})
for i in range(len(out)):
flag=0
for j in range(len(out[i])):
if flag==2:
out[i][j]=start_token
continue
if out[i][j]==start_token:
flag+=1
return out
else:
if layer_id==-1:
layer_id=len(Dis.model)
if layer_id in tf_sample_dict:
tf_sample=tf_sample_dict[layer_id]
else:
tf_sample = sample_link.sample_sequence_SMC(
Dis=Dis,
layer=layer_id,
hparams=hparams,
length=args.seq_len,
context=context,
batch_size=args.batch_size,
temperature=1.0,
top_k=args.top_k,
top_p=args.top_p,
start_token=enc.encoder['<|endoftext|>'])
tf_sample_dict[layer_id]=tf_sample
sample = data_sampler.sample(args.batch_size)[0][:,0:1]
out = sess.run(
tf_sample,
feed_dict={context: sample})
for i in range(len(out)):
flag=0
for j in range(len(out[i])):
if flag==2:
out[i][j]=start_token
continue
if out[i][j]==start_token:
flag+=1
return out
def validation():
print('Calculating validation loss...')
start_time=time.time()
losses = []
rates=[]
for batch in tqdm.tqdm(val_batches):
losses.append(sess.run(val_loss_mean, feed_dict={val_context: batch[0], val_context_len: batch[1]}))
v_val_loss = np.mean(losses)
v_summary = sess.run(val_loss_summary, feed_dict={val_loss_mean: v_val_loss})
summary_log.add_summary(v_summary, counter)
summary_log.flush()
print(
'[{counter} | {time:2.2f}] validation loss = {loss:2.2f}'
.format(
counter=counter,
time=time.time() - start_time,
loss=v_val_loss))
return v_val_loss
def validation_cut(NLL_bias_0=0):
print('Calculating validation loss...')
losses = []
rates=[]
for batch in tqdm.tqdm(val_batches):
losses.append(sess.run(val_loss_cut_mean, feed_dict={val_context: batch[0], val_context_len: batch[1], NLL_bias:NLL_bias_0}))
v_val_loss = np.mean(losses)
#v_summary = sess.run(val_loss_summary, feed_dict={val_loss_mean: v_val_loss})
#summary_log.add_summary(v_summary, counter)
#summary_log.flush()
print(
'[{counter} | {time:2.2f}] validation cut loss = {loss:2.2f}'
.format(
counter=counter,
time=time.time() - start_time,
loss=v_val_loss))
return v_val_loss
def sample_batch():
return [data_sampler.sample(1024) for _ in range(args.batch_size)]
def train_gpt():
val_loss_old=10000.0
avg_loss = (0.0, 0.0)
start_time = time.time()
counter=0
while True:
#pretraining
if counter % args.save_every == 0:
pass
#save()
if counter % args.sample_every == 0:
pass
#generate_samples()
if args.val_every > 0 and (counter % args.val_every == 0 or counter == 1):
val_loss_1=validation()
#generate_sample2(save_path='./samples/finetune/step/'+str(counter //args.val_every)+'.txt')
print(str(counter //args.val_every))
if val_loss_1>=val_loss_old:
print('pre-training ends!')
#break
else:
val_loss_old=val_loss_1
saver.save(sess, args.gpt_save_path+'a')
print('save succeed!')
if args.accumulate_gradients > 1:
sess.run(opt_reset)
for _ in range(args.accumulate_gradients):
batch, batch_len=data_sampler.sample(args.batch_size)
sess.run(
opt_compute, feed_dict={context: batch, context_len:batch_len})
(v_loss, v_summary) = sess.run((opt_apply, summaries))
else:
batch, batch_len=data_sampler.sample(args.batch_size)
(_, v_loss, v_summary) = sess.run(
(opt_apply, loss, summaries),
feed_dict={context: batch, context_len:batch_len})
summary_log.add_summary(v_summary, counter)
avg_loss = (avg_loss[0] * 0.9 + v_loss,
avg_loss[1] * 0.9 + 1.0)
print(
'[{counter} | {time:2.2f}] loss={loss:2.2f} avg={avg:2.2f}'
.format(
counter=counter,
time=time.time() - start_time,
loss=v_loss,
avg=avg_loss[0] / avg_loss[1]))
counter += 1
try:
if args.finetune:
train_gpt()
if True:
save_path=tf.train.latest_checkpoint(args.gpt_save_path)
saver.restore(sess, save_path)
print('Load gpt2 succeeded!')
if args.evaluate_finetune:
generate_sample2()
sample_path=args.gpt_sample_dir2+'sample.txt'
rev_ppl=train.file_f(train_data_path=sample_path, val_data_path=args.eval_data_path)
print('Rev_ppl for finetuning:{}'.format(rev_ppl))
##Begin cutting
counter=0
if args.train_tailor:
train_discri(500, 10)
if True:
for layer_id in range(len(Dis.model)):
save_path=args.dis_save_path+str(layer_id)+'/'
save_path=args.dis_save_path
save_path=tf.train.latest_checkpoint(save_path)
Dis.model[layer_id]['saver_discri'].restore(sess, save_path)
print('Load dis model succeeded!')
if args.evaluate_tailor:
generate_discri_sample2()
sample_path=save_path=args.dis_sample_dir2+'sample.txt'
rev_ppl=train.file_f(train_data_path=sample_path, val_data_path=args.eval_data_path)
print('Rev_ppl for Tailor-SMC:{}'.format(rev_ppl))
except KeyboardInterrupt:
print('interrupted')
def get_predictor(checkpoint_path):
logger.info('loading model')
import tensorflow as tf
import model
from icdar import restore_rectangle
import lanms
from eval import resize_image, sort_poly, detect
input_images = tf.placeholder(tf.float32,
shape=[None, None, None, 3],
name='input_images')
global_step = tf.get_variable('global_step', [],
initializer=tf.constant_initializer(0),
trainable=False)
f_score, f_geometry = model.model(input_images, is_training=False)
variable_averages = tf.train.ExponentialMovingAverage(0.997, global_step)
saver = tf.train.Saver(variable_averages.variables_to_restore())
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
ckpt_state = tf.train.get_checkpoint_state(checkpoint_path)
model_path = os.path.join(
checkpoint_path, os.path.basename(ckpt_state.model_checkpoint_path))
logger.info('Restore from {}'.format(model_path))
saver.restore(sess, model_path)
def predictor(img):
"""
:return: {
'text_lines': [
{
'score': ,
'x0': ,
'y0': ,
'x1': ,
...
'y3': ,
}
],
'rtparams': { # runtime parameters
'image_size': ,
'working_size': ,
},
'timing': {
'net': ,
'restore': ,
'nms': ,
'cpuinfo': ,
'meminfo': ,
'uptime': ,
}
}
"""
start_time = time.time()
rtparams = collections.OrderedDict()
rtparams['start_time'] = datetime.datetime.now().isoformat()
rtparams['image_size'] = '{}x{}'.format(img.shape[1], img.shape[0])
timer = collections.OrderedDict([('net', 0), ('restore', 0),
('nms', 0)])
im_resized, (ratio_h, ratio_w) = resize_image(img)
rtparams['working_size'] = '{}x{}'.format(im_resized.shape[1],
im_resized.shape[0])
start = time.time()
score, geometry = sess.run(
[f_score, f_geometry],
feed_dict={input_images: [im_resized[:, :, ::-1]]})
timer['net'] = time.time() - start
boxes, timer = detect(score_map=score, geo_map=geometry, timer=timer)
logger.info('net {:.0f}ms, restore {:.0f}ms, nms {:.0f}ms'.format(
timer['net'] * 1000, timer['restore'] * 1000, timer['nms'] * 1000))
if boxes is not None:
scores = boxes[:, 8].reshape(-1)
boxes = boxes[:, :8].reshape((-1, 4, 2))
boxes[:, :, 0] /= ratio_w
boxes[:, :, 1] /= ratio_h
duration = time.time() - start_time
timer['overall'] = duration
logger.info('[timing] {}'.format(duration))
text_lines = []
if boxes is not None:
text_lines = []
for box, score in zip(boxes, scores):
box = sort_poly(box.astype(np.int32))
if np.linalg.norm(box[0] -
box[1]) < 5 or np.linalg.norm(box[3] -
box[0]) < 5:
continue
tl = collections.OrderedDict(
zip(['x0', 'y0', 'x1', 'y1', 'x2', 'y2', 'x3', 'y3'],
map(float, box.flatten())))
tl['score'] = float(score)
text_lines.append(tl)
ret = {
'text_lines': text_lines,
'rtparams': rtparams,
'timing': timer,
}
ret.update(get_host_info())
return ret
return predictor
import model
import numpy as np
from keras.models import Model
from keras.models import load_model
xTrain = ip.imgprocess(trainImg, trainDir) / 255
yTrain = np.array(trainData[['MIDDLE', 'OLD', 'YOUNG']])
#print(type(xTrain))
#print(yTrain[:5])
'''
for i in range(11):
print(xTrain[i].dtype)
cv2.imshow(trainData['Class'][i], xTrain[i])
cv2.waitKey(0)
'''
trainModel = model.model()
trainModel.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
print(trainModel.summary())
trainModel.fit(xTrain, yTrain, epochs=150, batch_size=1000)
preds = trainModel.evaluate(xTrain, yTrain)
print("Loss = " + str(preds[0]))
print("Train Accuracy = " + str(preds[1]))
trainModel.save('../trainModel.h5')
import tensorflow as tf
from model import model
import ujson as json
import utils
from reader import reader
with open('./params.json', 'rb') as f:
params = json.load(f)
vocab, vocab_map, reverse_map = utils.vocab_maker(params['raw_data_dir'])
params['vocab_size'] = max(vocab_map.values()) + 1
reader = reader()
model = model(params)
model.forward()
model.backward()
fp = open('./data/dataset.pkl', 'rb')
init_op = [tf.local_variables_initializer(), tf.global_variables_initializer()]
sess = tf.Session()
sess.run(init_op)
for epoch in range(params['n_epochs']):
batch = reader.read_batches(fp)
inps, trgts, weights = utils.process_batches(batch, vocab_map)
feed_dict = {model.inp: inps, model.trgt: trgts, model.weights: weights}
logits, loss, _ = sess.run([model.logits, model.loss, model.train_op],
feed_dict)
print(loss)
def train(model,optimizer,epoch,save_dir):
dataParser = DataParser(args.batch_size)
batch_time = Averagvalue()
data_time = Averagvalue()
losses = Averagvalue()
# switch to train mode
model.train()
end = time.time()
epoch_loss = []
counter = 0
for batch_index ,(images,labels_numpy) in enumerate(generate_minibatches(dataParser,True)):
# measure data loading time
data_time.update(time.time()-end)
labels = []
if torch.cuda.is_available():
images = torch.from_numpy(images).cuda()
for item in labels_numpy:
labels.append(torch.from_numpy(item).cuda())
else:
images = torch.from_numpy(images)
for item in labels_numpy:
labels.append(torch.from_numpy(item))
if torch.cuda.is_available():
loss =torch.zeros(1).cuda()
else:
loss = torch.zeros(1)
optimizer.zero_grad()
outputs = model(images)
# 四张GT监督
for o in outputs[9:]: # o2 o3 o4
t_loss = cross_entropy_loss(o, labels[-1])
loss = loss +t_loss
counter +=1
for c_index,c in enumerate(outputs[:8]):
loss = loss + cross_entropy_loss(c, labels[c_index])
loss = loss/11
loss.backward()
acc_scroe = my_accuracy_score(outputs[9].cpu().detach().numpy(),labels[-1].cpu().detach().numpy())
print('the acc is :',acc_scroe)
# 下面应该是用来解决batch size 过下的问题
# if counter == args.itersize:
# optimizer.step()
# optimizer.zero_grad()
# counter = 0
optimizer.step()
optimizer.zero_grad()
# measure the accuracy and record loss
losses.update(loss.item(),images.size(0))
epoch_loss.append(loss.item())
batch_time.update(time.time()-end)
end = time.time()
# display and logging
if not isdir(save_dir):
os.makedirs(save_dir)
if batch_index % args.print_freq ==0:
info = 'Epoch: [{0}/{1}][{2}/{3}] '.format(epoch, args.maxepoch, batch_index, dataParser.steps_per_epoch) + \
'Time {batch_time.val:.3f} (avg:{batch_time.avg:.3f}) '.format(batch_time=batch_time) + \
'Loss {loss.val:f} (avg:{loss.avg:f}) '.format(
loss=losses)
print(info)
# torch.save(model,join(save_dir,"checkpoint.pth"))
# 每一轮保存一次参数
save_checkpoint({'epoch': epoch,'state_dict':model.state_dict(), 'optimizer': optimizer.state_dict()},filename=join(save_dir,"epooch-%d-checkpoint.pth" %epoch))
return losses.avg,epoch_loss
def train_main(dataset,
model_name='345M',
seed=None,
batch_size=1,
sample_length=1023,
sample_num=1,
sample_every=100,
run_name='run1',
restore_from='latest',
stop_after=None,
learning_rate=0.001,
beta1=0.9,
beta2=0.999,
epsilon=1e-08,
save_every=1000):
enc = encoder.get_encoder(model_name)
hparams = model.default_hparams()
with open(os.path.join('models', model_name, 'hparams.json')) as f:
hparams.override_from_dict(json.load(f))
if sample_length is None:
sample_length = hparams.n_ctx // 2
elif sample_length > hparams.n_ctx:
raise ValueError("Can't get samples longer than window size: %s" %
hparams.n_ctx)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
context = tf.placeholder(tf.int32, [batch_size, None])
np.random.seed(seed)
tf.set_random_seed(seed)
output = model.model(hparams=hparams, X=context)
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=context[:, 1:], logits=output['logits'][:, :-1]))
tf_sample = sample.sample_sequence(hparams=hparams,
length=sample_length,
context=context,
batch_size=batch_size,
temperature=1.0,
top_k=40)
train_vars = [v for v in tf.trainable_variables() if 'model' in v.name]
opt = tf.train.AdamOptimizer(learning_rate=learning_rate,
beta1=beta1,
beta2=beta2,
epsilon=epsilon).minimize(
loss, var_list=train_vars)
saver = tf.train.Saver(var_list=train_vars,
max_to_keep=5,
keep_checkpoint_every_n_hours=2)
sess.run(tf.global_variables_initializer())
if restore_from == 'latest':
ckpt = tf.train.latest_checkpoint(
os.path.join(CHECKPOINT_DIR, run_name))
if ckpt is None:
# Get fresh GPT weights if new run.
ckpt = tf.train.latest_checkpoint(
os.path.join('models', model_name))
elif restore_from == 'fresh':
ckpt = tf.train.latest_checkpoint(
os.path.join('models', model_name))
else:
ckpt = tf.train.latest_checkpoint(restore_from)
print('Loading checkpoint', ckpt)
saver.restore(sess, ckpt)
print('Loading dataset...')
chunks = load_dataset(enc, dataset)
data_sampler = Sampler(chunks)
print('dataset has', data_sampler.total_size, 'tokens')
print('Training...')
counter = 1
if os.path.exists(os.path.join(CHECKPOINT_DIR, run_name, 'counter')):
# Load the step number if we're resuming a run
# Add 1 so we don't immediately try to save again
with open(os.path.join(CHECKPOINT_DIR, run_name, 'counter'),
'r') as fp:
counter = int(fp.read()) + 1
def save():
maketree(os.path.join(CHECKPOINT_DIR, run_name))
print(
'Saving',
os.path.join(CHECKPOINT_DIR, run_name,
'model-{}').format(counter))
saver.save(sess,
os.path.join(CHECKPOINT_DIR, run_name, 'model'),
global_step=counter)
with open(os.path.join(CHECKPOINT_DIR, run_name, 'counter'),
'w') as fp:
fp.write(str(counter) + '\n')
def generate_samples():
context_tokens = data_sampler.sample(1)
all_text = []
index = 0
while index < sample_num:
out = sess.run(
tf_sample,
feed_dict={context: batch_size * [context_tokens]})
for i in range(min(sample_num - index, batch_size)):
text = enc.decode(out[i])
text = '======== SAMPLE {} ========\n{}\n'.format(
index + 1, text)
all_text.append(text)
index += 1
print(text)
maketree(os.path.join(SAMPLE_DIR, run_name))
with open(
os.path.join(SAMPLE_DIR, run_name,
'samples-{}').format(counter), 'w') as fp:
fp.write('\n'.join(all_text))
avg_loss = (0.0, 0.0)
start_time = time.time()
try:
while counter != stop_after:
if counter % save_every == 0:
save()
if counter % sample_every == 0:
generate_samples()
batch = [data_sampler.sample(1024) for _ in range(batch_size)]
_, lv = sess.run((opt, loss), feed_dict={context: batch})
avg_loss = (avg_loss[0] * 0.99 + lv, avg_loss[1] * 0.99 + 1.0)
print(
'[{counter} | {time:2.2f}] loss={loss:2.2f} avg={avg:2.2f}'
.format(counter=counter,
time=time.time() - start_time,
loss=lv,
avg=avg_loss[0] / avg_loss[1]))
counter += 1
except KeyboardInterrupt:
print('interrupted')
finally:
save()
def __init__(self):
mixer.init()
self.my_model = model.model()
# coding:utf-8
import time
from glob import glob
import numpy as np
from PIL import Image
import model
paths = glob('./test/*.*')
if __name__ == '__main__':
im = Image.open("./test/010.png")
img = np.array(im.convert('RGB'))
t = time.time()
result, img, angle = model.model(img,
model='keras',
adjust=True,
detectAngle=True)
print("It takes time:{}s".format(time.time() - t))
print("---------------------------------------")
for key in result:
print(result[key][1])
def do_train(args):
# create model
dnn_model = model(args.architecture, args.num_classes)
if args.num_gpus == 1:
dnn_model = dnn_model.cuda()
else:
dnn_model = torch.nn.DataParallel(dnn_model, device_ids = range(0, args.num_gpus)).cuda()
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
lr = utils.get_policy(args.LR_policy, args.LR_details)
wd = utils.get_policy(args.WD_policy, args.WD_details)
optimizer = utils.get_optimizer(args.optimizer, dnn_model.parameters(), 0.01)
train_loader = data_loader.CSVDataset(args.train_info, args.delimiter, args.raw_size, args.processed_size, args.batch_size, args.num_workers, args.path_prefix, True, shuffle = True).load()
start_epoch = 0
if args.retrain_from is not None:
checkpoint = torch.load(utils.smart_load(args.retrain_from))
dnn_model.module.load_state_dict(checkpoint['model'])
if args.transfer_mode[0] == 0:
optimizer.load_state_dict(checkpoint['optimizer'])
start_epoch = checkpoint['epoch']
if args.transfer_mode[0] == 1 or args.transfer_mode[0] == 3:
dnn_model.freeze()
if args.run_validation:
val_loader= data_loader.CSVDataset(args.val_info, args.delimiter, args.raw_size, args.processed_size, args.batch_size, args.num_workers, args.path_prefix, False, shuffle = False).load()
for epoch in range(start_epoch, start_epoch + args.num_epochs):
if args.optimizer not in ['adam', 'adadelta']:
utils.adjust_param(optimizer, 'lr', lr, epoch)
utils.adjust_param(optimizer, 'weight_decay', wd, epoch)
if args.transfer_mode[0] == 3 and epoch==args.transfer_mode[1]:
dnn_model.unfreeze()
batch_time = utils.AverageMeter()
data_time = utils.AverageMeter()
losses = utils.AverageMeter()
top1 = utils.AverageMeter()
topn = utils.AverageMeter()
# switch to train mode
dnn_model.train()
end = time.time()
for step, (input, target, _) in islice(enumerate(train_loader), args.num_batches):
# measure data loading time
data_time.update(time.time() - end)
input = input.cuda(non_blocking=True)
target = target.cuda(non_blocking=True)
# compute output
output = dnn_model(input)
#print(output,target)
loss = criterion(output, target)
# measure accuracy and record loss
prec1, precn = utils.accuracy(output, target, topk=(1, args.top_n))
losses.update(loss.item(), input.size(0))
top1.update(prec1[0], input.size(0))
topn.update(precn[0], input.size(0))
#print(loss.item(), prec1[0], precn[0])
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if step % 10 == 0:
format_str = ('%s: epoch %d, step %d, loss = %.2f, Top-1 = %.2f Top-' + str(args.top_n) + ' = %.2f')
print(format_str % (datetime.now(), epoch, step, losses.val, top1.val, topn.val))
sys.stdout.flush()
state= {'epoch': epoch + 1,
'arch': args.architecture,
'num_classes': args.num_classes,
'model': dnn_model.module.state_dict(),
'optimizer': optimizer.state_dict()
}
torch.save(state, utils.smart_save(os.path.join(args.log_dir, 'checkpoint%04d.pth.tar'%(epoch)), max_to_keep = args.max_to_keep))
# if validation data are provided, evaluate accuracy on the validation set after the end of each epoch
if args.run_validation:
valbatch_time = utils.AverageMeter()
vallosses = utils.AverageMeter()
valtop1 = utils.AverageMeter()
valtop5 = utils.AverageMeter()
# switch to evaluate mode
dnn_model.eval()
with torch.no_grad():
end = time.time()
for i, (input, target, _) in enumerate(val_loader):
input = input.cuda(non_blocking = True)
target = target.cuda(non_blocking = True)
# compute output
output = dnn_model(input)
loss = criterion(output, target)
# measure accuracy and record loss
prec1, prec5 = utils.accuracy(output, target, topk = (1, 5))
vallosses.update(loss.item(), input.size(0))
valtop1.update(prec1[0], input.size(0))
valtop5.update(prec5[0], input.size(0))
# measure elapsed time
valbatch_time.update(time.time() - end)
end = time.time()
print('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
i, len(val_loader), batch_time = valbatch_time, loss = vallosses,
top1 = valtop1, top5 = valtop5))
sys.stdout.flush()
print('Training finished')
import model, readin
# Reading config file
path = r'./_init/config.ini'
input_config = readin.readin(path)
# Create model
thismodel = model.model(input_config['location'], input_config['number_of_cars'], \
input_config['number_of_CPs'], input_config['firstday'], \
input_config['lastday'], input_config['trainingpercentage'], \
input_config['timestep'], input_config['EIT_radius'], \
input_config['Comb_diameter'], input_config['ip_address'], \
input_config['minimum cars'], input_config['maximum cars'])
# Run simulations
thismodel.run()
# Save results
thismodel.save_results()
def do_evaluate(args):
# if we want to do inference only (i.e. no label is provided) we only load images and their paths
val_loader = data_loader.CSVDataset(args.val_info, args.delimiter, args.raw_size, args.processed_size, args.batch_size, args.num_workers, args.path_prefix, False,
shuffle = False, inference_only= args.inference_only).load()
checkpoint = torch.load(utils.smart_load(args.log_dir))
dnn_model = model(checkpoint['arch'], checkpoint['num_classes'])
if args.num_gpus == 1:
dnn_model = dnn_model.cuda()
# Load pretrained parameters from disk
dnn_model.load_state_dict(checkpoint['model'])
else:
dnn_model = torch.nn.DataParallel(dnn_model, device_ids = range(0, args.num_gpus)).cuda()
# Load pretrained parameters from disk
dnn_model.module.load_state_dict(checkpoint['model'])
criterion = nn.CrossEntropyLoss().cuda()
# evaluation
if not args.inference_only:
losses = utils.AverageMeter()
top1 = utils.AverageMeter()
topn = utils.AverageMeter()
# Open an output file to write predictions
out_file = open(args.save_predictions, 'w')
predictions_format_str = ('%d, %s, %d, %s, %s\n')
for step,(input, target, info) in enumerate(val_loader):
input = input.cuda(non_blocking = True)
target = target.cuda(non_blocking = True)
# Load a batch of data
output= softmax(dnn_model(input))
loss = criterion(output, target)
# measure accuracy and record loss
prec1, prec5 = utils.accuracy(output, target, topk=(1, args.top_n))
losses.update(loss.item(), input.size(0))
top1.update(prec1[0], input.size(0))
topn.update(prec5[0], input.size(0))
print('Batch Number: %d, Top-1 Hit: %d, Top-%d Hit: %d, Loss %.2f, Top-1 Accuracy: %.3f, Top-%d Accuracy: %.3f'%
(step, top1.val, args.top_n, topn.val, losses.avg, top1.avg, args.top_n, topn.avg))
# log results into an output file
topnconf, topnguesses = output.topk(args.top_n, 1, True, True)
for i in range(0, len(info)):
out_file.write(predictions_format_str % (step * args.batch_size + i + 1, str(info[i]).encode('utf-8'), target[i],
', '.join('%d' % item for item in topnguesses[i]),
', '.join('%.4f' % item for item in topnconf[i])))
out_file.flush()
sys.stdout.flush()
out_file.close()
#inference
else:
# Open an output file to write predictions
out_file = open(args.save_predictions, 'w')
predictions_format_str = ('%d, %s, %s, %s\n')
for step,(input, info) in enumerate(val_loader):
# Load a batch of data
input = input.cuda(non_blocking = True)
# Load a batch of data
output= softmax(dnn_model(input))
# Run the network on the loaded batch
topnconf,topnguesses = output.topk(args.top_n,1, True, True)
print('Batch Number: %d of %d is done'%(step, args.num_val_batches))
# Log to an output file
for i in range(0, len(info)):
out_file.write(predictions_format_str % (step * args.batch_size + i + 1, str(info[i]).encode('utf-8'),
', '.join('%d' % item for item in topnguesses[i]),
', '.join('%.4f' % item for item in topnconf[i])))
out_file.flush()
out_file.close()
def main(_):
# Print FLAGS values
pprint(FLAGS.flag_values_dict())
# Define GPU configuration
os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpu
gpu_config = tf.ConfigProto()
gpu_config.gpu_options.allow_growth = True
# Define model name
if not FLAGS.phase:
setup_list = [
f"ngf_{FLAGS.ngf}",
f"ndf_{FLAGS.ndf}",
f"nz_{FLAGS.nz}",
f"lrD_{FLAGS.lrD}",
f"lrG_{FLAGS.lrG}",
# f"dg_{FLAGS.dg}",
# f"aug_{FLAGS.aug}",
f"lw_{FLAGS.alpha}",
f"ow_{FLAGS.beta}",
f"var_{FLAGS.var}",
f"phase_{FLAGS.phase}",
f"da_{FLAGS.da}",
f"clip_{FLAGS.clip}"
# f"wgan_{FLAGS.wgan}"
# f"nosig"
]
else:
setup_list = [
f"ngf_{FLAGS.ngf}",
f"ndf_{FLAGS.ndf}",
f"nz_{FLAGS.nz}",
f"lrD_{FLAGS.lrD}",
f"lrG_{FLAGS.lrG}",
# f"dg_{FLAGS.dg}",
f"lw_{FLAGS.alpha}",
f"ow_{FLAGS.beta}",
f"var_{FLAGS.var}",
f"phase_{FLAGS.phase}",
f"da_{FLAGS.da}",
f"clip_{FLAGS.clip}",
# f"wgan_{FLAGS.wgan}",
f"nhl_{FLAGS.nhl}",
f"nhw_{FLAGS.nhw}"
]
model_name = '_'.join(setup_list)
print(f"Model name: {model_name}")
M = model(FLAGS, gpu_config)
M.sess.run(tf.global_variables_initializer())
if FLAGS.phase:
# Previously learned autoencoder model name
setup_list = [
f"ngf_{FLAGS.ngf}",
f"ndf_{FLAGS.ndf}",
f"nz_{FLAGS.nz}",
f"lrD_{FLAGS.lrD}",
f"lrG_{FLAGS.lrG}",
# f"dg_{FLAGS.dg}",
# f"aug_{FLAGS.aug}",
f"lw_{FLAGS.alpha}",
f"ow_{FLAGS.beta}",
f"var_{FLAGS.var}",
f"phase_0",
f"da_{FLAGS.da}",
f"clip_{FLAGS.clip}"
# f"wgan_{FLAGS.wgan}"
# f"nosig"
]
lgan_name = '_'.join(setup_list)
# just for now
# lgan_name = 'ngf_64_ndf_64_nz_64_lrD_5e-05_lrG_0.001_dg_1_aug_0_lw_20.0_ow_0.01_var_3.0_phase_0_nosig'
lgan_name = 'ngf_64_ndf_64_nz_16_lw_20.0_ow_0.01_var_3.0_phase_0'
var_lgan = tf.get_collection('trainable_variables', 'lgan/gen')
path = tf.train.latest_checkpoint(
os.path.join(FLAGS.ckptdir, lgan_name))
tf.train.Saver(var_lgan).restore(M.sess, path)
print(colored(f"LGAN model is restored from {path}", "blue"))
saver = tf.train.Saver()
# Train the main model
train(M, FLAGS, saver=saver, model_name=model_name)
def main():
args = parser.parse_args()
enc = encoder.get_encoder(args.model_name)
hparams = model.default_hparams()
with open(os.path.join('models', args.model_name, 'hparams.json')) as f:
hparams.override_from_dict(json.load(f))
if args.sample_length > hparams.n_ctx:
raise ValueError("Can't get samples longer than window size: %s" %
hparams.n_ctx)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
context = tf.placeholder(tf.int32, [args.batch_size, None])
output = model.model(hparams=hparams, X=context)
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=context[:, 1:], logits=output['logits'][:, :-1]))
tf_sample = sample.sample_sequence(hparams=hparams,
length=args.sample_length,
context=context,
batch_size=args.batch_size,
temperature=1.0,
top_k=40)
train_vars = [v for v in tf.trainable_variables() if 'model' in v.name]
if args.accumulate_gradients > 1:
opt = AccumulatingOptimizer(
opt=tf.train.AdamOptimizer(learning_rate=args.learning_rate),
var_list=train_vars)
opt_reset = opt.reset()
opt_compute = opt.compute_gradients(loss)
opt_apply = opt.apply_gradients()
summary_loss = tf.summary.scalar('loss', opt_apply)
else:
opt_apply = tf.train.AdamOptimizer(
learning_rate=args.learning_rate).minimize(loss,
var_list=train_vars)
summary_loss = tf.summary.scalar('loss', loss)
summary_log = tf.summary.FileWriter(
os.path.join(CHECKPOINT_DIR, args.run_name))
saver = tf.train.Saver(var_list=train_vars,
max_to_keep=5,
keep_checkpoint_every_n_hours=2)
sess.run(tf.global_variables_initializer())
if args.restore_from == 'latest':
ckpt = tf.train.latest_checkpoint(
os.path.join(CHECKPOINT_DIR, args.run_name))
if ckpt is None:
# Get fresh GPT weights if new run.
ckpt = tf.train.latest_checkpoint(
os.path.join('models', args.model_name))
elif args.restore_from == 'fresh':
ckpt = tf.train.latest_checkpoint(
os.path.join('models', args.model_name))
else:
ckpt = tf.train.latest_checkpoint(args.restore_from)
print('Loading checkpoint', ckpt)
saver.restore(sess, ckpt)
print('Loading dataset...')
chunks = load_dataset(enc, args.dataset, args.combine)
# sample_chunks = load_dataset(enc, args.sampledataset, args.combine)
data_sampler = Sampler(chunks)
print('dataset has', data_sampler.total_size, 'tokens')
print('Training...')
token_test = np.stack(
enc.encode(
"Great Android Phone. This is a very nice phone. The screen is easy to read and the phone performs flawlessly."
))
print(test_sample)
counter = 1
counter_path = os.path.join(CHECKPOINT_DIR, args.run_name, 'counter')
if os.path.exists(counter_path):
# Load the step number if we're resuming a run
# Add 1 so we don't immediately try to save again
with open(counter_path, 'r') as fp:
counter = int(fp.read()) + 1
def save():
maketree(os.path.join(CHECKPOINT_DIR, args.run_name))
print(
'Saving',
os.path.join(CHECKPOINT_DIR, args.run_name,
'model-{}').format(counter))
saver.save(sess,
os.path.join(CHECKPOINT_DIR, args.run_name, 'model'),
global_step=counter)
with open(counter_path, 'w') as fp:
fp.write(str(counter) + '\n')
def generate_samples():
context_tokens = token_test
all_text = []
index = 0
while index < args.sample_num:
out = sess.run(
tf_sample,
feed_dict={context: args.batch_size * [context_tokens]})
for i in range(min(args.sample_num - index, args.batch_size)):
text = enc.decode(out[i])
text = '================ Generated version {} ================\n{}\n'.format(
index + 1, text)
all_text.append(text)
index += 1
print("start sentense :" + enc.decode(token_test))
print(text)
maketree(os.path.join(SAMPLE_DIR, args.run_name))
with open(
os.path.join(SAMPLE_DIR, args.run_name,
'samples-{}').format(counter), 'w') as fp:
fp.write('\n'.join(all_text))
def sample_batch():
return [data_sampler.sample(1024) for _ in range(args.batch_size)]
avg_loss = (0.0, 0.0)
try:
while True:
start_time = time.time()
if counter % args.save_every == 0:
save()
if counter % args.sample_every == 0:
generate_samples()
if args.accumulate_gradients > 1:
sess.run(opt_reset)
for _ in range(args.accumulate_gradients):
sess.run(opt_compute,
feed_dict={context: sample_batch()})
(v_loss, v_summary) = sess.run((opt_apply, summary_loss))
else:
(_, v_loss, v_summary) = sess.run(
(opt_apply, loss, summary_loss),
feed_dict={context: sample_batch()})
summary_log.add_summary(v_summary, counter)
avg_loss = (avg_loss[0] * 0.99 + v_loss,
avg_loss[1] * 0.99 + 1.0)
if counter % args.sample_every == 0:
print(
'[{counter} | {time:2.2f}] loss={loss:2.2f} avg={avg:2.2f}'
.format(counter=counter,
time=time.time() - start_time,
loss=v_loss,
avg=avg_loss[0] / avg_loss[1]))
start_time = time.time()
counter += 1
except KeyboardInterrupt:
print('interrupted')
save()
# if d_cnt < self.args.d_count:
# self.update_d(loss_)
# a = [self.d_r_loss.cpu().view(-1),self.d_f_loss.cpu().view(-1),self.d_cost.cpu().view(-1),self.wasserstein.cpu().view(-1)]
# a = np.array([[l.detach().numpy()[0] for l in a]])
# self.save_d_loss = util.add_loss(self.save_d_loss,a)
# #print(
# # 'batch:{}/{}--d_real_loss = {:0.6f}, d_fake_loss = {:0.6f},d_cost = {:0.6f}, wasserstein = {:0.6f}\n' \
# # .format(n_batch, self.args.n_train // self.args.batch_size + 1, self.d_r_loss,
# # self.d_f_loss, self.d_cost, self.wasserstein)
# #)
# else:
# d_cnt = 0
self.update_g(loss_)
# a=self.g_cost.cpu().view(-1).detach().numpy()
# self.save_g_loss = util.add_loss(self.save_g_loss,a)
# self.save_g_loss = torch.cat([self.save_g_loss, a], 0)
# del(a)
print('p_loss={:0.6f}'.format(self.p_loss))
# d_cnt += 1
util.save_mdoel(self.args.result_dir, self.model.g,
'single_generator')
if __name__ == '__main__':
data_ = data.Data(args)
model_ = model(args)
trainer_ = trainer(args, model_, data_)
trainer_.train()
import numpy as np
import matplotlib.pyplot as plt
import sklearn
import sklearn.datasets
from misc import predict, load_dataset, plot_decision_boundary, predict_dec
from model import model
# %matplotlib inline
plt.rcParams['figure.figsize'] = (7.0, 4.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'
# load image dataset: blue/red dots in circles
train_X, train_Y, test_X, test_Y = load_dataset()
parameters = model(train_X, train_Y, initialization="zeros")
print("On the train set:")
predictions_train = predict(train_X, train_Y, parameters)
print("On the test set:")
predictions_test = predict(test_X, test_Y, parameters)
plt.title("Model with Zeros initialization")
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
train_Y)
parameters = model(train_X, train_Y, initialization="random")
print("On the train set:")
predictions_train = predict(train_X, train_Y, parameters)
print("On the test set:")
def training(model, loss_fn, optimizer, filename):
try:
os.makedirs(args.SCNN_checkpoint_path)
except:
pass
saved_epoch = 0
min_valid_loss = 1e10
loss_hist = {'train loss': [], 'valid loss': []}
with tqdm(range(1, args.epoch + 1)) as pbar1, \
tqdm(total=len(args.train_noise_type)) as pbar2, \
tqdm(total=len(args.train_noise_type)) as pbar3:
for epoch in pbar1:
loss_hist['train loss'] = []
pbar2.reset()
for n, noise_type in enumerate(args.train_noise_type):
bs = 0
loss = 0
pbar2.set_description_str(
f'(Epoch {epoch}) noise type: {noise_type}')
for sample in range(1, 224):
if sample == 103:
continue
Sx, _, elec, clean = load_data(noise_type, sample)
noisy = torch.Tensor([Sx.T]).to(args.device)
enhan = model(noisy, elec)
loss += model.get_loss(loss_fn, enhan, clean)
bs += 1
if bs >= args.batch_size:
loss /= bs
loss_item = loss.item()
optimizer.zero_grad()
loss.backward()
optimizer.step()
loss_hist['train loss'].append(loss_item)
loss = 0
bs = 0
pbar2.refresh()
pbar1.refresh()
if bs != 0:
loss /= bs
loss_item = loss.item()
optimizer.zero_grad()
loss.backward()
optimizer.step()
loss_hist['train loss'].append(loss_item)
loss = 0
bs = 0
pbar2.set_postfix(loss=loss_item)
pbar2.update()
# ===== Validation =====
pbar3.reset()
valid_loss = 0
valid_sample = 0
for n, noise_type in enumerate(args.train_noise_type):
pbar3.set_description_str(f'noise type: {noise_type}')
for sample in range(224, 251):
Sx, _, elec, clean = load_data(noise_type,
sample,
norm=model.use_norm)
noisy = torch.Tensor([Sx.T]).to(args.device)
with torch.no_grad():
enhan = model(noisy, elec)
valid_loss += model.get_loss(loss_fn, enhan,
clean).item()
valid_sample += 1
pbar3.set_postfix(valid_loss=valid_loss / valid_sample)
pbar1.refresh()
pbar3.update()
valid_loss /= valid_sample
loss_hist['valid loss'].append(valid_loss)
if valid_loss < min_valid_loss:
min_valid_loss = valid_loss
saved_epoch = epoch
model.save_model(args.SCNN_checkpoint_path,
f'{filename} (Epoch {args.epoch}).pt')
pbar3.set_description_str(
f'Saved Epoch: {saved_epoch}, min valid loss: {min_valid_loss}'
)
# plt.figure()
plt.plot(loss_hist['train loss'])
plt.plot(
np.linspace(0, len(loss_hist['train loss']),
len(loss_hist['valid loss'])),
loss_hist['valid loss'])
# plt.xlabel('iteration')
# plt.ylabel('loss')
plt.legend(['Train', 'Valid'])
plt.tight_layout(pad=0.2)
# if epoch == 1:
# plt.savefig(args.SCNN_checkpoint_path + f'{filename} (Epoch 1).svg')
plt.show()
#可调整的参数
test_batch_size = 8#如炸显存请调小
title = 'final_test'
test_path = './test_nii/'#存放测试数据路径，根据实际情况修改
Model_path = './7.pth'#存放权重路径，根据实际情况修改
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
transform = transforms.Compose([
transforms.ToTensor()
])
save_path = './'+title+'_result/'#保存结果路径，根据实际情况修改
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
#####
file_list = os.listdir(test_path)
file_list.sort()
Model = model().to(device)
utils.path_checker(save_path)
Model.load_state_dict(torch.load(Model_path))
for name in file_list:
if name.split('.')[-1]!='nii':
break
test_set = Dataset(path=test_path+name, transform=transform)
test_loader = DataLoader(test_set, batch_size=test_batch_size, shuffle=False)
output = []
for index, img in enumerate(test_loader):
if index==len(test_loader)-1:
sys.stdout.write("\r[{}] [Batch {}/{}]\n".format(name, index+1, len(test_loader)))
else:
sys.stdout.write("\r[{}] [Batch {}/{}]".format(name, index+1, len(test_loader)))
def construct_model(self, ckpt_dir=None):
# Define inputs
self.X = tf.placeholder(tf.float32, [None, 32, 32, 3])
self.y = tf.placeholder(tf.int64, [None])
self.is_training = tf.placeholder(tf.bool)
self.global_step = tf.train.get_or_create_global_step()
# ================================================================ #
# YOUR CODE HERE:
# define our model
# save output of the model to self.y_out
# ================================================================ #
self.y_out = model(self.X, self.y)
# ================================================================ #
# END YOUR CODE HERE
# ================================================================ #
# Define our loss
total_loss = tf.losses.softmax_cross_entropy(tf.one_hot(self.y, 10),
logits=self.y_out)
self.mean_loss = tf.reduce_mean(total_loss)
# Define our optimizer
self.optimizer = tf.train.AdamOptimizer(
5e-4) # select optimizer and set learning rate
# split train_step = optimizer.minimize(self.mean_loss)
train_gradient = self.optimizer.compute_gradients(self.mean_loss)
# initialize or load model parameters
self.saver = tf.train.Saver(max_to_keep=10)
if ckpt_dir is not None:
self.saver.restore(self.sess, tf.train.latest_checkpoint(ckpt_dir))
print('Pre-trained model restored from %s' % (ckpt_dir))
else:
self.sess.run(tf.global_variables_initializer())
print('Initialize variables')
# ================================================================ #
# YOUR CODE HERE:
# implement in prune_utils.py
# 1.prune parameters based on your threshold
# (make sure pruning is effectively applied in step 1)
# 2.get pruned gradient update operator accordingly, save to prune_gradient
# ================================================================ #
prune_gradient = get_prune_op(self.sess,
train_gradient,
percentage=0.6)
# ================================================================ #
# END YOUR CODE HERE
# ================================================================ #
# save pruned parameters
tmp_dir = '__tmp__'
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
self.saver.save(self.sess, os.path.join(tmp_dir, 'pruned_model.ckpt'))
# define gradients and initialize optimizer parameters
self.train_op = self.optimizer.apply_gradients(
prune_gradient, global_step=self.global_step)
self.sess.run(tf.global_variables_initializer())
# reload pruned parameters
self.saver.restore(self.sess, tf.train.latest_checkpoint(tmp_dir))
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
remove_terms = punctuation + '0123456789'
norm_bible = [[word.lower() for word in sent if word not in remove_terms]
for sent in bible]
norm_bible = [' '.join(tok_sent) for tok_sent in norm_bible]
norm_bible = filter(None, normalize_corpus(norm_bible))
norm_bible = [tok_sent for tok_sent in norm_bible if len(tok_sent.split()) > 2]
tokenizer = text.Tokenizer()
word2id, id2word, sequences = tokenization(tokenizer, norm_bible)
vocab_size = len(word2id)
embed_size = 100
window_size = 2
cbow = model(vocab_size, embed_size, window_size)
cbow.summary()
for epoch in range(1, 6):
loss = 0
i = 0
for x, y in generate_context_word_pairs(corpus=sequences,
window_size=window_size,
vocab_size=vocab_size):
i += 1
loss += cbow.train_on_batch(x, y)
if i % 100000 == 0:
print('Processed {} (context, word) pairs'.format(i))
print('Epoch:', epoch, '\tloss:', loss)
print()
# coding:utf-8
import time
from glob import glob
import numpy as np
from PIL import Image
import model
# ces
paths = glob('./test/*.*')
if __name__ == '__main__':
im = Image.open("./test/a001.png")
img = np.array(im.convert('RGB'))
t = time.time()
'''
result,img,angel分别对应-识别结果，图像的数组，文字旋转角度
'''
result, img, angle = model.model(img,
model='pytorch',
adjust=True,
detectAngle=True)
print("It takes time:{}s".format(time.time() - t))
print("---------------------------------------")
for key in result:
print(result[key][1])
