def __init__(self):
super(OptionsScreen, self).__init__()
self.font = utils.getFont('VolterGoldfish', 44)
self.infofont = utils.getFont('VolterGoldfish', 18)
self.keyfont = utils.getFont('VolterGoldfish', 24)
self.background = pygame.image.load("../img/backgrounds/options.png").convert()
self.soundBar = pygame.image.load("../img/backgrounds/soundBar.png").convert_alpha()
self.musicBar = pygame.image.load("../img/backgrounds/soundBar.png").convert_alpha()
self.keyImg = pygame.image.load("../img/backgrounds/key.png").convert_alpha()
self.volumeTest = pygame.mixer.Sound("../sounds/volumeTest.wav")
self.menuEntries = ["Music","Sound","Controls","Back"]
self.menuPositions = [(200,100),(200,200),(200,300),(200,500)]
self.menuChoice = 0
self.activeColor = THECOLORS["black"]
self.inactiveColor = THECOLORS["grey64"]
self.soundLevel = 0
if exist('soundLevel'):
self.soundLevel = load('soundLevel')
self.musicLevel = 0
if exist('musicLevel'):
self.musicLevel = load('musicLevel')
self.actionKeys = ['slow time','shield','trap','left','jump','right']
self.keys = ['w','x','c','<','^','>']
self.actionKeysPos = [(200,450),(330,450),(430,450),(530,450),(630,450),(730,450)]
def test():
B0 = utils.load('blk/0.pkl')
B1 = utils.load('blk/1.pkl')
B2 = utils.load('blk/4.pkl')
B3 = utils.load('blk/5.pkl')
BM = [[B0,B1],[B2,B3]]
P = Block()
P.merge_from_matrix(BM)
utils.save(P, 'mb.pkl')
print P
def prepare(self, block_path, imgs_path, camera_list_path, plane_path): self.block = utils.load(block_path) for i in self.block.cameras: self.imgs[i] = np.array(Image.open(imgs_path + str(i)+'.jpg')) self.camera_list = utils.load(camera_list_path) self.plane = utils.load(plane_path) self.h, self.w = self.block.height * self.scale, self.block.width*self.scale self.h, self.w = int(self.h), int(self.w) self.labels = cp.deepcopy(self.block.labels) self.img = zeros((self.h,self.w,3),uint8)
def test():
#camera_list = CameraList(10, (4912,7360))
#camera_list.read_from_bundle('bundle.out')
#print camera_list
#saveCameraList(camera_list, 'camera_list.pkl')
C = utils.load('camera_list.pkl')
print C
def make_tests(test_descr=TRANSFORMS_TESTINFO):
"""generate tests classes from test info
return the list of generated test classes
"""
tests = []
for _transform, tr_input, tr_output, _normalize, _subobjects in test_descr:
# load transform if necessary
if type(_transform) is type(''):
try:
_transform = load(_transform).register()
except MissingBinary:
# we are not interessted in tests with missing binaries
continue
except:
import traceback
traceback.print_exc()
continue
if TR_NAMES is not None and not _transform.name() in TR_NAMES:
print 'skip test for', _transform.name()
continue
class TransformTestSubclass(TransformTest):
input = input_file_path(tr_input)
output = output_file_path(tr_output)
transform = _transform
normalize = lambda x, y: _normalize(y)
subobjects = _subobjects
tests.append(TransformTestSubclass)
tests.append(PILTransformsTest)
return tests
def f_load():
try:
return load(sess, saver, checkpoint_dir, name)
except:
print("Loading weights via Keras")
G.load_weights(checkpoint_dir+"/G_weights.keras")
D.load_weights(checkpoint_dir+"/D_weights.keras")
E.load_weights(checkpoint_dir+"/E_weights.keras")
def codebook_to_csv(k=128, des_name=constants.ORB_FEAT_NAME):
if not os.path.exists(constants.FILES_DIR_NAME):
os.makedirs(constants.FILES_DIR_NAME)
codebook = utils.load(filenames.codebook(k, des_name))
filename = "{0}/codebook_{1}_{2}.csv".format(constants.FILES_DIR_NAME, k, des_name)
utils.save_csv(filename, codebook)
print("Copied codebook into the file with name {0}. Press any key to exit...".format(filename))
cv2.waitKey()
def test_imageMatch(self):
img_prot = load('orig/kueche.jpg')
rgb_prot = img_prot.getdata()
hsv_prot = rgb2hsv(rgb_prot)
nc = 16
filt = Filter(nc)
hc = clust1d([x[0] for x in hsv_prot], nc, 1000)
add_palettes(img_prot, hc)
img_prot.show()
filt.hues = hc
filt.update()
def train(self, svm_kernel, codebook, des_option=constants.ORB_FEAT_OPTION, is_interactive=True):
"""
Gets the descriptors for the training set and then calculates the SVM for them.
Args:
svm_kernel (constant): The kernel of the SVM that will be created.
codebook (NumPy float matrix): Each row is a center of a codebook of Bag of Words approach.
des_option (integer): The option of the feature that is going to be used as local descriptor.
is_interactive (boolean): If it is the user can choose to load files or generate.
Returns:
cv2.SVM: The Support Vector Machine obtained in the training phase.
"""
des_name = constants.ORB_FEAT_NAME if des_option == constants.ORB_FEAT_OPTION else constants.SIFT_FEAT_NAME
k = len(codebook)
x_filename = filenames.vlads_train(k, des_name)
if is_interactive:
data_option = input("Enter [1] to calculate VLAD vectors for the training set or [2] to load them.\n")
else:
data_option = constants.GENERATE_OPTION
if data_option == constants.GENERATE_OPTION:
# Getting the global vectors for all of the training set
print("Getting global descriptors for the training set.")
start = time.time()
x, y = self.get_data_and_labels(self.dataset.get_train_set(), codebook, des_option)
utils.save(x_filename, x)
end = time.time()
print("VLADs training vectors saved on file {0}".format(x_filename))
self.log.train_vlad_time(end - start)
else:
# Loading the global vectors for all of the training set
print("Loading global descriptors for the training set.")
x = utils.load(x_filename)
y = self.dataset.get_train_y()
x = np.matrix(x, dtype=np.float32)
svm = cv2.SVM()
svm_filename = filenames.svm(k, des_name, svm_kernel)
if is_interactive:
svm_option = input("Enter [1] for generating a SVM or [2] to load one\n")
else:
svm_option = constants.GENERATE_OPTION
if svm_option == constants.GENERATE_OPTION:
# Calculating the Support Vector Machine for the training set
print("Calculating the Support Vector Machine for the training set...")
svm_params = dict(kernel_type=svm_kernel, svm_type=cv2.SVM_C_SVC, C=1)
start = time.time()
svm.train_auto(x, y, None, None, svm_params)
end = time.time()
self.log.svm_time(end - start)
# Storing the SVM in a file
svm.save(svm_filename)
else:
svm.load(svm_filename)
return svm
def _process_path(self, subfile, current_encoding=None) :
try:
text = load(subfile)
except (URLError, HTTPError, IOError) as e:
print '[SubsLoader] "{0}" ,cannot load: {1}'.format(subfile, e)
raise LoadError(subfile)
try:
decoded_list, encoding = decode(text, self._encodings, current_encoding)
except Exception as e:
print '[SubsLoader] "{0}" cannot decode: {1}'.format(subfile, e)
raise DecodeError(subfile)
return decoded_list, encoding
def loadOutput(self, name):
""" Load output from dataset.
Usage: JS = loadOutput(img_name)
Input:
img - Full path of image to load. It's useful to use something
like: img_name = getName (basename, 'img')
Output:
JS - JS structure for this image
"""
return utils.load(name, 'JS')
def loadSummary (self, name):
""" Load scene summary from dataset.
Usage: mask = loadSummary(name)
Input:
name - Full path of Summary to load. It's useful to use something
like: mask_name = getName (basename, 'mask')
Output:
JS - JS structure containing only a summary of regions in
JS.CompactReg.
"""
return utils.load(name, 'JS')
def loadSeq (self, name):
""" Load data Sequence containing info about a subset of images.
Usage: seq = loadSeq(name)
Input:
name - Full path of sequence to load. It's useful to use something
like: name = getName (basename, 'seq')
Output:
seq - RGBDSequence() object containing a list of files and their
poses.
"""
return utils.load(name, 'seq')
def load(self, name = None, path = None):
""" load data from mask, and update name/path if necessary. """
if name is not None:
self.name = name
if path is not None:
self.path = path
# BaseMask assumes we're reading masks from an image type
file = os.path.join(self.path, self.name)
if os.path.exists(file):
self.data = utils.load(file, 'mask', Matlab_compatible = True)
def load_output(self, count):
""" Load particle data from dumped output file.
Parameters
----------
count : string
The iteration time from which to load the data. If time is
'?' then list of available data files is returned else
the latest available data file is used
Notes
-----
Data is loaded from the :py:attr:`output_directory` using the same format
as stored by the :py:meth:`dump_output` method.
Proper functioning required that all the relevant properties of arrays be
dumped
"""
# get the list of available files
available_files = [i.rsplit('_',1)[1][:-4] for i in os.listdir(self.output_directory) if i.startswith(self.fname) and i.endswith('.npz')]
if count == '?':
return sorted(set(available_files), key=int)
else:
if not count in available_files:
msg = """File with iteration count `%s` does not exist"""%(count)
msg += "\nValid iteration counts are %s"%(sorted(set(available_files), key=int))
#print msg
raise IOError(msg)
array_names = [pa.name for pa in self.particles.arrays]
# load the output file
data = load(os.path.join(self.output_directory,
self.fname+'_'+str(count)+'.npz'))
arrays = [ data["arrays"][i] for i in array_names ]
# set the Particle's arrays
self.particles.arrays = arrays
# call the particle's initialize
self.particles.initialize()
self.t = float(data["solver_data"]['t'])
self.count = int(data["solver_data"]['count'])
def __init__(self, levelInd):
super(GameScreen, self).__init__()
self.fadeScreen = pygame.image.load("../img/backgrounds/blackscreen.png").convert()
self.font = getFont("VolterGoldfish", 20)
self.nextColorIcon = pygame.image.load("../img/hud/nextColor23.png").convert_alpha()
self.progressIcon = pygame.image.load("../img/hud/progress.png").convert_alpha()
self.timebar = pygame.image.load("../img/hud/timebar.png").convert_alpha()
self.shieldbar = pygame.image.load("../img/hud/shieldbar.png").convert_alpha()
self.lifebar_ = pygame.image.load("../img/hud/lifebar_.png").convert()
self.level = Level(levelInd)
self.camera = Camera(640, 800, 3200)
# running = True
self.retry = False
self.frame_number = 0
self.anims = []
self.stopDelay = 0
self.stopped = False
self.killFragments = []
self.dust = []
self.starting = True
self.ending = False
self.alpha = 255
# Music load
# pygame.mixer.music.load("../sounds/piano.wav")
# pygame.mixer.music.play(-1)
pygame.mixer.music.set_volume(0.5)
# Player
self.player = Player()
self.fire = Rect((0,3200),(800,5))
self.playerDown = False
self.answerPos = [(100,300),(100,400),(100,500)]
self.answerChoice = 0
self.activecolor = THECOLORS['white']
self.inactivecolor = THECOLORS['grey21']
if exist('powers'):
self.player.timePower, self.player.mines, self.player.shields = load('powers')
def extrapolation(what, **kwargs):
what = 'extrap_'+what
fname = utils.filename(what, **kwargs)
if path.exists(fname):
return utils.load(fname)
else:
import pairfield_correlations, density_correlations, density, energy
if what == 'extrap_pairfield':
return utils.load_or_evaluate(what, pairfield_correlations.evaluate_extrap_at, **kwargs)
elif what == 'extrap_densdens':
return utils.load_or_evaluate(what, density_correlations.evaluate_extrap_at, **kwargs)
elif what == 'extrap_density':
return utils.load_or_evaluate(what, density.evaluate_extrap_at, **kwargs)
elif what == 'extrap_energy':
return utils.load_or_evaluate(what, energy.evaluate_extrap, **kwargs)
else:
raise Exception('`%s` is not a valid measurement.' % what)
def _process_path(self, subfile, current_encoding=None) :
filename = os.path.basename(subfile)
size = getFileSize(subfile)
if size and size > SUBTITLES_FILE_MAX_SIZE:
self.log.error("<%s> not supported subtitles size ({%d}KB > {%d}KB)!", filename, size / 1024, SUBTITLES_FILE_MAX_SIZE / 1024)
raise LoadError('"%s" - not supported subtitles size: "%dKB"' % (toString(os.path.basename(subfile)), size / 1024))
try:
text = load(subfile)
except (URLError, HTTPError, IOError) as e:
self.log.error("<%s> %s", filename, str(e))
raise LoadError(subfile)
try:
decoded_text, encoding = decode(text, self._encodings, current_encoding)
except Exception as e:
self.log.error("<%s> %s", filename, "cannot decode")
raise DecodeError(subfile)
return decoded_text, encoding
def test(self, codebook, svm, des_option = constants.ORB_FEAT_OPTION, is_interactive=True):
"""
Gets the descriptors for the testing set and use the svm given as a parameter to predict all the elements
Args:
codebook (NumPy matrix): Each row is a center of a codebook of Bag of Words approach.
svm (cv2.SVM): The Support Vector Machine obtained in the training phase.
des_option (integer): The option of the feature that is going to be used as local descriptor.
is_interactive (boolean): If it is the user can choose to load files or generate.
Returns:
NumPy float array: The result of the predictions made.
NumPy float array: The real labels for the testing set.
"""
des_name = constants.ORB_FEAT_NAME if des_option == constants.ORB_FEAT_OPTION else constants.SIFT_FEAT_NAME
k = len(codebook)
x_filename = filenames.vlads_test(k, des_name)
if is_interactive:
data_option = input("Enter [1] to calculate VLAD vectors for the testing set or [2] to load them.\n")
else:
data_option = constants.GENERATE_OPTION
if data_option == constants.GENERATE_OPTION:
# Getting the global vectors for all of the testing set
print("Getting global descriptors for the testing set...")
start = time.time()
x, y = self.get_data_and_labels(self.dataset.get_test_set(), codebook, des_option)
utils.save(x_filename, x)
end = time.time()
print("VLADs testing vectors saved on file {0}".format(x_filename))
self.log.test_vlad_time(end - start)
else:
# Loading the global vectors for all of the testing set
print("Loading global descriptors for the testing set.")
x = utils.load(x_filename.format(des_name))
y = self.dataset.get_test_y()
x = np.matrix(x, dtype=np.float32)
# Predicting the testing set using the SVM
start = time.time()
result = svm.predict_all(x)
end = time.time()
self.log.predict_time(end - start)
mask = result == y
correct = np.count_nonzero(mask)
accuracy = (correct * 100.0 / result.size)
self.log.accuracy(accuracy)
return result, y
def load():
import config
try:
data = utils.load(cfgfilename())
except:
a = ConfigWindow()
a.exec_loop()
return
lines = data.split('\n')
for line in lines:
pair = line.split('=')
if pair:
if pair[0] == "name":
config.name = pair[1]
elif pair[0] == "email":
config.email = pair[1]
elif pair[0] == "pspec_folder":
config.pspec_folder = pair[1]
def result(what, **kwargs):
fname = utils.filename(what, **kwargs)
if path.exists(fname):
return utils.load(fname)
else:
try:
import pyalps_dset
import pairfield_correlations, density_correlations, density, amplitudes
if what == 'pairfield':
return utils.load_or_evaluate(what, pairfield_correlations.evaluate, **kwargs)
elif what == 'densdens':
return utils.load_or_evaluate(what, density_correlations.evaluate, **kwargs)
elif what == 'density':
return utils.load_or_evaluate(what, density.evaluate, **kwargs)
elif what == 'density_fit':
return utils.load_or_evaluate(what, density.evaluate_fit, **kwargs)
elif what == 'density_amplitudes':
return utils.load_or_evaluate(what, amplitudes.evaluate, **kwargs)
else:
raise Exception('`%s` is not a valid measurement.' % what)
except ImportError, e:
print 'To execute new evaluations you need the ALPS.Python library.'
print e
def run(self, restore=False):
g.init()
for event in pygame.event.get():
if event.type == pygame.QUIT:
return
elif event.type == pygame.VIDEORESIZE:
pygame.display.set_mode(event.size, pygame.RESIZABLE)
break
if not self.journal:
utils.load()
self.trip = trip.Trip(self.sugar, self.label, self.colors[0])
self.trip.setup()
load_save.retrieve()
self.buttons_setup()
if self.canvas is not None:
self.canvas.grab_focus()
ctrl = False
pygame.key.set_repeat(600, 120)
key_ms = pygame.time.get_ticks()
going = True
while going:
if self.journal:
# Pump GTK messages.
while Gtk.events_pending():
Gtk.main_iteration()
# Pump PyGame messages.
for event in pygame.event.get():
if event.type == pygame.QUIT:
# only in standalone version
if not self.journal:
utils.save()
going = False
elif event.type == pygame.MOUSEMOTION:
g.pos = event.pos
self.trip.check_mouse()
g.redraw = True
if self.canvas is not None:
self.canvas.grab_focus()
elif event.type == pygame.MOUSEBUTTONDOWN:
g.redraw = True
if event.button == 1:
bu = buttons.check()
if bu != '':
self.do_button(bu)
else:
if self.trip.check_mouse():
self.do_click()
self.flush_queue()
elif event.type == pygame.KEYDOWN:
# throttle keyboard repeat
if pygame.time.get_ticks() - key_ms > 110:
key_ms = pygame.time.get_ticks()
if ctrl:
if event.key == pygame.K_q:
if not self.journal:
utils.save()
going = False
break
else:
ctrl = False
if event.key in (pygame.K_LCTRL, pygame.K_RCTRL):
ctrl = True
break
self.do_key(event.key)
g.redraw = True
self.flush_queue()
elif event.type == pygame.KEYUP:
ctrl = False
if not going:
break
if g.redraw:
self.display()
if g.version_display:
utils.version_display()
g.screen.blit(g.pointer, g.pos)
pygame.display.flip()
g.redraw = False
if self.trip.delay:
pygame.time.delay(1000)
self.trip.delay = False
self.trip.gone()
g.redraw = True
self.trip.complete()
g.clock.tick(40)
# c_svm_param
# knn
# naive_bayes
# ols
# ridge_reg
# lasso
# adaboost
# voting
# random_forest
# bow
start = time.time()
print time.ctime()
X_tr, X_te, y_tr, y_te, sorted_files_tr, sorted_files_te = \
load(load_from_folder, display_collage, submission)
N_tr = X_tr.shape[0]
N_te = X_te.shape[0]
D = X_tr.shape[1]
print "Training : [Inputs x Features ] = [%d x %d]" % (N_tr,D)
print "Test : [Inputs x Features ] = [%d x %d]" % (N_te,D)
####################### Feature Expansion ################################
if classifier!="nn" and classifier!="bow":
X_tr = feature_exp(X_tr)
X_te = feature_exp(X_te)
D = X_tr.shape[1]
print "After Feature Expansion: Training : [Inputs x Features ] = [%d x %d]" % (N_tr,D)
print "After Feature Expansion: Test : [Inputs x Features ] = [%d x %d]" % (N_te,D)
"""
script_dir = Path(os.path.abspath(__file__)).parent
data_dir = script_dir.parent / "data"
models_path = data_dir / "models"
if not models_path.exists():
print("please download models to %s." % models_path)
sys.exit(1)
# Load all models into a list
loaded = load([
(NetPredictor, "v3/sep/1"),
(NetPredictor, "v3/sep/2"),
(NetPredictor, "v3/sep/4"),
(NetPredictor, "v3/blkx/2"),
(NetPredictor, "v3/blkx/4"),
(NetPredictor, "v3/blkx/5"),
],
models_path=models_path)
sep_predictor = VotingPredictor(loaded["v3/sep/1"], loaded["v3/sep/2"],
loaded["v3/sep/4"])
blkx_predictor = VotingPredictor(loaded["v3/blkx/2"], loaded["v3/blkx/4"],
loaded["v3/blkx/5"])
for page_path in (data_dir / "pages").iterdir():
if page_path.suffix == ".jpg":
page = Page.Page(page_path)
sep_res = sep_predictor(page)
def main():
def _str_to_bool(s):
"""Convert string to bool (in argparse context)."""
if s.lower() not in ['true', 'false']:
raise ValueError('Argument needs to be a boolean, got {}'.format(s))
return {'true': True, 'false': False}[s.lower()]
parser = argparse.ArgumentParser(description='WaveNet example network')
DATA_DIRECTORY = '.\\data\\moon,.\\data\\son'
parser.add_argument('--data_dir', type=str, default=DATA_DIRECTORY, help='The directory containing the VCTK corpus.')
LOGDIR = None
#LOGDIR = './/logdir-wavenet//train//2018-12-21T22-58-10'
parser.add_argument('--logdir', type=str, default=LOGDIR,help='Directory in which to store the logging information for TensorBoard. If the model already exists, it will restore the state and will continue training. Cannot use with --logdir_root and --restore_from.')
parser.add_argument('--logdir_root', type=str, default=None,help='Root directory to place the logging output and generated model. These are stored under the dated subdirectory of --logdir_root. Cannot use with --logdir.')
parser.add_argument('--restore_from', type=str, default=None,help='Directory in which to restore the model from. This creates the new model under the dated directory in --logdir_root. Cannot use with --logdir.')
CHECKPOINT_EVERY = 1000 # checkpoint 저장 주기
parser.add_argument('--checkpoint_every', type=int, default=CHECKPOINT_EVERY,help='How many steps to save each checkpoint after. Default: ' + str(CHECKPOINT_EVERY) + '.')
config = parser.parse_args() # command 창에서 입력받을 수 있는 조건
config.data_dir = config.data_dir.split(",")
try:
directories = validate_directories(config,hparams)
except ValueError as e:
print("Some arguments are wrong:")
print(str(e))
return
logdir = directories['logdir']
restore_from = directories['restore_from']
# Even if we restored the model, we will treat it as new training
# if the trained model is written into an arbitrary location.
is_overwritten_training = logdir != restore_from
log_path = os.path.join(logdir, 'train.log')
infolog.init(log_path, logdir)
global_step = tf.Variable(0, name='global_step', trainable=False)
# Create coordinator.
coord = tf.train.Coordinator()
num_speakers = len(config.data_dir)
# Load raw waveform from VCTK corpus.
with tf.name_scope('create_inputs'):
# Allow silence trimming to be skipped by specifying a threshold near
# zero.
silence_threshold = hparams.silence_threshold if hparams.silence_threshold > EPSILON else None
gc_enable = num_speakers > 1
# AudioReader에서 wav 파일을 잘라 input값을 만든다. receptive_field길이만큼을 앞부분에 pad하거나 앞조각에서 가져온다. (receptive_field+ sample_size)크기로 자른다.
reader = DataFeederWavenet(coord,config.data_dir,batch_size=hparams.wavenet_batch_size,receptive_field=WaveNetModel.calculate_receptive_field(hparams.filter_width, hparams.dilations,hparams.scalar_input, hparams.initial_filter_width),
gc_enable= gc_enable)
if gc_enable:
audio_batch, lc_batch, gc_id_batch = reader.inputs_wav, reader.local_condition, reader.speaker_id
else:
audio_batch, lc_batch = reader.inputs_wav, self.local_condition
# Create network.
net = WaveNetModel(
batch_size=hparams.wavenet_batch_size,
dilations=hparams.dilations,
filter_width=hparams.filter_width,
residual_channels=hparams.residual_channels,
dilation_channels=hparams.dilation_channels,
quantization_channels=hparams.quantization_channels,
out_channels =hparams.out_channels,
skip_channels=hparams.skip_channels,
use_biases=hparams.use_biases, # True
scalar_input=hparams.scalar_input,
initial_filter_width=hparams.initial_filter_width,
global_condition_channels=hparams.gc_channels,
global_condition_cardinality=num_speakers,
local_condition_channels=hparams.num_mels,
upsample_factor=hparams.upsample_factor,
train_mode=True)
if hparams.l2_regularization_strength == 0:
hparams.l2_regularization_strength = None
net.add_loss(input_batch=audio_batch,local_condition=lc_batch, global_condition_batch=gc_id_batch, l2_regularization_strength=hparams.l2_regularization_strength)
net.add_optimizer(hparams,global_step)
run_metadata = tf.RunMetadata()
# Set up session
sess = tf.Session(config=tf.ConfigProto(log_device_placement=False)) # log_device_placement=False --> cpu/gpu 자동 배치.
init = tf.global_variables_initializer()
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=hparams.max_checkpoints) # 최대 checkpoint 저장 갯수 지정
try:
start_step = load(saver, sess, restore_from) # checkpoint load
if is_overwritten_training or start_step is None:
# The first training step will be saved_global_step + 1,
# therefore we put -1 here for new or overwritten trainings.
zero_step_assign = tf.assign(global_step, 0)
sess.run(zero_step_assign)
except:
print("Something went wrong while restoring checkpoint. We will terminate training to avoid accidentally overwriting the previous model.")
raise
###########
start_step = sess.run(global_step)
last_saved_step = start_step
try:
reader.start_in_session(sess,start_step)
while not coord.should_stop():
start_time = time.time()
if hparams.store_metadata and step % 50 == 0:
# Slow run that stores extra information for debugging.
log('Storing metadata')
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
step, loss_value, _ = sess.run([global_step, net.loss, net.optimize],options=run_options,run_metadata=run_metadata)
tl = timeline.Timeline(run_metadata.step_stats)
timeline_path = os.path.join(logdir, 'timeline.trace')
with open(timeline_path, 'w') as f:
f.write(tl.generate_chrome_trace_format(show_memory=True))
else:
step, loss_value, _ = sess.run([global_step,net.loss, net.optimize])
duration = time.time() - start_time
log('step {:d} - loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, duration))
if step % config.checkpoint_every == 0:
save(saver, sess, logdir, step)
last_saved_step = step
if step >= hparams.num_steps:
# error message가 나오지만, 여기서 멈춘 것은 맞다.
raise Exception('End xxx~~~yyy')
except Exception as e:
print('finally')
#if step > last_saved_step:
# save(saver, sess, logdir, step)
coord.request_stop(e)
# START !!! {} PID: {}
#==============================================================================
""".format(__file__, os.getpid()))
import pandas as pd
import utils
from nltk import stem
pt = stem.PorterStemmer().stem
from gensim.models import Doc2Vec
d2v = Doc2Vec.load('../nlp_source/d2v/enwiki_dbow/doc2vec.bin')
from gensim.models import KeyedVectors
w2v = KeyedVectors.load_word2vec_format('../nlp_source/w2v/GoogleNews-vectors-negative300.bin.gz', binary=True)
train, test = utils.load(4,1)
stop_verbs = {'do','does','did','done','is','am','are','be','been',
'being','was','were',"'s","'m"}
"""
nltk.pos_tag(nltk.word_tokenize('How can I be a good geologist ?'))
s='How do I use Twitter as a business source ? '
s='What is the best way to gain confidence ? '
s='How can I find an IT job in Japan ? '
s='How I can speak English fluently ? '
s='Should I switch from Spotify to Apple Music ? '
s='What`s the best way to get rid of p**n addiction ? '
def main():
#if not torch.cuda.is_available():
# logging.info('no gpu device available')
# sys.exit(1)
np.random.seed(args.seed)
'''
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
'''
device = torch.device("cpu")
torch.manual_seed(args.seed)
logging.info('use cpu')
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
param_list = list(model.parameters())
# print(model)
# model = model.cuda()
model.to(device)
utils.load(model, args.model_path)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
#criterion = criterion.cuda()
criterion.to(device)
_, test_transform = utils._data_transforms_cifar10(args)
test_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=test_transform)
test_queue = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
model.drop_path_prob = args.drop_path_prob
METRICS = Metrics(list(model.parameters()), p=1)
PERFORMANCE_STATISTICS = {}
ARCH_STATISTICS = {}
metrics_path = './metrics_stat_test.xlsx'
weights_path = './weights_stat_test.xlsx'
epoch = 0
io_metrics = METRICS.evaluate(epoch)
PERFORMANCE_STATISTICS[f'in_S_epoch_{epoch}'] = io_metrics.input_channel_S
PERFORMANCE_STATISTICS[
f'out_S_epoch_{epoch}'] = io_metrics.output_channel_S
PERFORMANCE_STATISTICS[
f'mode12_S_epoch_{epoch}'] = io_metrics.mode_12_channel_S
PERFORMANCE_STATISTICS[f'fc_S_epoch_{epoch}'] = io_metrics.fc_S
PERFORMANCE_STATISTICS[
f'in_rank_epoch_{epoch}'] = io_metrics.input_channel_rank
PERFORMANCE_STATISTICS[
f'out_rank_epoch_{epoch}'] = io_metrics.output_channel_rank
PERFORMANCE_STATISTICS[
f'mode12_rank_epoch_{epoch}'] = io_metrics.mode_12_channel_rank
PERFORMANCE_STATISTICS[f'fc_rank_epoch_{epoch}'] = io_metrics.fc_rank
PERFORMANCE_STATISTICS[
f'in_condition_epoch_{epoch}'] = io_metrics.input_channel_condition
PERFORMANCE_STATISTICS[
f'out_condition_epoch_{epoch}'] = io_metrics.output_channel_condition
PERFORMANCE_STATISTICS[
f'mode12_condition_epoch_{epoch}'] = io_metrics.mode_12_channel_condition
# write metrics data to xls file
metrics_df = pd.DataFrame(data=PERFORMANCE_STATISTICS)
metrics_df.to_excel(metrics_path)
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
y.append(pred)
prob.append(joint_prob[pred] / np.sum(joint_prob))
input_prob.append(np.sum(joint_prob))
return np.array(apply_index2raw(y, self.feat2index_dict_y)), prob, input_prob
if __name__ == "__main__":
p_x_train = os.path.join("data", "x_train.pkl")
p_y_train = os.path.join("data", "y_train.pkl")
p_x_test = os.path.join("data", "x_test.pkl")
p_y_test = os.path.join("data", "y_test.pkl")
X_train = load(p_x_train)
y_train = load(p_y_train)
X_test = load(p_x_test)
y_test = load(p_y_test)
model = Naive_bayes()
# we are going to split dimension of age, BMI and charges
model.fit(X_train, y_train, discretize_dim=(0, 2, 5), split_num=(5, 5, 5))
pred, _, _ = model.predict(X_train)
train_acc = np.sum((pred == y_train)) / y_train.shape[0]
pred, _, _ = model.predict(X_test)
test_acc = np.sum((pred == y_test)) / y_test.shape[0]
print("train accuracy: {}, test accuracy: {}".format(train_acc, test_acc))
def print_max_significance(discovery_sig_df):
max_sig_idx = discovery_sig_df['ZA'].idxmax()
max_sig_data = discovery_sig_df.iloc[max_sig_idx]
max_sig = max_sig_data['ZA']
max_sig_cut = max_sig_data['t_cut']
max_sig_str = 'Max significance {:.2f} for cut value {}'.format(max_sig, max_sig_cut)
print max_sig_str
if __name__ == '__main__':
DATA_F = 'atlas-higgs-challenge-2014-v2.csv'
CLF_F = 'BDT.pkl'
data_path = os.path.join('data', DATA_F)
log.info('Reading data from {}'.format(data_path))
data, labels, weights = get_data_labels_weights(data_path)
log.info('Splitting into signal and background samples')
X_sig, X_bkg, w_sig, w_bkg = split_sig_bkg(labels, data, weights)
clf = load(CLF_F)
log.info('Calculating classifier predictions')
p_sig = clf.decision_function(X_sig)
p_bkg = clf.decision_function(X_bkg)
log.info('Scanning discovery significance over cut values')
ds = discovery_significance(p_sig, p_bkg, w_sig, w_bkg)
print_max_significance(ds)
import task2
import utils
from matplotlib import pyplot as plt
import numpy as np
tcases = utils.load('testcases.pkl')
for tcase in tcases:
img_patches, shape, reconstructed = tcases[tcase]
cleaned_img = task2.reconstruct_from_noisy_patches(img_patches, shape)
#if tcase == 3 :
#print(img_patches)
#print(shape)
#print(reconstructed.shape)
#key = [(197, 207, 214, 222), (167, 207, 197, 222), (153, 207, 167, 222), (137, 207, 153, 222)]
#for i in key:
# utils.make_fig(img_patches[i], cmap='gray', title='Yours')
try:
if np.allclose(cleaned_img, reconstructed):
print('testcase# {} passed'.format(tcase))
utils.make_fig(reconstructed, cmap='gray', title='Correct')
utils.make_fig(cleaned_img, cmap='gray', title='Yours')
plt.show()
else:
print('testcase# {} failed'.format(tcase))
utils.make_fig(reconstructed, cmap='gray', title='Correct')
utils.make_fig(cleaned_img, cmap='gray', title='Yours')
plt.show()
except Exception as e:
print('testcase# {} failed'.format(tcase))
import linear_model as lm
import utils
'''
聽說取二次也算 linear regression
'''
train_path = os.path.join(os.path.dirname(__file__), "./data/train.csv")
test_path = os.path.join(os.path.dirname(__file__), "./data/test.csv")
output_path = os.path.join(os.path.dirname(__file__), "./ans.csv")
model_path = os.path.join(os.path.dirname(__file__), "./model")
scaler_path = os.path.join(os.path.dirname(__file__), "./scaler_sqr")
fea_select, y_pos = (0, 4, 5, 6, 7, 8, 9, 16), 70
x, y = utils.load(train_path, mode='train', fea_select=fea_select,
y_pos=y_pos) # 讀出所有 data 、擷取 feature 、劃分 9 天成一筆
x = np.concatenate((x, x**2), axis=1) # 加入平方當特徵
x, max, min = utils.rescaling(x) # 作 rescaling ， 在 [0, 1] 間
x, y = utils.shuffle(x, y)
x_train, y_train, x_val, y_val = utils.validation(x, y, ratio=0.1)
b, w = lm.LinearRegression(x,
y,
lr=100000,
epoch=1000000,
lr_method='adagrad',
x_val=x_val,
y_val=y_val)
x_test = utils.load(test_path, mode='test', fea_select=fea_select, y_pos=y_pos)
x_test = np.concatenate((x_test, x_test**2), axis=1)
def knndis(t, X_M):
Mg = X_M.shape[1]
dist = np.sum((np.transpose(np.matlib.repmat(t, Mg, 1)) - X_M)**2, 0)
dist = np.sort(dist)
return sum(dist[0:10])
sess = tf.compat.v1.Session(config=config)
# Initialize for Tensorflow
restore_var = [v for v in tf.global_variables()]
loader = tf.train.Saver(var_list=restore_var)
ckpt = const.SNAPSHOT_DIR
load(loader, sess, ckpt)
# Yolo Initializer
config_file = "/content/test/cfg/yolov4.cfg"
weights = "/content/test/yolov4.weights"
data_file = "/content/test/cfg/coco.data"
batch_size = 1
thresh = 0.25 # remove object with confidence score < 0.25
save_labels = False # do not export output to .txt file
random.seed(3) # deterministic bbox colors
network, class_names, class_colors = darknet.load_network(
config_file, data_file, weights, batch_size=batch_size)
count = 4
def run(self):
g.init()
if not self.journal:
utils.load()
else:
g.ok = self.ok
g.rule = self.rule
self.cards = cards.Cards()
self.cards.ok = g.ok
self.cards.rule = g.rule
self.cards.redeal()
bx = g.sx(17.4)
by1 = g.sy(15.5)
by2 = g.sy(18.5)
self.yes_bu = []
self.no_bu = []
for i in range(3):
bu = buttons.Button("yes", (bx, by1), True)
bu.active = False
self.yes_bu.append(bu)
bu = buttons.Button("no", (bx, by2), True)
bu.active = False
self.no_bu.append(bu)
bx += self.cards.width + g.sy(.5)
bx = g.sx(29.9)
by = g.sy(14.2)
buttons.Button("redeal", (bx, by), True)
if self.journal: # Sugar only
a, b, c, d = pygame.cursors.load_xbm('my_cursor.xbm',
'my_cursor_mask.xbm')
pygame.mouse.set_cursor(a, b, c, d)
going = True
while going:
# Pump GTK messages.
while gtk.events_pending():
gtk.main_iteration()
# Pump PyGame messages.
for event in pygame.event.get():
if event.type == pygame.QUIT:
if not self.journal:
g.ok = self.cards.ok
g.rule = self.cards.rule
utils.save()
going = False
elif event.type == pygame.MOUSEMOTION:
g.redraw = True
elif event.type == pygame.MOUSEBUTTONDOWN:
g.redraw = True
if event.button == 2: # centre button
if not self.journal:
g.version_display = not g.version_display
break
bu = buttons.check()
if bu <> '':
self.button(bu)
break
self.click()
if not going: break
self.update()
if g.redraw:
self.display()
if g.version_display: utils.version_display()
pygame.display.flip()
g.redraw = False
tf = False
if pygame.mouse.get_focused(): tf = True
pygame.mouse.set_visible(tf)
g.clock.tick(40)
# be ready for xo quit at any time
self.ok = self.cards.ok
self.rule = self.cards.rule
def main():
if not torch.cuda.is_available():
logging.info('No GPU device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('GPU device = %d' % args.gpu)
logging.info("args = %s", args)
gpu_logger = GpuLogThread([args.gpu], writer, seconds=15 if not args.test else 1)
gpu_logger.start()
logging.debug(locals())
model = None
# prepare dataset
train_transform, valid_transform = utils.data_transforms(args.dataset, args.cutout, args.cutout_length)
if args.dataset == "CIFAR100":
train_data = dset.CIFAR100(root=args.tmp_data_dir, train=True, download=True, transform=train_transform)
elif args.dataset == "CIFAR10":
train_data = dset.CIFAR10(root=args.tmp_data_dir, train=True, download=True, transform=train_transform)
elif args.dataset == 'MIT67':
dset_cls = dset.ImageFolder
data_path = '%s/MIT67/train' % args.tmp_data_dir # 'data/MIT67/train'
val_path = '%s/MIT67/test' % args.tmp_data_dir # 'data/MIT67/val'
train_data = dset_cls(root=data_path, transform=train_transform)
valid_data = dset_cls(root=val_path, transform=valid_transform)
elif args.dataset == 'Sport8':
dset_cls = dset.ImageFolder
data_path = '%s/Sport8/train' % args.tmp_data_dir # 'data/Sport8/train'
val_path = '%s/Sport8/test' % args.tmp_data_dir # 'data/Sport8/val'
train_data = dset_cls(root=data_path, transform=train_transform)
valid_data = dset_cls(root=val_path, transform=valid_transform)
elif args.dataset == "flowers102":
dset_cls = dset.ImageFolder
data_path = '%s/flowers102/train' % args.tmp_data_dir
val_path = '%s/flowers102/test' % args.tmp_data_dir
train_data = dset_cls(root=data_path, transform=train_transform)
valid_data = dset_cls(root=val_path, transform=valid_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
import random;random.shuffle(indices)
train_iterator = utils.DynamicBatchSizeLoader(torch.utils.data.DataLoader(
train_data, batch_size=args.batch_multiples,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True, num_workers=args.workers), args.batch_size_min)
valid_iterator = utils.DynamicBatchSizeLoader(torch.utils.data.DataLoader(
train_data, batch_size=args.batch_multiples,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]),
pin_memory=True, num_workers=args.workers), args.batch_size_min)
# build Network
logging.debug('building network')
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
num_graph_edges = sum(list(range(2, 2 + args.blocks)))
switches_normal = SwitchManager(num_graph_edges, copy.deepcopy(PRIMITIVES), 'normal')
switches_reduce = SwitchManager(num_graph_edges, copy.deepcopy(PRIMITIVES), 'reduce')
total_epochs = 0
for cycle in parse_cycles():
logging.debug('new cycle %s' % repr(cycle))
print('\n' * 3, '-' * 100)
print(cycle)
print('', '-' * 100, '\n')
writer.add_scalar('cycle/net_layers', cycle.net_layers, cycle.num)
writer.add_scalar('cycle/net_init_c', cycle.net_init_c, cycle.num)
writer.add_scalar('cycle/net_dropout', cycle.net_dropout, cycle.num)
writer.add_scalar('cycle/ops_keep', cycle.ops_keep, cycle.num)
writer.add_scalar('cycle/epochs', cycle.epochs, cycle.num)
writer.add_scalar('cycle/grace_epochs', cycle.grace_epochs, cycle.num)
writer.add_scalar('cycle/morphs', cycle.morphs, cycle.num)
switches_normal.plot_ops(logging.info, writer, cycle.num)
switches_reduce.plot_ops(logging.info, writer, cycle.num)
# rebuild the model in each cycle, clean up the cache...
logging.debug('building model')
del model
torch.cuda.empty_cache()
if args.dataset == "CIFAR100":
CLASSES = 100
elif args.dataset == "CIFAR10":
CLASSES = 10
elif args.dataset == 'MIT67':
dset_cls = dset.ImageFolder
CLASSES = 67
elif args.dataset == 'Sport8':
dset_cls = dset.ImageFolder
CLASSES = 8
elif args.dataset == "flowers102":
dset_cls = dset.ImageFolder
CLASSES = 102
model = Network(cycle.net_init_c,
CLASSES,
cycle.net_layers,
criterion,
switches_normal=switches_normal,
switches_reduce=switches_reduce,
steps=args.blocks,
p=cycle.net_dropout,
largemode=True if args.dataset in utils.LARGE_DATASETS else False)
gpu_logger.reset_recent()
if cycle.load:
utils.load(model, model_path)
if args.reset_alphas:
model.reset_alphas()
if args.test:
model.randomize_alphas()
if cycle.init_morphed:
model.init_morphed(switches_normal, switches_reduce)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
logging.debug('building optimizers')
optimizer = torch.optim.SGD(model.net_parameters,
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
optimizer_a = torch.optim.Adam(model.arch_parameters,
lr=args.arch_learning_rate, betas=(0.5, 0.999),
weight_decay=args.arch_weight_decay)
logging.debug('building scheduler')
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, int(cycle.epochs),
eta_min=args.learning_rate_min)
if args.batch_size_max > args.batch_size_min:
train_iterator.set_batch_size(args.batch_size_min)
valid_iterator.set_batch_size(args.batch_size_min)
sm_dim = -1
scale_factor = 0.2
for epoch in range(cycle.epochs):
lr = scheduler.get_lr()[0]
logging.info('Epoch: %d lr: %e', epoch, lr)
epoch_start = time.time()
# training
if epoch < cycle.grace_epochs:
model.update_p(cycle.net_dropout * (cycle.epochs - epoch - 1) / cycle.epochs)
else:
model.update_p(cycle.net_dropout * np.exp(-(epoch - cycle.grace_epochs) * scale_factor))
train_acc, train_obj = train(train_iterator, valid_iterator, model, criterion, optimizer, optimizer_a,
gpu_logger, train_arch=epoch >= cycle.grace_epochs)
epoch_duration = time.time() - epoch_start
# log info
logging.info('Train_acc %f', train_acc)
logging.info('Epoch time: %ds', epoch_duration)
writer.add_scalar('train/accuracy', train_acc, total_epochs)
writer.add_scalar('train/loss', train_obj, total_epochs)
writer.add_scalar('epoch/lr', lr, total_epochs)
writer.add_scalar('epoch/seconds', epoch_duration, total_epochs)
writer.add_scalar('epoch/model.p', model.p, total_epochs)
writer.add_scalar('epoch/batch_size', train_iterator.batch_size, total_epochs)
# validation, only for the last 5 epochs in a cycle
if cycle.epochs - epoch < 5:
valid_acc, valid_obj = infer(valid_iterator, model, criterion)
logging.info('Valid_acc %f', valid_acc)
writer.add_scalar('valid/accuracy', valid_acc, total_epochs)
writer.add_scalar('valid/loss', valid_obj, total_epochs)
total_epochs += 1
gpu_logger.reset_recent()
scheduler.step()
utils.save(model, model_path)
print('\n' * 2, '------Dropping/morphing paths------')
# Save switches info for s-c refinement.
if cycle.is_last:
switches_normal_copy = switches_normal.copy()
switches_reduce_copy = switches_reduce.copy()
# drop operations with low architecture weights, add morphed ones
arch_param = model.arch_parameters
normal_prob = F.softmax(arch_param[0], dim=sm_dim).data.cpu().numpy()
switches_normal.drop_and_morph(normal_prob, cycle.ops_keep, writer, cycle.num, num_morphs=cycle.morphs,
no_zero=cycle.is_last and args.restrict_zero, keep_morphable=not cycle.is_last)
reduce_prob = F.softmax(arch_param[1], dim=sm_dim).data.cpu().numpy()
switches_reduce.drop_and_morph(reduce_prob, cycle.ops_keep, writer, cycle.num, num_morphs=cycle.morphs,
no_zero=cycle.is_last and args.restrict_zero, keep_morphable=not cycle.is_last)
logging.info('switches_normal = \n%s', switches_normal)
logging.info('switches_reduce = \n%s', switches_reduce)
# end last cycle with shortcut/zero pruning and save the genotype
if cycle.is_last:
#import ipdb;ipdb.set_trace()
arch_param = model.arch_parameters
normal_prob = F.softmax(arch_param[0], dim=sm_dim).data.cpu().numpy()
reduce_prob = F.softmax(arch_param[1], dim=sm_dim).data.cpu().numpy()
normal_final = [0 for _ in range(num_graph_edges)]
reduce_final = [0 for _ in range(num_graph_edges)]
# Generate Architecture
keep_normal = [0, 1]
keep_reduce = [0, 1]
n = 3
start = 2
for i in range(3):
end = start + n
tbsn = normal_final[start:end]
tbsr = reduce_final[start:end]
edge_n = sorted(range(n), key=lambda x: tbsn[x])
keep_normal.append(edge_n[-1] + start)
keep_normal.append(edge_n[-2] + start)
edge_r = sorted(range(n), key=lambda x: tbsr[x])
keep_reduce.append(edge_r[-1] + start)
keep_reduce.append(edge_r[-2] + start)
start = end
n = n + 1
for i in range(num_graph_edges):
if i not in keep_normal:
for j in range(len(switches_normal.current_ops)):
switches_normal[i][j] = False
if i not in keep_reduce:
for j in range(len(switches_reduce.current_ops)):
switches_reduce[i][j] = False
switches_normal.keep_2_branches(normal_prob)
switches_reduce.keep_2_branches(reduce_prob)
switches_normal.plot_ops(logging.info, writer, cycle.num + 1)
switches_reduce.plot_ops(logging.info, writer, cycle.num + 1)
genotype = parse_network(switches_normal, switches_reduce)
logging.info(genotype)
save_genotype(args.save + 'genotype.json', genotype)
with open(args.save + "/best_genotype.txt", "w") as f:
f.write(str(genotype))
gpu_logger.stop()
def create_zips(args):
def get_filepaths(subset):
filepaths = []
tids = tracks.index[tracks['set', 'subset'] <= subset]
for tid in tids:
filepaths.append(utils.get_audio_path('', tid))
return filepaths
def get_checksums(base_dir, filepaths):
"""Checksums are assumed to be stored in order for efficiency."""
checksums = []
with open(os.path.join(args.path, base_dir, 'checksums')) as f:
for filepath in filepaths:
exist = False
for line in f:
if filepath == line[42:-1]:
exist = True
break
if not exist:
raise ValueError('checksum not found: {}'.format(filepath))
checksums.append(line)
return checksums
def create_zip(zip_filename, base_dir, filepaths):
# Audio: all compressions are the same.
# CSV: stored > deflated > BZIP2 > LZMA.
# LZMA is close to BZIP2 and too recent to be widely available (unzip).
compression = zipfile.ZIP_BZIP2
zip_filepath = os.path.join(args.path, zip_filename)
with zipfile.ZipFile(zip_filepath, 'x', compression) as zf:
def info(name):
name = os.path.join(zip_filename[:-4], name)
info = zipfile.ZipInfo(name, (2017, 4, 1, 0, 0, 0))
info.external_attr = 0o444 << 16 | 0o2 << 30
return info
zf.writestr(info('README.txt'), README, compression)
checksums = get_checksums(base_dir, filepaths)
zf.writestr(info('checksums'), ''.join(checksums), compression)
for filepath in tqdm(filepaths):
src = os.path.join(args.path, base_dir, filepath)
dst = os.path.join(zip_filename[:-4], filepath)
zf.write(src, dst)
os.chmod(zip_filepath, 0o444)
os.utime(zip_filepath, (TIME, TIME))
METADATA = [
'not_found.pickle',
'raw_genres.csv',
'raw_albums.csv',
'raw_artists.csv',
'raw_tracks.csv',
'tracks.csv',
'genres.csv',
'raw_echonest.csv',
'echonest.csv',
'features.csv',
]
create_zip('fma_metadata.zip', 'fma_metadata', METADATA)
tracks = utils.load('tracks.csv')
create_zip('fma_small.zip', 'fma_large', get_filepaths('small'))
create_zip('fma_medium.zip', 'fma_large', get_filepaths('medium'))
create_zip('fma_large.zip', 'fma_large', get_filepaths('large'))
create_zip('fma_full.zip', 'fma_full', get_filepaths('large'))
def run(self):
g.init()
if not self.journal: utils.load()
self.didg = didg.Didg()
g.carry_shape = didg.shapes[0]
self.didg.make_carry()
load_save.retrieve()
self.buttons_setup()
if self.canvas <> None: self.canvas.grab_focus()
ctrl = False
pygame.key.set_repeat(600, 120)
key_ms = pygame.time.get_ticks()
need_carry = False
going = True
while going:
if self.journal:
# Pump GTK messages.
while Gtk.events_pending():
Gtk.main_iteration()
# Pump PyGame messages.
for event in pygame.event.get():
if event.type == pygame.QUIT:
if not self.journal: utils.save()
going = False
elif event.type == pygame.MOUSEMOTION:
g.pos = event.pos
need_carry = True
if self.canvas <> None: self.canvas.grab_focus()
elif event.type == pygame.MOUSEBUTTONDOWN:
g.redraw = True
if event.button == 1:
if self.do_click(1):
pass
else:
bu = buttons.check()
if bu != '':
self.do_button(bu)
self.flush_queue()
if event.button == 3:
self.do_click(3)
elif event.type == pygame.KEYDOWN:
# throttle keyboard repeat
if pygame.time.get_ticks() - key_ms > 110:
key_ms = pygame.time.get_ticks()
if ctrl:
if event.key == pygame.K_q:
if not self.journal: utils.save()
going = False
break
else:
ctrl = False
if event.key in (pygame.K_LCTRL, pygame.K_RCTRL):
ctrl = True
break
self.do_key(event.key)
g.redraw = True
self.flush_queue()
elif event.type == pygame.KEYUP:
ctrl = False
if not going: break
if g.redraw:
g.carry_save = False
self.display()
if g.version_display: utils.version_display()
self.carry()
need_carry = False
pygame.display.flip()
g.redraw = False
elif need_carry:
self.carry()
pygame.display.flip()
need_carry = False
g.clock.tick(40)
import os
print(
"""#==============================================================================
# START !!! {} PID: {}
#==============================================================================
""".format(__file__, os.getpid()))
#import pandas as pd
#import numpy as np
from itertools import product
#from collections import Counter
import multiprocessing as mp
total_proc = 2
import utils
train, test = utils.load(3, 1)
#==============================================================================
# def
#==============================================================================
def set_q(s):
return set(s.lower().split()) - utils.kigou
def _and_(s1, s2):
return s1 & s2
def set_diff(s1, s2):
return s1 - s2
def main():
''' set default hyperparams in default_hyperparams.py '''
parser = argparse.ArgumentParser()
# Required arguments
parser.add_argument('-d',
'--dataset',
choices=supported.datasets,
required=True)
parser.add_argument('--algorithm',
required=True,
choices=supported.algorithms)
parser.add_argument(
'--root_dir',
required=True,
help=
'The directory where [dataset]/data can be found (or should be downloaded to, if it does not exist).'
)
# Dataset
parser.add_argument(
'--split_scheme',
help=
'Identifies how the train/val/test split is constructed. Choices are dataset-specific.'
)
parser.add_argument('--dataset_kwargs',
nargs='*',
action=ParseKwargs,
default={})
parser.add_argument(
'--download',
default=False,
type=parse_bool,
const=True,
nargs='?',
help=
'If true, tries to downloads the dataset if it does not exist in root_dir.'
)
parser.add_argument(
'--frac',
type=float,
default=1.0,
help=
'Convenience parameter that scales all dataset splits down to the specified fraction, for development purposes.'
)
# Loaders
parser.add_argument('--loader_kwargs',
nargs='*',
action=ParseKwargs,
default={})
parser.add_argument('--train_loader', choices=['standard', 'group'])
parser.add_argument('--uniform_over_groups',
type=parse_bool,
const=True,
nargs='?')
parser.add_argument('--distinct_groups',
type=parse_bool,
const=True,
nargs='?')
parser.add_argument('--n_groups_per_batch', type=int)
parser.add_argument('--batch_size', type=int)
parser.add_argument('--eval_loader',
choices=['standard'],
default='standard')
# Model
parser.add_argument('--model', choices=supported.models)
parser.add_argument(
'--model_kwargs',
nargs='*',
action=ParseKwargs,
default={},
help=
'keyword arguments for model initialization passed as key1=value1 key2=value2'
)
# Transforms
parser.add_argument('--train_transform', choices=supported.transforms)
parser.add_argument('--eval_transform', choices=supported.transforms)
parser.add_argument(
'--target_resolution',
nargs='+',
type=int,
help=
'target resolution. for example --target_resolution 224 224 for standard resnet.'
)
parser.add_argument('--resize_scale', type=float)
parser.add_argument('--max_token_length', type=int)
# Objective
parser.add_argument('--loss_function', choices=supported.losses)
# Algorithm
parser.add_argument('--groupby_fields', nargs='+')
parser.add_argument('--group_dro_step_size', type=float)
parser.add_argument('--coral_penalty_weight', type=float)
parser.add_argument('--irm_lambda', type=float)
parser.add_argument('--irm_penalty_anneal_iters', type=int)
parser.add_argument('--algo_log_metric')
parser.add_argument('--hsic_beta', type=float)
parser.add_argument('--sd_penalty_lamb', type=float)
parser.add_argument('--grad_penalty_lamb', type=float)
parser.add_argument(
'--params_regex',
type=str,
help='Regular expression specifying which gradients to penalize.')
parser.add_argument('--label_cond',
type=parse_bool,
const=True,
nargs='?',
default=False)
parser.add_argument('--dann_lamb', type=float)
parser.add_argument('--dann_dc_name', type=str)
# Model selection
parser.add_argument('--val_metric')
parser.add_argument('--val_metric_decreasing',
type=parse_bool,
const=True,
nargs='?')
# Optimization
parser.add_argument('--n_epochs', type=int)
parser.add_argument('--optimizer', choices=supported.optimizers)
parser.add_argument('--lr', type=float)
parser.add_argument('--weight_decay', type=float)
parser.add_argument('--max_grad_norm', type=float)
parser.add_argument('--optimizer_kwargs',
nargs='*',
action=ParseKwargs,
default={})
# Scheduler
parser.add_argument('--scheduler', choices=supported.schedulers)
parser.add_argument('--scheduler_kwargs',
nargs='*',
action=ParseKwargs,
default={})
parser.add_argument('--scheduler_metric_split',
choices=['train', 'val'],
default='val')
parser.add_argument('--scheduler_metric_name')
# Evaluation
parser.add_argument('--evaluate_all_splits',
type=parse_bool,
const=True,
nargs='?',
default=True)
parser.add_argument('--eval_splits', nargs='+', default=[])
parser.add_argument('--eval_only',
type=parse_bool,
const=True,
nargs='?',
default=False)
parser.add_argument('--eval_epoch', default=None, type=int)
parser.add_argument('--save_z',
type=parse_bool,
const=True,
nargs='?',
default=False)
# Misc
parser.add_argument('--device', type=int, default=0)
parser.add_argument('--seed', type=int, default=0)
parser.add_argument('--log_dir', default='./logs')
parser.add_argument('--log_every', default=50, type=int)
parser.add_argument('--save_step', type=int)
parser.add_argument('--save_best',
type=parse_bool,
const=True,
nargs='?',
default=True)
parser.add_argument('--save_last',
type=parse_bool,
const=True,
nargs='?',
default=True)
parser.add_argument('--no_group_logging',
type=parse_bool,
const=True,
nargs='?')
parser.add_argument('--use_wandb',
type=parse_bool,
const=True,
nargs='?',
default=False)
parser.add_argument('--progress_bar',
type=parse_bool,
const=True,
nargs='?',
default=False)
parser.add_argument('--resume',
type=parse_bool,
const=True,
nargs='?',
default=False)
config = parser.parse_args()
config = populate_defaults(config)
# set device
config.device = torch.device("cuda:" + str(
config.device)) if torch.cuda.is_available() else torch.device("cpu")
## Initialize logs
if os.path.exists(config.log_dir) and config.resume:
resume = True
mode = 'a'
elif os.path.exists(config.log_dir) and config.eval_only:
resume = False
mode = 'a'
else:
resume = False
mode = 'w'
if not os.path.exists(config.log_dir):
os.makedirs(config.log_dir)
logger = Logger(os.path.join(config.log_dir, 'log.txt'), mode)
# Record config
log_config(config, logger)
# Set random seed
set_seed(config.seed)
# Data
full_dataset = supported.datasets[config.dataset](
root_dir=config.root_dir,
download=config.download,
split_scheme=config.split_scheme,
**config.dataset_kwargs)
# To implement data augmentation (i.e., have different transforms
# at training time vs. test time), modify these two lines:
train_transform = initialize_transform(
transform_name=config.train_transform,
config=config,
dataset=full_dataset)
eval_transform = initialize_transform(transform_name=config.eval_transform,
config=config,
dataset=full_dataset)
train_grouper = CombinatorialGrouper(dataset=full_dataset,
groupby_fields=config.groupby_fields)
datasets = defaultdict(dict)
for split in full_dataset.split_dict.keys():
if split == 'train':
transform = train_transform
verbose = True
elif split == 'val':
transform = eval_transform
verbose = True
else:
transform = eval_transform
verbose = False
# Get subset
datasets[split]['dataset'] = full_dataset.get_subset(
split, frac=config.frac, transform=transform)
if split == 'train':
datasets[split]['loader'] = get_train_loader(
loader=config.train_loader,
dataset=datasets[split]['dataset'],
batch_size=config.batch_size,
uniform_over_groups=config.uniform_over_groups,
grouper=train_grouper,
distinct_groups=config.distinct_groups,
n_groups_per_batch=config.n_groups_per_batch,
**config.loader_kwargs)
else:
datasets[split]['loader'] = get_eval_loader(
loader=config.eval_loader,
dataset=datasets[split]['dataset'],
grouper=train_grouper,
batch_size=config.batch_size,
**config.loader_kwargs)
# Set fields
datasets[split]['split'] = split
datasets[split]['name'] = full_dataset.split_names[split]
datasets[split]['verbose'] = verbose
# Loggers
# Loggers
datasets[split]['eval_logger'] = BatchLogger(
os.path.join(config.log_dir, f'{split}_eval.csv'),
mode=mode,
use_wandb=(config.use_wandb and verbose))
datasets[split]['algo_logger'] = BatchLogger(
os.path.join(config.log_dir, f'{split}_algo.csv'),
mode=mode,
use_wandb=(config.use_wandb and verbose))
if config.use_wandb:
initialize_wandb(config)
# Logging dataset info
if config.no_group_logging and full_dataset.is_classification and full_dataset.y_size == 1:
log_grouper = CombinatorialGrouper(dataset=full_dataset,
groupby_fields=['y'])
elif config.no_group_logging:
log_grouper = None
else:
log_grouper = train_grouper
log_group_data(datasets, log_grouper, logger)
## Initialize algorithm
algorithm = initialize_algorithm(config=config,
datasets=datasets,
train_grouper=train_grouper)
if not config.eval_only:
## Load saved results if resuming
resume_success = False
if resume:
save_path = os.path.join(config.log_dir, 'last_model.pth')
if not os.path.exists(save_path):
epochs = [
int(file.split('_')[0])
for file in os.listdir(config.log_dir)
if file.endswith('.pth')
]
if len(epochs) > 0:
latest_epoch = max(epochs)
save_path = os.path.join(config.log_dir,
f'{latest_epoch}_model.pth')
try:
prev_epoch, best_val_metric = load(algorithm, save_path)
epoch_offset = prev_epoch + 1
logger.write(
f'Resuming from epoch {epoch_offset} with best val metric {best_val_metric}'
)
resume_success = True
except FileNotFoundError:
pass
if resume_success == False:
epoch_offset = 0
best_val_metric = None
train(algorithm=algorithm,
datasets=datasets,
general_logger=logger,
config=config,
epoch_offset=epoch_offset,
best_val_metric=best_val_metric)
else:
if config.eval_epoch is None:
eval_model_path = os.path.join(config.log_dir, 'best_model.pth')
else:
eval_model_path = os.path.join(config.log_dir,
f'{config.eval_epoch}_model.pth')
best_epoch, best_val_metric = load(algorithm, eval_model_path)
if config.eval_epoch is None:
epoch = best_epoch
else:
epoch = config.eval_epoch
evaluate(algorithm=algorithm,
datasets=datasets,
epoch=epoch,
general_logger=logger,
config=config)
logger.close()
for split in datasets:
datasets[split]['eval_logger'].close()
datasets[split]['algo_logger'].close()
def test_SdA(finetune_lr=0.1, pretraining_epochs=15,
pretrain_lr=0.001, training_epochs=1000,
dataset='mnist.pkl.gz', batch_size=32, finetuning= False, selvar=False):
"""
:type learning_rate: float
:param learning_rate: learning rate used in the finetune stage
(factor for the stochastic gradient)
:type pretraining_epochs: int
:param pretraining_epochs: number of epoch to do pretraining
:type pretrain_lr: float
:param pretrain_lr: learning rate to be used during pre-training
:type n_iter: int
:param n_iter: maximal number of iterations ot run the optimizer
:type dataset: string
:param dataset: path the the pickled dataset
"""
#datasets = load_data(dataset)
datasets = load_otto_data("data/train.csv", selected=selvar)
data = load_otto_data('data/test.csv', train=False, selected=selvar)
data_x, data_y = data[0]
print data_x.get_value().shape
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_train_batches /= batch_size
n_data_batches = data_x.get_value(borrow=True).shape[0]
n_data_batches /= batch_size
# numpy random generator
# start-snippet-3
numpy_rng = numpy.random.RandomState(89677)
newmodel = True
## check if model exists and resume otherwise rebuild
if os.path.isfile("./tmp/sda"):
print ("Loading existing model...")
sda = ut.load("sda", "./tmp/")
newmodel = False
print ("done.")
return sda
else:
# construct the stacked denoising autoencoder class
print '... building the model'
sda = SdA(
numpy_rng=numpy_rng,
n_ins=93, #784, # 28 * 28,
hidden_layers_sizes=[60], # [1000, 1000, 1000],
corruption_levels=[0.1, 0.2, 0.2, 0.2],
n_outs=10 #9
)
# train only if we just created a new model
if newmodel:
#########################
# PRETRAINING THE MODEL #
#########################
print '... getting the pretraining functions'
if finetuning:
pretraining_fns = sda.pretraining_functions(train_set_x=train_set_x,
batch_size=batch_size)
else:
pretraining_fns = sda.pretraining_functions(train_set_x=data_x,
batch_size=batch_size)
print '... pre-training the model'
start_time = time.clock()
## Pre-train layer-wise
corruption_levels = [0.1, 0.1, 0.2] #[.1, .2, .3, .2, .2, .2]
for i in xrange(sda.n_layers): # go through pretraining epochs
for epoch in xrange(pretraining_epochs): # go through the training set
c = []
#for batch_index in xrange(n_train_batches):
for batch_index in xrange(n_data_batches):
c.append(pretraining_fns[i](index=batch_index,
corruption=corruption_levels[i],
lr=pretrain_lr))
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
end_time = time.clock()
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
########################
# FINETUNING THE MODEL #
########################
if finetuning:
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
train_fn, validate_model, test_model = sda.build_finetune_functions(
datasets=datasets,
batch_size=batch_size,
learning_rate=finetune_lr
)
print '... finetunning the model'
# early-stopping parameters
patience = 20 * n_train_batches # look as this many examples regardless
patience_increase = 2. # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
test_score = 0.
start_time = time.clock()
done_looping = False
epoch = 0
while (epoch < training_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_fn(minibatch_index)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
validation_losses = validate_model()
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (
this_validation_loss < best_validation_loss *
improvement_threshold
):
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = test_model()
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
if patience <= iter:
done_looping = True
break
end_time = time.clock()
print(
(
'Optimization complete with best validation score of %f %%, '
'on iteration %i, '
'with test performance %f %%'
)
% (best_validation_loss * 100., best_iter + 1, test_score * 100.)
)
print >> sys.stderr, ('The training code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
print "Saving model to disk"
ut.save(sda, "sda")
return sda
#%%
import re
from utils import load
raw = load("day24.txt")
direction = {
"e": 1.0,
"se": 0.5 - 1j,
"sw": -0.5 - 1j,
"w": -1.0,
"nw": -0.5 + 1j,
"ne": 0.5 + 1j,
}
def run_init() -> set[complex]:
# black: 0 white: 1
locs: set[complex] = set()
for line in raw:
locs ^= {
sum(direction[step] for step in re.findall(r"([ns]?[ew])", line))
}
return locs
def test1() -> None:
locs = run_init()
assert len(locs) == 528
def main(argv):
learnModel = True # by default we learn each model again
try:
opts, args = getopt.getopt(argv,"hl:",["load="])
except getopt.GetoptError:
print 'analyticSuite.py -l <modelpath>'
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print 'analyticSuite.py -l <path>'
sys.exit()
elif opt in ("-l", "--load"):
modelpath = arg
learnModel = False # do not learn, load an existing model
################################################
# load_data returns data in tuple form (x,y)
################################################
if not learnModel:
unseen_data = md.load_data('data/test.csv', datatype="test", verbose=True)
################################################
# GradientBoosting classifier
################################################
if learnModel:
# load training dataset and split
train,test,valid = md.load_data( 'data/train.csv', datatype="train", verbose=True)
x_train,y_train = train
x_test, y_test = test
x_valid, y_valid = valid
params = {'n_estimators': 800,
'max_depth': 3,
'subsample': 0.5,
'learning_rate': 0.1,
'min_samples_leaf': 1}
# train Gradient Boosting Classifier
gbc = ensemble.GradientBoostingClassifier(**params)
gbc.fit(x_train, y_train)
train_acc, test_acc = evalError(gbc, (x_train,y_train), (x_test,y_test) )
print "GradientBoostingClassifier\ntrain accuracy %f\ntest accuracy %f\n\n" %(train_acc, test_acc)
ut.save(gbc, "GradientBoosting", verbose=True)
else:
print "Loading Gradient Boosting model"
gbc = ut.load("GradientBoosting", modelpath)
#ut.writePredictionFile(gbc.predict(unseen_data), modeltype="gbc")
ut.writePredictionFile(gbc.predict_proba(unseen_data), modeltype="gbc")
# free memory
del gbc
gc.collect()
##############################################
## Support Vector Machine ##
##############################################
if learnModel:
svmc = svm.SVC(probability=True)
svmc.fit(x_train, y_train)
ut.save(svmc, "SupportVectorMachine", verbose=True)
train_acc, test_acc = evalError(svmc, (x_train,y_train), (x_test,y_test))
print "ExtraTreeClassifier\ntrain accuracy %f\ntest accuracy %f\n\n" %(train_acc, test_acc)
#svmscores = cross_val_score(svmc, x_train, y_train)
#print "SupportVectorMachine mean score %f" %svmscores.mean()
else:
print "Loading SVM model"
svmc = ut.load("SupportVectorMachine", modelpath)
ut.writePredictionFile(svmc.predict(unseen_data), modeltype="svm")
# free memory
del svmc
gc.collect()
############################################
# ExtraTreesClassifier
############################################
if learnModel:
etc = ensemble.ExtraTreesClassifier(n_estimators=600,
max_depth=None,
min_samples_split=1,
random_state=0,
n_jobs=-1)
etc.fit(x_train, y_train)
ut.save(etc, "ExtraTreesClassifier", verbose=True)
train_acc, test_acc = evalError(etc, (x_train,y_train), (x_test,y_test))
print "ExtraTreeClassifier\ntrain accuracy %f\ntest accuracy %f\n\n" %(train_acc, test_acc)
#scores = cross_val_score(etc, x_train, y_train)
else:
print "Loading ExtraTrees Classifier model"
etc = ut.load("ExtraTreesClassifier", modelpath)
#ut.writePredictionFile(etc.predict(unseen_data), modeltype="etc")
ut.writePredictionFile(etc.predict_proba(unseen_data), modeltype="etc")
# free memory
del etc
gc.collect()
def run(self, restore=False):
self.g_init()
if not self.journal:
utils.load()
load_save.retrieve()
if restore:
self.restore_pattern()
else:
g.delay = (3 - g.level) * 400
self.tu = my_turtle.TurtleClass()
self.tu.current = [1, 1, 1, 3, 2]
self.get_goal()
if g.pattern == 1:
self.tu.current = utils.copy_list(g.goal)
self.tu.setup(self.colors[0])
g.numbers = utils.copy_list(self.tu.current)
#buttons
x = g.sx(7.3)
y = g.sy(16.5)
dx = g.sy(2.6)
if not self.sugar:
buttons.Button("cyan", (x, y), True)
x += dx
buttons.off('cyan')
buttons.Button("green", (x, y), True)
x += dx
buttons.Button("red", (x, y), True)
x += dx
buttons.Button("black", (x, y), True)
x += dx
self.slider = slider.Slider(g.sx(23.5), g.sy(21), 3, utils.YELLOW)
self.mouse_1st_no() # to 1st number
if self.canvas is not None:
self.canvas.grab_focus()
ctrl = False
pygame.key.set_repeat(600, 120)
key_ms = pygame.time.get_ticks()
going = True
while going:
if self.journal:
# Pump GTK messages.
while gtk.events_pending():
gtk.main_iteration()
# Pump PyGame messages.
for event in pygame.event.get():
if event.type == pygame.QUIT:
if not self.journal:
utils.save()
going = False
elif event.type == pygame.MOUSEMOTION:
g.pos = event.pos
g.redraw = True
if self.canvas is not None:
self.canvas.grab_focus()
elif event.type == pygame.MOUSEBUTTONDOWN:
g.redraw = True
if g.big:
g.big = False
else:
bu = buttons.check()
if bu != '':
self.do_button(bu)
self.flush_queue()
elif not self.sugar:
if utils.mouse_on_img1(g.magician, g.magician_c):
self.help2()
elif utils.mouse_in(g.x1, g.y0, g.x1 + g.bw,
g.y0 + g.bw):
self.big_pic()
elif self.slider.mouse():
g.delay = (3 - g.level) * 400
else:
g.show_help = False
self.check_nos(event.button)
else:
g.show_help = False
self.check_nos(event.button)
elif event.type == pygame.KEYDOWN:
# throttle keyboard repeat
if pygame.time.get_ticks() - key_ms > 110:
key_ms = pygame.time.get_ticks()
if ctrl:
if event.key == pygame.K_q:
if not self.journal:
utils.save()
going = False
break
else:
ctrl = False
if event.key in (pygame.K_LCTRL, pygame.K_RCTRL):
ctrl = True
break
self.do_key(event.key)
g.redraw = True
self.flush_queue()
elif event.type == pygame.KEYUP:
ctrl = False
if not going:
break
if self.tu.running:
self.tu.move()
if not g.crash_drawn:
g.crash_drawn = True
g.redraw = True
if g.redraw:
self.display()
if g.version_display:
utils.version_display()
g.screen.blit(g.pointer, g.pos)
pygame.display.flip()
g.redraw = False
g.clock.tick(40)
def _create_particles(self, particle_factory, *args, **kw):
""" Create particles given a callable `particle_factory` and any
arguments to it.
This will also automatically distribute the particles among
processors if this is a parallel run. Returns a list of particle
arrays that are created.
"""
solver = self._solver
num_procs = self.num_procs
options = self.options
rank = self.rank
comm = self.comm
# particle array info that is used to create dummy particles
# on non-root processors
particles_info = {}
# Only master creates the particles.
if rank == 0:
if options.restart_file is not None:
data = load(options.restart_file)
arrays = data["arrays"]
solver_data = data["solver_data"]
# arrays and particles
particles = []
for array_name in arrays:
particles.append(arrays[array_name])
# save the particles list
self.particles = particles
# time, timestep and solver iteration count at restart
t, dt, count = solver_data["t"], solver_data["dt"], solver_data["count"]
# rescale dt at restart
dt *= options.rescale_dt
solver.t, solver.dt, solver.count = t, dt, count
else:
self.particles = particle_factory(*args, **kw)
# get the array info which will be b'casted to other procs
particles_info = utils.get_particles_info(self.particles)
# Broadcast the particles_info to other processors for parallel runs
if self.num_procs > 1:
particles_info = self.comm.bcast(particles_info, root=0)
# now all processors other than root create dummy particle arrays
if rank != 0:
self.particles = utils.create_dummy_particles(particles_info)
# Instantiate the Parallel Manager here and do an initial LB
self.pm = None
if num_procs > 1:
options = self.options
if options.with_zoltan:
if not (Has_Zoltan and Has_MPI):
raise RuntimeError("Cannot run in parallel!")
else:
raise ValueError(
"""Sorry. You're stuck with Zoltan for now
use the option '--with_zoltan' for parallel runs
"""
)
# create the parallel manager
obj_weight_dim = "0"
if options.zoltan_weights:
obj_weight_dim = "1"
zoltan_lb_method = options.zoltan_lb_method
zoltan_debug_level = options.zoltan_debug_level
zoltan_obj_wgt_dim = obj_weight_dim
# ghost layers
ghost_layers = options.ghost_layers
# radius scale for the parallel update
radius_scale = options.parallel_scale_factor * solver.kernel.radius_scale
self.pm = pm = ZoltanParallelManagerGeometric(
dim=solver.dim,
particles=self.particles,
comm=comm,
lb_method=zoltan_lb_method,
obj_weight_dim=obj_weight_dim,
ghost_layers=ghost_layers,
update_cell_sizes=options.update_cell_sizes,
radius_scale=radius_scale,
)
### ADDITIONAL LOAD BALANCING FUNCTIONS FOR ZOLTAN ###
# RCB lock directions
if options.zoltan_rcb_lock_directions:
pm.set_zoltan_rcb_lock_directions()
if options.zoltan_rcb_reuse:
pm.set_zoltan_rcb_reuse()
if options.zoltan_rcb_rectilinear:
pm.set_zoltan_rcb_rectilinear_blocks()
if options.zoltan_rcb_set_direction > 0:
pm.set_zoltan_rcb_directions(str(options.zoltan_rcb_set_direction))
# set zoltan options
pm.pz.Zoltan_Set_Param("DEBUG_LEVEL", options.zoltan_debug_level)
pm.pz.Zoltan_Set_Param("DEBUG_MEMORY", "0")
# do an initial load balance
pm.update()
pm.initial_update = False
# set subsequent load balancing frequency
lb_freq = options.lb_freq
if lb_freq < 1:
raise ValueError("Invalid lb_freq %d" % lb_freq)
pm.set_lb_freq(lb_freq)
# wait till the initial partition is done
comm.barrier()
# set the solver's parallel manager
solver.set_parallel_manager(self.pm)
return self.particles
LR = 0.01
GPUS = 1
RAW_MODEL = None
TASK = "CNN@%s_AR@%d_DISC@%d_CTC@%d_MTO@%s_DLOSS@%s_MARGIN@%.2f_INIT@%d_BN@%d"%(RES_TYPE,AR_EN,DISC_EN,CTC_EN,MANY_TO_ONE,
METRIC_LOSS,MARGIN,1 if RAW_MODEL else 0,BN_DIM)
MODEL_DIR = "exp/%s"%TASK
if not os.path.isdir(MODEL_DIR):os.mkdir(MODEL_DIR)
lr_reducer = ReduceLROnPlateau(factor=0.3, cooldown=0, patience=2, min_lr=1e-5,
monitor='val_y_accent_acc', mode='max', min_delta=0.001, verbose=1)
early_stopper = EarlyStopping(patience=3,
monitor='val_y_accent_acc', mode='max', min_delta=0.001, verbose=1)
csv_logger = CSVLogger('%s/train.csv' % MODEL_DIR)
# generator and data
DATA_DCT = us.load("array/data_scp.pkl")
ACCENT_DCT = us.load("array/accent_scp.pkl")
TRANS_DCT = us.load("array/trans_scp.pkl")
train_lst = us.read_lines("train.lst")
dev_lst = us.read_lines("dev.lst")[:100]
N_BATCHS = len(train_lst)//BATCH_SIZE
train_generator = us.data_generator(train_lst,
batch_size=BATCH_SIZE,
ctc_enable=CTC_EN,
ar_enable=AR_EN,
disc_enable=DISC_EN,
data_dct=DATA_DCT,
accent_dct=ACCENT_DCT,
trans_dct=TRANS_DCT,
max_input_len=MAX_INPUT_LEN,
def run_trials(args):
"""
Run args["conf_trials"] trials and survive args["max_attempts"] failed
attempts.
"""
print(f"Arguments: {args}")
if args["no_rand"]:
utils.set_rand_seed()
out_dir = args["out_dir"]
if not path.isdir(out_dir):
print(f"Output directory does not exist. Creating it: {out_dir}")
os.makedirs(out_dir)
net_tmp = models.MODELS[args["model"]]()
# Verify that the necessary supplemental parameters are present.
for param in net_tmp.params:
assert param in args, f"\"{param}\" not in args: {args}"
# Assemble the output filepath.
out_flp = path.join(
args["out_dir"],
(utils.args_to_str(args, order=sorted(defaults.DEFAULTS.keys()))) + (
# Determine the proper extension based on the type of
# model.
".pickle"
if isinstance(net_tmp, models.SvmSklearnWrapper) else ".pth"))
# If custom features are specified, then overwrite the model's
# default features.
fets = args["features"]
if fets:
net_tmp.in_spc = fets
else:
assert "arrival time us" not in args["features"]
args["features"] = net_tmp.in_spc
# If a trained model file already exists, then delete it.
if path.exists(out_flp):
os.remove(out_flp)
# Load or geenrate training data.
dat_flp = path.join(out_dir, "data.npz")
scl_prms_flp = path.join(out_dir, "scale_params.json")
# Check for the presence of both the data and the scaling
# parameters because the resulting model is useless without the
# proper scaling parameters.
if (not args["regen_data"] and path.exists(dat_flp)
and path.exists(scl_prms_flp)):
print("Found existing data!")
dat_in, dat_out, dat_out_raw, dat_out_oracle, num_flws = utils.load(
dat_flp)
dat_in_shape = dat_in.shape
dat_out_shape = dat_out.shape
assert dat_in_shape[0] == dat_out_shape[0], \
f"Data has invalid shapes! in: {dat_in_shape}, out: {dat_out_shape}"
else:
print("Regenerating data...")
dat_in, dat_out, dat_out_raw, dat_out_oracle, num_flws = (gen_data(
net_tmp, args, dat_flp, scl_prms_flp))
print(f"Number of input features: {len(dat_in.dtype.names)}")
# Visualaize the ground truth data.
utils.visualize_classes(net_tmp, dat_out)
# TODO: Parallelize attempts.
trls = args["conf_trials"]
apts = 0
apts_max = args["max_attempts"]
ress = []
while trls > 0 and apts < apts_max:
apts += 1
res = (run_sklearn if isinstance(net_tmp, models.SvmSklearnWrapper)
else run_torch)(args, dat_in, dat_out, dat_out_raw,
dat_out_oracle, num_flws, out_dir, out_flp)
if res[0] == 100:
print((
f"Training failed (attempt {apts}/{apts_max}). Trying again!"))
else:
ress.append(res)
trls -= 1
if ress:
print(("Resulting accuracies: "
f"{', '.join([f'{acc:.2f}' for acc, _ in ress])}"))
max_acc, tim_s = max(ress, key=lambda p: p[0])
print(f"Maximum accuracy: {max_acc:.2f}")
# Return the minimum error instead of the maximum accuracy.
return 1 - max_acc, tim_s
print(f"Model cannot be trained with args: {args}")
return float("NaN"), float("NaN")
def FID(img):
with torch.no_grad():
batch_size = 32
cfg['batch_size']['train'] = batch_size
dataset = fetch_dataset(cfg['data_name'], cfg['subset'], verbose=False)
real_data_loader = make_data_loader(dataset)['train']
generated_data_loader = DataLoader(img, batch_size=batch_size)
if cfg['data_name'] in ['COIL100', 'Omniglot']:
model = models.classifier().to(cfg['device'])
model_tag = ['0', cfg['data_name'], cfg['subset'], 'classifier']
model_tag = '_'.join(filter(None, model_tag))
checkpoint = load(
'./metrics_tf/res/classifier/{}_best.pt'.format(model_tag))
model.load_state_dict(checkpoint['model_dict'])
model.train(False)
real_feature = []
for i, input in enumerate(real_data_loader):
input = collate(input)
input = to_device(input, cfg['device'])
real_feature_i = model.feature(input)
real_feature.append(real_feature_i.cpu().numpy())
real_feature = np.concatenate(real_feature, axis=0)
generated_feature = []
for i, input in enumerate(generated_data_loader):
input = {
'img': input,
'label': input.new_zeros(input.size(0)).long()
}
input = to_device(input, cfg['device'])
generated_feature_i = model.feature(input)
generated_feature.append(generated_feature_i.cpu().numpy())
generated_feature = np.concatenate(generated_feature, axis=0)
else:
model = inception_v3(pretrained=True,
transform_input=False).to(cfg['device'])
up = nn.Upsample(size=(299, 299),
mode='bilinear',
align_corners=False)
model.feature = nn.Sequential(*[
up, model.Conv2d_1a_3x3, model.Conv2d_2a_3x3,
model.Conv2d_2b_3x3,
nn.MaxPool2d(kernel_size=3, stride=2), model.Conv2d_3b_1x1,
model.Conv2d_4a_3x3,
nn.MaxPool2d(kernel_size=3, stride=2), model.Mixed_5b,
model.Mixed_5c, model.Mixed_5d, model.Mixed_6a, model.Mixed_6b,
model.Mixed_6c, model.Mixed_6d, model.Mixed_6e, model.Mixed_7a,
model.Mixed_7b, model.Mixed_7c,
nn.AdaptiveAvgPool2d(1),
nn.Flatten()
])
model.train(False)
real_feature = []
for i, input in enumerate(real_data_loader):
input = collate(input)
input = to_device(input, cfg['device'])
real_feature_i = model.feature(input['img'])
real_feature.append(real_feature_i.cpu().numpy())
real_feature = np.concatenate(real_feature, axis=0)
generated_feature = []
for i, input in enumerate(generated_data_loader):
input = to_device(input, cfg['device'])
generated_feature_i = model.feature(input)
generated_feature.append(generated_feature_i.cpu().numpy())
generated_feature = np.concatenate(generated_feature, axis=0)
mu1 = np.mean(real_feature, axis=0)
sigma1 = np.cov(real_feature, rowvar=False)
mu2 = np.mean(generated_feature, axis=0)
sigma2 = np.cov(generated_feature, rowvar=False)
mu1 = np.atleast_1d(mu1)
mu2 = np.atleast_1d(mu2)
sigma1 = np.atleast_2d(sigma1)
sigma2 = np.atleast_2d(sigma2)
assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths"
assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions"
diff = mu1 - mu2
# product might be almost singular
covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
if not np.isfinite(covmean).all():
offset = np.eye(sigma1.shape[0]) * 1e-6
covmean = linalg.sqrtm((sigma1 + offset).dot(sigma2 + offset))
# numerical error might give slight imaginary component
if np.iscomplexobj(covmean):
if not np.allclose(np.diagonal(covmean).imag, 0, atol=1e-3):
m = np.max(np.abs(covmean.imag))
raise ValueError("Imaginary component {}".format(m))
covmean = covmean.real
tr_covmean = np.trace(covmean)
fid = diff.dot(diff) + np.trace(sigma1) + np.trace(
sigma2) - 2 * tr_covmean
fid = fid.item()
return fid
def load(self):
utils.load(self.model, os.path.join(self.args.save, 'weights.pt'))
#%%
import re
from typing import Callable
from utils import load
raw = load("day04.txt", split="\n\n")
# %%
def test1() -> list[str]:
valid: list[str] = list()
for line in raw:
test = (
len(re.findall("[bie]yr", line)) == 3,
re.findall("hgt", line),
len(re.findall("[he]cl", line)) == 2,
re.findall("pid", line),
)
if all(test):
valid.append(line)
assert len(valid) == 190
return valid
# %%
def proc_hgt(x: str) -> bool:
if x.endswith("cm"):
return 150 <= int(x[:-2]) <= 193
elif x.endswith("in"):
return 59 <= int(x[:-2]) <= 76
else:
raise NotImplementedError
return y, x
if __name__ == '__main__':
import os
import anyconfig
from utils import parse_config, load, get_parameter_number
config = anyconfig.load(
open("config/imagedataset_None_VGG_RNN_CTC.yaml", 'rb'))
if 'base' in config:
config = parse_config(config)
if os.path.isfile(config['dataset']['alphabet']):
config['dataset']['alphabet'] = load(config['dataset']['alphabet'])
device = torch.device('cpu')
config['arch']['backbone']['in_channels'] = 3
config['arch']['head']['n_class'] = 95
net = Model(config['arch']).to(device)
print(net.name, len(config['dataset']['alphabet']))
a = torch.randn(2, 3, 32, 320).to(device)
import time
text_for_pred = torch.LongTensor(2, 25 + 1).fill_(0)
tic = time.time()
for i in range(1):
b = net(a, text_for_pred)[0]
print(b.shape)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Nov 27 23:44:00 2018
@author: dhruvinpatel
"""
from sklearn.cluster import DBSCAN
import numpy as np
from utils import load, cal_sse
import matplotlib.pyplot as plt
text_vectors = load('text_vectors.p')
clustering = DBSCAN(eps=.5, min_samples=10,metric='cosine').fit(text_vectors)
labels = clustering.labels_
n_labels = set(labels)
clusters = {}
for n in n_labels:
if n == -1:
continue
else:
clusters[n] = []
for i in range(len(labels)):
if labels[i] == -1:
continue
else:
clist = clusters[labels[i]]
clist.append(text_vectors[i])
import os
from os import path
import utils
dumpdir = path.join('..', 'dumps')
try:
data = utils.load(path.join(dumpdir, 'datadict.pkl'))
ingredient_dict = utils.load(
path.join(dumpdir, 'ingredient_dictionary.pkl'))
except Exception as e:
print(
"pkl files not found. Please run 'ingredient_dictionary.py' first. Exiting..."
)
exit(1)
def ranker(input_ingredients):
def printer(*args):
print("ranker: ", end="")
print(*args)
unigrams = set(input_ingredients)
candidates = [(ingredient_dict[uni], len(ingredient_dict[uni]))
for uni in unigrams if uni in ingredient_dict]
candidate_scores = {}
for candidate_list, score in candidates:
for candidate in candidate_list:
if candidate not in candidate_scores:
candidate_scores[candidate] = 0.0
candidate_scores[candidate] += 1.0 / score
def load_model(self, model_path):
self.Q = utils.load(model_path)
Command: THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python kaggle_otto_nn.py
On EC2 g2.2xlarge instance: 19s/epoch. 6-7 minutes total training time.
Best validation score at epoch 21: 0.4881
Try it at home:
- with/without BatchNormalization (BatchNormalization helps!)
- with ReLU or with PReLU (PReLU helps!)
- with smaller layers, largers layers
- with more layers, less layers
- with different optimizers (SGD+momentum+decay is probably better than Adam!)
'''
np.random.seed(1337) # for reproducibility
sort_idx = ut.load("sort_idx")
sort_idx = sort_idx[-93:]
print(type(sort_idx))
print(sort_idx.shape)
## check if raw data exist
if os.path.isfile("./tmp/X-extra-features-0"):
print ("Loading existing data (with extra features)...")
#print(X.shape)
for i in range(6):
print("Loading subset %d"%i)
objname = "X-extra-features-%d"%i
labelsname = "labels-%d"%i
if i==0:
raw_path = params['RAW_MODEL_PATH']
spectro_path = params['SPECTRUM_MODEL_PATH']
test_dataset_path = params['TEST_PICKLE']
batch_size = BATCH_SIZE
try:
with open(params['DICT_JSON'], mode='r', encoding='utf-8') as fin:
class_dict = json.load(fin)
except Exception as e:
print(e)
print("We have lost correpsondances, aborting...")
sys.exit(0)
test_set = load(test_dataset_path)
if test_set is None:
print("No Test data, aborting...")
sys.exit(0)
random.shuffle(test_set)
test_lens = get_class_numbers(test_set, class_dict)
test_data = get_reduced_set(test_set, test_lens, 'min')
def _DScnn(x, rawnet, spectronet):
c1 = rawnet(x[0])
c2 = spectronet(x[1])
c1 = tf.nn.softmax(c1)
c2 = tf.nn.softmax(c2)
def run(self, restore=False):
self.g_init()
if not self.journal:
utils.load()
load_save.retrieve()
self.aim = simon.Simon(1200) # arg is glow time
if self.sugar:
self.set_delay(800)
else:
self.set_delay()
self.player = simon.Simon(200)
if restore:
self.restore_pattern()
self.aim.started = True
if self.sugar:
self.green_button.set_sensitive(True)
self.back_button.set_sensitive(False)
else:
bx = g.sx(22.42)
by = g.sy(20.8)
buttons.Button('green', (bx, by), True)
buttons.Button('back', (bx, by), True)
buttons.off('back')
self.slider = slider.Slider(g.sx(9), g.sy(20.8), 3, utils.BLUE)
self.rc = rc_skip_last.RC(3, 5)
if self.canvas is not None:
self.canvas.grab_focus()
ctrl = False
pygame.key.set_repeat(600, 120)
key_ms = pygame.time.get_ticks()
going = True
while going:
if self.journal:
# Pump GTK messages.
while gtk.events_pending():
gtk.main_iteration()
# Pump PyGame messages.
for event in pygame.event.get():
if event.type == pygame.QUIT:
if not self.journal:
utils.save()
going = False
elif event.type == pygame.MOUSEMOTION:
g.pos = event.pos
g.redraw = True
if self.canvas is not None:
self.canvas.grab_focus()
self.mouse_set()
elif event.type == pygame.MOUSEBUTTONDOWN:
g.redraw = True
if event.button == 1:
self.do_click()
self.flush_queue()
elif event.type == pygame.KEYDOWN:
# throttle keyboard repeat
if pygame.time.get_ticks() - key_ms > 110:
key_ms = pygame.time.get_ticks()
if ctrl:
if event.key == pygame.K_q:
if not self.journal:
utils.save()
going = False
break
else:
ctrl = False
if event.key in (pygame.K_LCTRL, pygame.K_RCTRL):
ctrl = True
break
self.do_key(event.key)
g.redraw = True
self.flush_queue()
elif event.type == pygame.KEYUP:
ctrl = False
if not going:
break
if self.sugar:
if g.player_n == 0 and not self.green_button.get_sensitive():
self.back_button.set_sensitive(True)
else:
if g.player_n == 0 and not buttons.active('green'):
buttons.on('back')
self.player.do()
self.aim.do()
if g.redraw:
self.display()
if g.version_display:
utils.version_display()
if self.aim.running or self.aim.glow_active:
pass
else:
g.screen.blit(g.pointer, g.pos)
pygame.display.flip()
g.redraw = False
g.clock.tick(40)
def run(self, restore=False):
self.black = False
g.init()
if not self.journal:
utils.load()
load_save.retrieve()
x = g.sx(26)
y = g.sy(11.2)
if not self.sugar:
buttons.Button("new", (x, y))
x, y = g.cxy2
dx = g.sy(4)
self.back_button = buttons.Button("back", (x, y))
x += dx
buttons.Button("plus", (x, y))
x += dx
buttons.Button("times", (x, y))
x += dx
buttons.Button("equals", (x, y))
self.ops = ['back', 'plus', 'times', 'equals']
if not self.sugar:
self.slider = slider.Slider(g.sx(22.4), g.sy(20.5), 10,
utils.GREEN)
self.mouse_auto = True
self.anim_ms = None
self.level1() # initial animation
self.scored = False
if restore:
g.score = self.save_score
ctrl = False
pygame.key.set_repeat(600, 120)
key_ms = pygame.time.get_ticks()
going = True
if self.canvas is not None:
self.canvas.grab_focus()
while going:
if self.journal:
# Pump GTK messages.
while Gtk.events_pending():
Gtk.main_iteration()
# Pump PyGame messages.
for event in pygame.event.get():
if event.type == pygame.QUIT:
if not self.journal:
utils.save()
going = False
elif event.type == pygame.MOUSEMOTION:
g.pos = event.pos
g.redraw = True
if self.canvas is not None:
self.canvas.grab_focus()
elif event.type == pygame.MOUSEBUTTONDOWN:
g.redraw = True
self.anim_end()
if event.button == 1:
bu = buttons.check()
if bu == '':
if not self.check_numbers():
if not self.sugar:
if self.slider.mouse():
self.level1()
else:
self.do_button(bu) # eg do_button('plus')
self.flush_queue()
elif event.type == pygame.KEYDOWN:
self.anim_end()
# throttle keyboard repeat
if pygame.time.get_ticks() - key_ms > 110:
key_ms = pygame.time.get_ticks()
if ctrl:
if event.key == pygame.K_q:
if not self.journal:
utils.save()
going = False
break
else:
ctrl = False
if event.key in (pygame.K_LCTRL, pygame.K_RCTRL):
ctrl = True
break
self.do_key(event.key)
g.redraw = True
self.flush_queue()
elif event.type == pygame.KEYUP:
ctrl = False
if not going:
break
self.animation()
if g.redraw:
self.display()
if g.version_display:
utils.version_display()
if not self.black:
g.screen.blit(g.pointer, g.pos)
pygame.display.flip()
g.redraw = False
g.clock.tick(40)
#!/usr/bin/env python
from utils import (load, take, show, bgr, image, like, bounds,
channels, crop, scale, color, avail, colorPicker)
from proto import alias, sharpen, group, find, edge, center, distance
from PIL import Image
print "# fast stuff"
img = load('samples/abstract/colors.png')
#b = take()
show(img)
b, g, r = bgr(img)
img = image(b,b,b)
test = like(img)
bound = bounds(b)
channel = channels(b)
coord = (0,0,50,50)
closer = crop(img, coord)
bigger = scale(closer, 2.0)
eyedrop = color(img, 0, 30)
pallet = avail(img)
colorPicker(img,0,30)
print "# slow stuff"
res1 = alias(img, .3)
res2 = sharpen(img, .3)
blob1 = group(img)
mask = Image.new("RGB", (50, 10), "white")
blob3 = find(img,mask,(3,3))
coords1 = edge(img)
coords2 = center(blob1)
dist = distance(0,3)
def _parse_embedding(self):
return load(self.store_folder / self.pkl_file)
def main(args):
# loading configurations
with open(args.config) as f:
config = yaml.safe_load(f)["configuration"]
name = config["Name"]
# Construct or load embeddings
print("Initializing embeddings ...")
vocab_size = config["embeddings"]["vocab_size"]
embed_size = config["embeddings"]["embed_size"]
embeddings = init_embeddings(vocab_size, embed_size, name=name)
print("\tDone.")
# Build the model and compute losses
source_ids = tf.placeholder(tf.int32, [None, None], name="source")
target_ids = tf.placeholder(tf.int32, [None, None], name="target")
sequence_mask = tf.placeholder(tf.bool, [None, None], name="mask")
attn_wrappers = {
"None": None,
"Attention": AttentionWrapper,
}
attn_wrapper = attn_wrappers.get(config["decoder"]["wrapper"])
(enc_num_layers, enc_num_units, enc_cell_type, enc_bidir,
dec_num_layers, dec_num_units, dec_cell_type, state_pass,
infer_batch_size, infer_type, beam_size, max_iter,
attn_num_units, l2_regularize) = get_model_config(config)
print("Building model architecture ...")
CE, loss, logits, infer_outputs = compute_loss(
source_ids, target_ids, sequence_mask, embeddings,
enc_num_layers, enc_num_units, enc_cell_type, enc_bidir,
dec_num_layers, dec_num_units, dec_cell_type, state_pass,
infer_batch_size, infer_type, beam_size, max_iter,
attn_wrapper, attn_num_units, l2_regularize, name)
print("\tDone.")
# Set up session
restore_from = config["training"]["restore_from"]
gpu_fraction = config["training"]["gpu_fraction"]
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=gpu_fraction)
sess = tf.Session(config=tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options))
init = tf.global_variables_initializer()
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.trainable_variables())
try:
saved_global_step = load(saver, sess, restore_from)
if saved_global_step is None:
raise ValueError("Cannot find the checkpoint to restore from.")
except Exception:
print("Something went wrong while restoring checkpoint. ")
raise
# ##### Inference #####
# Load data
print("Loading inference data ...")
# id_0, id_1, id_2 preserved for SOS, EOS, constant zero padding
embed_shift = 3
filename = config["inference"]["infer_source_file"]
max_leng = config["inference"]["infer_source_max_length"]
source_data = loadfile(filename, is_source=True,
max_length=max_leng) + embed_shift
print("\tDone.")
# Inference
print("Start inferring ...")
final_result = []
n_data = source_data.shape[0]
n_pad = n_data % infer_batch_size
if n_pad > 0:
n_pad = infer_batch_size - n_pad
pad = np.zeros((n_pad, max_leng), dtype=np.int32)
source_data = np.concatenate((source_data, pad))
for ith in range(int(len(source_data) / infer_batch_size)):
start = ith * infer_batch_size
end = (ith + 1) * infer_batch_size
batch = source_data[start:end]
result = sess.run(infer_outputs, feed_dict={source_ids: batch})
result = result.ids[:, :, 0]
if result.shape[1] < max_iter:
l_pad = max_iter - result.shape[1]
result = np.concatenate(
(result, np.ones((infer_batch_size, l_pad))), axis=1)
final_result.append(result)
final_result = np.concatenate(final_result)[:n_data] - embed_shift
final_result[final_result < 0] = -1
final_result = final_result.astype(str).tolist()
final_result = list(map(lambda t: " ".join(t), final_result))
df = pd.DataFrame(data={"0": final_result})
df.to_csv(config["inference"]["output_path"], header=None, index=None)
print("\tDone.")
import os
import utils
from processing import calculate_relations
from models import Theme, Teacher, Student
if __name__ == '__main__':
os.makedirs('output', exist_ok=True)
# Загрузка тем
themes_data = utils.load('../data/themes.csv')
themes = utils.extract_models(Theme, themes_data)
utils.print_multiline(themes)
# Загрузка преподавателей
teachers_data = utils.load('../data/teachers.csv')
teachers = utils.extract_models(Teacher, teachers_data)
utils.print_multiline(teachers[:10])
# Загрузка студентов
students_data = utils.load('../data/students.csv')
students = utils.extract_models(Student, students_data)
utils.print_multiline(students[:10])
students, teachers, lines = calculate_relations(students, teachers)
utils.dump(lines, 'output/relations.csv')
print('- ' * 30)
for idx, _teacher in enumerate(teachers):