def main(): (x_train, y_train), (x_validation, y_validation) = load_data() model = Model(*juxt(identity, computational_graph(y_train.shape[1]))(Input( shape=x_train.shape[1:]))) model.compile(loss='categorical_crossentropy', optimizer=SGD(momentum=0.9), metrics=['accuracy' ]) # 論文にはnesterov=Trueだと書いてあったけど、コードだとFalseだった……。 model.summary() # plot_model(model, to_file='./results/model.png') train_data = ImageDataGenerator(featurewise_center=True, featurewise_std_normalization=True, width_shift_range=0.125, height_shift_range=0.125, horizontal_flip=True) validation_data = ImageDataGenerator(featurewise_center=True, featurewise_std_normalization=True) for data in (train_data, validation_data): data.fit(x_train) # 実用を考えると、x_validationでのfeaturewiseのfitは無理だと思う……。 batch_size = 128 epoch_size = 200 results = model.fit_generator( train_data.flow(x_train, y_train, batch_size=batch_size), steps_per_epoch=x_train.shape[0] // batch_size, epochs=epoch_size, callbacks=[ LearningRateScheduler( partial( getitem, tuple( take( epoch_size, concat(repeat(0.1, 60), repeat(0.02, 60), repeat(0.004, 40), repeat(0.0008)))))) ], validation_data=validation_data.flow(x_validation, y_validation, batch_size=batch_size), validation_steps=x_validation.shape[0] // batch_size) with open('./results/history.pickle', 'wb') as f: pickle.dump(results.history, f) save_model(model, './results/model.h5') del model
def feedback(self, inputs, outputs, initials=None, latches=None, keep_outputs=False): import warnings warnings.warn("deprecated", DeprecationWarning) def create_wire(val): iname, oname, lname, init = val return { 'input': iname, 'output': oname, 'latch': lname, 'init': init, 'keep_output': keep_outputs } if initials is None: initials = fn.repeat(False) if latches is None: assert (set(inputs) & self.latches) == set() latches = inputs vals = zip(inputs, outputs, latches, initials) return self.loopback(*map(create_wire, vals))
def get_interesting_repos(g: Github, session: Any) -> List[Repository]: repos: List[Repository] = [] grepos = g.search_repositories(query='stars:>250 forks:>50', sort='stars', order='desc') records = zip(grepos, fy.repeat('most_stars')) grepos = g.search_repositories(query='forks:>5 topic:kaggle-competition', sort='stars', order='desc') records = fy.concat(records, zip(grepos, fy.repeat('kaggle'))) grepos = g.search_repositories(query='forks:>5 topic:tensorflow-model', sort='stars', order='desc') records = fy.concat(records, zip(grepos, fy.repeat('tensorflow-model'))) grepos = g.search_repositories( query='cookiecutterdatascience in:readme forks:>5 stars:>0 fork:true', sort='stars', order='desc') records = fy.concat(records, zip(grepos, fy.repeat('cookiecutterdatascience'))) for grepo, search_method in tqdm(records): repo = (session.query(Repository).filter( Repository.id == grepo.full_name).one_or_none()) if repo is None: repo = Repository( id=grepo.full_name, owner=grepo.owner.login, name=grepo.name, description=grepo.description, search_method=search_method, ) repos.append(repo) return repos
def create_file_obj_dict(namedtup): # create fields with all the info we are saving from the diff text change_dict = dict(zip(namedtup.__dict__['_fields'], funcy.repeat(None))) change_dict['raw_diff'] = [] change_dict['filename_old'] = '' change_dict['filename_new'] = '' change_dict['functions_changed'] = [] change_dict['locations_changed'] = [] change_dict['list_changes'] = [] change_dict['num_changes'] = 0 change_dict['is_rename'] = False change_dict['is_new'] = False change_dict['is_deletion'] = False return change_dict
def step(self): self.elapse += 1 self.actions = [] for car, player in zip(self.cars, concat(self.players, repeat(None))): # アクションを取得します。 acceleration, braking, steering = player.get_action( self.create_observation(car)) if player else (0, 0, 0) # アクションを正規化します。 acceleration = self._clip(acceleration, -1, 1) braking = self._clip(braking, 0, 1) # noqa: E221, E241 steering = self._clip(steering, -1, 1) # noqa: E221, E241 # 正規化したアクションを記録します。 self.actions.append((acceleration, braking, steering)) # 衝突して故障した車は、修理が終わるまでは行動できません。 if car.crash_energy > 0: car.crash_energy = max(car.crash_energy - 100000, 0) continue # ゆらぎを出すために、アクションに小さな正規乱数を加えます。スターの次の出現位置が変わると強化学習が難しくなりそうなので、別のRandomインスタンスを使用します。 acceleration = self._clip( acceleration + self.control_random.gauss(0, 0.05), -1, 1) braking = self._clip(braking + self.control_random.gauss(0, 0.05), 0, 1) # noqa: E221, E241 steering = self._clip(steering + self.control_random.gauss(0, 0.05), -1, 1) # noqa: E221 # アクションを実行します。 car.accelerate(acceleration * 20000) car.brake(braking * 200000) car.steer(steering * 20000) self.space.step(1 / FPS) for star in filter(lambda star: star.is_catched, self.stars): self._reset_star_position(star) star.is_catched = False return self.elapse >= GAME_PERIOD_SEC * FPS # ゲームはGAME_PERIOD_SECで終了します。
def busy_account_following(account_name, following): """ Fetch users followers or followings and their metadata. Returned list is ordered by follow time (newest followers first). \n Usage: `GET /busy/<string:account_name>/<string:following>`\n `following` must be 'following' or 'followers'.\n """ if following not in ['following', 'followers']: raise ParseError(detail='Please specify following or followers.') acc = mongo.db['Accounts'].find_one({'name': account_name}, { following: 1, '_id': 0 }) if not acc: raise NotFound(detail='Could not find STEEM account %s' % account_name) # if follower list is empty if not acc[following]: return [] allowed_fields = { '_id': 0, 'name': 1, 'sp': 1, 'rep': 1, 'followers_count': 1, 'following_count': 1, 'post_count': 1, } accounts_w_meta = list(mongo.db['Accounts'].find( {'name': { '$in': acc[following] }}, allowed_fields)) # return in LIFO order (last to follow is listed first) accounts_ordered = list(repeat('', len(acc[following]))) for a in accounts_w_meta: with suppress(ValueError): accounts_ordered[acc[following].index(a.get('name', None))] = a return [x for x in accounts_ordered if x][::-1]
def __init__(self, players, seed=None): self.game_random = Random(seed) self.control_random = Random(seed) self.players = players self.elapse = 0 self.actions = repeat((0, 0, 0), len(players)) self.space = pymunk.Space() self.space.add_wildcard_collision_handler(1).post_solve = self._crash self.space.add_wildcard_collision_handler(2).begin = self._catch self.cars = [] self.obstacles = [] self.stars = [] for a, b in ((-1000, 1000), (1000, 1000)), ((1000, 1000), (1000, -1000)), ((1000, -1000), (-1000, -1000)), ((-1000, -1000), (-1000, 1000)): self._append_wall(a, b) for position, angle in map( lambda i: (pymunk.Vec2d(80, 0).rotated(pi * 2 / 8 * i), pi * 2 / 8 * i), range(8)): self._append_car(position, angle) for _ in range(OBSTACLE_COUNT): self._append_obstacle() for _ in range(STAR_COUNT): self._append_star()
class IXY(Structure): _fields_ = list(zip(IXY_props, F.repeat(c_int)))
class NODE(Structure): _fields_ = list(zip(NODE_props, F.repeat(c_int)))
parser.add_argument( "--no-wl2", action="store_true", help="Do not compute WL2 encodings.") parser.add_argument( "-d", "--dataset", action="append", help="Prepare this dataset.") args = parser.parse_args() p = mp.cpu_count() if args.parallel else 1 if args.dataset is None or len(args.dataset) == 0: ds = datasets.stored else: ds = args.dataset dsl = len(ds) print(args) print(f"Will prepare {dsl} stored datasets with parallelism {p}:") for d in ds: print(f"- {d}") print("Starting...") with mp.Pool(p) as p: p.starmap( prepare_ds, zip(ds, fy.repeat(not args.no_wl2), fy.repeat(not args.no_gram))) print("Prepared all stored datasets.")
def run(): batch_size = 32 num_classes = 10 epochs = 200 with tf.device("/cpu:0"): (x_train, y_train), (x_test, y_test) = cifar10.load_data() # Convert class vectors to binary class matrices. y_train = to_categorical(y_train, num_classes) y_test = to_categorical(y_test, num_classes) x_train = x_train.astype('float32', copy=False) x_test = x_test.astype('float32', copy=False) x_train /= 255 x_test /= 255 optimizer = Adam(lr=0.001) model = SqueezeNet(classes=num_classes) squeezenet_model_file = './sqz_log/model.h5' if os.path.exists(squeezenet_model_file): model.layers.pop() model = Model(name="sqzn_no_softmax", inputs=model.input, outputs=model.layers[-1].output) model.load_weights(squeezenet_model_file, by_name=True) # model.load_weights(squeezenet_model_file, by_name=True) else: # train a new SqueezeNet model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) # train_data = ImageDataGenerator(featurewise_center=True, featurewise_std_normalization=True, # width_shift_range=0.125, height_shift_range=0.125, horizontal_flip=True) # validation_data = ImageDataGenerator(featurewise_center=True, featurewise_std_normalization=True) train_data = ImageDataGenerator() validation_data = ImageDataGenerator() for data in (train_data, validation_data): data.fit(x_train) callbacks = [ LearningRateScheduler( partial( getitem, tuple( take( epochs, concat(repeat(0.01, 1), repeat(0.1, 99), repeat(0.01, 50), repeat(0.001)))))), ModelCheckpoint(filepath=squeezenet_model_file), TensorBoard(log_dir="./sqz_log", batch_size=batch_size) ] results = model.fit_generator( train_data.flow(x_train, y_train, batch_size=batch_size), steps_per_epoch=x_train.shape[0] // batch_size, epochs=epochs, callbacks=callbacks, validation_data=validation_data.flow(x_test, y_test, batch_size=batch_size), validation_steps=x_test.shape[0] // batch_size) with open('./sqz_log/history.pickle', 'wb') as f: pickle.dump(results.history, f) save_model(model, squeezenet_model_file) # Build the siamese architecture # model_cut = Model(name="sqzn_no_softmax", inputs=model.input, outputs=model.layers[-1].output) # model_cut.load_weights(squeezenet_model_file, by_name=True) # with tf.device("/cpu:0"): # model_cut.summary() input_shape = x_train.shape[1:] im_in1 = Input(shape=input_shape) im_in2 = Input(shape=input_shape) feat_x1 = model(im_in1) feat_x2 = model(im_in2) lambda_merge = Lambda(euclidean_distance, output_shape=(1, ))([feat_x1, feat_x2]) siamese = Model(name="siamese", inputs=[im_in1, im_in2], outputs=lambda_merge) with tf.device("/cpu:0"): siamese.summary() optimizer = RMSprop() # SGD(momentum=0.9) siamese.compile(optimizer=optimizer, loss=contrastive_loss, metrics=[accuracy]) def make_img_pair(identical, from_train): """Select the image pairs""" label = np.random.randint(0, num_classes) if identical: if from_train: idx = np.nonzero(y_train[:, label] == 1)[0] else: idx = np.nonzero(y_test[:, label] == 1)[0] # pick any two indexes randomly id1 = np.random.randint(0, idx.shape[0]) id2 = np.random.randint(0, idx.shape[0]) while id1 == id2: id2 = np.random.randint(0, idx.shape[0]) else: if from_train: idx1 = np.nonzero(y_train[:, label] == 1)[0] idx2 = np.nonzero(y_train[:, (label + 1) % num_classes] == 1)[0] else: idx1 = np.nonzero(y_test[:, label] == 1)[0] idx2 = np.nonzero(y_train[:, (label + 1) % num_classes] == 1)[0] # pick any two indexes randomly id1 = np.random.randint(0, idx1.shape[0]) id2 = np.random.randint(0, idx2.shape[0]) if from_train: return np.array([x_train[id1], x_train[id2]]) else: return np.array([x_test[id1], x_test[id2]]) def generator(from_train): while True: X = [[None, None]] * batch_size y = [[None]] * batch_size indexes = np.arange(batch_size) identical = True for i in indexes: X[i] = make_img_pair(identical, from_train) y[i] = [1 if identical else 0] identical = not identical np.random.shuffle(indexes) X = np.asarray(X)[indexes] y = np.asarray(y)[indexes] # print("generator: from_train:", from_train, " - X:", X.shape, "- y:", y.shape) yield [X[:, 0], X[:, 1]], y siamese_model_file = "./siam_log/siamese.h5" epochs = 100 callbacks = [ LearningRateScheduler( partial( getitem, tuple( take( epochs, concat(repeat(0.01, 1), repeat(0.1, 99), repeat(0.01, 50), repeat(0.001)))))), ModelCheckpoint(filepath=siamese_model_file), TensorBoard(log_dir="./siam_log", batch_size=batch_size) ] outputs = siamese.fit_generator( generator(from_train=True), initial_epoch=0, steps_per_epoch=x_train.shape[0] // batch_size, epochs=epochs, validation_data=generator(from_train=False), validation_steps=x_test.shape[0] // batch_size, callbacks=callbacks) with open('./siam_log/history.pickle', 'wb') as f: pickle.dump(outputs.history, f) save_model(siamese, siamese_model_file)
def main(): import os with tf.device("/cpu:0"): (x_train, y_train), (x_validation, y_validation) = load_data() batch_size = 32 epochs = 200 input_shape = Input(shape=x_train.shape[1:]) model_file = './results/model.h5' if os.path.exists(model_file): model = load_model(model_file) # with tf.device("/cpu:0"): # validation_data = ImageDataGenerator(featurewise_center=True, featurewise_std_normalization=True) else: model = Model(*juxt(identity, computational_graph(y_train.shape[1]))( input_shape)) model.compile(loss='categorical_crossentropy', optimizer=SGD(momentum=0.9), metrics=['accuracy']) with tf.device("/cpu:0"): train_data = ImageDataGenerator(featurewise_center=True, featurewise_std_normalization=True, width_shift_range=0.125, height_shift_range=0.125, horizontal_flip=True) validation_data = ImageDataGenerator( featurewise_center=True, featurewise_std_normalization=True) for data in (train_data, validation_data): data.fit( x_train) # 実用を考えると、x_validationでのfeaturewiseのfitは無理だと思う……。 results = model.fit_generator( train_data.flow(x_train, y_train, batch_size=batch_size), steps_per_epoch=x_train.shape[0] // batch_size, epochs=epochs, callbacks=[ LearningRateScheduler( partial( getitem, tuple( take( epochs, concat(repeat(0.01, 1), repeat(0.1, 99), repeat(0.01, 50), repeat(0.001)))))) ], validation_data=validation_data.flow(x_validation, y_validation, batch_size=batch_size), validation_steps=x_validation.shape[0] // batch_size) with open('./results/history.pickle', 'wb') as f: pickle.dump(results.history, f) save_model(model, model_file) try: with tf.device("/cpu:0"): # model.summary() # print("=== AFTER POPPING THE LAST ===") model.layers.pop() # model.summary() # generate_confusion_matrix(model, x_validation, y_validation, batch_size) # plot_model(model, to_file='./results/model.png') except Exception as ex: print("plot_model failed with error:", repr(ex), "\nMoving on...") siamese(input_shape, model)
def main(): # # CIFAR-10 # cifar = CIFAR_10() # # x_train.shape = (50000, 32, 32, 3) # y_train.shape = (50000, 10) # x_validation.shape= (10000, 32, 32, 3) # y_validation.shape= (10000, 10) # data = cifar.load_data() x_train = data['training_data'] y_train = data['training_label'] x_validation = data['validation_data'] y_validation = data['validation_label'] print("x_train.shape=", x_train.shape) print("y_train.shape=", y_train.shape) print("x_validation.shape=", x_validation.shape) print("y_validation.shape=", y_validation.shape) # # SqueezeNet # squeeze = SqueezeNet() i = Input(shape=x_train.shape[1:]) o = squeeze.make_graph(y_train.shape[1])(i) # # model # model = Model(inputs=i, outputs=o) # # compile model # model.compile( loss='categorical_crossentropy', optimizer=SGD(momentum=0.9), metrics=['accuracy'] ) # # generator in ImageDataGenerator by keras # train_data = ImageDataGenerator( featurewise_center=True, featurewise_std_normalization=True, width_shift_range=0.125, height_shift_range=0.125, horizontal_flip=True ) validation_data = ImageDataGenerator( featurewise_center=True, featurewise_std_normalization=True ) for data in (train_data, validation_data): data.fit(x_train) # 実用を考えると、x_validationでのfeaturewiseのfitは無理だと思う… … 。 # # check pickle # # file_pickle = "./results/history.pickle" model_path = "./results" model_file = model_path + "/model.h5" model_weights = model_path + "/weights.h5" print(f"models: model={model_file}, weight={model_weights}" ) # print(f"models: arch =", options['file_arch']) # print(f"models: weight=", options['model_weights']) if not path.exists(model_path): os.mkdir(model_path) # # print model # from lib_utils import print_model_summary print_model_summary(model, "./results/network.txt", "model.png") # # check model, if not exist trained model, we have to make trained parameters for model. # if not path.exists(model_file): # # fit generator # batch_size = 1000 # 100 epochs = 1 # 200 results = model.fit_generator( # # generate train data (ImageDataGenerator by keras) # train_data.flow(x_train, y_train, batch_size=batch_size), # # steps/epoch # steps_per_epoch=x_train.shape[0] // batch_size, # # epoch # epochs=epochs, # # callbacks # callbacks = [ LearningRateScheduler( partial( getitem, tuple(take(epochs, concat(repeat(0.010, 1), repeat(0.100, 99), repeat(0.010, 50), repeat(0.001)))) ) ) ], # # generate validation data (ImageDataGenerator by keras) # validation_data=validation_data.flow(x_validation, y_validation, batch_size=batch_size), # # validation step # validation_steps=x_validation.shape[0] // batch_size, # # max_queue_size # max_queue_size=4 ) # # save keras model # from lib_utils import save_model_by_keras save_model_by_keras(model, model_file, model_weights) # del model else: # # load keras model # if path.exists(model_file): print("load model...") from lib_utils import load_model_by_keras model = load_model_by_keras(model_file, model_weights) print("load model...done") else: print("load model...: not found=", model_file, model_weights ) # # check version # from lib_utils import get_version get_version(model_file) # # evaluate # """ print("model evaluate...") score = lmodel.evaluate(x_validation, y_validation, verbose=1) print("model evaluate: loss=", score[0]) print("model evaluate: accuracy=", score[1]) """ # # prediction # print("model prediction...") # lmodel.predict(y_validation.shape[1]) # lmodel.predict(x_train.shape[1:]) print("x_validation.shape=", x_validation.shape) print("x_validation.shape[0]=", x_validation.shape[0]) print("x_validation.shape[1]=", x_validation.shape[1]) print("x_validation.shape[2]=", x_validation.shape[2]) print("x_validation.shape[3]=", x_validation.shape[3]) i0 = x_validation[0:1] i1 = x_validation.reshape(10000,32,32,3) i2 = i1[0] print("i0.shape=", i0.shape) print("i1.shape=", i1.shape) print("i2.shape=", i2.shape) # lmodel.predict(i0, verbose=1) predo = model.predict(x_validation, verbose=1)[0] print(predo) """ """ preds = model.predict(x_validation, verbose=1) # for pre in preds: # y = pre.argmax() # print("label: ", y_validation[y]) print('done')