def get(self, headers): user = g.user group = g.group query = self.select().group(group) query = query.min(epoch(group.get_day_end() - timedelta(days=8))) query = query.max(epoch(group.get_day_end() + timedelta(hours=4))) count, page, events = query.execute() if count == 0 and group.status == 'initializing': tries = int(request.args.get('tries', 0)) if tries < 5: sleep(1) # clients seem to not obey Retry-After, so artificially delay request_arguments = request.args.copy().to_dict() request_arguments['tries'] = tries response = redirect('%s?%s' % (request.path, url_encode(request_arguments))) response.headers.add('Retry-After', 1) return response else: # return without "headers" so this response isn't cached return self.serialize_list(self.model, events, count, page), 200 attending_id = g.user.get_attending_id() if attending_id: try: attending = Event.find(attending_id) if attending in events: events.remove(attending) if 'page' not in request.args: events.insert(0, attending) attending.current_user_attending = True except DoesNotExist: logger.warn('Event {} not found'.format(attending_id)) return self.serialize_list(self.model, events, count, page), 200, headers
def get(self, headers): user = g.user group = g.group query = self.select().group(group).order('asc') query = query.min(epoch(group.get_day_start())) query = query.max(epoch(group.get_day_end() + timedelta(days=10))) count, page, events = query.execute() return self.serialize_list(self.model, events, count, page), 200, headers
def __get_page(self, key): start_page = self._page min = self._min or '-inf' max = self._max or '+inf' count = self.db.get_sorted_set_size(key, min, max) if count == 0: return 0, start_page, [] if self._model_class == Event and self._group: if min is None: min = epoch(self._group.get_day_start() - timedelta(days=7)) if max is None: # add 1 hour to account for sub-scoring max = epoch(self._group.get_day_end() + timedelta(hours=1)) if self._order == 'desc': range_f = self.db.sorted_set_rrange_by_score rank_f = self.db.sorted_set_rrank min, max = max, min else: range_f = self.db.sorted_set_range_by_score rank_f = self.db.sorted_set_rank pages = int(math.ceil(float(count) / self._limit)) if self._start: position = rank_f(key, self._start) if position is not None: start_page = int((float(position) / self._limit) + 1) collected = [] page = start_page for page in range(start_page, pages + 1): start = (page - 1) * self._limit model_ids = range_f(key, min, max, start, self._limit, withscores=True) instances = self._model_class.find([m[0] for m in model_ids]) for index, instance in enumerate(instances): if instance is not None: instance.score = model_ids[index][1] secured = self.__clean_results(instances) collected.extend(secured) # if we have page size results, break if len(collected) >= self._limit: break return count, page, collected
def index(self): super(EventMessage, self).index() event = self.event # record to the global by_votes sort with self.db.transaction(commit_on_select=False): num_votes = self.db.get_sorted_set_size(skey(self, 'votes')) sub_sort = epoch(self.created) / epoch(event.expires + timedelta(days=365)) by_votes_key = skey(event, 'messages', 'by_votes') self.db.sorted_set_add(by_votes_key, self.id, num_votes + sub_sort) self.db.expire(by_votes_key, self.ttl()) self.record_for_user(self.user)
def index(self): super(EventMessageVote, self).index() user = self.user message = self.message event = message.event # record the vote into the global "by_votes" sort order with self.db.transaction(commit_on_select=False): num_votes = self.db.get_sorted_set_size(skey(message, 'votes')) sub_sort = epoch() / epoch(event.expires + timedelta(days=365)) by_votes = skey(event, 'messages', 'by_votes') self.db.sorted_set_add(by_votes, self.message_id, num_votes + sub_sort, replicate=False) self.db.expire(by_votes, self.ttl())
def get(self, headers): group = g.group meta = {} dates = [(group.get_day_end() + timedelta(days=i)) for i in range(10)] p = wigo_db.redis.pipeline() for d in dates: p.zcount(skey(group, 'events'), epoch(d), epoch(d + timedelta(hours=3))) counts = p.execute() for index, d in enumerate(dates): meta[(d - timedelta(days=1)).date().isoformat()] = counts[index] return meta, 200, headers
def new_friend(user_id, friend_id): user = User.find(user_id) friend = User.find(friend_id) if not user.is_friend(friend): return min = epoch(datetime.utcnow() - timedelta(days=8)) # tells each friend about the event history of the other def capture_history(u, f): # capture each of the users posted photos with wigo_db.transaction(commit_on_select=False): for message in EventMessage.select().key(skey( u, 'event_messages')).min(min): if message.user and message.event: message.record_for_user(f) # capture the events being attended for event in Event.select().user(u).min(min): if u.is_attending(event) and f.can_see_event(event): event.add_to_user_attending(f, u) capture_history(user, friend) capture_history(friend, user)
def index(self): super(EventAttendee, self).index() user = self.user event = self.event # check if the user is switching events for the date the event is on current_event_id = user.get_attending_id(event) if current_event_id and current_event_id != event.id: EventAttendee({'event_id': current_event_id, 'user_id': user.id}).delete() with self.db.transaction(commit_on_select=False): # first update the global state of the event attendees_key = skey(event, 'attendees') self.db.sorted_set_add(attendees_key, user.id, epoch(self.created)) self.db.expire(attendees_key, event.ttl()) # now update the users view of the events # record the exact event the user is currently attending user.set_attending(event) # record current user as an attendee attendees_key = user_attendees_key(user, event) self.db.sorted_set_add(attendees_key, user.id, 'inf') self.db.expire(attendees_key, event.ttl()) # record the event into the events the user can see with self.db.transaction(commit_on_select=False): event.update_global_events() event.update_user_events(user)
def rate_limit(key, expires, lock_timeout=30): if Configuration.ENVIRONMENT in ('dev', 'test'): yield False else: # TODO this is legacy and can be removed at some point if redis.exists('rate_limit:{}'.format(key)): yield True return if isinstance(expires, datetime): expires = expires - datetime.utcnow() rate_limit_set = 'rate_limits_{}'.format( wigo_db.redis.get_server_name(key)) score = wigo_db.sorted_set_get_score(rate_limit_set, key) if score and float(score) > time(): yield True else: lock = redis.lock('locks:{}'.format(key), timeout=lock_timeout) if lock.acquire(blocking=False): score = wigo_db.sorted_set_get_score(rate_limit_set, key) if score and float(score) > time(): yield True else: try: yield False wigo_db.sorted_set_add( rate_limit_set, key, epoch(datetime.utcnow() + expires)) finally: lock.release() else: yield True
def expire(self, key, expires, long_term_expires=None): redis = self.get_redis(True) expires = check_expires(expires) redis.zadd(self.get_expire_key(key), key, epoch(datetime.utcnow() + expires)) if self.queued_db and long_term_expires: self.queued_db.expire(key, expires, long_term_expires)
def record_for_user(self, user): event = self.event # record into users list of messages by time with self.db.transaction(commit_on_select=False): user_emessages_key = user_eventmessages_key(user, event) self.db.sorted_set_add(user_emessages_key, self.id, epoch(self.created)) self.db.expire(user_emessages_key, self.ttl()) # record into users list by vote count num_votes = self.db.get_sorted_set_size(skey(self, 'votes')) sub_sort = epoch() / epoch(event.expires + timedelta(days=365)) by_votes_key = user_eventmessages_key(user, event, True) self.db.sorted_set_add(by_votes_key, self.id, num_votes + sub_sort, replicate=False) self.db.expire(by_votes_key, self.ttl()) user.track_meta('last_event_change')
def get_score_key(time, distance, num_attending): if num_attending > 999: num_attending = 999 targets = [0, 20, 50, 70, 100] distance_bucket = next(reversed([t for t in targets if t <= distance]), None) + 10 adjustment = (1 - (distance_bucket / 1000.0)) + (num_attending / 10000.0) return str(Decimal(epoch(time)) + Decimal(adjustment))
def get_tapped_ids(self): from server.db import wigo_db return wigo_db.sorted_set_range_by_score( skey(self, 'tapped'), epoch(self.group.get_day_start()), 'inf', limit=5000)
def index(self): super(Notification, self).index() key = skey('user', self.user_id, 'notifs') primitive = self.to_primitive() self.db.sorted_set_add(key, primitive, epoch(self.created), dt=dict, replicate=False) self.db.clean_old(key, self.TTL)
def get(self, user_id, headers): user = g.user group = g.group query = self.select().user(user) query = query.min(epoch(group.get_day_end() - timedelta(days=15))) query = query.max(time()) count, page, instances = query.execute() return self.serialize_list(self.model, instances, count, page), 200, headers
def record_for_user(self, user): message = self.message event = message.event with self.db.transaction(commit_on_select=False): user_votes = user_votes_key(user, self.message) self.db.sorted_set_add(user_votes, self.user.id, epoch(self.created), replicate=False) self.db.expire(user_votes, self.ttl()) # this forces the message to update its indexes for the user message.record_for_user(user)
def setup(u1, u2): self.db.sorted_set_add(skey(u1, 'friends'), u2.id, epoch(self.created)) self.db.sorted_set_add(skey(u1, 'friends', 'top'), u2.id, 10000) self.db.sorted_set_add(skey(u1, 'friends', 'alpha'), u2.id, prefix_score(u2.full_name.lower()), replicate=False) if u2.privacy == 'private': self.db.set_add(skey(u1, 'friends', 'private'), u2.id, replicate=False)
def set(self, key, value, expires=None, long_term_expires=None): redis = self.get_redis(True) expires = check_expires(expires) result = redis.set(key, self.encode(value, dict)) if expires: redis.zadd(self.get_expire_key(key), key, epoch(datetime.utcnow() + expires)) if self.queued_db and long_term_expires != 0: self.queued_db.set(key, value, expires, long_term_expires) return result
def get(self, user_id, headers): user = g.user group = g.group query = self.select().user(user) query = query.min(epoch(group.get_day_end() - timedelta(days=8))) query = query.max(epoch(group.get_day_end() + timedelta(hours=4))) count, page, events = query.execute() attending_id = user.get_attending_id() if attending_id: try: attending = Event.find(attending_id) if attending in events: events.remove(attending) if 'page' not in request.args: events.insert(0, attending) attending.current_user_attending = True except DoesNotExist: logger.warn('Event {} not found'.format(attending_id)) return self.serialize_list(self.model, events, count, page), 200, headers
def notifications_model_listener(sender, instance, created): if isinstance(instance, User): if is_new_user(instance, created): new_user.delay(instance.id) if not created and (instance.was_changed('status') and instance.get_previous_old_value('status') == 'waiting'): notify_unlocked.delay(instance.id) elif isinstance(instance, EventAttendee) and created: notify_on_attendee.delay(instance.event_id, instance.user_id) elif isinstance(instance, EventMessage) and created: notify_on_eventmessage.delay(instance.id) elif isinstance(instance, EventMessageVote) and created: notify_on_eventmessage_vote.delay(instance.user_id, instance.message_id) elif isinstance(instance, Message) and created: notify_on_message.delay(instance.id) elif isinstance(instance, Friend) and created: notify_on_friend.delay(instance.user_id, instance.friend_id, instance.accepted) elif isinstance(instance, Tap) and created: notify_on_tap.delay(instance.user_id, instance.tapped_id) elif isinstance(instance, Invite) and created: notify_on_invite.delay(instance.user_id, instance.invited_id, instance.event_id) elif isinstance(instance, Notification) and created: instance.user.track_meta('last_notification', epoch(instance.created))
def save_data(data): observations = data["observations"] now_epoch = epoch(datetime.utcnow()) out = [] for i in observations: # NOTE: getting the user does not have to happen here (as the total response time for the json post should be < 500ms) # It could be off loaded to do a daily batch. This will be something like "user" below will be the email address, id, username, etc... of the device, or None device_owner = get_client_owner(i["clientMac"]) user = device_owner["user"] if device_owner else None location = i.get("location") out.append({ "ap": data["apMac"], "client_mac": i["clientMac"], "lat": location.get("lat") if location else None, "lng": location.get("lng") if location else None, "identity": user, "time_present": i["seenEpoch"], "time_now": now_epoch }) # printing here - can send to api, log file, etc... # to_log_file(out) # to_firehose(out) pprint(out)
def new_group(group_id): group = Group.find(group_id) logger.info('new group {} created, importing events'.format( group.name.encode('utf-8'))) num_imported = 0 imported = set() min = epoch(group.get_day_end() - timedelta(days=7)) with wigo_db.transaction(commit_on_select=False): for close_group in get_close_groups(group.latitude, group.longitude, 100): if close_group.id == group.id: continue for event in Event.select().group(close_group).min(min): # only import the events the group actually owns if event.group_id != close_group.id: continue # no double imports if event.id not in imported: event.update_global_events(group=group) imported.add(event.id) num_imported += 1 for event in Event.select().key(skey('global', 'events')).min(min): if event.id not in imported: event.update_global_events(group=group) imported.add(event.id) num_imported += 1 logger.info('imported {} events into group {}'.format( num_imported, group.name.encode('utf-8'))) group.track_meta('last_event_change', expire=None) group.status = 'active' group.save()
def main(): parser = argparse.ArgumentParser(description='Parameter Processing') parser.add_argument('--dataset', type=str, default='CIFAR10', help='dataset') parser.add_argument('--model', type=str, default='ConvNet', help='model') parser.add_argument('--ipc', type=int, default=1, help='image(s) per class') parser.add_argument( '--eval_mode', type=str, default='S', help='eval_mode' ) # S: the same to training model, M: multi architectures, W: net width, D: net depth, A: activation function, P: pooling layer, N: normalization layer, parser.add_argument('--num_exp', type=int, default=5, help='the number of experiments') parser.add_argument( '--num_eval', type=int, default=10, help='the number of evaluating randomly initialized models') parser.add_argument('--epoch_eval_train', type=int, default=300, help='epochs to train a model with synthetic data') parser.add_argument('--Iteration', type=int, default=1000, help='training iterations') parser.add_argument('--lr_img', type=float, default=0.1, help='learning rate for updating synthetic images') parser.add_argument('--lr_net', type=float, default=0.01, help='learning rate for updating network parameters') parser.add_argument('--batch_real', type=int, default=256, help='batch size for real data') parser.add_argument('--batch_train', type=int, default=256, help='batch size for training networks') parser.add_argument( '--init', type=str, default='noise', help= 'initialization of synthetic data, noise/real: initialize from random noise or real images. The two initializations will get similar performances.' ) parser.add_argument('--clip_syn', type=str, default='False', help='clip the pixel values of synthetic data') parser.add_argument('--data_path', type=str, default='data', help='dataset path') parser.add_argument('--save_path', type=str, default='result', help='path to save results') parser.add_argument('--dis_metric', type=str, default='ours', help='distance metric') # The experimental results in the paper were obtained with --init noise and --clip_syn False. # For speeding up, we can decrease the Iteration and epoch_eval_train, which will not cause significant performance decrease. args = parser.parse_args() args.clip_syn = True if args.clip_syn == 'True' else False args.outer_loop, args.inner_loop = get_loops(args.ipc) args.device = 'cuda' if torch.cuda.is_available() else 'cpu' print('Hyper-parameters: \n', args.__dict__) if not os.path.exists(args.data_path): os.mkdir(args.data_path) if not os.path.exists(args.save_path): os.mkdir(args.save_path) eval_it_pool = np.arange( 0, args.Iteration + 1, 100).tolist() if args.eval_mode == 'S' else [ args.Iteration ] # The list of iterations when we evaluate models and record results. channel, im_size, num_classes, class_names, mean, std, dst_train, dst_test, testloader = get_dataset( args.dataset, args.data_path) model_eval_pool = get_eval_pool(args.eval_mode, args.model, args.model) print('Evaluation model pool: ', model_eval_pool) accs_all_exps = dict() # record performances of all experiments for key in model_eval_pool: accs_all_exps[key] = [] data_save = [] for exp in range(args.num_exp): print('\n================== Exp %d ==================\n ' % exp) ''' organize the real dataset ''' images_all = [] labels_all = [] indices_class = [[] for c in range(num_classes)] images_all = [ torch.unsqueeze(dst_train[i][0], dim=0) for i in range(len(dst_train)) ] labels_all = [dst_train[i][1] for i in range(len(dst_train))] for i, lab in enumerate(labels_all): indices_class[lab].append(i) images_all = torch.cat(images_all, dim=0).to(args.device) labels_all = torch.tensor(labels_all, dtype=torch.long, device=args.device) for c in range(num_classes): print('class c = %d: %d real images' % (c, len(indices_class[c]))) def get_images(c, n): # get random n images from class c idx_shuffle = np.random.permutation(indices_class[c])[:n] return images_all[idx_shuffle] for ch in range(channel): print('real images channel %d, mean = %.4f, std = %.4f' % (ch, torch.mean( images_all[:, ch]), torch.std(images_all[:, ch]))) ''' initialize the synthetic data ''' image_syn = torch.randn(size=(num_classes * args.ipc, channel, im_size[0], im_size[1]), dtype=torch.float, requires_grad=True, device=args.device) label_syn = torch.tensor( [np.ones(args.ipc) * i for i in range(num_classes)], dtype=torch.long, requires_grad=False, device=args.device).view(-1) # [0,0,0, 1,1,1, ..., 9,9,9] if args.init == 'real': print('initialize synthetic data from random real images') for c in range(num_classes): image_syn.data[c * args.ipc:(c + 1) * args.ipc] = get_images( c, args.ipc).detach().data else: print('initialize synthetic data from random noise') ''' training ''' optimizer_img = torch.optim.SGD( [ image_syn, ], lr=args.lr_img, momentum=0.5) # optimizer_img for synthetic data optimizer_img.zero_grad() criterion = nn.CrossEntropyLoss().to(args.device) print('%s training begins' % get_time()) for it in range(args.Iteration + 1): ''' Evaluate synthetic data ''' if it in eval_it_pool: for model_eval in model_eval_pool: print( '-------------------------\nEvaluation\nmodel_train = %s, model_eval = %s, iteration = %d' % (args.model, model_eval, it)) param_augment = get_daparam(args.dataset, args.model, model_eval, args.ipc) if param_augment['strategy'] != 'none': epoch_eval_train = 1000 # More training epochs for augmentation would be better. print('data augmentation = %s' % param_augment) else: epoch_eval_train = args.epoch_eval_train accs = [] for it_eval in range(args.num_eval): net_eval = get_network( model_eval, channel, num_classes, im_size).to(args.device) # get a random model image_syn_eval, label_syn_eval = copy.deepcopy( image_syn.detach()), copy.deepcopy( label_syn.detach() ) # avoid any unaware modification _, acc_train, acc_test = evaluate_synset( it_eval, net_eval, image_syn_eval, label_syn_eval, testloader, args.lr_net, args.batch_train, param_augment, args.device, epoch_eval_train) accs.append(acc_test) print( 'Evaluate %d random %s, mean = %.4f std = %.4f\n-------------------------' % (len(accs), model_eval, np.mean(accs), np.std(accs))) if it == args.Iteration: # record the final results accs_all_exps[model_eval] += accs ''' visualize and save ''' save_name = os.path.join( args.save_path, 'vis_%s_%s_%dipc_exp%d_iter%d.png' % (args.dataset, args.model, args.ipc, exp, it)) image_syn_vis = copy.deepcopy(image_syn.detach().cpu()) for ch in range(channel): image_syn_vis[:, ch] = image_syn_vis[:, ch] * std[ch] + mean[ch] image_syn_vis[image_syn_vis < 0] = 0.0 image_syn_vis[image_syn_vis > 1] = 1.0 save_image(image_syn_vis, save_name, nrow=args.ipc) # The generated images would be slightly different from the visualization results in the paper, because of the clip and normalization of pixels. ''' Train synthetic data ''' net = get_network(args.model, channel, num_classes, im_size).to(args.device) # get a random model net.train() net_parameters = list(net.parameters()) optimizer_net = torch.optim.SGD( net.parameters(), lr=args.lr_net, momentum=0.5) # optimizer_img for synthetic data optimizer_net.zero_grad() loss_avg = 0 for ol in range(args.outer_loop): ''' freeze the running mu and sigma for BatchNorm layers ''' # Synthetic data batch, e.g. only 1 image/batch, is too small to obtain stable mu and sigma. # So, we calculate and freeze mu and sigma for BatchNorm layer with real data batch ahead. # This would make the model with BatchNorm layers easier to train. BN_flag = False BNSizePC = 16 # for batch normalization for module in net.modules(): if 'BatchNorm' in module._get_name(): #BatchNorm BN_flag = True if BN_flag: img_real = torch.cat( [get_images(c, BNSizePC) for c in range(num_classes)], dim=0) net.train() # for updating the mu, sigma of BatchNorm output_real = net(img_real) # get running mu, sigma for module in net.modules(): if 'BatchNorm' in module._get_name(): #BatchNorm module.eval( ) # fix mu and sigma of every BatchNorm layer ''' update synthetic data ''' loss = torch.tensor(0.0).to(args.device) for c in range(num_classes): img_real = get_images(c, args.batch_real) lab_real = torch.ones((img_real.shape[0], ), device=args.device, dtype=torch.long) * c output_real = net(img_real) loss_real = criterion(output_real, lab_real) gw_real = torch.autograd.grad(loss_real, net_parameters) gw_real = list((_.detach().clone() for _ in gw_real)) img_syn = image_syn[c * args.ipc:(c + 1) * args.ipc].reshape( (args.ipc, channel, im_size[0], im_size[1])) lab_syn = torch.ones( (args.ipc, ), device=args.device, dtype=torch.long) * c output_syn = net(img_syn) loss_syn = criterion(output_syn, lab_syn) gw_syn = torch.autograd.grad(loss_syn, net_parameters, create_graph=True) loss += match_loss(gw_syn, gw_real, args) optimizer_img.zero_grad() loss.backward() optimizer_img.step() loss_avg += loss.item() if args.clip_syn: # clip synthetic data for ch in range(channel): image_syn_ch = image_syn[:, ch] clip_min, clip_max = (0.0 - mean[ch]) / std[ch], ( 1.0 - mean[ch]) / std[ch] image_syn_ch[ image_syn_ch > clip_max].data[:] = clip_max image_syn_ch[ image_syn_ch < clip_min].data[:] = clip_min if ol == args.outer_loop - 1: break ''' update network to be unified with optimizer_img xxxxxxxxxxxxxxxxx ''' image_syn_train, label_syn_train = copy.deepcopy( image_syn.detach()), copy.deepcopy( label_syn.detach()) # avoid any unaware modification dst_train = TensorDataset(image_syn_train, label_syn_train) trainloader = torch.utils.data.DataLoader(dst_train, batch_size=256, shuffle=True, num_workers=0) for il in range(args.inner_loop): epoch('train', trainloader, net, optimizer_net, criterion, None, args.device) loss_avg /= (num_classes * args.outer_loop) if it % 10 == 0: print('%s iter = %04d, loss = %.4f' % (get_time(), it, loss_avg)) if it == args.Iteration: # only record the final results data_save.append([ copy.deepcopy(image_syn.detach().cpu()), copy.deepcopy(label_syn.detach().cpu()) ]) torch.save( { 'data': data_save, 'accs_all_exps': accs_all_exps, }, os.path.join( args.save_path, 'res_%s_%s_%dipc.pt' % (args.dataset, args.model, args.ipc))) print('\n==================== Final Results ====================\n') for key in model_eval_pool: accs = accs_all_exps[key] print( 'Run %d experiments, train on %s, evaluate %d random %s, mean = %.2f%% std = %.2f%%' % (args.num_exp, args.model, len(accs), key, np.mean(accs) * 100, np.std(accs) * 100))
def train_mlp(args, loaders, model, criterion, optimizer, scheduler, l1_factor=0.0, clr_factor=0.0, model_name='model'): train_loader = loaders['train_loader'] valid_loader = loaders['valid_loader'] test_loader = loaders['test_loader'] if args.cuda: model = model.cuda() best_valid_loss = sys.float_info.max train_losses = [] valid_losses = [] if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) for i_epoch in tqdm(range(args.epochs), desc='Epochs'): # Train train_labels, train_preds, train_loss = epoch(train_loader, model, train=True, criterion=criterion, optimizer=optimizer, l1_factor=l1_factor, clr_factor=clr_factor) train_losses.append(train_loss) # Validation valid_labels, valid_preds, valid_loss = epoch(valid_loader, model, criterion=criterion) # Learning rate decay if scheduler is not None: scheduler.step(valid_loss) valid_losses.append(valid_loss) # remember best valid loss and save checkpoint is_best = valid_loss < best_valid_loss if is_best: best_valid_loss = valid_loss # evaluate on test set test_labels, test_preds, test_loss = epoch(test_loader, model, criterion=criterion) with open(args.output_dir + model_name + '_result.txt', 'w') as f: f.write('Best Validation Epoch: {}\n'.format(i_epoch)) f.write('Best Validation Loss: {}\n'.format(best_valid_loss)) f.write('Train Loss: {}\n'.format(train_loss)) f.write('Test Loss: {}\n'.format(test_loss)) # Save entire model torch.save(model, args.output_dir + model_name + '.pth') # Save model params torch.save(model.state_dict(), args.output_dir + model_name + '_params.pth') # plot plt.figure() plt.plot(np.arange(len(train_losses)), np.array(train_losses), label='Training Loss') plt.plot(np.arange(len(valid_losses)), np.array(valid_losses), label='Validation Loss') plt.xlabel('epoch') plt.ylabel('Loss') plt.legend(loc="best") plt.savefig(args.output_dir + model_name + '_loss.eps', format='eps')
def fetchData(requestContext, pathExpr): seriesList = [] startTime = int(epoch(requestContext['startTime'])) endTime = int(epoch(requestContext['endTime'])) def _fetchData(pathExpr, startTime, endTime, requestContext, seriesList): matching_nodes = STORE.find(pathExpr, startTime, endTime, local=requestContext['localOnly']) fetches = [(node, node.fetch(startTime, endTime)) for node in matching_nodes if node.is_leaf] for node, results in fetches: if isinstance(results, FetchInProgress): results = results.waitForResults() if not results: log.info( "render.datalib.fetchData :: no results for %s.fetch(%s, %s)" % (node, startTime, endTime)) continue try: (timeInfo, values) = results except ValueError as e: raise Exception( "could not parse timeInfo/values from metric '%s': %s" % (node.path, e)) (start, end, step) = timeInfo series = TimeSeries(node.path, start, end, step, values) series.pathExpression = pathExpr # hack to pass expressions through to render functions seriesList.append(series) # Prune empty series with duplicate metric paths to avoid showing empty graph elements for old whisper data names = set([s.name for s in seriesList]) for name in names: series_with_duplicate_names = [ s for s in seriesList if s.name == name ] empty_duplicates = [ s for s in series_with_duplicate_names if not nonempty(s) ] if series_with_duplicate_names == empty_duplicates and len( empty_duplicates) > 0: # if they're all empty empty_duplicates.pop() # make sure we leave one in seriesList for series in empty_duplicates: seriesList.remove(series) return seriesList retries = 1 # start counting at one to make log output and settings more readable while True: try: seriesList = _fetchData(pathExpr, startTime, endTime, requestContext, seriesList) return seriesList except Exception, e: if retries >= settings.MAX_FETCH_RETRIES: log.exception("Failed after %s retry! Root cause:\n%s" % (settings.MAX_FETCH_RETRIES, format_exc())) raise e else: log.exception( "Got an exception when fetching data! Try: %i of %i. Root cause:\n%s" % (retries, settings.MAX_FETCH_RETRIES, format_exc())) retries += 1
def clean_old(self, key, ttl=None): if ttl is None: ttl = timedelta(days=10) up_to = datetime.utcnow() - ttl self.sorted_set_remove_by_score(key, '-inf', epoch(up_to))
def get_index_score(self): return epoch(self.created)
def data_save_listener(sender, instance, created): if isinstance(instance, User): if is_new_user(instance, created): new_user(instance.id) if instance.status == 'deleted': delete_user.delay(instance.id, instance.group_id) publish_model_change(instance) elif isinstance(instance, Group): if created: new_group.delay(instance.id) publish_model_change(instance) elif isinstance(instance, Event): event_related_change.delay(instance.group_id, instance.id, instance.is_global) if not created: publish_model_change(instance) elif isinstance(instance, Friend) and created: if instance.accepted: new_friend.delay(instance.user_id, instance.friend_id) else: instance.friend.track_meta('last_friend_request', epoch(instance.created)) instance.user.track_meta('last_friend_change') instance.friend.track_meta('last_friend_change') elif isinstance(instance, Tap): instance.user.track_meta('last_tap_change') elif isinstance(instance, Block): instance.user.track_meta('last_block_change') elif isinstance(instance, Invite): user_invited.delay(instance.event_id, instance.user_id, instance.invited_id) elif isinstance(instance, EventAttendee): event = instance.event event_related_change.delay(event.group_id, event.id, event.is_global) tell_friends_user_attending.delay(instance.user_id, instance.event_id) elif isinstance(instance, EventMessage): event = instance.event event_related_change.delay(event.group_id, event.id, event.is_global) tell_friends_event_message.delay(instance.id) elif isinstance(instance, EventMessageVote): event = instance.message.event event_related_change.delay(event.group_id, event.id, event.is_global) # tell_friends_about_vote.delay(instance.message_id, instance.user_id) elif isinstance(instance, Message): instance.user.track_meta('last_message_change') instance.to_user.track_meta('last_message_change') instance.to_user.track_meta('last_message_received', epoch(instance.created))
with open(args.csr_path, 'rb') as f: X_test = pickle.load(f) X_test = X_test.todense() test_set = TensorDataset(torch.from_numpy(X_test.astype('float32')), torch.from_numpy(y_test.astype('int'))) test_loader = DataLoader(dataset=test_set, batch_size=args.eval_batch_size, shuffle=False) model = torch.load(args.model_path) if args.cuda: model = model.cuda() _, scores, _ = epoch(test_loader, model, output_activation=F.softmax) if args.cuda: scores = scores.cpu() scores = scores.numpy() preds = np.argmax(scores, axis=1) y_scores = scores[:, 1] y_preds = preds auroc = roc_auc_score(y_test, y_scores) aupr = average_precision_score(y_test, y_scores) f1 = f1_score(y_test, y_preds) accuracy = accuracy_score(y_test, y_preds) print("AUROC: {}, AUPR: {}, F1: {}, ACC: {}".format(
def index(self): super(Friend, self).index() with self.db.transaction(commit_on_select=False): if self.accepted: def setup(u1, u2): self.db.sorted_set_add(skey(u1, 'friends'), u2.id, epoch(self.created)) self.db.sorted_set_add(skey(u1, 'friends', 'top'), u2.id, 10000) self.db.sorted_set_add(skey(u1, 'friends', 'alpha'), u2.id, prefix_score(u2.full_name.lower()), replicate=False) if u2.privacy == 'private': self.db.set_add(skey(u1, 'friends', 'private'), u2.id, replicate=False) setup(self.user, self.friend) setup(self.friend, self.user) for type in ('friend_requests', 'friend_requested'): self.db.sorted_set_remove(skey('user', self.user_id, type), self.friend_id) self.db.sorted_set_remove( skey('user', self.friend_id, type), self.user_id) else: def teardown(u1, u2): self.db.sorted_set_remove(skey(u1, 'friends'), u2.id) self.db.sorted_set_remove(skey(u1, 'friends', 'top'), u2.id) self.db.sorted_set_remove(skey(u1, 'friends', 'alpha'), u2.id, replicate=False) self.db.set_remove(skey(u1, 'friends', 'private'), u2.id, replicate=False) teardown(self.user, self.friend) teardown(self.friend, self.user) f_reqed_key = skey('user', self.user_id, 'friend_requested') self.db.sorted_set_add(f_reqed_key, self.friend_id, epoch(self.created)) f_req_key = skey('user', self.friend_id, 'friend_requests') self.db.sorted_set_add(f_req_key, self.user_id, epoch(self.created)) f_req_in_common_key = skey('user', self.friend_id, 'friend_requests', 'common') f_in_common = self.user.get_num_friends_in_common( self.friend_id) self.db.sorted_set_add(f_req_in_common_key, self.user_id, f_in_common) # clean out old friend requests self.db.clean_old(f_reqed_key, timedelta(days=30)) self.db.clean_old(f_req_key, timedelta(days=30)) self.db.clean_old(f_req_in_common_key, timedelta(days=30))