def update_or_create_criterion(data, criterion=None): ''' Takes a dict of data where the keys are fields of the criterion model. Valid keys are subcategory_id, title, recommendation_text, help_text, adverse, and active. The 'active' key only uses a False value. Once created, a criterion's subcategory cannot be changed. ''' subcategory_id = data.get('subcategory_id') if criterion is None: criterion = Criterion(subcategory_id=subcategory_id) elif subcategory_id is not None and subcategory_id != criterion.subcategory_id: raise ValueError(strings.cannot_change_subcategory) if 'title' in data: criterion.title = data['title'] if 'recommendation_text' in data: criterion.recommendation_text = data['recommendation_text'] if 'help_text' in data: criterion.help_text = data['help_text'] if 'adverse' in data: criterion.adverse = data['adverse'] # You cannot reactivate a criterion after deactivating it if 'active' in data and not data['active']: criterion.deactivate() return criterion.save()
def create_criterion(fbid, criterion): """ TODO: write description """ user = get_user_by_id(fbid) is_created = False to_delete_criterion = -1 if user is not None: for i in range(len(user.current_search.criteria)): if user.current_search.criteria[i]["name"] == criterion["name"]: is_created = True if criterion["value"] is None: to_delete_criterion = i else: user.current_search.criteria[i]["value"] = criterion[ "value"] user.update(current_search=user.current_search) if (to_delete_criterion != -1): del user.current_search.criteria[to_delete_criterion] user.update(current_search=user.current_search) if not is_created: if criterion["value"] is not None: cr = Criterion(criterion["name"], criterion["value"]) user.current_search.criteria.append(cr) user.save()
def get(self): for c in Criterion.all(): params = { "screen_name": c.screen_name, "term": c.term, "list_id": c.list_id, } taskqueue.add(url = "/collect", params = params)
def setUp(self): self.client = app.test_client() self.category = create_category() self.subcategory = create_subcategory(self.category.id) self.criterion = Criterion( subcategory_id=self.subcategory.id, title='Includes economic abuse framework', recommendation_text=( "The state's definition of domestic violence should include a framework of " 'economic abuse' ), help_text=( 'This means that the state acknowledges the role that economic control and abuse ' 'can play in domestic violence' ), adverse=False, ).save()
def post(self): cookies = Cookies(self, max_age = COOKIE_LIFE) user, screen_name = get_user_status(cookies) if user is None: self.error(401) return term = self.request.get("term") list_id = self.request.get("list_id") # delete the criterion if both are empty. if len(term) == 0 and len(list_id) == 0: c = Criterion.gql("WHERE screen_name=:name", name=screen_name).get() if c is not None: c.delete() return # return fail if some fields are empty. if len(term) == 0 or len(list_id) == 0: self.error(400) return # convert list_id from string to long try: list_id = long(list_id) # long except: self.error(400) return # create or update criterion c = Criterion.gql("WHERE screen_name=:name", name=user.screen_name).get() if c is not None: c.term = term c.list_id = list_id else: c = Criterion( screen_name = user.screen_name, term = term, list_id = list_id) c.put()
def benchmark_torch_snn_forward(name, num_tables, num_embeddings, embedding_dim, dense_features_dim, batch_size, bag_size, iters): net = SNN(num_tables, num_embeddings, embedding_dim, dense_features_dim) dense_features = torch.randn(batch_size, dense_features_dim).cuda() dense_features.requires_grad = False sparse_features = [ torch.randint(low=0, high=embedding_dim - 1, size=(batch_size, bag_size)).cuda() for _ in range(num_tables) ] for sf in sparse_features: sf.requires_grad = False labels = torch.rand(size=(batch_size, 1), device=torch.cuda.current_device()) weights = torch.rand(size=(batch_size, 1), device=torch.cuda.current_device()) weights.requires_grad = False logits = net(dense_features, sparse_features) net = torch.jit.trace(net, example_inputs=(dense_features, sparse_features)) def forward(dense_features, sparse_features): return net(dense_features, sparse_features) time_per_batch = benchmark_torch_function(iters, forward, dense_features, sparse_features) logging.info( f"{name}, DLRM, FORWARD: Batch Size: {batch_size}, Bag Size: {bag_size}, Num Tables: {num_tables}, Embedding Dim: {embedding_dim}, Dense Features Dim: {dense_features_dim}, Time per batch: {time_per_batch * 1.0e6}us, QPS: {batch_size * 1.0 / time_per_batch:.2e}" ) print(json.dumps(dict(name=name, implementation="baseline", method="forward", B=batch_size, L=bag_size, T=num_tables, D=embedding_dim, dense_D=dense_features_dim, time_per_batch=time_per_batch, qps=batch_size * 1.0 / time_per_batch))) criterion = Criterion().cuda() criterion = torch.jit.trace(criterion, example_inputs=(logits, labels, weights)) optimizer = torch.optim.SGD(net.parameters(), lr=0.05) def forward_backward_update(dense_features, sparse_features, labels): optimizer.zero_grad() logits = net(dense_features, sparse_features) loss = criterion(logits.float(), labels, weights) loss.backward() optimizer.step() time_per_batch = benchmark_torch_function(iters, forward_backward_update, dense_features, sparse_features, labels) logging.info( f"{name}, DLRM, FORWARDBACKWARDUPDATE: Batch Size: {batch_size}, Bag Size: {bag_size}, Num Tables: {num_tables}, Embedding Dim: {embedding_dim}, Dense Features Dim: {dense_features_dim}, Time per batch: {time_per_batch * 1.0e6}us, QPS: {batch_size * 1.0 / time_per_batch:.2e}" ) print(json.dumps(dict(name=name, implementation="baseline", method="forwardbackwardupdate", B=batch_size, L=bag_size, T=num_tables, D=embedding_dim, dense_D=dense_features_dim, time_per_batch=time_per_batch, qps=batch_size * 1.0 / time_per_batch)))
def __init__(self): self.train_step = 0 self.epoch = 0 self.lr = cfg.lr self.model = EndToEndNet() self.criterion = Criterion() lrs = [{"params": self.model.srm.parameters(), "lr": cfg.lrs[0]}, {"params": self.model.efficient_net.parameters(), "lr": cfg.lrs[1]}, ] self.optimizer = AdamW(lrs) self.scheduler = lr_scheduler.ReduceLROnPlateau(self.optimizer, **cfg.lr_scheduler_params) logs_dir = os.path.join(cfg.log_dir, cfg.train_name) self.writer = TensorboardLogging(logs_dir)
def get_Crtierion(): data = request.get_json() idPlan = data['idPlan'] print(platform.system()) criterions = Criterion.get_by_Plan(idPlan) criterions_list = [] for criterion in criterions: criterion_dict = {} criterion_dict['idCriterion'] = criterion.idCriterion criterion_dict['code'] = criterion.code criterion_dict['name'] = criterion.name criterion_dict['description'] = criterion.description criterions_list.append(criterion_dict) return jsonify({'criterions': criterions_list})
def test_init_invalid_subcategory(self): with self.assertRaises(ValueError) as e: Criterion( subcategory_id=0, title='Includes economic abuse framework', recommendation_text=( "The state's definition of domestic violence should include a framework of " 'economic abuse' ), help_text=( 'This means that the state acknowledges the role that economic control and ' 'abuse can play in domestic violence' ), adverse=False, ) self.assertEqual(str(e.exception), subcategory_not_found)
def test_get_criterion(self): criterion = Criterion( subcategory_id=self.subcategory.id, title='Includes economic abuse framework', recommendation_text= ("The state's definition of domestic violence should include a framework of " 'economic abuse'), help_text= ('This means that the state acknowledges the role that economic control and abuse ' 'can play in domestic violence'), adverse=False, ).save() response = self.client.get('/criteria/%i' % criterion.id) self.assertEqual(response.status_code, 200) json_response = json.loads(response.data.decode('utf-8')) self.assertEqual( json_response, { 'id': criterion.id, 'subcategory_id': criterion.subcategory_id, 'title': 'Includes economic abuse framework', 'recommendation_text': "The state's definition of domestic violence should include a framework of " 'economic abuse', 'help_text': 'This means that the state acknowledges the role that economic control and abuse ' 'can play in domestic violence', 'active': True, 'deactivated_at': None, 'adverse': False, })
def create_criterion(name='best offer', description=None, score_delta=1): criterion = Criterion() criterion.name = name criterion.description = description criterion.scoreDelta = score_delta return criterion
class CriterionTestCase(unittest.TestCase): def setUp(self): self.client = app.test_client() self.category = create_category() self.subcategory = create_subcategory(self.category.id) self.criterion = Criterion( subcategory_id=self.subcategory.id, title='Includes economic abuse framework', recommendation_text=( "The state's definition of domestic violence should include a framework of " 'economic abuse' ), help_text=( 'This means that the state acknowledges the role that economic control and abuse ' 'can play in domestic violence' ), adverse=False, ).save() def tearDown(self): clear_database(db) def test_init(self): self.assertEqual(self.criterion.subcategory_id, self.subcategory.id) self.assertEqual(self.criterion.title, 'Includes economic abuse framework') self.assertEqual( self.criterion.recommendation_text, "The state's definition of domestic violence should include a framework of economic " 'abuse', ) self.assertEqual( self.criterion.help_text, 'This means that the state acknowledges the role that economic control and abuse can ' 'play in domestic violence', ), self.assertTrue(self.criterion.active) self.assertFalse(self.criterion.adverse) def test_init_default_adverse_value(self): criterion = Criterion(subcategory_id=self.subcategory.id) self.assertFalse(criterion.adverse) def test_init_invalid_subcategory(self): with self.assertRaises(ValueError) as e: Criterion( subcategory_id=0, title='Includes economic abuse framework', recommendation_text=( "The state's definition of domestic violence should include a framework of " 'economic abuse' ), help_text=( 'This means that the state acknowledges the role that economic control and ' 'abuse can play in domestic violence' ), adverse=False, ) self.assertEqual(str(e.exception), subcategory_not_found) def test_serialize(self): self.assertEqual( { 'id': self.criterion.id, 'subcategory_id': self.subcategory.id, 'title': 'Includes economic abuse framework', 'recommendation_text': "The state's definition of domestic violence should include a framework of " 'economic abuse', 'help_text': 'This means that the state acknowledges the role that economic control and ' 'abuse can play in domestic violence', 'active': True, 'deactivated_at': None, 'adverse': False, }, self.criterion.serialize() ) def test_deactivate(self): self.criterion.deactivate() self.assertFalse(self.criterion.active) self.assertTrue(isinstance(self.criterion.deactivated_at, datetime.datetime)) self.assertTrue(self.criterion.deactivated_at < datetime.datetime.utcnow())
def test_init_default_adverse_value(self): criterion = Criterion(subcategory_id=self.subcategory.id) self.assertFalse(criterion.adverse)
def benchmark_torch_uniform_snn_forward(name, num_tables, num_embeddings, embedding_dim, dense_features_dim, batch_size, bag_size, iters, fp16=0): assert batch_size % hvd.size() == 0 assert num_embeddings % hvd.size() == 0 assert bag_size % hvd.size() == 0 net = DistributedUniformShardedSNN(num_tables, num_embeddings, embedding_dim, dense_features_dim).cuda() dense_features = torch.randn(batch_size // hvd.size(), dense_features_dim, device=torch.cuda.current_device()) dense_features.requires_grad = False sharded_sparse_features = torch.randint( low=0, high=num_embeddings // hvd.size(), size=(batch_size, num_tables, bag_size // hvd.size()), device=torch.cuda.current_device()).int() sharded_sparse_features.requires_grad = False labels = torch.rand(size=(batch_size // hvd.size(), 1), device=torch.cuda.current_device()) weights = torch.rand(size=(batch_size // hvd.size(), 1), device=torch.cuda.current_device()) weights.requires_grad = False logits = net(dense_features, sharded_sparse_features) net = torch.jit.trace(net, example_inputs=(dense_features, sharded_sparse_features)) if fp16: net = apex.amp.initialize(net, opt_level=FP16_LEVEL, verbosity=0) # net_fp16 = torch.jit.trace(net_fp16, # example_inputs=(dense_features, # sharded_sparse_features)) def forward(dense_features, partitioned_sparse_features): return net( dense_features, sharded_sparse_features.random_(0, num_embeddings // hvd.size())) MPI.COMM_WORLD.Barrier() time_per_batch = benchmark_torch_function(iters, forward, dense_features, sharded_sparse_features) print( json.dumps( dict(name=name, method="forward", implementation="uniform", rank=hvd.rank(), workers=hvd.size(), B=batch_size, L=bag_size, T=num_tables, D=embedding_dim, dense_D=dense_features_dim, fp16=fp16, time_per_batch=time_per_batch, qps=batch_size * 1.0 / time_per_batch))) if hvd.rank() == 0: logging.info( f"{name}, UNIFORM, FORWARD: rank={hvd.rank()}, Workers: {hvd.size()}, Batch Size: {batch_size}, Bag Size: {bag_size}, Num Tables: {num_tables}, Embedding Dim: {embedding_dim}, fp16: {fp16}, Time per batch: {time_per_batch * 1.0e6}us, QPS: {batch_size * 1.0 / time_per_batch:.2e}" ) criterion = Criterion().cuda() criterion = torch.jit.trace(criterion, example_inputs=(logits.float(), labels, weights)) def forward_backward(dense_features, sharded_sparse_features, labels): logits = net( dense_features, sharded_sparse_features.random_(0, num_embeddings // hvd.size())) loss = criterion(logits.float(), labels, weights) loss.backward() MPI.COMM_WORLD.Barrier() time_per_batch = benchmark_torch_function(iters, forward_backward, dense_features, sharded_sparse_features, labels) print( json.dumps( dict(name=name, method="forwardbackward", implementation="uniform", rank=hvd.rank(), workers=hvd.size(), B=batch_size, L=bag_size, T=num_tables, D=embedding_dim, dense_D=dense_features_dim, fp16=fp16, time_per_batch=time_per_batch, qps=batch_size * 1.0 / time_per_batch))) if hvd.rank() == 0: logging.info( f"{name}, UNIFORM, FORWARDBACKWARD: rank={hvd.rank()}, Workers: {hvd.size()}, Batch Size: {batch_size}, Bag Size: {bag_size}, Num Tables: {num_tables}, Embedding Dim: {embedding_dim}, fp16: {fp16}, Time per batch: {time_per_batch * 1.0e6}us, QPS: {batch_size * 1.0 / time_per_batch:.2e}" ) # optimizer = apex.optimizers.FusedSGD([v for (_, v) in dense_named_parameters], lr=0.05) net = DistributedUniformShardedSNN(num_tables, num_embeddings, embedding_dim, dense_features_dim).cuda() # TODO: Broken, PyTorch JIT bug. # net = torch.jit.trace(net, # example_inputs=(dense_features, # sharded_sparse_features)) dense_named_parameters = [(k, v) for (k, v) in net.named_parameters() if "embedding" not in k] optimizer = FastZeroFusedSGD([v for (_, v) in dense_named_parameters], lr=0.05) if fp16: net, optimizer = apex.amp.initialize(net, optimizer, opt_level=FP16_LEVEL, verbosity=0) # net = torch.jit.trace(net, # example_inputs=(dense_features, # sharded_sparse_features)) optimizer.zero_grad = optimizer.amp_zero_grad net.dense_arch = SingleGPUDDP(net.dense_arch, device_ids=[torch.cuda.current_device()]) net.over_arch = SingleGPUDDP(net.over_arch, device_ids=[torch.cuda.current_device()]) # hvd.broadcast_parameters(dense_named_parameters, root_rank=0) # hvd.broadcast_optimizer_state(optimizer, root_rank=0) def forward_backward_update(dense_features, sharded_sparse_features, labels): optimizer.zero_grad() logits = net( dense_features, sharded_sparse_features.random_(0, num_embeddings // hvd.size())) loss = criterion(logits, labels, weights) loss.backward() optimizer.step() MPI.COMM_WORLD.Barrier() import os if os.environ.get('BIGADS_PROFILE') and hvd.rank() == 0: with torch.autograd.profiler.profile(use_cuda=True, record_shapes=True) as prof: time_per_batch = benchmark_torch_function(iters, forward_backward_update, dense_features, sharded_sparse_features, labels) prof.export_chrome_trace( f"{'fp16' if fp16 else 'fp32'}-{hvd.size()}-uniform_{os.environ.get('BIGADS_PROFILE')}" ) else: time_per_batch = benchmark_torch_function(iters, forward_backward_update, dense_features, sharded_sparse_features, labels) print( json.dumps( dict(name=name, method="forwardbackwardupdate", implementation="uniform", rank=hvd.rank(), workers=hvd.size(), B=batch_size, L=bag_size, T=num_tables, D=embedding_dim, dense_D=dense_features_dim, fp16=fp16, time_per_batch=time_per_batch, qps=batch_size * 1.0 / time_per_batch))) if hvd.rank() == 0: logging.info( f"{name}, UNIFORM, FORWARDBACKWARDUPDATE: rank={hvd.rank()}, Workers: {hvd.size()}, Batch Size: {batch_size}, Bag Size: {bag_size}, Num Tables: {num_tables}, Embedding Dim: {embedding_dim}, fp16: {fp16}, Time per batch: {time_per_batch * 1.0e6}us, QPS: {batch_size * 1.0 / time_per_batch:.2e}" )
def benchmark_torch_uniform_snn_forward(name, num_tables, num_embeddings, embedding_dim, dense_features_dim, batch_size, bag_size, iters, fp16=0): net = UniformShardedSNN(num_tables, num_embeddings, embedding_dim, dense_features_dim).cuda() dense_features = torch.randn(batch_size, dense_features_dim, device=torch.cuda.current_device()) dense_features.requires_grad = False sharded_sparse_features = torch.randint(low=0, high=num_embeddings, size=(batch_size, num_tables, bag_size), device=torch.cuda.current_device()).int() # hack - compute grad for sparse-features to avoid having to allocate a grad for weights. sharded_sparse_features.requires_grad = False labels = torch.rand(size=(batch_size, 1), device=torch.cuda.current_device()) weights = torch.rand(size=(batch_size, 1), device=torch.cuda.current_device()) weights.requires_grad = False if fp16: net_fp16 = apex.amp.initialize(net, opt_level="O2", verbosity=0) # net_fp16 = torch.jit.trace(net_fp16, # example_inputs=(dense_features, # sharded_sparse_features)) net = torch.jit.trace(net, example_inputs=(dense_features, sharded_sparse_features)) def forward(dense_features, partitioned_sparse_features): return (net if not fp16 else net_fp16)(dense_features, sharded_sparse_features.random_(0, num_embeddings)) if os.environ.get('BIGADS_PROFILE_FORWARD'): with torch.autograd.profiler.profile(use_cuda=True, record_shapes=True) as prof: time_per_batch = benchmark_torch_function(iters, forward, dense_features, sharded_sparse_features) prof.export_chrome_trace(("fp16-" if fp16 else "fp32-") + os.environ.get('BIGADS_PROFILE_FORWARD')) else: time_per_batch = benchmark_torch_function(iters, forward, dense_features, sharded_sparse_features) if fp16: del net_fp16 logging.info( f"{name}, UNIFORM, FORWARD: Batch Size: {batch_size}, Bag Size: {bag_size}, Num Tables: {num_tables}, Embedding Dim: {embedding_dim}, Dense Features Dim: {dense_features_dim}, fp16: {fp16}, Time per batch: {time_per_batch * 1.0e6}us, QPS: {batch_size * 1.0 / time_per_batch:.2e}" ) print(json.dumps(dict(name=name, implementation="fused", method="forward", B=batch_size, L=bag_size, T=num_tables, D=embedding_dim, dense_D=dense_features_dim, time_per_batch=time_per_batch, qps=batch_size * 1.0 / time_per_batch))) criterion = Criterion().cuda() logits = net(dense_features, sharded_sparse_features) criterion = torch.jit.trace(criterion, example_inputs=(logits, labels, weights)) if fp16: net = UniformShardedSNN(num_tables, num_embeddings, embedding_dim, dense_features_dim).cuda() dense_named_parameters = [(k, v) for (k, v) in net.named_parameters() if "embedding" not in k] # optimizer = apex.optimizers.FusedSGD([v for (_, v) in dense_named_parameters], lr=0.05) optimizer = FastZeroFusedSGD([v for (_, v) in dense_named_parameters], lr=0.05) if fp16: net, optimizer = apex.amp.initialize(net, optimizer, opt_level="O2", verbosity=0) # net = torch.jit.trace(net, # example_inputs=(dense_features, # sharded_sparse_features)) optimizer.zero_grad = optimizer.amp_zero_grad def forward_backward_update(dense_features, sharded_sparse_features, labels): optimizer.zero_grad() logits = net(dense_features, sharded_sparse_features.random_(0, num_embeddings)) loss = criterion(logits.float(), labels, weights) loss.backward() optimizer.step() if os.environ.get('BIGADS_PROFILE'): with torch.autograd.profiler.profile(use_cuda=True, record_shapes=True) as prof: time_per_batch = benchmark_torch_function(iters, forward_backward_update, dense_features, sharded_sparse_features, labels) prof.export_chrome_trace(("fp16-" if fp16 else "fp32-") + os.environ.get('BIGADS_PROFILE')) else: time_per_batch = benchmark_torch_function(iters, forward_backward_update, dense_features, sharded_sparse_features, labels) logging.info( f"{name}, UNIFORM, FORWARDBACKWARDUPDATE: Batch Size: {batch_size}, Bag Size: {bag_size}, Num Tables: {num_tables}, Embedding Dim: {embedding_dim}, Dense Features Dim: {dense_features_dim}, fp16: {fp16}, Time per batch: {time_per_batch * 1.0e6}us, QPS: {batch_size * 1.0 / time_per_batch:.2e}" ) print(json.dumps(dict(name=name, implementation="fused", method="forwardbackwardupdate", B=batch_size, L=bag_size, T=num_tables, D=embedding_dim, dense_D=dense_features_dim, time_per_batch=time_per_batch, qps=batch_size * 1.0 / time_per_batch)))
def test_get_criteria(self): criterion1 = Criterion( subcategory_id=self.subcategory.id, title='Includes economic abuse framework', recommendation_text= ("The state's definition of domestic violence should include a framework of " 'economic abuse'), help_text= ('This means that the state acknowledges the role that economic control and abuse ' 'can play in domestic violence'), adverse=False, ) criterion2 = Criterion( subcategory_id=self.subcategory.id, title='Uses coercive control framework', recommendation_text= ("The state's definition of domestic violence should use a framework of coercive " 'control'), help_text= ('This means that the state acknowledges the role that coercion can play in ' 'domestic violence'), adverse=True, ) criterion2.deactivate() Criterion.save_all([criterion1, criterion2]) response = self.client.get('/criteria') self.assertEqual(response.status_code, 200) json_response = json.loads(response.data.decode('utf-8')) self.assertEqual(len(json_response), 2) self.assertEqual( json_response[0], { 'id': criterion1.id, 'subcategory_id': criterion1.subcategory_id, 'title': 'Includes economic abuse framework', 'recommendation_text': "The state's definition of domestic violence should include a framework of " 'economic abuse', 'help_text': 'This means that the state acknowledges the role that economic control and abuse ' 'can play in domestic violence', 'active': True, 'deactivated_at': None, 'adverse': False, }) criterion2_expected = { 'id': criterion2.id, 'subcategory_id': criterion2.subcategory_id, 'title': 'Uses coercive control framework', 'recommendation_text': "The state's definition of domestic violence should use a framework of coercive " 'control', 'help_text': 'This means that the state acknowledges the role that coercion can play in ' 'domestic violence', 'active': False, 'adverse': True, } # Assert that the expected results are a subset of the actual results self.assertTrue( criterion2_expected.items() <= json_response[1].items()) self.assertTrue(isinstance(json_response[1]['deactivated_at'], str))
def create_industrial_criteria() -> dict: logger.info('create_industrial_criteria') criteria_by_name = {} criterion1 = Criterion() criterion1.name = 'Bonne offre d’appel' criterion1.description = 'Offre déjà beaucoup réservée par les autres jeunes' criterion1.scoreDelta = 1 criteria_by_name[criterion1.name] = criterion1 criterion2 = Criterion() criterion2.name = 'Mauvaise accroche' criterion2.description = 'Offre ne possédant pas une accroche de qualité suffisante' criterion2.scoreDelta = -1 criteria_by_name[criterion2.name] = criterion2 criterion3 = Criterion() criterion3.name = 'Offre de médiation spécifique' criterion3.description = 'Offre possédant une médiation orientée pour les jeunes de 18 ans' criterion3.scoreDelta = 2 criteria_by_name[criterion3.name] = criterion3 PcObject.save(*criteria_by_name.values()) logger.info('created {} criteria'.format(len(criteria_by_name))) return criteria_by_name