def online_update(self, path, ts_list, input_shape, criterion, optimizer, logger, ga_gen): if len(ts_list) == 0: return 1 self.train() train_data = utils.load_disc_update_data(path, ts_list, input_shape) train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) logger.info( '****************************************************************') logger.info( '****************************************************************') logger.info( '****************************************************************') logger.info('time = %s, ga gen %d, online update discriminator', str(utils.get_unix_timestamp()), ga_gen) for epoch in range(args.update_epochs): logger.info('time = %s, epoch %d', str(utils.get_unix_timestamp()), epoch) print('time = {}, epoch {}'.format(str(utils.get_unix_timestamp()), epoch)) self.train() train_loss, train_acc = train(train_queue, self, criterion, optimizer, logger) logger.info('time = %s, train_loss %f train_acc %f', str(utils.get_unix_timestamp()), train_loss, train_acc) print('time = {}, train_loss {} train_acc {}'.format( str(utils.get_unix_timestamp()), train_loss, train_acc))
def execute_pull_outbound_oracle(): r"""Wrapps the evalution logic for the pull outbound oracle. Note that we always retrieve the same state such that this is not a varying factor, and hence it can not influence the performance. This is without loss of generality. """ pull_outbound_oracle = PullOutboundOracle( public_address=config.PUBLIC_ADDRESS, private_address=config.PRIVATE_ADDRESS, smart_contract_address=config.ARRIVAL_SMART_CONTRACT_ADDRESS, web_socket=config.WEB_SOCKET, abi=config.ARRIVAL_ABI) start_timestamp = get_unix_timestamp() retrieved_state = pull_outbound_oracle.retrieve_state_from_transaction_hash( transaction_hash= "0xf15753a9ef3d83e6d9974a936795039605449cdf2530545c8b339deaa6a7641f") end_timestamp = get_unix_timestamp() save_to_mongo( db="pullOutboundOracle", collection="arrival", document={ "start_timestamp": start_timestamp, "end_timestamp": end_timestamp, "transaction_hash": "0xf15753a9ef3d83e6d9974a936795039605449cdf2530545c8b339deaa6a7641f", "state": retrieved_state[1] }) print(retrieved_state)
def train(train_queue, model, criterion, optimizer, logger): objs = utils.AvgrageMeter() top1 = utils.AvgrageMeter() model.train() for step, (input, target) in enumerate(train_queue): n = input.size(0) input = Variable(input.float(), requires_grad=False).to(device) target = Variable(target, requires_grad=False).to(device) logits = model(input) loss = criterion(logits, target) optimizer.zero_grad() loss.backward() nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) optimizer.step() prec1, prec5 = utils.accuracy(logits, target, topk=(1, 5)) objs.update(loss.item(), n) top1.update(prec1.item(), n) if step % args.report_freq == 0: logger.info('time = %s, train %03d %e %f', str(utils.get_unix_timestamp()), step, objs.avg, top1.avg) print('time = {}, train {} {}'.format( str(utils.get_unix_timestamp()), step, objs.avg)) return objs.avg, top1.avg
def process_new_event(self, new_event): verify_customer_received_timestamp = get_unix_timestamp() # Respond to event via transaction self.count += 1 self.state = { "is_verified": True, "order_ID": self.count, "error_code": 0 } self.encoded_abi = self.encoded_abi_order(self.state) start_timestamp = get_unix_timestamp() transaction_hash = web3.eth.to_hex(self.send_raw_transaction()) end_timestamp = get_unix_timestamp() save_to_mongo(db="pullInboundOracle", collection="order", document={ "transaction_hash": transaction_hash, "order": self.state, "order_start_timestamp": start_timestamp, "order_end_timestamp": end_timestamp, "verify_customer": new_event, "verify_customer_received_timestamp": verify_customer_received_timestamp }) print( f'(Order) Timestamp: {convert_unix_timesamp_to_datetime(get_unix_timestamp())} | Transaction hash: {transaction_hash} |' )
def test(test_queue, model, criterion, logger): objs = utils.AvgrageMeter() top1 = utils.AvgrageMeter() model.eval() with torch.no_grad(): for step, (input, target) in enumerate(test_queue): n = input.size(0) input = Variable(input.float()).to(device) target = Variable(target).to(device) logits = model(input) loss = criterion(logits, target) prec1, prec5 = utils.accuracy(logits, target, topk=(1, 5)) objs.update(loss.item(), n) top1.update(prec1.item(), n) if step % args.report_freq == 0: logger.info('time = %s, test %03d %e %f', str(utils.get_unix_timestamp()), step, objs.avg, top1.avg) print('time = {}, test {} {}'.format( str(utils.get_unix_timestamp()), step, objs.avg)) return objs.avg, top1.avg
def execute_push_inbound_oracle(): r""" Sends a transaction to the smart contract which triggers an event, which then triggers the pull inbound oracle. """ random_verify_customer_state = RandomVerifyCustomerGenerator( ).get_random_verify_customer() push_inbound_oracle = VerifyCustomerState( verify_customer=random_verify_customer_state, public_address=config.PUBLIC_ADDRESS, private_address=config.PRIVATE_ADDRESS, smart_contract_address=config.CUSTOMER_SMART_CONTRACT_ADDRESS_TWO, abi=config.CUSTOMER_ABI, web_socket=config.WEB_SOCKET) start_timestamp = get_unix_timestamp() transaction_hash = web3.eth.to_hex( push_inbound_oracle.send_raw_transaction()) end_timestamp = get_unix_timestamp() save_to_mongo(db="pullInboundOracle", collection="verifyCustomer", document={ "transaction_hash": transaction_hash, "start_timestamp": start_timestamp, "end_timestamp": end_timestamp, "document": random_verify_customer_state }) print(f'(Verify Customer) Timestamp: {convert_unix_timesamp_to_datetime(get_unix_timestamp())} | Transaction hash: {transaction_hash} |'\ f'Verify Customer: {random_verify_customer_state}')
def execute_push_inbound_oracle(): r"""Wrapps the logic that is necessary to execute the push inbound oracle and the associated measuring logic. The measuring logic includes taking timestamps and storing the generatred random arrival states with the timestamps into MongoDB. """ random_arrival_state = RandomArrivalGenerator().get_random_arrival() push_inbound_oracle = ArrivalState( public_address=config.PUBLIC_ADDRESS, private_address=config.PRIVATE_ADDRESS, smart_contract_address=config.ARRIVAL_SMART_CONTRACT_ADDRESS, web_socket=config.WEB_SOCKET, abi=config.ARRIVAL_ABI, arrival=random_arrival_state) start_timestamp = get_unix_timestamp() transaction_hash = web3.eth.to_hex( push_inbound_oracle.send_raw_transaction()) end_timestamp = get_unix_timestamp() save_to_mongo(db="pushInboundOracle", collection="arrival", document={ "transaction_hash": transaction_hash, "start_timestamp": start_timestamp, "end_timestamp": end_timestamp, "document": push_inbound_oracle.state }) return transaction_hash, push_inbound_oracle.state
def make_query(payload): date_range = payload['range'] date_from = datetime.strptime(date_range['from'], '%Y-%m-%dT%H:%M:%S.%fZ') date_to = datetime.strptime(date_range['to'], '%Y-%m-%dT%H:%M:%S.%fZ') targets = [] dynamodb = boto3.resource('dynamodb', aws_access_key_id=os.getenv('ACCESS_KEY'), aws_secret_access_key=os.getenv('SECRET_KEY'), region_name=os.getenv('REGION_NAME')) table = dynamodb.Table(os.getenv('DYNAMODB_TABLE')) fields = list(map(lambda x: x['target'], payload['targets'])) date_from_str = date_from.strftime("%Y/%m/%dT%H:%M:%S.%fZ") date_to_str = date_to.strftime("%Y/%m/%dT%H:%M:%S.%fZ") query = table.scan(FilterExpression=Attr('date').between( date_from_str, date_to_str), Limit=payload['maxDataPoints']) query = query['Items'] for field in fields: target = { 'target': field, 'datapoints': [[item.get(field), get_unix_timestamp(item['date'])] for item in query] } target['datapoints'].sort(key=lambda x: x[1]) targets.append(target) return json.dumps(targets, default=decimal_encoder)
def process_new_event(self, new_event): received_timestamp = get_unix_timestamp() save_to_mongo(db="pushOutboundOracle", collection="arrival", document={ "received_timestamp": received_timestamp, "event": new_event })
def log_flock_kinematics(self, logger, iteration): if not self.data_header_made: logger.info(self.make_log_data_header(self.sorted_log_data_name_list, self.axes)) self.data_header_made = True os_timestamp = 0 est_timestamp = {} sim_kinematics = {} est_kinematics = {} os_timestamp = utils.get_unix_timestamp() msg = '{},{}'.format(iteration, os_timestamp) # get simGroundTruthKinematics and multiRotorState.kinematics for drone in self.drones: sim_kinematics[drone.vehicle_name] = self.client.simGetGroundTruthKinematics(vehicle_name=drone.vehicle_name) multi_rotor_state = self.client.getMultirotorState(vehicle_name=drone.vehicle_name) est_timestamp[drone.vehicle_name] = multi_rotor_state.timestamp est_kinematics[drone.vehicle_name] = multi_rotor_state.kinematics_estimated # sort the data for drone in self.drones: est_ts_made = False for data_type in self.sorted_log_data_name_list: func = self.log_data_list[data_type] if 'sim' in data_type: sim_data = getattr(sim_kinematics[drone.vehicle_name], func) for axis in self.axes: if axis == 'w' and "orient" not in data_type: continue msg += ',' data = getattr(sim_data, axis+'_val') if 'pos' in data_type: msg += str(data + drone.origin[self.axes.index(axis)-1]) else: msg += str(data) else: est_data = getattr(est_kinematics[drone.vehicle_name], func) if not est_ts_made: msg += ',' msg += str(est_timestamp[drone.vehicle_name]) est_ts_made = True for axis in self.axes: if axis == 'w' and "orient" not in data_type: continue msg += ',' msg += str(getattr(est_data, axis+'_val')) for drone_name in self.leader_list: drone_index = int(drone_name.replace("Drone", "")) msg += ',' msg += str(self.drones[drone_index].track_err) logger.info(msg)
def main(): seed = args.seed np.random.seed(seed) cudnn.benchmark = True torch.manual_seed(seed) cudnn.enabled = True torch.cuda.manual_seed(seed) timestamp = str(utils.get_unix_timestamp()) utils.makedirs(args.save) path = os.path.join(args.save, timestamp) utils.create_exp_dir(path, scripts_to_save=glob.glob('../*.py')) logger = utils.get_logger(args.save, timestamp, file_type='txt') utils.makedirs(os.path.join(path, 'logs')) logger.info("time = %s, args = %s", str(utils.get_unix_timestamp()), args) input_shape = [ 11, 9, 3 ] # MANUALLY SET NUMBER OF CHANNELS (11) ACCORDING TO PRETRAINING os.system('cp -f ../pretrain-weights.pt {}'.format( os.path.join(path, 'weights.pt'))) utils.makedirs(os.path.join(path, 'scripts')) os.system('cp -f ./for-copy/parse-ga.py {}'.format( os.path.join(path, 'scripts', 'parse-ga.py'))) os.system('cp -f ./for-copy/parse-ga.py {}'.format( os.path.join(path, 'scripts', 'parse-log.py'))) os.system('cp -f ./for-copy/parse_data.py {}'.format( os.path.join(path, 'scripts', 'parse_data.py'))) os.system('cp -f ./for-copy/optimization-plots.sh {}'.format( os.path.join(path, 'scripts', '1_optimization-plots.sh'))) # PyTorch criterion = nn.CrossEntropyLoss() criterion = criterion.to(device) model = Network(input_shape, args.num_drones, criterion, path) model = model.to(device) utils.load(model, os.path.join(path, 'weights.pt')) optimizer = torch.optim.SGD(model.parameters(), lr=args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) # PyGMO prob = pg.problem(genetic_algo.Flocking(path, timestamp, model)) pop = pg.population(prob, size=10, seed=24601) algo = pg.algorithm( pg.sga(gen=1, cr=.90, m=0.02, param_s=3, crossover="single", mutation="uniform", selection="truncated")) algo.set_verbosity(1) for i in range(29): logger.info( "time = %s gen = %d \n champ_f = %s \n champ_x = %s \n f_s = %s \n x_s = %s \n id_s = %s", str(utils.get_unix_timestamp()), i + 1, str(np.array(pop.champion_f).tolist()), str(np.array(pop.champion_x).tolist()), str(np.array(pop.get_f()).tolist()), str(np.array(pop.get_x()).tolist()), str(np.array(pop.get_ID()).tolist())) pop = algo.evolve(pop) model.online_update(path, genetic_algo.TS_LIST[-100:], input_shape, criterion, optimizer, logger, i) utils.save(model, os.path.join(path, 'weights.pt'))