def online_update(self, path, ts_list, input_shape, criterion, optimizer,
logger, ga_gen):
if len(ts_list) == 0:
return 1
self.train()
train_data = utils.load_disc_update_data(path, ts_list, input_shape)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
logger.info(
'****************************************************************')
logger.info(
'****************************************************************')
logger.info(
'****************************************************************')
logger.info('time = %s, ga gen %d, online update discriminator',
str(utils.get_unix_timestamp()), ga_gen)
for epoch in range(args.update_epochs):
logger.info('time = %s, epoch %d', str(utils.get_unix_timestamp()),
epoch)
print('time = {}, epoch {}'.format(str(utils.get_unix_timestamp()),
epoch))
self.train()
train_loss, train_acc = train(train_queue, self, criterion,
optimizer, logger)
logger.info('time = %s, train_loss %f train_acc %f',
str(utils.get_unix_timestamp()), train_loss, train_acc)
print('time = {}, train_loss {} train_acc {}'.format(
str(utils.get_unix_timestamp()), train_loss, train_acc))
def execute_pull_outbound_oracle():
r"""Wrapps the evalution logic for the pull outbound oracle.
Note that we always retrieve the same state such that this is not a varying factor, and hence it can
not influence the performance. This is without loss of generality.
"""
pull_outbound_oracle = PullOutboundOracle(
public_address=config.PUBLIC_ADDRESS,
private_address=config.PRIVATE_ADDRESS,
smart_contract_address=config.ARRIVAL_SMART_CONTRACT_ADDRESS,
web_socket=config.WEB_SOCKET,
abi=config.ARRIVAL_ABI)
start_timestamp = get_unix_timestamp()
retrieved_state = pull_outbound_oracle.retrieve_state_from_transaction_hash(
transaction_hash=
"0xf15753a9ef3d83e6d9974a936795039605449cdf2530545c8b339deaa6a7641f")
end_timestamp = get_unix_timestamp()
save_to_mongo(
db="pullOutboundOracle",
collection="arrival",
document={
"start_timestamp": start_timestamp,
"end_timestamp": end_timestamp,
"transaction_hash":
"0xf15753a9ef3d83e6d9974a936795039605449cdf2530545c8b339deaa6a7641f",
"state": retrieved_state[1]
})
print(retrieved_state)
def train(train_queue, model, criterion, optimizer, logger):
objs = utils.AvgrageMeter()
top1 = utils.AvgrageMeter()
model.train()
for step, (input, target) in enumerate(train_queue):
n = input.size(0)
input = Variable(input.float(), requires_grad=False).to(device)
target = Variable(target, requires_grad=False).to(device)
logits = model(input)
loss = criterion(logits, target)
optimizer.zero_grad()
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip)
optimizer.step()
prec1, prec5 = utils.accuracy(logits, target, topk=(1, 5))
objs.update(loss.item(), n)
top1.update(prec1.item(), n)
if step % args.report_freq == 0:
logger.info('time = %s, train %03d %e %f',
str(utils.get_unix_timestamp()), step, objs.avg,
top1.avg)
print('time = {}, train {} {}'.format(
str(utils.get_unix_timestamp()), step, objs.avg))
return objs.avg, top1.avg
def process_new_event(self, new_event):
verify_customer_received_timestamp = get_unix_timestamp()
# Respond to event via transaction
self.count += 1
self.state = {
"is_verified": True,
"order_ID": self.count,
"error_code": 0
}
self.encoded_abi = self.encoded_abi_order(self.state)
start_timestamp = get_unix_timestamp()
transaction_hash = web3.eth.to_hex(self.send_raw_transaction())
end_timestamp = get_unix_timestamp()
save_to_mongo(db="pullInboundOracle",
collection="order",
document={
"transaction_hash":
transaction_hash,
"order":
self.state,
"order_start_timestamp":
start_timestamp,
"order_end_timestamp":
end_timestamp,
"verify_customer":
new_event,
"verify_customer_received_timestamp":
verify_customer_received_timestamp
})
print(
f'(Order) Timestamp: {convert_unix_timesamp_to_datetime(get_unix_timestamp())} | Transaction hash: {transaction_hash} |'
)
def test(test_queue, model, criterion, logger):
objs = utils.AvgrageMeter()
top1 = utils.AvgrageMeter()
model.eval()
with torch.no_grad():
for step, (input, target) in enumerate(test_queue):
n = input.size(0)
input = Variable(input.float()).to(device)
target = Variable(target).to(device)
logits = model(input)
loss = criterion(logits, target)
prec1, prec5 = utils.accuracy(logits, target, topk=(1, 5))
objs.update(loss.item(), n)
top1.update(prec1.item(), n)
if step % args.report_freq == 0:
logger.info('time = %s, test %03d %e %f',
str(utils.get_unix_timestamp()), step, objs.avg,
top1.avg)
print('time = {}, test {} {}'.format(
str(utils.get_unix_timestamp()), step, objs.avg))
return objs.avg, top1.avg
def execute_push_inbound_oracle():
r""" Sends a transaction to the smart contract which triggers an event, which then
triggers the pull inbound oracle.
"""
random_verify_customer_state = RandomVerifyCustomerGenerator(
).get_random_verify_customer()
push_inbound_oracle = VerifyCustomerState(
verify_customer=random_verify_customer_state,
public_address=config.PUBLIC_ADDRESS,
private_address=config.PRIVATE_ADDRESS,
smart_contract_address=config.CUSTOMER_SMART_CONTRACT_ADDRESS_TWO,
abi=config.CUSTOMER_ABI,
web_socket=config.WEB_SOCKET)
start_timestamp = get_unix_timestamp()
transaction_hash = web3.eth.to_hex(
push_inbound_oracle.send_raw_transaction())
end_timestamp = get_unix_timestamp()
save_to_mongo(db="pullInboundOracle",
collection="verifyCustomer",
document={
"transaction_hash": transaction_hash,
"start_timestamp": start_timestamp,
"end_timestamp": end_timestamp,
"document": random_verify_customer_state
})
print(f'(Verify Customer) Timestamp: {convert_unix_timesamp_to_datetime(get_unix_timestamp())} | Transaction hash: {transaction_hash} |'\
f'Verify Customer: {random_verify_customer_state}')
def execute_push_inbound_oracle():
r"""Wrapps the logic that is necessary to execute the push inbound oracle and the associated
measuring logic. The measuring logic includes taking timestamps and storing the generatred
random arrival states with the timestamps into MongoDB.
"""
random_arrival_state = RandomArrivalGenerator().get_random_arrival()
push_inbound_oracle = ArrivalState(
public_address=config.PUBLIC_ADDRESS,
private_address=config.PRIVATE_ADDRESS,
smart_contract_address=config.ARRIVAL_SMART_CONTRACT_ADDRESS,
web_socket=config.WEB_SOCKET,
abi=config.ARRIVAL_ABI,
arrival=random_arrival_state)
start_timestamp = get_unix_timestamp()
transaction_hash = web3.eth.to_hex(
push_inbound_oracle.send_raw_transaction())
end_timestamp = get_unix_timestamp()
save_to_mongo(db="pushInboundOracle",
collection="arrival",
document={
"transaction_hash": transaction_hash,
"start_timestamp": start_timestamp,
"end_timestamp": end_timestamp,
"document": push_inbound_oracle.state
})
return transaction_hash, push_inbound_oracle.state
def make_query(payload):
date_range = payload['range']
date_from = datetime.strptime(date_range['from'], '%Y-%m-%dT%H:%M:%S.%fZ')
date_to = datetime.strptime(date_range['to'], '%Y-%m-%dT%H:%M:%S.%fZ')
targets = []
dynamodb = boto3.resource('dynamodb',
aws_access_key_id=os.getenv('ACCESS_KEY'),
aws_secret_access_key=os.getenv('SECRET_KEY'),
region_name=os.getenv('REGION_NAME'))
table = dynamodb.Table(os.getenv('DYNAMODB_TABLE'))
fields = list(map(lambda x: x['target'], payload['targets']))
date_from_str = date_from.strftime("%Y/%m/%dT%H:%M:%S.%fZ")
date_to_str = date_to.strftime("%Y/%m/%dT%H:%M:%S.%fZ")
query = table.scan(FilterExpression=Attr('date').between(
date_from_str, date_to_str),
Limit=payload['maxDataPoints'])
query = query['Items']
for field in fields:
target = {
'target':
field,
'datapoints': [[item.get(field),
get_unix_timestamp(item['date'])]
for item in query]
}
target['datapoints'].sort(key=lambda x: x[1])
targets.append(target)
return json.dumps(targets, default=decimal_encoder)
def process_new_event(self, new_event):
received_timestamp = get_unix_timestamp()
save_to_mongo(db="pushOutboundOracle",
collection="arrival",
document={
"received_timestamp": received_timestamp,
"event": new_event
})
def log_flock_kinematics(self, logger, iteration):
if not self.data_header_made:
logger.info(self.make_log_data_header(self.sorted_log_data_name_list, self.axes))
self.data_header_made = True
os_timestamp = 0
est_timestamp = {}
sim_kinematics = {}
est_kinematics = {}
os_timestamp = utils.get_unix_timestamp()
msg = '{},{}'.format(iteration, os_timestamp)
# get simGroundTruthKinematics and multiRotorState.kinematics
for drone in self.drones:
sim_kinematics[drone.vehicle_name] = self.client.simGetGroundTruthKinematics(vehicle_name=drone.vehicle_name)
multi_rotor_state = self.client.getMultirotorState(vehicle_name=drone.vehicle_name)
est_timestamp[drone.vehicle_name] = multi_rotor_state.timestamp
est_kinematics[drone.vehicle_name] = multi_rotor_state.kinematics_estimated
# sort the data
for drone in self.drones:
est_ts_made = False
for data_type in self.sorted_log_data_name_list:
func = self.log_data_list[data_type]
if 'sim' in data_type:
sim_data = getattr(sim_kinematics[drone.vehicle_name], func)
for axis in self.axes:
if axis == 'w' and "orient" not in data_type:
continue
msg += ','
data = getattr(sim_data, axis+'_val')
if 'pos' in data_type:
msg += str(data + drone.origin[self.axes.index(axis)-1])
else:
msg += str(data)
else:
est_data = getattr(est_kinematics[drone.vehicle_name], func)
if not est_ts_made:
msg += ','
msg += str(est_timestamp[drone.vehicle_name])
est_ts_made = True
for axis in self.axes:
if axis == 'w' and "orient" not in data_type:
continue
msg += ','
msg += str(getattr(est_data, axis+'_val'))
for drone_name in self.leader_list:
drone_index = int(drone_name.replace("Drone", ""))
msg += ','
msg += str(self.drones[drone_index].track_err)
logger.info(msg)
def main():
seed = args.seed
np.random.seed(seed)
cudnn.benchmark = True
torch.manual_seed(seed)
cudnn.enabled = True
torch.cuda.manual_seed(seed)
timestamp = str(utils.get_unix_timestamp())
utils.makedirs(args.save)
path = os.path.join(args.save, timestamp)
utils.create_exp_dir(path, scripts_to_save=glob.glob('../*.py'))
logger = utils.get_logger(args.save, timestamp, file_type='txt')
utils.makedirs(os.path.join(path, 'logs'))
logger.info("time = %s, args = %s", str(utils.get_unix_timestamp()), args)
input_shape = [
11, 9, 3
] # MANUALLY SET NUMBER OF CHANNELS (11) ACCORDING TO PRETRAINING
os.system('cp -f ../pretrain-weights.pt {}'.format(
os.path.join(path, 'weights.pt')))
utils.makedirs(os.path.join(path, 'scripts'))
os.system('cp -f ./for-copy/parse-ga.py {}'.format(
os.path.join(path, 'scripts', 'parse-ga.py')))
os.system('cp -f ./for-copy/parse-ga.py {}'.format(
os.path.join(path, 'scripts', 'parse-log.py')))
os.system('cp -f ./for-copy/parse_data.py {}'.format(
os.path.join(path, 'scripts', 'parse_data.py')))
os.system('cp -f ./for-copy/optimization-plots.sh {}'.format(
os.path.join(path, 'scripts', '1_optimization-plots.sh')))
# PyTorch
criterion = nn.CrossEntropyLoss()
criterion = criterion.to(device)
model = Network(input_shape, args.num_drones, criterion, path)
model = model.to(device)
utils.load(model, os.path.join(path, 'weights.pt'))
optimizer = torch.optim.SGD(model.parameters(),
lr=args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
# PyGMO
prob = pg.problem(genetic_algo.Flocking(path, timestamp, model))
pop = pg.population(prob, size=10, seed=24601)
algo = pg.algorithm(
pg.sga(gen=1,
cr=.90,
m=0.02,
param_s=3,
crossover="single",
mutation="uniform",
selection="truncated"))
algo.set_verbosity(1)
for i in range(29):
logger.info(
"time = %s gen = %d \n champ_f = %s \n champ_x = %s \n f_s = %s \n x_s = %s \n id_s = %s",
str(utils.get_unix_timestamp()), i + 1,
str(np.array(pop.champion_f).tolist()),
str(np.array(pop.champion_x).tolist()),
str(np.array(pop.get_f()).tolist()),
str(np.array(pop.get_x()).tolist()),
str(np.array(pop.get_ID()).tolist()))
pop = algo.evolve(pop)
model.online_update(path, genetic_algo.TS_LIST[-100:], input_shape,
criterion, optimizer, logger, i)
utils.save(model, os.path.join(path, 'weights.pt'))
