def __init__(self,n):
#for pygame
self.running = True
self.screen = None
#for agent and environment
self.car = Car()
self.checkpoints = Checkpoints()
#for training
self.timer = 0
self.ep_count = 1
self.test_ep_count = 1
self.epsilon = 0.01
self.a = 0.01
self.n = n
self.discount = 1
self.max_steps = 8000
self.Q = np.random.sample([2,3,3,3,3,3,3,9])+10 #optimistic
self.state = []
self.next_state = []
self.action = 0
self.state_path = []
self.action_path = []
self.reward_path = []
self.ep_rewards = []
self.T = float('inf')
#use to control whether or not to render a pygame window
self.show = 1
def main():
# parse the arguments
args = parser.parse_args()
random.seed(args.manual_seed)
torch.manual_seed(args.manual_seed)
utils.saveargs(args)
# initialize the checkpoint class
checkpoints = Checkpoints(args)
# Create Model
models = Model(args)
model, criterion = models.setup(checkpoints)
# Data Loading
dataloader = Dataloader(args)
loaders = dataloader.create()
# The trainer handles the training loop
trainer = Trainer(args, model, criterion)
# The trainer handles the evaluation on validation set
tester = Tester(args, model, criterion)
# start training !!!
loss_best = 1e10
for epoch in range(args.nepochs):
# train for a single epoch
loss_train = trainer.train(epoch, loaders)
loss_test = tester.test(epoch, loaders)
if loss_best > loss_test:
model_best = True
loss_best = loss_test
checkpoints.save(epoch, model, model_best)
def __init__(self, T, suffix_array, cp_step, sa_step):
if T[-1] != '$':
T += '$'
full_sa = suffix_array(T)
self.bwt = bwt(T, full_sa)
self.sa = self.cut_suffix_array(full_sa, sa_step)
self.cps = Checkpoints(self.bwt, cp_step)
self.first_col = self.calc_first_col(self.bwt)
def reset(self,track=1):
self.car = Car(track)
self.checkpoints = Checkpoints(track)
self.timer = 0
self.state = []
self.next_state = []
self.state_path = [] #some initial state
self.action = 4
self.action_path = []
self.reward_path = []
self.T = float('inf')
def __init__(self, nrun=-1):
self.args = Namespace(
cuda=True,
ndf=8,
nef=8,
wkld=0.01,
gbweight=100,
nlatent=9,
nechannels=6,
ngchannels=1,
resume="",
save="mini-save",
loader_train="h5py",
loader_test="h5py",
dataset_test=None,
dataset_train="filelist",
split_test=0.0,
split_train=1.0,
filename_test="./data/data.txt",
filename_train="./data/data.txt",
batch_size=64,
resolution_high=512,
resolution_wide=512,
nthreads=32,
images="mini-save/images",
pre_name="save",
)
latest_save = sorted(list(Path("results").iterdir()))[nrun]
self.rundate = latest_save.name
latest_save = latest_save.joinpath("Save")
latest_save = {"netG": latest_save}
self.args.resume = latest_save
checkpoints = Checkpoints(self.args)
# Create model
models = Model(self.args)
self.model, self.criterion = models.setup(checkpoints)
# Data loading
self.dataloader = Dataloader(self.args)
self.loader = self.dataloader.create(flag="Test")
print("\t\tBatches:\t", len(self.loader))
self.resolution_high = self.args.resolution_high
self.resolution_wide = self.args.resolution_wide
self.batch_size = self.args.batch_size
self.ngchannels = self.args.ngchannels
self.nechannels = self.args.nechannels
self.nlatent = self.args.nlatent
self.composition = torch.FloatTensor(self.batch_size, self.ngchannels, self.resolution_high, self.resolution_wide)
self.metadata = torch.FloatTensor(self.batch_size, self.nechannels, self.resolution_high, self.resolution_wide)
if self.args.cuda:
self.composition = self.composition.cuda()
self.metadata = self.metadata.cuda()
self.composition = Variable(self.composition)
self.metadata = Variable(self.metadata)
self.imgio = plugins.ImageIO(self.args.images, self.args.pre_name)
def main():
# parse the arguments
random.seed(args.manual_seed)
torch.manual_seed(args.manual_seed)
if args.save_results:
utils.saveargs(args, config_file)
# initialize the checkpoint class
checkpoints = Checkpoints(args)
# Create Model
models = Model(args)
model, model_dict, evaluation = models.setup(checkpoints)
print('Model:\n\t{model}\nTotal params:\n\t{npar:.2f}M'.format(
model=args.model_type,
npar=sum(p.numel() for p in model.parameters()) / 1000000.0))
#### get kernel information ####
ndemog = args.ndemog
ndemog = list(range(ndemog))
demog_combs = list(combinations(ndemog, 2))
#### get kernel information ####
dist_list = get_att_dist(model)
pdb.set_trace()
def main():
# parse the arguments
args = config.parse_args()
if (args.ngpu > 0 and torch.cuda.is_available()):
device = "cuda:0"
else:
device = "cpu"
args.device = torch.device(device)
random.seed(args.manual_seed)
torch.manual_seed(args.manual_seed)
if args.save_results:
utils.saveargs(args)
# initialize the checkpoint class
checkpoints = Checkpoints(args)
# Create Model
models = Model(args)
model, criterion, evaluation = models.setup(checkpoints)
print('Model:\n\t{model}\nTotal params:\n\t{npar:.2f}M'.format(
model=args.model_type,
npar=sum(p.numel() for p in model.parameters()) / 1000000.0))
# Data Loading
dataloader = Dataloader(args)
loaders = dataloader.create()
# The trainer handles the training loop
trainer = Trainer(args, model, criterion, evaluation)
# The trainer handles the evaluation on validation set
tester = Tester(args, model, criterion, evaluation)
# start training !!!
loss_best = 1e10
for epoch in range(args.nepochs):
print('\nEpoch %d/%d\n' % (epoch + 1, args.nepochs))
# train for a single epoch
loss_train = trainer.train(epoch, loaders)
loss_test = tester.test(epoch, loaders)
if loss_best > loss_test:
model_best = True
loss_best = loss_test
if args.save_results:
checkpoints.save(epoch, model, model_best)
def main():
# Parse the Arguments
args = config.parse_args()
random.seed(args.manual_seed)
tf.set_random_seed(args.manual_seed)
now = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S/')
args.save = os.path.join(args.result_path, now, 'save')
args.logs = os.path.join(args.result_path, now, 'logs')
if args.save_results:
utils.saveargs(args)
# Initialize the Checkpoints Class
checkpoints = Checkpoints(args)
# Create Model
models = Model(args)
model, criterion, evaluation = models.setup(checkpoints)
# Print Model Summary
print('Model summary: {}'.format(model.name))
print(model.summary())
# Data Loading
dataloader_obj = Dataloader(args)
dataloader = dataloader_obj.create()
# Initialize Trainer and Tester
trainer = Trainer(args, model, criterion, evaluation)
tester = Tester(args, model, criterion, evaluation)
# Start Training !!!
loss_best = 1e10
for epoch in range(args.nepochs):
print('\nEpoch %d/%d' % (epoch + 1, args.nepochs))
# Train and Test for a Single Epoch
loss_train = trainer.train(epoch, dataloader["train"])
loss_test = tester.test(epoch, dataloader["test"])
if loss_best > loss_test:
model_best = True
loss_best = loss_test
if args.save_results:
checkpoints.save(epoch, model, model_best)
def main():
demog_type = 'race'
demog_target = {'gender': 1, 'age': 2, 'race': 3}
demog_refer = {'gender': [2, 3], 'age': [1, 3], 'race': [1, 2]}
# initialize the checkpoint class
checkpoints = Checkpoints(args)
# Create Model
models = Model(args)
model, model_dict, evaluation = models.setup(checkpoints)
print('Model:\n\t{model}\nTotal params:\n\t{npar:.2f}M'.format(
model=args.model_type,
npar=sum(p.numel() for p in model['feat'].parameters()) / 1000000.0))
# The trainer handles the evaluation on validation set
tester = Tester(args, model, model_dict['loss'], evaluation)
test_freq = 1
dataloader = Dataloader(args)
dataset_options_test = args.dataset_options_test
resfilename = '/research/prip-gongsixu/codes/biasface/results/evaluation/demogbias/race.txt'
gdemog = get_demog_cohorts(demog_type)
with open(resfilename, 'w') as f:
for demog_group in gdemog:
dataset_options_test['target_ind'] = demog_target[demog_type]
dataset_options_test['refer_ind'] = demog_refer[demog_type]
dataset_options_test['demog_group'] = demog_group
args.dataset_options_test = dataset_options_test
loaders = dataloader.create(flag='Test')
acc_test = tester.test_demog(demog_type, loaders)
f.write(demog_group + '\t' + str(acc_test) + '\n')
print(acc_test)
class Model:
def __init__(self,n):
#for pygame
self.running = True
self.screen = None
#for agent and environment
self.car = Car()
self.checkpoints = Checkpoints()
#for training
self.timer = 0
self.ep_count = 1
self.test_ep_count = 1
self.epsilon = 0.01
self.a = 0.01
self.n = n
self.discount = 1
self.max_steps = 8000
self.Q = np.random.sample([2,3,3,3,3,3,3,9])+10 #optimistic
self.state = []
self.next_state = []
self.action = 0
self.state_path = []
self.action_path = []
self.reward_path = []
self.ep_rewards = []
self.T = float('inf')
#use to control whether or not to render a pygame window
self.show = 1
#used to start pygame window
def on_init(self):
pygame.init()
self.screen = pygame.display.set_mode([640, 400], pygame.HWSURFACE | pygame.DOUBLEBUF)
self.running = True
#I only use this to lose pygame window
def on_event(self, event):
if event.type == pygame.QUIT:
self.running = False
def loop(self):
#used to train the agent
if(self.timer == 0):
#intializing
self.render_train() if self.show == 1 else self.norender_train()
self.car.update_state()
self.state = self.car.state
self.state_path.append(self.car.state)
self.action = self.choose_action()
self.action_path.append(self.action)
self.act()
self.reward_path.append(0)
self.timer += 1
else:
#Sometimes episode continues
if(self.timer < self.T):
terminal = (self.timer > self.max_steps or self.check_collision())
self.render_train() if self.show == 1 else self.norender_train()
self.car.update_state()
self.next_state = self.car.state
reward = self.current_reward()
#terminal state
if(terminal):
self.T = self.timer
else:
self.action = self.choose_action(1)
self.reward_path.append(reward)
self.action_path.append(self.action)
self.state_path.append(self.car.state)
#Sometimes the episode has ended and the agent's policy is updated
toa = self.timer-self.n
if(toa >= 0):
#Updating is done using the model properties
gain = 0.0
for t in range(toa, min(self.T, toa+self.n)):
gain += (self.discount**(t-toa-1))*self.reward_path[t]
if(toa + self.n < self.T):
q_index = copy.copy(self.state_path[toa+self.n])
q_index.append(self.action_path[toa+self.n])
gain += (self.discount**(self.n))*self.Q[tuple(q_index)]
q_update = copy.copy(self.state_path[toa])
q_update.append(self.action_path[toa])
if(self.reward_path[toa] != neg_reward):
self.Q[tuple(q_update)] += self.a*(gain-self.Q[tuple(q_update)])
if(toa == self.T-1):
#end of episode
print(f'Episode: {self.ep_count}, res: {sum(self.reward_path)}')
self.ep_count+=1
self.ep_rewards.append(sum(self.reward_path))
self.reset()
else:
#not end of episode
self.timer += 1
#Setup next time step
self.state = self.next_state
self.act()
#test_loop() is pretty similar to the loop() function except that it doesn't update self.Q
#Therefore, it can be used to run multiple episodes to evaluate the performace of the current Q
def test_loop(self, track):
if(self.timer == 0):
self.render_test(track) if self.show == 1 else self.norender_test(track)
self.car.update_state()
self.state = self.car.state
self.state_path.append(self.car.state)
self.action = self.choose_action()
self.action_path.append(self.action)
self.act()
self.reward_path.append(0)
self.timer += 1
return 0
else:
if(self.timer < self.T):
terminal = (self.timer > self.max_steps or self.check_collision(track))
self.render_test(track) if self.show == 1 else self.norender_test(track)
self.car.update_state()
self.next_state = self.car.state
reward = 0 if self.reward_path[len(self.reward_path)-1] == neg_reward else self.current_reward(track)
#terminal
if(terminal):
self.T = self.timer
else:
self.action = self.choose_action(1)
self.reward_path.append(reward)
self.action_path.append(self.action)
self.state_path.append(self.car.state)
else:
reward = 0
toa = self.timer-self.n
if(toa == self.T-1):
print(f'Test Episode: {self.test_ep_count}, res: {sum(self.reward_path)}')
self.test_ep_count+=1
self.reset(track)
else:
self.timer += 1
self.state = self.next_state
self.act()
return reward
#quits pygame
def clean(self):
pygame.quit()
#run() indefinitly trains the agent and renders a pygame window
def run(self):
if self.on_init() == False:
self.running = False
while(self.running):
for event in pygame.event.get():
self.on_event(event)
self.loop()
pygame.display.update()
self.clean()
#train_wr() trains the agent for a set number of episodes and renders a pygame window
def train_wr(self, num_eps):
self.reset()
run_lim = self.ep_count+num_eps
if self.on_init() == False:
self.running = False
while(self.running and self.ep_count < run_lim):
for event in pygame.event.get():
self.on_event(event)
self.loop()
pygame.display.update()
self.clean()
#train() trains the agent for a set number of episodes and does not render a pygame window
def train(self, num_eps):
self.reset()
self.show = 0
run_lim = self.ep_count+num_eps
while(self.ep_count<run_lim):
self.loop()
self.show = 1
#test_wr() tests the agent for a set number of episodes and renders a pygame window
#test_wr() returns the average sum of rewards for a episode
#It also uses test_loop() so self.Q is never updated which allows it to test the current epsilon greedy policy w.r.t. self.Q
def test_wr(self,num_eps,track):
self.reset(track)
self.test_ep_count = 1
reward_total = 0
if self.on_init() == False:
self.running = False
while( self.running and self.test_ep_count <= num_eps):
for event in pygame.event.get():
self.on_event(event)
r = self.test_loop(track)
reward_total += r
pygame.display.update()
self.clean()
return reward_total/num_eps
#test() is the same as test_wr() except that it renders a pygame window
def test(self,num_eps,track):
self.reset(track)
self.show = 0
temp = copy.copy(self.epsilon)
self.test_ep_count = 1
reward_total = 0
while(self.test_ep_count <= num_eps):
r = self.test_loop(track)
reward_total+=r
self.epsilon = temp
self.show = 1
return reward_total/num_eps
#This just resets everything at the begining of an episode.
#The car has to go back to the start, checkpoints have to be reset, actions and rewards are cleared, etc.
def reset(self,track=1):
self.car = Car(track)
self.checkpoints = Checkpoints(track)
self.timer = 0
self.state = []
self.next_state = []
self.state_path = [] #some initial state
self.action = 4
self.action_path = []
self.reward_path = []
self.T = float('inf')
#This renders elements for training (Always track 1)
def render_train(self):
#render the game elements
self.screen.fill(background_color)
self.render_track1()
self.checkpoints.render(self.screen)
self.car.move()
self.car.render(self.screen)
#This sets up the the current time step for training, but nothing is rendered to a pygame window
def norender_train(self):
self.car.move()
#This renders elements for testing (Sometimes track 1, sometimes track 2)
def render_test(self,track):
#render the game elements
self.screen.fill(background_color)
if(track == 2):
self.render_track2()
else:
self.render_track1()
self.checkpoints.render(self.screen)
self.car.move(track)
self.car.render(self.screen)
#This sets up the the current time step for testing, but nothing is rendered to a pygame window
def norender_test(self,track):
self.car.move(track)
#Simply renders the rectangles to show where track 1 is
def render_track1(self):
pygame.draw.rect(self.screen, track_color, [50,50,60,300])
pygame.draw.rect(self.screen, track_color, [50,50,540,60])
pygame.draw.rect(self.screen, track_color, [50,290,240,60])
pygame.draw.rect(self.screen, track_color, [230,170,60,180])
pygame.draw.rect(self.screen, track_color, [230,170,180,60])
pygame.draw.rect(self.screen, track_color, [530,50,60,300])
pygame.draw.rect(self.screen, track_color, [350,170,60,180])
pygame.draw.rect(self.screen, track_color, [350,290,240,60])
#Same as render_track1() but its for track 2
def render_track2(self):
pygame.draw.rect(self.screen, track_color, [50,50,540,60])
pygame.draw.rect(self.screen, track_color, [530,50,60,300])
pygame.draw.rect(self.screen, track_color, [410,170,60,180])
pygame.draw.rect(self.screen, track_color, [410,290,180,60])
pygame.draw.rect(self.screen, track_color, [50,170,420,60])
pygame.draw.rect(self.screen, track_color, [50,50,60,180])
#It checks to see if the car hit a wall. If it did, then the neg_reward is returned and the car turns red
#It also checks to see if the car is passing a checkpoint, and gives pos_reward if it has.
def current_reward(self,track=1):
reward = 0
if(self.check_collision(track)):
self.car.color = [255,0,0]
reward = neg_reward
else:
self.car.color = [10,46,73]
reward = 0
if(self.checkpoints.check_pass(self.car.corners)):
reward = reward+pos_reward
return reward
#Check collisions just looks to see that all 4 corners of the car are on the track. If they are, then no collision
def check_collision(self,track=1):
if(on_track(self.car.corners[0],track) and on_track(self.car.corners[1],track) and on_track(self.car.corners[2],track) and on_track(self.car.corners[3],track)):
return False
else:
return True
#Choosing random action with probability self.epsilon
#Choosing greedy action w.r.t. self.Q with probablilty (1-self.epsilon)
def choose_action(self, next=0):
if(np.random.sample(1) < self.epsilon):
action = np.random.randint(0,9)
else:
i = self.state if next == 0 else self.next_state
action_choices = self.Q[i[0],i[1],i[2],i[3],i[4],i[5],i[6],:]
action = np.argmax(action_choices)
return action
#uses the selected action to accelerate and steer the car
def act(self):
accel = math.floor(self.action/3)
turn = self.action%3
self.car.pedal(accel)
self.car.steer(turn)
import torch import random import torchvision from model import Model from config import parser from dataloader import Dataloader from checkpoints import Checkpoints from train import Trainer import utils # parse the arguments args = parser.parse_args() random.seed(args.manual_seed) torch.manual_seed(args.manual_seed) utils.saveargs(args) # initialize the checkpoint class checkpoints = Checkpoints(args) # Create Model models = Model(args) model, criterion = models.setup(checkpoints) # Data Loading dataloader = Dataloader(args) loader_train, loader_test = dataloader.create() # The trainer handles the training loop and evaluation on validation set trainer = Trainer(args, model, criterion) loss_train = trainer.visualize(loader_train)
class FMIndex:
def cut_suffix_array(self, full_sa, sa_step):
"""Sizes down suffix array taking every sa_step-th element, the rest are removed"""
res_sa = {}
for i in range(len(full_sa)):
if full_sa[i] % sa_step == 0:
res_sa[i] = full_sa[i]
return res_sa
def calc_first_col(self, bwt):
"""
Returns first column of BWT matrix.
It is enough to keep only number of characters less than current, because it is sorted column.
"""
cnts = {}
for c in bwt:
cnts[c] = cnts.get(c, 0) + 1
ret = {}
total = 0
for c, cnt in sorted(cnts.items()):
ret[c] = total
total += cnt
return ret
def __init__(self, T, suffix_array, cp_step, sa_step):
if T[-1] != '$':
T += '$'
full_sa = suffix_array(T)
self.bwt = bwt(T, full_sa)
self.sa = self.cut_suffix_array(full_sa, sa_step)
self.cps = Checkpoints(self.bwt, cp_step)
self.first_col = self.calc_first_col(self.bwt)
def count(self, c):
"""Return number of characters less than c"""
if c not in self.first_col:
for cc in sorted(self.first_col.keys()):
if c < cc:
return self.first_col[cc]
return self.first_col[cc]
else:
return self.first_col[c]
def interval(self, p):
"""Returns inclusive interval of BWM rows where p is prefix"""
l, r = 0, len(self.bwt) - 1
print(self.bwt)
print(self.first_col)
for i in range(len(p) - 1, -1,
-1): #start from last character and go backwards
l = self.cps.rank(self.bwt, p[i], l - 1) + self.count(p[i])
r = self.cps.rank(self.bwt, p[i], r) + self.count(p[i]) - 1
if r < l:
break
return l, r + 1
def get_offset(self, r):
"""Given row in matrix returns its real offset in text"""
steps = 0
while r not in self.sa:
c = self.bwt[r]
r = self.cps.rank(self.bwt, c, r - 1) + self.count(c)
steps += 1
return self.sa[r] + steps
def find_occurences(self, p):
"""Returns all occurences of pattern p"""
l, r = self.interval(p)
return [self.get_offset(i) for i in range(l, r)]
def deploy():
ret_code = 0
# noinspection PyBroadException
logger.debug("=================================")
logger.info("=== Starting up ...")
logger.debug("=================================")
try:
settings, args = get_settings()
director_vm = Director()
sah_node = Sah()
if args.list_edge_sites:
list_edge_sites(settings, director_vm)
os._exit(0)
is_deploy_edge_site = bool(args.edge_sites) or args.edge_sites_all
logger.info("Deploying edge site(s) from CLI argument? %s",
str(is_deploy_edge_site))
edge_sites = None
# deploying edge site(s)?, validate sites are in settings, if
# valid return a list of sites based on args.
if is_deploy_edge_site:
edge_sites = validate_edge_sites_in_settings(args, settings)
logger.info("Edge sites after validation: {}".format(str(edge_sites)))
if args.validate_only is True:
logger.info("Only validating ini/properties config values")
else:
if args.overcloud_only is True:
logger.info("Only redeploying the overcloud")
logger.info("Settings .ini: " + settings.settings_file)
logger.info("Settings .properties " + settings.network_conf)
settings.get_version_info()
logger.info("source version # : " +
settings.source_version.decode('utf-8'))
tester = Checkpoints()
tester.verify_deployer_settings()
if args.validate_only is True:
logger.info("Settings validated")
os._exit(0)
tester.retreive_switches_config()
tester.sah_health_check()
# mutually exclusive command, configure tempest and quit.
if args.tempest_config_only:
logger.info("Only (re-)generating tempest.conf")
director_vm.configure_tempest()
os._exit(0)
# mutually exclusive command, run tempest and quit.
if args.run_tempest_only:
logger.info("Only running tempest, will configure " +
"tempest.conf if needed.")
director_vm.run_tempest()
os._exit(0)
logger.info("Uploading configs/iso/scripts.")
sah_node.clear_known_hosts()
sah_node.handle_lock_files()
sah_node.upload_iso()
sah_node.upload_director_scripts()
sah_node.upload_powerflexgw_scripts()
sah_node.upload_powerflexmgmt_scripts()
sah_node.enable_chrony_ports()
if args.overcloud_only is False and is_deploy_edge_site is False:
deploy_undercloud(settings, sah_node, tester, director_vm)
if args.undercloud_only:
return
elif is_deploy_edge_site is False:
logger.info("=== Skipped Director VM/Undercloud install")
logger.debug("Deleting overcloud stack")
director_vm.delete_overcloud()
if is_deploy_edge_site:
_is_oc_failed, _error_oc = tester.verify_overcloud_deployed()
if _is_oc_failed:
logger.error("Attempted to deploy edge site(s) but the "
"overcloud has not been deployed, "
"or failed to deploy. "
"Edge sites cannot be deployed without an "
"existing overcloud, exiting")
os._exit(0)
deploy_edge_sites(sah_node, director_vm, edge_sites)
os._exit(0)
else: # no edge sites arguments, just deploy overcloud
deploy_overcloud(director_vm)
_is_oc_failed, _err_oc = tester.verify_overcloud_deployed()
if _is_oc_failed:
raise _err_oc
# lastly, if there are edge sites defined in .ini
# and deploy_edge_sites is set to true in ini deploy the sites
if settings.deploy_edge_sites and settings.edge_sites:
logger.info(
"Automatic edge site deployment is set to true "
"in the ini (deploy_edge_sites==true), "
"deploying all defined edge sites: %s",
str(settings.edge_sites))
deploy_edge_sites(sah_node, director_vm, settings.edge_sites)
if settings.hpg_enable:
logger.info("HugePages has been successfully configured "
"with size: " + settings.hpg_size)
if settings.numa_enable:
logger.info("NUMA has been successfully configured "
"with hostos_cpus count: " + settings.hostos_cpu_count)
director_vm.summarize_deployment()
tester.verify_computes_virtualization_enabled()
tester.verify_backends_connectivity()
director_vm.enable_fencing()
director_vm.run_sanity_test()
external_sub_guid = director_vm.get_sanity_subnet()
if external_sub_guid:
director_vm.configure_tempest()
run_tempest(director_vm)
if len(settings.powerflex_nodes) > 0:
powerflexgw_vm = Powerflexgw()
deploy_powerflex_gw(settings, sah_node, tester, powerflexgw_vm)
if settings.enable_powerflex_mgmt:
powerflexmgmt_vm = Powerflexmgmt()
deploy_powerflex_mgmt(settings, sah_node, tester,
powerflexmgmt_vm)
logger.info("Deployment summary info; useful ip's etc.. " +
"/auto_results/deployment_summary.log")
except: # noqa: E722
logger.error(traceback.format_exc())
e = sys.exc_info()[0]
logger.error(e)
print(e)
print(traceback.format_exc())
ret_code = 1
logger.info("log : /auto_results/ ")
sys.exit(ret_code)
def main():
# parse the arguments
random.seed(args.manual_seed)
torch.manual_seed(args.manual_seed)
if args.save_results:
utils.saveargs(args, config_file)
# initialize the checkpoint class
checkpoints = Checkpoints(args)
# Create Model
models = Model(args)
model, model_dict, evaluation = models.setup(checkpoints)
print('Model:\n\t{model}\nTotal params:\n\t{npar:.2f}M'.format(
model=args.model_type,
npar=sum(p.numel() for p in model['feat'].parameters()) / 1000000.0))
# The trainer handles the training loop
trainer = Trainer(args, model, model_dict['loss'], evaluation)
# The trainer handles the evaluation on validation set
tester = Tester(args, model, model_dict['loss'], evaluation)
test_freq = 1
dataloader = Dataloader(args)
if args.extract_feat:
loaders = dataloader.create(flag='Test')
tester.extract_features(loaders)
# tester.extract_features_h5py(loaders, len(dataloader.dataset_test))
elif args.just_test:
loaders = dataloader.create(flag='Test')
acc_test = tester.test(args.epoch_number, loaders)
print(acc_test)
else:
loaders = dataloader.create()
if args.dataset_train == 'ClassSamplesDataLoader':
loaders['train'] = dataloader.dataset_train
# start training !!!
acc_best = 0
loss_best = 999
stored_models = {}
for epoch in range(args.nepochs - args.epoch_number):
epoch += args.epoch_number
print('\nEpoch %d/%d\n' % (epoch + 1, args.nepochs))
# train for a single epoch
# loss_train = 3.0
loss_train = trainer.train(epoch, loaders, checkpoints, acc_best)
if float(epoch) % test_freq == 0:
acc_test = tester.test(epoch, loaders)
if loss_best > loss_train:
model_best = True
loss_best = loss_train
acc_best = acc_test
if args.save_results:
stored_models['model'] = model
stored_models['loss'] = trainer.criterion
checkpoints.save(acc_best, stored_models, epoch, 'final',
model_best)
def main():
# parse the arguments
args = config.parse_args()
random.seed(args.manual_seed)
torch.manual_seed(args.manual_seed)
args.save = os.path.join(args.result_path, 'save')
args.logs = os.path.join(args.result_path, 'logs')
utils.saveargs(args)
# initialize the checkpoint class
checkpoints = Checkpoints(args)
# Create Model
models = Model(args)
rankgan_model, criterion = models.setup(checkpoints)
modelD = rankgan_model[0]
modelG = rankgan_model[1]
Encoder = rankgan_model[2]
prevD, prevG = None, None
if args.netD is not '':
checkpointD = checkpoints.load(args.netD)
modelD.load_state_dict(checkpointD)
if args.netG is not '':
checkpointG = checkpoints.load(args.netG)
modelG.load_state_dict(checkpointG)
if args.netE is not '':
checkpointE = checkpoints.load(args.netE)
Encoder.load_state_dict(checkpointE)
if args.prevD is not '':
prevD = copy.deepcopy(modelD)
checkpointDprev = checkpoints.load(args.prevD)
prevD.load_state_dict(checkpointDprev)
if args.prevG is not '':
prevG = copy.deepcopy(modelG)
checkpointGprev = checkpoints.load(args.prevG)
prevG.load_state_dict(checkpointGprev)
# Data Loading
dataloader = Dataloader(args)
loader_train = dataloader.create(flag="Train")
loader_test = dataloader.create(flag="Test")
# The trainer handles the training loop and evaluation on validation set
trainer = Trainer(args, modelD, modelG, Encoder, criterion, prevD, prevG)
for epoch in range(args.nepochs):
# train for a single epoch
# cur_time = time.time()
# if stage == 2:
loss_train = trainer.train(epoch, loader_train)
# if stage > 0:
# disc_acc = trainer.test(stage, epoch, loader_test)
# print("Time taken = {}".format(time.time() - cur_time))
try:
torch.save(modelD.state_dict(), '%s/netD.pth' % (args.save, stage))
for i in range(args.nranks - 1):
torch.save(modelG.state_dict(),
'%s/order_%d_netG.pth' % (i + 1, stage))
except Exception as e:
print(e)
import os
import datetime
import utils
import copy
import config
# parse the arguments
args = config.parse_args()
random.seed(args.manual_seed)
torch.manual_seed(args.manual_seed)
args.save = os.path.join(args.result_path, 'save')
args.logs = os.path.join(args.result_path, 'logs')
utils.saveargs(args)
# initialize the checkpoint class
checkpoints = Checkpoints(args)
# Create Model
models = Model(args)
gogan_model, criterion = models.setup(checkpoints)
netD = gogan_model[0]
netG = gogan_model[1]
netE = gogan_model[2]
if args.netD is not '':
checkpointD = checkpoints.load(args.netD)
netD.load_state_dict(checkpointD)
if args.netG is not '':
checkpointG = checkpoints.load(args.netG)
netG.load_state_dict(checkpointG)
if args.netE is not '':
def main():
# parse the arguments
args = config.parse_args()
random.seed(args.manual_seed)
torch.manual_seed(args.manual_seed)
args.save = os.path.join(args.result_path, 'save')
args.logs = os.path.join(args.result_path, 'logs')
utils.saveargs(args)
# initialize the checkpoint class
checkpoints = Checkpoints(args)
# Create Model
models = Model(args)
rankgan_model, criterion = models.setup(checkpoints)
modelD = rankgan_model[0]
modelG = rankgan_model[1]
Encoder = rankgan_model[2]
prevD, prevG = None, None
if args.netD is not '':
checkpointD = checkpoints.load(args.netD)
modelD.load_state_dict(checkpointD)
if args.netG is not '':
checkpointG = checkpoints.load(args.netG)
modelG.load_state_dict(checkpointG)
if args.netE is not '':
checkpointE = checkpoints.load(args.netE)
Encoder.load_state_dict(checkpointE)
if args.prevD is not '':
prevD = copy.deepcopy(modelD)
checkpointDprev = checkpoints.load(args.prevD)
prevD.load_state_dict(checkpointDprev)
if args.prevG is not '':
prevG = copy.deepcopy(modelG)
checkpointGprev = checkpoints.load(args.prevG)
prevG.load_state_dict(checkpointGprev)
# Data Loading
dataloader = Dataloader(args)
loader_train = dataloader.create(flag="Train")
loader_test = dataloader.create(flag="Test")
# The trainer handles the training loop and evaluation on validation set
trainer = Trainer(args, modelD, modelG, Encoder, criterion, prevD, prevG)
# start training !!!
num_stages = args.num_stages
stage_epochs = args.stage_epochs
for stage in range(args.start_stage, num_stages):
# check whether ready to start new stage and if not, optimize discriminator
# if stage > 2:
# print("Optimizing Discriminator")
# trainer.setup_stage(stage, loader_test)
# opt_disc_flag = True
# epoch = 0
# while opt_disc_flag:
# opt_disc_flag = trainer.optimize_discriminator(stage-1, epoch, loader_train)
# epoch += 1
# setup trainer for the stage
trainer.setup_stage(stage, loader_test)
print("Training for Stage {}".format(stage))
for epoch in range(stage_epochs[stage]):
# train for a single epoch
# cur_time = time.time()
# if stage == 2:
loss_train = trainer.train(stage, epoch, loader_train)
# if stage > 0:
# disc_acc = trainer.test(stage, epoch, loader_test)
# print("Time taken = {}".format(time.time() - cur_time))
try:
torch.save(modelD.state_dict(),
'%s/stage_%d_netD.pth' % (args.save, stage))
torch.save(modelG.state_dict(),
'%s/stage_%d_netG.pth' % (args.save, stage))
torch.save(Encoder.state_dict(),
'%s/stage_%d_netE.pth' % (args.save, stage))
except Exception as e:
print(e)
def deploy():
ret_code = 0
# noinspection PyBroadException
logger.debug("=================================")
logger.info("=== Starting up ...")
logger.debug("=================================")
try:
settings, args = get_settings()
if args.validate_only is True:
logger.info("Only validating ini/properties config values")
else:
if args.overcloud_only is True:
logger.info("Only redeploying the overcloud")
if args.skip_dashboard_vm is True:
logger.info("Skipping Dashboard VM install")
logger.info("Settings .ini: " + settings.settings_file)
logger.info("Settings .properties " + settings.network_conf)
settings.get_version_info()
logger.info("source version # : " + settings.source_version)
tester = Checkpoints()
tester.verify_deployer_settings()
if args.validate_only is True:
logger.info("Settings validated")
os._exit(0)
tester.retreive_switches_config()
non_sah_nodes = (settings.controller_nodes + settings.compute_nodes +
settings.ceph_nodes)
sah_node = Sah()
tester.sah_health_check()
# mutually exclusive command, configure tempest and quit.
if args.tempest_config_only:
logger.info("Only (re-)generating tempest.conf")
director_vm = Director()
director_vm.configure_tempest()
os._exit(0)
# mutually exclusive command, run tempest and quit.
if args.run_tempest_only:
logger.info("Only running tempest, will configure " +
"tempest.conf if needed.")
director_vm = Director()
director_vm.run_tempest()
os._exit(0)
logger.info("Uploading configs/iso/scripts.")
sah_node.clear_known_hosts()
sah_node.handle_lock_files()
sah_node.upload_iso()
sah_node.upload_director_scripts()
director_ip = settings.director_node.public_api_ip
if args.overcloud_only is False:
Ssh.execute_command(director_ip, "root",
settings.director_node.root_password,
"subscription-manager remove --all")
Ssh.execute_command(director_ip, "root",
settings.director_node.root_password,
"subscription-manager unregister")
sah_node.delete_director_vm()
logger.info("=== create the director vm")
sah_node.create_director_vm()
tester.director_vm_health_check()
logger.info("Preparing the Director VM")
director_vm = Director()
director_vm.apply_internal_repos()
logger.debug("=== Uploading & configuring undercloud.conf . "
"environment yaml ===")
director_vm.upload_update_conf_files()
logger.info("=== installing the director & undercloud ===")
director_vm.inject_ssh_key()
director_vm.upload_cloud_images()
director_vm.install_director()
tester.verify_undercloud_installed()
if args.undercloud_only:
return
else:
logger.info("=== Skipped Director VM/Undercloud install")
director_vm = Director()
logger.debug("Deleting overcloud stack")
director_vm.delete_overcloud()
if args.skip_dashboard_vm is False:
logger.debug("Delete the Dashboard VM")
dashboard_ip = settings.dashboard_node.public_api_ip
logger.debug(
Ssh.execute_command(dashboard_ip, "root",
settings.dashboard_node.root_password,
"subscription-manager remove --all"))
Ssh.execute_command(dashboard_ip, "root",
settings.dashboard_node.root_password,
"subscription-manager unregister")
sah_node.delete_dashboard_vm()
logger.info("=== creating Dashboard VM")
sah_node.create_dashboard_vm()
tester.dashboard_vm_health_check()
else:
logger.info("Skipped the Dashboard VM install")
logger.info("=== Preparing the overcloud ===")
# The network-environment.yaml must be setup for use during DHCP
# server configuration
director_vm.setup_net_envt()
director_vm.configure_dhcp_server()
director_vm.node_discovery()
director_vm.configure_idracs()
director_vm.import_nodes()
director_vm.node_introspection()
director_vm.update_sshd_conf()
director_vm.assign_node_roles()
director_vm.revert_sshd_conf()
director_vm.setup_templates()
logger.info("=== Installing the overcloud ")
logger.debug("installing the overcloud ... this might take a while")
director_vm.deploy_overcloud()
cmd = "source ~/stackrc; openstack stack list | grep " \
+ settings.overcloud_name + " | awk '{print $8}'"
overcloud_status = \
Ssh.execute_command_tty(director_ip,
settings.director_install_account_user,
settings.director_install_account_pwd,
cmd)[0]
logger.debug("=== Overcloud stack state : " + overcloud_status)
if settings.hpg_enable:
logger.info(
" HugePages has been successfully configured with size: " +
settings.hpg_size)
if settings.numa_enable:
logger.info(" NUMA has been successfully configured"
" with hostos_cpus count: " +
settings.hostos_cpu_count)
logger.info("====================================")
logger.info(" OverCloud deployment status: " + overcloud_status)
logger.info(" log : /auto_results/ ")
logger.info("====================================")
if "CREATE_COMPLETE" not in overcloud_status:
raise AssertionError("OverCloud did not install properly : " +
overcloud_status)
director_vm.summarize_deployment()
tester.verify_computes_virtualization_enabled()
tester.verify_backends_connectivity()
if args.skip_dashboard_vm is False:
director_vm.configure_dashboard()
director_vm.enable_fencing()
director_vm.run_sanity_test()
external_sub_guid = director_vm.get_sanity_subnet()
if external_sub_guid:
director_vm.configure_tempest()
run_tempest()
logger.info("Deployment summary info; useful ip's etc.. " +
"/auto_results/deployment_summary.log")
except: # noqa: E722
logger.error(traceback.format_exc())
e = sys.exc_info()[0]
logger.error(e)
print e
print traceback.format_exc()
ret_code = 1
logger.info("log : /auto_results/ ")
sys.exit(ret_code)
def main():
# load pop for extrapolation
pop = pickle.load(open("Results/CIFAR10_baseline/Run5/pop_extra.pkl",
"rb"))
for i in range(len(pop)):
genome = pop[i].genome
# parse the arguments
args = parser.parse_args()
args.save = os.path.join("Extrapolation_results",
"Model_ID_{}".format(pop[i].id))
random.seed(args.manual_seed)
torch.manual_seed(args.manual_seed)
utils.saveargs(args)
# initialize the checkpoint class
checkpoints = Checkpoints(args)
# Create Model
# genome = [[[0], [0, 0], [0, 0, 0], [0, 0, 0, 1], [0, 0, 0, 0, 0], [0]],
# [[0], [0, 0], [0, 1, 0], [0, 1, 0, 1], [0, 1, 1, 1, 1], [0]],
# [[1], [0, 0], [0, 0, 0], [0, 1, 1, 1], [0, 0, 0, 0, 1], [1]]]
# genome = [[[0], [0, 1], [1, 0, 0], [0, 0, 1, 1], [0, 0, 1, 1, 1], [0]],
# [[0], [0, 1], [0, 0, 0], [0, 0, 1, 1], [1, 1, 1, 1, 1], [0]],
# [[0], [0, 0], [0, 0, 1], [1, 1, 0, 1], [1, 1, 0, 1, 1], [1]]]
models = Model(args, genome)
model, criterion, num_params = models.setup()
model = calculate_flops.add_flops_counting_methods(model)
# print(model)
# Data Loading
dataloader = Dataloader(args)
loaders = dataloader.create()
# The trainer handles the training loop
trainer = Trainer(args, model, criterion)
# The trainer handles the evaluation on validation set
tester = Tester(args, model, criterion)
# start training !!!
loss_best = 1e10
acc_test_list = []
acc_best = 0
for epoch in range(args.nepochs):
# train for a single epoch
start_time_epoch = time.time()
if epoch == 0:
model.start_flops_count()
loss_train, acc_train = trainer.train(epoch, loaders)
loss_test, acc_test = tester.test(epoch, loaders)
acc_test_list.append(acc_test)
if acc_test > acc_best:
model_best = True
# update the best test accu found so found
acc_best = acc_test
loss_best = loss_test
checkpoints.save(epoch, model, model_best)
time_elapsed = np.round((time.time() - start_time_epoch), 2)
if epoch == 0:
n_flops = (model.compute_average_flops_cost() / 1e6 / 2)
if np.isnan(np.average(loss_train)):
break
print(
"Epoch {:d}:, test error={:0.2f}, FLOPs={:0.2f}M, n_params={:0.2f}M, {:0.2f} sec"
.format(epoch, 100.0 - acc_test, n_flops, num_params / 1e6,
time_elapsed))
# save the final model parameter
# torch.save(model.state_dict(),
# "model_file/model%d.pth" % int(args.genome_id - 1))
pop[i].fitness[0] = acc_best
pop[i].fitness[1] = n_flops
pop[i].n_params = num_params
# error = 100 - np.mean(acc_test_list[-3:])
# accuracy = acc_best
# fitness = [acc_best, n_flops, num_params]
# with open("output_file/output%d.pkl" % int(args.genome_id - 1), "wb") as f:
# pickle.dump(fitness, f)
with open("Results/CIFAR10_baseline/Run5/pop_extra_evaluated.pkl",
"wb") as f:
pickle.dump(pop, f)
import torchvision from model import Model from config import parser from dataloader import Dataloader from checkpoints import Checkpoints from train import Trainer import utils # parse the arguments args = parser.parse_args() random.seed(args.manual_seed) torch.manual_seed(args.manual_seed) utils.saveargs(args) # initialize the checkpoint class checkpoints = Checkpoints(args) # Create Model models = Model(args) model, criterion = models.setup(checkpoints) # Data Loading dataloader = Dataloader(args) loader_train, loader_test = dataloader.create() # The trainer handles the training loop and evaluation on validation set trainer = Trainer(args, model, criterion) # start training !!! acc_best = 0 for epoch in range(args.nepochs):
import time
import datetime
import copy
import os
import config
# parse the arguments
args = config.parse_args()
random.seed(args.manual_seed)
torch.manual_seed(args.manual_seed)
args.save = os.path.join(args.result_path, 'save')
args.logs = os.path.join(args.result_path, 'logs')
utils.saveargs(args)
# initialize the checkpoint class
checkpoints = Checkpoints(args)
# Create Model
models = Model(args)
gogan_model, criterion = models.setup(checkpoints)
modelD = gogan_model[0]
modelG = gogan_model[1]
Encoder = gogan_model[2]
prevD, prevG = None, None
if args.netD is not '':
checkpointD = checkpoints.load(args.netD)
modelD.load_state_dict(checkpointD)
if args.netG is not '':
checkpointG = checkpoints.load(args.netG)
modelG.load_state_dict(checkpointG)
def main():
# parse the arguments
random.seed(args.manual_seed)
torch.manual_seed(args.manual_seed)
if args.save_results:
utils.saveargs(args, config_file)
# initialize the checkpoint class
checkpoints = Checkpoints(args)
# Create Model
models = Model(args)
model_dict, evaluation = models.setup(checkpoints)
print('Model:\n\t{model}\nTotal params:\n\t{npar:.4f}M'.format(
model=args.model_type,
npar=sum(p.numel() for p in model_dict['model'].parameters()) / 1000000.0))
#### get kernel information ####
ndemog = args.ndemog
ndemog = list(range(ndemog))
demog_combs = list(combinations(ndemog, 2))
#### get kernel information ####
#### create writer for tensor boader ####
if args.save_results:
writer = SummaryWriter(args.tblog_dir)
else:
writer = None
#### create writer for tensor boader ####
# The trainer handles the training loop
trainer = Trainer(args, model_dict['model'], model_dict['loss'], model_dict['optimizer'], writer)
# The trainer handles the evaluation on validation set
tester = Tester(args, model_dict['model'], evaluation, writer)
test_freq = 1
dataloader = Dataloader(args)
if args.extract_feat:
loaders = dataloader.create(flag='Test')
tester.extract_features(loaders, 1)
elif args.just_test:
loaders = dataloader.create(flag='Test')
acc_test,acc_mean = tester.test(loaders, 1)
print(acc_test, acc_mean)
else:
loaders = dataloader.create()
if args.dataset_train == 'ClassSamplesDataLoader':
loaders['train'] = dataloader.dataset_train
# start training !!!
acc_best = 0
loss_best = 999
stored_models = {}
for epoch in range(args.nepochs-args.epoch_number):
epoch += args.epoch_number
print('\nEpoch %d/%d\n' % (epoch + 1, args.nepochs))
# train for a single epoch
loss_train = trainer.train(loaders, epoch)
acc_test=0
if float(epoch) % test_freq == 0:
acc_test,acc_mean = tester.test(loaders, epoch)
if loss_best > loss_train:
loss_best = loss_train
acc_best = acc_test
if float(epoch) % test_freq == 0 and args.save_results:
stored_models['model'] = trainer.model
stored_models['loss'] = trainer.criterion
stored_models['optimizer'] = trainer.optimizer
checkpoints.save(acc_test, stored_models, epoch)
if epoch == args.fuse_epoch:
update_kernels(args, trainer.model, demog_combs, ndemog)
if args.save_results:
writer.close()
def deploy():
ret_code = 0
# noinspection PyBroadException
try:
logger.debug("=================================")
logger.info("=== Starting up ...")
logger.debug("=================================")
parser = argparse.ArgumentParser(
description='JetPack 10.x deployer')
parser.add_argument('-s', '--settings',
help='ini settings file, e.g settings/acme.ini',
required=True)
parser.add_argument('-undercloud_only', '--undercloud_only',
help='Only reinstall the undercloud',
action='store_true', required=False)
parser.add_argument('-overcloud_only', '--overcloud_only',
help='Only reinstall the overcloud',
action='store_true', required=False)
parser.add_argument('-skip_dashboard_vm', '--skip_dashboard_vm',
help='Do not reinstall the Dashboard VM',
action='store_true',
required=False)
parser.add_argument('-validate_only', '--validate_only',
help='No deployment - just validate config values',
action='store_true',
required=False)
args, others = parser.parse_known_args()
if len(others) > 0:
parser.print_help()
msg = "Invalid argument(s) :"
for each in others:
msg += " " + each + ";"
raise AssertionError(msg)
if args.validate_only is True:
logger.info("Only validating ini/properties config values")
else:
if args.overcloud_only is True:
logger.info("Only redeploying the overcloud")
if args.skip_dashboard_vm is True:
logger.info("Skipping Dashboard VM install")
logger.debug("loading settings files " + args.settings)
settings = Settings(args.settings)
logger.info("Settings .ini: " + settings.settings_file)
logger.info("Settings .properties " + settings.network_conf)
settings.get_version_info()
logger.info("source version # : " + settings.source_version)
tester = Checkpoints()
tester.verify_deployer_settings()
if args.validate_only is True:
logger.info("Settings validated")
os._exit(0)
if settings.retreive_switches_config is True:
tester.retreive_switches_config()
non_sah_nodes = (settings.controller_nodes +
settings.compute_nodes +
settings.ceph_nodes)
sah_node = Sah()
tester.sah_health_check()
logger.info("Uploading configs/iso/scripts..")
sah_node.clear_known_hosts()
sah_node.handle_lock_files()
sah_node.upload_iso()
sah_node.upload_director_scripts()
director_ip = settings.director_node.public_api_ip
if args.overcloud_only is False:
Ssh.execute_command(director_ip,
"root",
settings.director_node.root_password,
"subscription-manager remove --all")
Ssh.execute_command(director_ip,
"root",
settings.director_node.root_password,
"subscription-manager unregister")
sah_node.delete_director_vm()
logger.info("=== create the director vm")
sah_node.create_director_vm()
tester.director_vm_health_check()
logger.info("Preparing the Director VM")
director_vm = Director()
director_vm.apply_internal_repos()
logger.debug(
"=== Uploading & configuring undercloud.conf . "
"environment yaml ===")
director_vm.upload_update_conf_files()
logger.info("=== installing the director & undercloud ===")
director_vm.inject_ssh_key()
director_vm.upload_cloud_images()
director_vm.install_director()
tester.verify_undercloud_installed()
if args.undercloud_only:
return
else:
logger.info("=== Skipped Director VM/Undercloud install")
director_vm = Director()
logger.debug("Deleting overcloud stack")
director_vm.delete_overcloud()
if args.skip_dashboard_vm is False:
logger.debug("Delete the Dashboard VM")
dashboard_ip = settings.dashboard_node.public_api_ip
logger.debug(
Ssh.execute_command(dashboard_ip,
"root",
settings.dashboard_node.root_password,
"subscription-manager remove --all"))
Ssh.execute_command(dashboard_ip,
"root",
settings.dashboard_node.root_password,
"subscription-manager unregister")
sah_node.delete_dashboard_vm()
logger.info("=== creating Dashboard VM")
sah_node.create_dashboard_vm()
tester.dashboard_vm_health_check()
else:
logger.info("Skipped the Dashboard VM install")
logger.info("=== Preparing the overcloud ===")
# The network-environment.yaml must be setup for use during DHCP
# server configuration
director_vm.setup_net_envt()
director_vm.configure_dhcp_server()
director_vm.node_discovery()
director_vm.configure_idracs()
director_vm.import_nodes()
director_vm.node_introspection()
director_vm.update_sshd_conf()
director_vm.assign_node_roles()
director_vm.revert_sshd_conf
director_vm.setup_templates()
logger.info("=== Installing the overcloud ")
logger.debug("installing the overcloud ... this might take a while")
director_vm.deploy_overcloud()
cmd = "source ~/stackrc; openstack stack list | grep " \
+ settings.overcloud_name + " | awk '{print $6}'"
overcloud_status = \
Ssh.execute_command_tty(director_ip,
settings.director_install_account_user,
settings.director_install_account_pwd,
cmd)[0]
logger.debug("=== Overcloud stack state : " + overcloud_status)
if settings.hpg_enable:
logger.info(
" HugePages has been successfully configured with size: " +
settings.hpg_size)
if settings.numa_enable:
logger.info(
" NUMA has been successfully configured"
" with hostos_cpus count: " +
settings.hostos_cpu_count)
logger.info("====================================")
logger.info(" OverCloud deployment status: " + overcloud_status)
logger.info(" log : /auto_results/ ")
logger.info("====================================")
if "CREATE_COMPLETE" not in overcloud_status:
raise AssertionError(
"OverCloud did not install properly : " + overcloud_status)
director_vm.summarize_deployment()
tester.verify_computes_virtualization_enabled()
tester.verify_backends_connectivity()
if args.skip_dashboard_vm is False:
director_vm.configure_dashboard()
director_vm.enable_fencing()
director_vm.enable_instance_ha()
director_vm.configure_tempest()
director_vm.run_sanity_test()
run_tempest()
logger.info("Deployment summary info; useful ip's etc.. " +
"/auto_results/deployment_summary.log")
except:
logger.error(traceback.format_exc())
e = sys.exc_info()[0]
logger.error(e)
print e
print traceback.format_exc()
ret_code = 1
logger.info("log : /auto_results/ ")
sys.exit(ret_code)
import random
from model import Model
from config import parser
from dataloader import Dataloader
from checkpoints import Checkpoints
from train import Trainer
import utils
# parse the arguments
args = parser.parse_args()
random.seed(args.manual_seed)
torch.manual_seed(args.manual_seed)
utils.saveargs(args)
# initialize the checkpoint class
checkpoints = Checkpoints(args)
# Create Model
models = Model(args)
model, criterion = models.setup(checkpoints)
# Data Loading
dataloader = Dataloader(args)
loader_train, loader_test = dataloader.create()
print("\t\tBatches:\t", len(loader_train))
print("\t\tBatches (Test):\t", len(loader_test))
# The trainer handles the training loop and evaluation on validation set
trainer = Trainer(args, model, criterion)
if args.oneshot == 0:
