def __init__(self, conf):
self.globalConf = conf
self.verdict = True
self.output = ""
self.output_list = []
self.name = ""
self.description = ""
self.specific = ""
self.isWorkaround = False
self.skip = False
self.backup_name = ""
self.defaultParameter = dict()
self.logger = LoggerModule.LoggerModule()
self.host = HostModule.HostModule()
self.relayCard = RelayCardModule.RelayCardModule()
self.device = DeviceModule.DeviceModule()
self.workaround = WorkaroundModule.WorkaroundModule()
self.misc = MiscModule.MiscModule()
self.osManager = OsManagerModule.OsManagerModule()
self.flashFile = FlashFileModule.FlashFileModule()
self.logs = LogsModule.LogsModule()
self.efiVar = EfiVarModule.EfiVarModule()
self.dediprog = DediprogModule.DediprogModule()
self.outputModule = OutputModule.OutputModule()
self.configuration = ConfigurationModule.ConfigurationModule()
self.bootData = BootDataModule.BootDataModule()
self.resetIrq = ResetIrqModule.ResetIrqModule()
self.watchdog = WatchdogModule.WatchdogModule()
self.download = DownloadModule.DownloadModule()
self.campaign = CampaignModule.CampaignModule()
self.flash = FlashModule.FlashModule()
self.events = EventsModule.EventsModule()
def init(self, globalConf=None, externalRelayCard=None):
self.globalConf = globalConf
self.__logger = LoggerModule.LoggerModule()
self.__host = HostModule.HostModule()
self.__output = OutputModule.OutputModule()
self.configuration = ConfigurationModule.ConfigurationModule()
self.cardConfiguration = self.globalConf.get(
"RELAY_CARD_CONFIGURATION", dict())
self.relayConfiguration = self.cardConfiguration.get("relays", dict())
self.__delayBetweenCommands = 1
self.__allow_usb_toggle_event = False
self.__usb_connected = "unknown"
if externalRelayCard:
self.__relay = externalRelayCard
else:
self.__relay = self.__internalRelayCard()
self.__relay.setConfig(self.cardConfiguration)
if self.relayConfiguration:
self.__relay.setDefaultState()
else:
self.relayConfiguration["SwitchOnOff"] = -1
self.relayConfiguration["UsbHostPcConnect"] = -1
self.relayConfiguration["VolumeUp"] = -1
self.relayConfiguration["VolumeDown"] = -1
self.relayConfiguration["Dediprog"] = -1
self.relayConfiguration["PowerSupply"] = -1
self.relayConfiguration["J6B2_relay"] = -1
self.SwitchOnOff = self.relayConfiguration.get("SwitchOnOff", -1)
self.UsbHostPcConnect = self.relayConfiguration.get(
"UsbHostPcConnect", -1)
self.VolumeUp = self.relayConfiguration.get("VolumeUp", -1)
self.VolumeDown = self.relayConfiguration.get("VolumeDown", -1)
self.Dediprog = self.relayConfiguration.get("Dediprog", -1)
self.PowerSupply = self.relayConfiguration.get("PowerSupply", -1)
self.J6B2_relay = self.relayConfiguration.get("J6B2_relay", -1)
def init(self, globalConf=None):
self.__globalConf = globalConf
self.__logger = LoggerModule.LoggerModule()
self.__host = HostModule.HostModule()
self.__relayCard = RelayCardModule.RelayCardModule()
self.__device = DeviceModule.DeviceModule()
self.__workaround = WorkaroundModule.WorkaroundModule()
self.__getEndTime = None
self.__misc = MiscModule.MiscModule()
self.__osManager = OsManagerModule.OsManagerModule()
self.__flashFile = FlashFileModule.FlashFileModule()
self.__output = OutputModule.OutputModule()
# table to match uefi var with offset in RCSI_table
self.__RSCI_dict = dict()
self.__RSCI_dict["wake_source"] = 36
self.__RSCI_dict["reset_source"] = 37
self.__RSCI_dict["reset_type"] = 38
self.__RSCI_dict["shutdown_source"] = 39
self.__RSCI_dict["reset_extra_information"] = None
# table to match uefi var to name in sysfs
self.efi_vars_matching = dict()
self.efi_vars_matching["wake_source"] = "WakeSource"
self.efi_vars_matching["reset_source"] = "ResetSource"
self.efi_vars_matching["reset_type"] = "ResetType"
self.efi_vars_matching["shutdown_source"] = "ShutdownSource"
self.efi_vars_matching["mode"] = "LoaderEntryLast"
self.efi_vars_matching["watchdog"] = "WdtCounter"
def init(self, globalConf=None):
self.__globalConf = globalConf
self.__logger = LoggerModule.LoggerModule()
self.__host = HostModule.HostModule()
self.__device = DeviceModule.DeviceModule()
self.__workaround = WorkaroundModule.WorkaroundModule()
self.__configuration = ConfigurationModule.ConfigurationModule()
self.__misc = MiscModule.MiscModule()
self.__osManager = OsManagerModule.OsManagerModule()
self.__output = OutputModule.OutputModule()
self.__report = ReportModule.ReportModule()
self.logsFiles = list()
self.__mandatoryLogsFiles = ["history_event"]
self.tc_name = ""
self.tc_name_number = ""
campaign_report_path = self.__globalConf.get("REPORT_PATH")
if campaign_report_path and os.path.isdir(campaign_report_path):
self.bootota_log_path = os.path.abspath(os.path.join(campaign_report_path, "logs_bootota"))
# if directory not created, create it for future use in TC
if not os.path.isdir(self.bootota_log_path):
os.makedirs(self.bootota_log_path)
self.__logger.printLog("WARNING", "LogsModule init: storing BootOta specific data into "
"{}".format(self.bootota_log_path))
else:
self.__logger.printLog("WARNING", "LogsModule init: invalid or empty report path "
"('{}')".format(campaign_report_path))
self.bootota_log_path = ""
self.tc_report_path = ""
def init(self, globalConf):
self.__globalConf = globalConf
self.__logger = LoggerModule.LoggerModule()
self.__output = OutputModule.OutputModule()
self.__host = HostModule.HostModule()
self.local_files_handler = self.__FilesHandler()
self.extra_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "Extras"))
self.otaBuild = ""
def init(self, globalConf=None):
self.__globalConf = globalConf
self.__logger = LoggerModule.LoggerModule()
self.__host = HostModule.HostModule()
self.__relayCard = RelayCardModule.RelayCardModule()
self.__output = OutputModule.OutputModule()
self.__misc = MiscModule.MiscModule()
self.__configuration = ConfigurationModule.ConfigurationModule()
def init(self, globalConf):
self.__globalConf = globalConf
self.__logger = LoggerModule.LoggerModule()
self.__host = HostModule.HostModule()
self.__output = OutputModule.OutputModule()
self.__efiVar = EfiVarModule.EfiVarModule()
self.__resetIrq = ResetIrqModule.ResetIrqModule()
self.__device = DeviceModule.DeviceModule()
self.__osManager = OsManagerModule.OsManagerModule()
self.__logs = LogsModule.LogsModule()
self.__workaround = WorkaroundModule.WorkaroundModule()
self.__configuration = ConfigurationModule.ConfigurationModule()
self.__RSCI_table = list()
def init(self, globalConf):
self.__globalConf = globalConf
self.__logger = LoggerModule.LoggerModule()
self.__host = HostModule.HostModule()
self.__output = OutputModule.OutputModule()
self.__device = DeviceModule.DeviceModule()
self.__osManager = OsManagerModule.OsManagerModule()
self.__misc = MiscModule.MiscModule()
self.__workaround = WorkaroundModule.WorkaroundModule()
self.__configuration = ConfigurationModule.ConfigurationModule()
self.__logs = LogsModule.LogsModule()
self.force_ramdump_removal = False
self.wd_handler = self.HistoryEventHandler(self.__globalConf)
def __init__(self, conf):
self.__globalConf = conf
self.__historyEvent = ""
self.__initial_content = []
self.__delta = ""
self.__crashlogs = None
self.__configuration = ConfigurationModule.ConfigurationModule()
self.__output = OutputModule.OutputModule()
self.__logger = LoggerModule.LoggerModule()
self.__host = HostModule.HostModule()
self.__device = DeviceModule.DeviceModule()
self.__workaround = WorkaroundModule.WorkaroundModule()
self.__logs = LogsModule.LogsModule()
self.__misc = MiscModule.MiscModule()
def __init__(self, conf):
self.globalConf = conf
self.logger = LoggerModule.LoggerModule()
self.host = HostModule.HostModule()
self.relayCard = RelayCardModule.RelayCardModule()
self.device = DeviceModule.DeviceModule()
self.workaround = WorkaroundModule.WorkaroundModule()
self.misc = MiscModule.MiscModule()
self.osManager = OsManagerModule.OsManagerModule()
self.flashFile = FlashFileModule.FlashFileModule()
self.logs = LogsModule.LogsModule()
self.efiVar = EfiVarModule.EfiVarModule()
self.dediprog = DediprogModule.DediprogModule()
self.output = OutputModule.OutputModule()
self.configuration = ConfigurationModule.ConfigurationModule()
self.bootData = BootDataModule.BootDataModule()
self.resetIrq = ResetIrqModule.ResetIrqModule()
self.download = DownloadModule.DownloadModule()
self.flash = FlashModule.FlashModule()
self.campaign = CampaignModule.CampaignModule()
self.watchdog = WatchdogModule.WatchdogModule()
self.report = ReportModule.ReportModule()
self.events = EventsModule.EventsModule()
self.step_list = list()
self.verdict = True
self.output = ""
self.name = ""
self.skipping = False
self.tc_crashlogs = None
self.allowed_TC_upon_success = []
self.removed_TC_upon_success = []
self.allowed_TC_upon_failure = []
self.removed_TC_upon_failure = []
self.allowed_TC_upon_skip = []
self.removed_TC_upon_skip = []
self.campaign_constraint_file = ""
self.campaign_constraint_file_name = "PupdrCampaignConstraints.json"
self.enable_init = True
self.enable_final = True
self.INVALID = "INVALID"
self.VALID = "VALID"
self.BLOCKED = "BLOCKED"
self.FAILURE = "FAILURE"
self.SUCCESS = "SUCCESS"
self.verdict_name = self.SUCCESS
self.step_number = 0
self.output_dict = {}
def init(self, globalConf=None):
self.__globalConf = globalConf
self.__logger = LoggerModule.LoggerModule()
self.__output = OutputModule.OutputModule()
self.flash = None
self.boot = None
self.logs = None
self.timeout = None
self.download = None
self.custom = None
self.boot_vars = None
self.workaround = None
self.board = None
self.branch = None
self.test_description = None
self.__configuration_list = dict()
self.__branchList = None
self.__boardList = None
self.__current_config_name = ""
def main():
t = Timer()
input_object = InputReader.InputReader(CSVPath, TXTPath)
t.tic()
part_type, init_r, init_v, mass, init_rho, init_energy, body_forces, init_sigma, normals = input_object.read_csv(
)
params_list = input_object.read_txt()
print()
t.toc("Loading simulation data")
t.tic()
output_object = OutputModule.Output(outPath)
solver_object = Solver.Solver(params_list, init_r, init_rho)
solver_object.solve(part_type, init_r, init_v, init_rho, mass, init_sigma,
init_energy, body_forces, output_object)
t.toc("Problem solution")
def init(self, globalConf):
self.__globalConf = globalConf
self.__logger = LoggerModule.LoggerModule()
self.__host = HostModule.HostModule()
self.__output = OutputModule.OutputModule()
self.__configuration = ConfigurationModule.ConfigurationModule()
def train():
# read configuration file
with open(FLAGS.config_json) as config_json:
config = json.load(config_json)
"""Train the Vnet model"""
with tf.Graph().as_default():
global_step = tf.train.get_or_create_global_step()
# patch_shape(batch_size, height, width, depth, channels)
input_channel_num = len(config['TrainingSetting']['Data']['ImageFilenames'])
input_batch_shape = (None, FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer, input_channel_num)
output_batch_shape = (None, FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer, 1)
if FLAGS.attention:
images_placeholder, labels_placeholder, distmap_placeholder = placeholder_inputs(input_batch_shape,output_batch_shape, attention=True)
else:
images_placeholder, labels_placeholder = placeholder_inputs(input_batch_shape,output_batch_shape, attention=False)
if FLAGS.image_log:
for batch in range(FLAGS.batch_size):
# plot images in tensorboard
for image_channel in range(input_channel_num):
images_log = tf.cast(images_placeholder[batch:batch+1,:,:,:,image_channel], dtype=tf.uint8)
tf.summary.image(config['TrainingSetting']['Data']['ImageFilenames'][image_channel], tf.transpose(images_log,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
if FLAGS.attention:
# dist map will be plot in color
distmap_log = grayscale_to_rainbow(tf.transpose(distmap_placeholder[batch:batch+1,:,:,:,0],[3,1,2,0]))
distmap_log = tf.cast(tf.scalar_mul(255,distmap_log), dtype=tf.uint8)
tf.summary.image("distmap", distmap_log,max_outputs=FLAGS.patch_layer)
# plot labels
labels_log = tf.cast(tf.scalar_mul(255,labels_placeholder[batch:batch+1,:,:,:,0]), dtype=tf.uint8)
tf.summary.image("label", tf.transpose(labels_log,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
# Get images and labels
train_data_dir = os.path.join(FLAGS.data_dir,'training')
test_data_dir = os.path.join(FLAGS.data_dir,'testing')
# support multiple image input, but here only use single channel, label file should be a single file with different classes
# Force input pipepline to CPU:0 to avoid operations sometimes ended up at GPU and resulting a slow down
with tf.device('/cpu:0'):
# create transformations to image and labels
trainTransforms = [
NiftiDataset3D.ExtremumNormalization(0.1),
# NiftiDataset.Normalization(),
NiftiDataset3D.Resample((0.25,0.25,0.25)),
NiftiDataset3D.Padding((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer)),
NiftiDataset3D.RandomCrop((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),FLAGS.drop_ratio,FLAGS.min_pixel),
# NiftiDataset.ConfidenceCrop((FLAGS.patch_size*3, FLAGS.patch_size*3, FLAGS.patch_layer*3),(0.0001,0.0001,0.0001)),
# NiftiDataset.BSplineDeformation(randomness=2),
# NiftiDataset.ConfidenceCrop((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),(0.5,0.5,0.5)),
# NiftiDataset.ConfidenceCrop2((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),rand_range=32,probability=0.8),
NiftiDataset3D.RandomFlip([True, False, False]),
NiftiDataset3D.RandomNoise()
]
TrainDataset = NiftiDataset3D.NiftiDataset(
data_dir=train_data_dir,
image_filenames=config['TrainingSetting']['Data']['ImageFilenames'],
label_filename=config['TrainingSetting']['Data']['LabelFilename'],
transforms=trainTransforms,
train=True,
distmap=FLAGS.attention
)
trainDataset = TrainDataset.get_dataset()
trainDataset = trainDataset.shuffle(buffer_size=5)
trainDataset = trainDataset.batch(FLAGS.batch_size,drop_remainder=True)
if FLAGS.testing:
# use random crop for testing
testTransforms = [
NiftiDataset3D.ExtremumNormalization(0.1),
# NiftiDataset.Normalization(),
NiftiDataset3D.Resample((0.25,0.25,0.25)),
NiftiDataset3D.Padding((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer)),
NiftiDataset3D.RandomCrop((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),FLAGS.drop_ratio,FLAGS.min_pixel),
# NiftiDataset.ConfidenceCrop((FLAGS.patch_size*2, FLAGS.patch_size*2, FLAGS.patch_layer*2),(0.0001,0.0001,0.0001)),
# NiftiDataset.BSplineDeformation(),
# NiftiDataset.ConfidenceCrop((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),(0.75,0.75,0.75)),
# NiftiDataset.ConfidenceCrop2((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),rand_range=32,probability=0.8),
NiftiDataset3D.RandomFlip([True, False, False]),
]
TestDataset = NiftiDataset3D.NiftiDataset(
data_dir=test_data_dir,
image_filenames=config['TrainingSetting']['Data']['ImageFilenames'],
label_filename=config['TrainingSetting']['Data']['LabelFilename'],
transforms=testTransforms,
train=True,
distmap=FLAGS.attention
)
testDataset = TestDataset.get_dataset()
testDataset = testDataset.shuffle(buffer_size=5)
testDataset = testDataset.batch(FLAGS.batch_size,drop_remainder=True)
train_iterator = trainDataset.make_initializable_iterator()
next_element_train = train_iterator.get_next()
if FLAGS.testing:
test_iterator = testDataset.make_initializable_iterator()
next_element_test = test_iterator.get_next()
# Initialize the model
with tf.name_scope("vnet"):
model = VNet.VNet(
num_classes=2, # binary for 2
keep_prob=1.0, # default 1
num_channels=16, # default 16
num_levels=3, # default 4
num_convolutions=(1,2,2), # default (1,2,3,3), size should equal to num_levels
bottom_convolutions=3, # default 3
activation_fn="prelu") # default relu
logits_vnet = model.network_fn(images_placeholder)
if FLAGS.attention:
with tf.name_scope("attention"):
attentionModule = attention.AttentionModule(
num_classes=2,
is_training=True,
activation_fn="relu",
keep_prob=1.0)
logits_attention = attentionModule.GetNetwork(logits_vnet)
softmax_attention = tf.nn.softmax(logits_attention, name="softmax_attention")
for batch in range(FLAGS.batch_size):
softmax_att_log_0 = grayscale_to_rainbow(tf.transpose(softmax_attention[batch:batch+1,:,:,:,0],[3,1,2,0]))
softmax_att_log_1 = grayscale_to_rainbow(tf.transpose(softmax_attention[batch:batch+1,:,:,:,1],[3,1,2,0]))
softmax_att_log_0 = tf.cast(tf.scalar_mul(255,softmax_att_log_0), dtype=tf.uint8)
softmax_att_log_1 = tf.cast(tf.scalar_mul(255,softmax_att_log_1), dtype=tf.uint8)
if FLAGS.image_log:
tf.summary.image("softmax_attention_0", softmax_att_log_0,max_outputs=FLAGS.patch_layer)
tf.summary.image("softmax_attention_1", softmax_att_log_1,max_outputs=FLAGS.patch_layer)
with tf.name_scope("masked_vnet"):
logits_masked = (1+softmax_attention)*logits_vnet
with tf.name_scope("output"):
outputModule = OutputModule.OutputModule(
num_classes=2,
is_training=True,
activation_fn="relu",
keep_prob=1.0)
logits_output = outputModule.GetNetwork(logits_masked)
else:
logits_output = logits_vnet
# for batch in range(FLAGS.batch_size):
# logits_max = tf.reduce_max(logits[batch:batch+1,:,:,:,:])
# logits_min = tf.reduce_min(logits[batch:batch+1,:,:,:,:])
# logits_log_0 = logits[batch:batch+1,:,:,:,0]
# logits_log_1 = logits[batch:batch+1,:,:,:,1]
# # normalize to 0-255 range
# logits_log_0 = tf.cast((logits_log_0-logits_min)*255./(logits_max-logits_min), dtype=tf.uint8)
# logits_log_1 = tf.cast((logits_log_1-logits_min)*255./(logits_max-logits_min), dtype=tf.uint8)
# tf.summary.image("logits_0", tf.transpose(logits_log_0,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
# tf.summary.image("logits_1", tf.transpose(logits_log_1,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
# learning rate
with tf.name_scope("learning_rate"):
learning_rate = tf.train.exponential_decay(FLAGS.init_learning_rate, global_step,
FLAGS.decay_steps,FLAGS.decay_factor,staircase=False)
tf.summary.scalar('learning_rate', learning_rate)
# softmax op for masked logit layer
with tf.name_scope("softmax"):
softmax_op = tf.nn.softmax(logits_output,name="softmax")
if FLAGS.image_log:
for batch in range(FLAGS.batch_size):
softmax_log_0 = grayscale_to_rainbow(tf.transpose(softmax_op[batch:batch+1,:,:,:,0],[3,1,2,0]))
softmax_log_1 = grayscale_to_rainbow(tf.transpose(softmax_op[batch:batch+1,:,:,:,1],[3,1,2,0]))
softmax_log_0 = tf.cast(tf.scalar_mul(255,softmax_log_0), dtype=tf.uint8)
softmax_log_1 = tf.cast(tf.scalar_mul(255,softmax_log_1), dtype=tf.uint8)
tf.summary.image("softmax_0", softmax_log_0,max_outputs=FLAGS.patch_layer)
tf.summary.image("softmax_1", softmax_log_1,max_outputs=FLAGS.patch_layer)
# Op for calculating loss
with tf.name_scope("loss"):
if (FLAGS.loss_function == "xent"):
loss_op = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits_output,
labels=tf.squeeze(labels_placeholder,
squeeze_dims=[4])))
elif(FLAGS.loss_function == "weighted_cross_entropy"):
class_weights = tf.constant([1.0, 1.0])
# deduce weights for batch samples based on their true label
onehot_labels = tf.one_hot(tf.squeeze(labels_placeholder,squeeze_dims=[4]),depth = 2)
weights = tf.reduce_sum(class_weights * onehot_labels, axis=-1)
# compute your (unweighted) softmax cross entropy loss
unweighted_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits_masked,
labels=tf.squeeze(labels_placeholder,
squeeze_dims=[4]))
# apply the weights, relying on broadcasting of the multiplication
weighted_loss = unweighted_loss * weights
# reduce the result to get your final loss
loss_op = tf.reduce_mean(weighted_loss)
elif(FLAGS.loss_function == "sorensen"):
# Dice Similarity, currently only for binary segmentation, here we provide two calculation methods, first one is closer to classical dice formula
sorensen = dice_coe(tf.expand_dims(softmax_op[:,:,:,:,1],-1),tf.cast(labels_placeholder,dtype=tf.float32), loss_type='sorensen')
# sorensen = dice_coe(softmax_op,tf.cast(tf.one_hot(labels_placeholder[:,:,:,:,0],depth=2),dtype=tf.float32), loss_type='sorensen', axis=[1,2,3])
loss_op = 1. - sorensen
elif(FLAGS.loss_function == "jaccard"):
# Dice Similarity, currently only for binary segmentation, here we provide two calculation methods, first one is closer to classical dice formula
# jaccard = dice_coe(tf.expand_dims(softmax_op[:,:,:,:,1],-1),tf.cast(labels_placeholder,dtype=tf.float32), loss_type='jaccard')
jaccard = dice_coe(softmax_op,tf.cast(tf.one_hot(labels_placeholder[:,:,:,:,0],depth=2),dtype=tf.float32), loss_type='jaccard', axis=[1,2,3])
loss_op = 1. - jaccard
tf.summary.scalar('loss',loss_op)
if FLAGS.attention:
# attention loss function
with tf.name_scope("attention_loss"):
if (FLAGS.attention_loss_function == "l2"):
distmap_0 = 1. - tf.squeeze(distmap_placeholder,axis=-1)
distmap_1 = tf.squeeze(distmap_placeholder,axis=-1)
# distmap = tf.stack([distmap_0,distmap_1],axis=-1)
# att_loss_op_ = tf.square(softmax_attention-distmap)*100 # attention softmax and distmap are between 0 and 1, 100 is for regularization
att_loss_op_ = tf.square(softmax_attention[:,:,:,:,1]-distmap_1)*100
att_loss_op_ = tf.expand_dims(att_loss_op_,axis=-1)
att_loss_op = tf.reduce_mean(att_loss_op_)
elif (FLAGS.attention_loss_function == "abs"):
distmap_0 = 1. - tf.squeeze(distmap_placeholder,axis=-1)
distmap_1 = tf.squeeze(distmap_placeholder,axis=-1)
distmap = tf.stack([distmap_0,distmap_1],axis=-1)
att_loss_op_ = tf.abs(softmax_attention-distmap)
att_loss_op = tf.reduce_mean(att_loss_op_)
else:
sys.exit("Invalid loss function");
tf.summary.scalar('attention_loss',att_loss_op)
if FLAGS.image_log:
for batch in range(FLAGS.batch_size):
att_loss_log_0 = grayscale_to_rainbow(tf.transpose(att_loss_op_[batch:batch+1,:,:,:,0],[3,1,2,0]))
# att_loss_log_1 = grayscale_to_rainbow(tf.transpose(att_loss_op_[batch:batch+1,:,:,:,1],[3,1,2,0]))
att_loss_log_0 = tf.cast(tf.scalar_mul(255,att_loss_log_0), dtype=tf.uint8)
# att_loss_log_1 = tf.cast(tf.scalar_mul(255,att_loss_log_1), dtype=tf.uint8)
# this two values is the same for binary classification
tf.summary.image("att_loss_0", att_loss_log_0,max_outputs=FLAGS.patch_layer)
# tf.summary.image("att_loss_1", att_loss_log_1,max_outputs=FLAGS.patch_layer)
# total loss
with tf.name_scope("total_loss"):
total_loss_op = att_loss_op + loss_op
tf.summary.scalar('total_loss',total_loss_op)
# Argmax Op to generate label from logits
with tf.name_scope("predicted_label"):
pred_op = tf.argmax(logits_output, axis=4 , name="prediction")
if FLAGS.image_log:
for batch in range(FLAGS.batch_size):
pred_log = tf.cast(tf.scalar_mul(255,pred_op[batch:batch+1,:,:,:]), dtype=tf.uint8)
tf.summary.image("pred", tf.transpose(pred_log,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
# Accuracy of model
with tf.name_scope("accuracy"):
correct_pred = tf.equal(tf.expand_dims(pred_op,-1), tf.cast(labels_placeholder,dtype=tf.int64))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# metrics of the model
with tf.name_scope("metrics"):
tp, tp_op = tf.metrics.true_positives(labels_placeholder, pred_op, name="true_positives")
tn, tn_op = tf.metrics.true_negatives(labels_placeholder, pred_op, name="true_negatives")
fp, fp_op = tf.metrics.false_positives(labels_placeholder, pred_op, name="false_positives")
fn, fn_op = tf.metrics.false_negatives(labels_placeholder, pred_op, name="false_negatives")
sensitivity_op = tf.divide(tf.cast(tp_op,tf.float32),tf.cast(tf.add(tp_op,fn_op),tf.float32))
specificity_op = tf.divide(tf.cast(tn_op,tf.float32),tf.cast(tf.add(tn_op,fp_op),tf.float32))
dice_op = 2.*tp_op/(2.*tp_op+fp_op+fn_op)
tf.summary.scalar('sensitivity', sensitivity_op)
tf.summary.scalar('specificity', specificity_op)
tf.summary.scalar('dice', dice_op)
# Training Op
with tf.name_scope("training"):
# optimizer
if FLAGS.optimizer == "sgd":
optimizer = tf.train.GradientDescentOptimizer(learning_rate=FLAGS.init_learning_rate)
elif FLAGS.optimizer == "adam":
optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.init_learning_rate)
elif FLAGS.optimizer == "momentum":
optimizer = tf.train.MomentumOptimizer(learning_rate=FLAGS.init_learning_rate, momentum=FLAGS.momentum)
elif FLAGS.optimizer == "nesterov_momentum":
optimizer = tf.train.MomentumOptimizer(learning_rate=FLAGS.init_learning_rate, momentum=FLAGS.momentum, use_nesterov=True)
else:
sys.exit("Invalid optimizer");
if FLAGS.attention:
train_op = optimizer.minimize(
loss = total_loss_op,
global_step=global_step)
else:
train_op = optimizer.minimize(
loss = loss_op,
global_step=global_step)
# the update op is required by batch norm layer: https://www.tensorflow.org/api_docs/python/tf/layers/batch_normalization
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
train_op = tf.group([train_op, update_ops])
# # epoch checkpoint manipulation
start_epoch = tf.get_variable("start_epoch", shape=[1], initializer= tf.zeros_initializer,dtype=tf.int32)
start_epoch_inc = start_epoch.assign(start_epoch+1)
# saver
summary_op = tf.summary.merge_all()
checkpoint_prefix = os.path.join(FLAGS.checkpoint_dir ,"checkpoint")
print("Setting up Saver...")
saver = tf.train.Saver(keep_checkpoint_every_n_hours=5)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# config.gpu_options.per_process_gpu_memory_fraction = 0.4
# training cycle
with tf.Session(config=config) as sess:
# Initialize all variables
sess.run(tf.global_variables_initializer())
print("{}: Start training...".format(datetime.datetime.now()))
# summary writer for tensorboard
train_summary_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph)
if FLAGS.testing:
test_summary_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test', sess.graph)
# restore from checkpoint
if FLAGS.restore_training:
# check if checkpoint exists
if os.path.exists(checkpoint_prefix+"-latest"):
print("{}: Last checkpoint found at {}, loading...".format(datetime.datetime.now(),FLAGS.checkpoint_dir))
latest_checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_dir,latest_filename="checkpoint-latest")
saver.restore(sess, latest_checkpoint_path)
print("{}: Last checkpoint epoch: {}".format(datetime.datetime.now(),start_epoch.eval()[0]))
print("{}: Last checkpoint global step: {}".format(datetime.datetime.now(),tf.train.global_step(sess, global_step)))
# loop over epochs
for epoch in np.arange(start_epoch.eval(), FLAGS.epochs):
# initialize iterator in each new epoch
sess.run(train_iterator.initializer)
if FLAGS.testing:
sess.run(test_iterator.initializer)
print("{}: Epoch {} starts".format(datetime.datetime.now(),epoch+1))
# training phase
while True:
try:
sess.run(tf.local_variables_initializer())
if FLAGS.attention:
[image, label, distMap] = sess.run(next_element_train)
distMap = distMap[:,:,:,:,np.newaxis]
else:
[image, label] = sess.run(next_element_train)
label = label[:,:,:,:,np.newaxis]
model.is_training = True;
if FLAGS.attention:
train, summary, loss, att_loss = sess.run([train_op, summary_op, loss_op, att_loss_op],
feed_dict={
images_placeholder: image,
labels_placeholder: label,
distmap_placeholder: distMap,
model.train_phase: True,
attentionModule.train_phase: False,
outputModule.train_phase: False})
print('{}: Training dice loss: {}'.format(datetime.datetime.now(), str(loss)))
print('{}: Training attention loss: {}'.format(datetime.datetime.now(), str(att_loss)))
else:
train, summary, loss = sess.run([train_op, summary_op, loss_op], feed_dict={images_placeholder: image, labels_placeholder: label, model.train_phase: True})
print('{}: Training dice loss: {}'.format(datetime.datetime.now(), str(loss)))
train_summary_writer.add_summary(summary, global_step=tf.train.global_step(sess, global_step))
train_summary_writer.flush()
except tf.errors.OutOfRangeError:
start_epoch_inc.op.run()
model.is_training = False;
# print(start_epoch.eval())
# save the model at end of each epoch training
print("{}: Saving checkpoint of epoch {} at {}...".format(datetime.datetime.now(),epoch+1,FLAGS.checkpoint_dir))
if not (os.path.exists(FLAGS.checkpoint_dir)):
os.makedirs(FLAGS.checkpoint_dir,exist_ok=True)
saver.save(sess, checkpoint_prefix,
global_step=tf.train.global_step(sess, global_step),
latest_filename="checkpoint-latest")
print("{}: Saving checkpoint succeed".format(datetime.datetime.now()))
break
if FLAGS.testing:
# testing phase
print("{}: Training of epoch {} finishes, testing start".format(datetime.datetime.now(),epoch+1))
while True:
try:
sess.run(tf.local_variables_initializer())
if FLAGS.attention:
[image, label, distMap] = sess.run(next_element_test)
distMap = distMap[:,:,:,:,np.newaxis]
else:
[image, label] = sess.run(next_element_test)
label = label[:,:,:,:,np.newaxis]
model.is_training = False;
if FLAGS.attention:
loss, summary, att_loss = sess.run([loss_op, summary_op, att_loss_op],
feed_dict={
images_placeholder: image,
labels_placeholder: label,
distmap_placeholder: distMap,
model.train_phase: False,
attentionModule.train_phase: False,
outputModule.train_phase: False})
else:
loss, summary = sess.run([loss_op, summary_op], feed_dict={images_placeholder: image, labels_placeholder: label, model.train_phase: False})
test_summary_writer.add_summary(summary, global_step=tf.train.global_step(sess, global_step))
test_summary_writer.flush()
except tf.errors.OutOfRangeError:
break
# close tensorboard summary writer
train_summary_writer.close()
if FLAGS.testing:
test_summary_writer.close()
loop = None # Embedded loop file
embeddedC = None # Embedded loop connection object
inputList = [] # List of connected Input devices on the GPIO pins
outputList = [] # List of connected Output devices on the GPIO pins
pwmList = [] # List of connected PWM devices on the GPIO pins
adcList = [] # List of connected ADC devices on the Analog pins
i2cList = [] # List of connected IMU devices on the I2C ports
connectionState = "" # Current connection state to be displayed
updateDuration = 0 # Cycle duration needed to update values
scanDuration = 0 # Cycle duration needed to scan for new devices
mux = MuxModule.GetMux() # Initialize the SoftWEAR Mux Module
input = InputModule.Input() # Initialize the SoftWEAR Input Module
output = OutputModule.Output() # Initialize the SoftWEAR Output Module
pwm = PWMModule.PWM() # Initialize the SoftWEAR PWM Module
adc = ADCModule.ADC() # Initialize the SoftWEAR ADC Module
i2c = I2CModule.I2C() # Initialize the SoftWEAR I2C Module
GPIO.setup(scanPin, GPIO.IN, GPIO.PUD_UP) # Setup scan pin
def muxScan():
"""Scan for new mux devices."""
global mux
mux.scan() # Scan devices on the mux pins
def inputScan():
"""Scan for new input devices."""
def init(self, conf):
# global conf
self.__globalConf = conf
# create all modules
# create logger
isExternal = self.__globalConf.get("EXTERNAL_LOGGER")
LoggerModule.LoggerModule().init(self.__globalConf,
externalLogger=isExternal)
self.Logger = LoggerModule.LoggerModule()
# create misc
MiscModule.MiscModule().init(self.__globalConf)
self.Misc = MiscModule.MiscModule()
# create configs
ConfigurationModule.ConfigurationModule().init(self.__globalConf)
self.Configuration = ConfigurationModule.ConfigurationModule()
# create workaround
WorkaroundModule.WorkaroundModule().init(self.__globalConf)
self.Workaround = WorkaroundModule.WorkaroundModule()
# create host
isExternal = self.__globalConf.get("EXTERNAL_LOCAL_EXEC", False)
HostModule.HostModule().init(self.__globalConf,
externalCmdExec=isExternal)
self.Host = HostModule.HostModule()
# create relay card
RelayCardModule.RelayCardModule().init(
self.__globalConf,
externalRelayCard=self.__globalConf.get("EXTERNAL_RELAY_CARD"))
self.RelayCard = RelayCardModule.RelayCardModule()
# create report
ReportModule.ReportModule().init(self.__globalConf)
self.Report = ReportModule.ReportModule()
# create output
OutputModule.OutputModule().init(self.__globalConf)
self.Output = OutputModule.OutputModule()
# create device
DeviceModule.DeviceModule().init(self.__globalConf)
self.Device = DeviceModule.DeviceModule()
# create campaign
CampaignModule.CampaignModule().init(self.__globalConf)
self.Campaign = CampaignModule.CampaignModule()
# create OsManager
OsManagerModule.OsManagerModule().init(self.__globalConf)
self.OsManager = OsManagerModule.OsManagerModule()
# create dediprog
DediprogModule.DediprogModule().init(self.__globalConf)
self.Dediprog = DediprogModule.DediprogModule()
# create FlashFile
FlashFileModule.FlashFileModule().init(self.__globalConf)
self.FlashFile = FlashFileModule.FlashFileModule()
# create logs
LogsModule.LogsModule().init(self.__globalConf)
self.Logs = LogsModule.LogsModule()
# create efi var
EfiVarModule.EfiVarModule().init(self.__globalConf)
self.EfiVar = EfiVarModule.EfiVarModule()
# create Flash
FlashModule.FlashModule().init(self.__globalConf)
self.Flash = FlashModule.FlashModule()
# create reset irq
ResetIrqModule.ResetIrqModule().init(self.__globalConf)
self.ResetIrq = ResetIrqModule.ResetIrqModule()
# create boot data
BootDataModule.BootDataModule().init(self.__globalConf)
self.BootData = BootDataModule.BootDataModule()
# create watchdog
WatchdogModule.WatchdogModule().init(self.__globalConf)
self.Watchdog = WatchdogModule.WatchdogModule()
# create download
DownloadModule.DownloadModule().init(self.__globalConf)
self.Download = DownloadModule.DownloadModule()
# create testing
TestingModule.TestingModule().init(self.__globalConf)
self.Testing = TestingModule.TestingModule()
# create DntEvents
EventsModule.EventsModule().init(self.__globalConf)
self.Events = EventsModule.EventsModule()
# scripts
scripts.COS_scripts.COS().init(self.__globalConf)
self.COS_scripts = scripts.COS_scripts.COS()
scripts.MOS_scripts.MOS().init(self.__globalConf)
self.MOS_scripts = scripts.MOS_scripts.MOS()
scripts.Watchdog_scripts.Watchdog().init(self.__globalConf)
self.Watchdog_scripts = scripts.Watchdog_scripts.Watchdog()
scripts.KeyPress_scripts.KeyPress().init(self.__globalConf)
self.KeyPress_scripts = scripts.KeyPress_scripts.KeyPress()
scripts.POS_scripts.POS().init(self.__globalConf)
self.POS_scripts = scripts.POS_scripts.POS()
scripts.RMA_scripts.RMA().init(self.__globalConf)
self.RMA_scripts = scripts.RMA_scripts.RMA()
scripts.ROS_scripts.ROS().init(self.__globalConf)
self.ROS_scripts = scripts.ROS_scripts.ROS()
scripts.DNX_scripts.DNX().init(self.__globalConf)
self.DNX_scripts = scripts.DNX_scripts.DNX()
scripts.Flash_scripts.Flash().init(self.__globalConf)
self.Flash_scripts = scripts.Flash_scripts.Flash()
scripts.Shell_scripts.Shell().init(self.__globalConf)
self.Shell_scripts = scripts.Shell_scripts.Shell()
scripts.Time_scripts.Time().init(self.__globalConf)
self.Time_scripts = scripts.Time_scripts.Time()
scripts.Debug_scripts.Debug().init(self.__globalConf)
self.Debug_scripts = scripts.Debug_scripts.Debug()
# print Library version
self.Logger.printLog(
"INFO", "BOOT/OTA LIBRARY Code Version: " + self.__version)
self.__globalConf["BOOTOTA_LIBRARY_CODE_VERSION"] = self.__version
if self.__framework_push:
self.Logger.printLog(
"INFO", "FRAMEWORK Code Version: " + self.__framework_push)
self.__globalConf["FRAMEWORK_CODE_VERSION"] = self.__framework_push