Пример #1
0
 def __init__(self, conf):
     self.globalConf = conf
     self.verdict = True
     self.output = ""
     self.output_list = []
     self.name = ""
     self.description = ""
     self.specific = ""
     self.isWorkaround = False
     self.skip = False
     self.backup_name = ""
     self.defaultParameter = dict()
     self.logger = LoggerModule.LoggerModule()
     self.host = HostModule.HostModule()
     self.relayCard = RelayCardModule.RelayCardModule()
     self.device = DeviceModule.DeviceModule()
     self.workaround = WorkaroundModule.WorkaroundModule()
     self.misc = MiscModule.MiscModule()
     self.osManager = OsManagerModule.OsManagerModule()
     self.flashFile = FlashFileModule.FlashFileModule()
     self.logs = LogsModule.LogsModule()
     self.efiVar = EfiVarModule.EfiVarModule()
     self.dediprog = DediprogModule.DediprogModule()
     self.outputModule = OutputModule.OutputModule()
     self.configuration = ConfigurationModule.ConfigurationModule()
     self.bootData = BootDataModule.BootDataModule()
     self.resetIrq = ResetIrqModule.ResetIrqModule()
     self.watchdog = WatchdogModule.WatchdogModule()
     self.download = DownloadModule.DownloadModule()
     self.campaign = CampaignModule.CampaignModule()
     self.flash = FlashModule.FlashModule()
     self.events = EventsModule.EventsModule()
Пример #2
0
 def init(self, globalConf=None, externalRelayCard=None):
     self.globalConf = globalConf
     self.__logger = LoggerModule.LoggerModule()
     self.__host = HostModule.HostModule()
     self.__output = OutputModule.OutputModule()
     self.configuration = ConfigurationModule.ConfigurationModule()
     self.cardConfiguration = self.globalConf.get(
         "RELAY_CARD_CONFIGURATION", dict())
     self.relayConfiguration = self.cardConfiguration.get("relays", dict())
     self.__delayBetweenCommands = 1
     self.__allow_usb_toggle_event = False
     self.__usb_connected = "unknown"
     if externalRelayCard:
         self.__relay = externalRelayCard
     else:
         self.__relay = self.__internalRelayCard()
         self.__relay.setConfig(self.cardConfiguration)
         if self.relayConfiguration:
             self.__relay.setDefaultState()
         else:
             self.relayConfiguration["SwitchOnOff"] = -1
             self.relayConfiguration["UsbHostPcConnect"] = -1
             self.relayConfiguration["VolumeUp"] = -1
             self.relayConfiguration["VolumeDown"] = -1
             self.relayConfiguration["Dediprog"] = -1
             self.relayConfiguration["PowerSupply"] = -1
             self.relayConfiguration["J6B2_relay"] = -1
     self.SwitchOnOff = self.relayConfiguration.get("SwitchOnOff", -1)
     self.UsbHostPcConnect = self.relayConfiguration.get(
         "UsbHostPcConnect", -1)
     self.VolumeUp = self.relayConfiguration.get("VolumeUp", -1)
     self.VolumeDown = self.relayConfiguration.get("VolumeDown", -1)
     self.Dediprog = self.relayConfiguration.get("Dediprog", -1)
     self.PowerSupply = self.relayConfiguration.get("PowerSupply", -1)
     self.J6B2_relay = self.relayConfiguration.get("J6B2_relay", -1)
Пример #3
0
 def init(self, globalConf=None):
     self.__globalConf = globalConf
     self.__logger = LoggerModule.LoggerModule()
     self.__host = HostModule.HostModule()
     self.__relayCard = RelayCardModule.RelayCardModule()
     self.__device = DeviceModule.DeviceModule()
     self.__workaround = WorkaroundModule.WorkaroundModule()
     self.__getEndTime = None
     self.__misc = MiscModule.MiscModule()
     self.__osManager = OsManagerModule.OsManagerModule()
     self.__flashFile = FlashFileModule.FlashFileModule()
     self.__output = OutputModule.OutputModule()
     # table to match uefi var with offset in RCSI_table
     self.__RSCI_dict = dict()
     self.__RSCI_dict["wake_source"] = 36
     self.__RSCI_dict["reset_source"] = 37
     self.__RSCI_dict["reset_type"] = 38
     self.__RSCI_dict["shutdown_source"] = 39
     self.__RSCI_dict["reset_extra_information"] = None
     # table to match uefi var to name in sysfs
     self.efi_vars_matching = dict()
     self.efi_vars_matching["wake_source"] = "WakeSource"
     self.efi_vars_matching["reset_source"] = "ResetSource"
     self.efi_vars_matching["reset_type"] = "ResetType"
     self.efi_vars_matching["shutdown_source"] = "ShutdownSource"
     self.efi_vars_matching["mode"] = "LoaderEntryLast"
     self.efi_vars_matching["watchdog"] = "WdtCounter"
Пример #4
0
 def init(self, globalConf=None):
     self.__globalConf = globalConf
     self.__logger = LoggerModule.LoggerModule()
     self.__host = HostModule.HostModule()
     self.__device = DeviceModule.DeviceModule()
     self.__workaround = WorkaroundModule.WorkaroundModule()
     self.__configuration = ConfigurationModule.ConfigurationModule()
     self.__misc = MiscModule.MiscModule()
     self.__osManager = OsManagerModule.OsManagerModule()
     self.__output = OutputModule.OutputModule()
     self.__report = ReportModule.ReportModule()
     self.logsFiles = list()
     self.__mandatoryLogsFiles = ["history_event"]
     self.tc_name = ""
     self.tc_name_number = ""
     campaign_report_path = self.__globalConf.get("REPORT_PATH")
     if campaign_report_path and os.path.isdir(campaign_report_path):
         self.bootota_log_path = os.path.abspath(os.path.join(campaign_report_path, "logs_bootota"))
         # if directory not created, create it for future use in TC
         if not os.path.isdir(self.bootota_log_path):
             os.makedirs(self.bootota_log_path)
         self.__logger.printLog("WARNING", "LogsModule init: storing BootOta specific data into "
                                           "{}".format(self.bootota_log_path))
     else:
         self.__logger.printLog("WARNING", "LogsModule init: invalid or empty report path "
                                           "('{}')".format(campaign_report_path))
         self.bootota_log_path = ""
     self.tc_report_path = ""
Пример #5
0
 def init(self, globalConf):
     self.__globalConf = globalConf
     self.__logger = LoggerModule.LoggerModule()
     self.__output = OutputModule.OutputModule()
     self.__host = HostModule.HostModule()
     self.local_files_handler = self.__FilesHandler()
     self.extra_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "Extras"))
     self.otaBuild = ""
Пример #6
0
 def init(self, globalConf=None):
     self.__globalConf = globalConf
     self.__logger = LoggerModule.LoggerModule()
     self.__host = HostModule.HostModule()
     self.__relayCard = RelayCardModule.RelayCardModule()
     self.__output = OutputModule.OutputModule()
     self.__misc = MiscModule.MiscModule()
     self.__configuration = ConfigurationModule.ConfigurationModule()
Пример #7
0
 def init(self, globalConf):
     self.__globalConf = globalConf
     self.__logger = LoggerModule.LoggerModule()
     self.__host = HostModule.HostModule()
     self.__output = OutputModule.OutputModule()
     self.__efiVar = EfiVarModule.EfiVarModule()
     self.__resetIrq = ResetIrqModule.ResetIrqModule()
     self.__device = DeviceModule.DeviceModule()
     self.__osManager = OsManagerModule.OsManagerModule()
     self.__logs = LogsModule.LogsModule()
     self.__workaround = WorkaroundModule.WorkaroundModule()
     self.__configuration = ConfigurationModule.ConfigurationModule()
     self.__RSCI_table = list()
Пример #8
0
 def init(self, globalConf):
     self.__globalConf = globalConf
     self.__logger = LoggerModule.LoggerModule()
     self.__host = HostModule.HostModule()
     self.__output = OutputModule.OutputModule()
     self.__device = DeviceModule.DeviceModule()
     self.__osManager = OsManagerModule.OsManagerModule()
     self.__misc = MiscModule.MiscModule()
     self.__workaround = WorkaroundModule.WorkaroundModule()
     self.__configuration = ConfigurationModule.ConfigurationModule()
     self.__logs = LogsModule.LogsModule()
     self.force_ramdump_removal = False
     self.wd_handler = self.HistoryEventHandler(self.__globalConf)
Пример #9
0
 def __init__(self, conf):
     self.__globalConf = conf
     self.__historyEvent = ""
     self.__initial_content = []
     self.__delta = ""
     self.__crashlogs = None
     self.__configuration = ConfigurationModule.ConfigurationModule()
     self.__output = OutputModule.OutputModule()
     self.__logger = LoggerModule.LoggerModule()
     self.__host = HostModule.HostModule()
     self.__device = DeviceModule.DeviceModule()
     self.__workaround = WorkaroundModule.WorkaroundModule()
     self.__logs = LogsModule.LogsModule()
     self.__misc = MiscModule.MiscModule()
Пример #10
0
 def __init__(self, conf):
     self.globalConf = conf
     self.logger = LoggerModule.LoggerModule()
     self.host = HostModule.HostModule()
     self.relayCard = RelayCardModule.RelayCardModule()
     self.device = DeviceModule.DeviceModule()
     self.workaround = WorkaroundModule.WorkaroundModule()
     self.misc = MiscModule.MiscModule()
     self.osManager = OsManagerModule.OsManagerModule()
     self.flashFile = FlashFileModule.FlashFileModule()
     self.logs = LogsModule.LogsModule()
     self.efiVar = EfiVarModule.EfiVarModule()
     self.dediprog = DediprogModule.DediprogModule()
     self.output = OutputModule.OutputModule()
     self.configuration = ConfigurationModule.ConfigurationModule()
     self.bootData = BootDataModule.BootDataModule()
     self.resetIrq = ResetIrqModule.ResetIrqModule()
     self.download = DownloadModule.DownloadModule()
     self.flash = FlashModule.FlashModule()
     self.campaign = CampaignModule.CampaignModule()
     self.watchdog = WatchdogModule.WatchdogModule()
     self.report = ReportModule.ReportModule()
     self.events = EventsModule.EventsModule()
     self.step_list = list()
     self.verdict = True
     self.output = ""
     self.name = ""
     self.skipping = False
     self.tc_crashlogs = None
     self.allowed_TC_upon_success = []
     self.removed_TC_upon_success = []
     self.allowed_TC_upon_failure = []
     self.removed_TC_upon_failure = []
     self.allowed_TC_upon_skip = []
     self.removed_TC_upon_skip = []
     self.campaign_constraint_file = ""
     self.campaign_constraint_file_name = "PupdrCampaignConstraints.json"
     self.enable_init = True
     self.enable_final = True
     self.INVALID = "INVALID"
     self.VALID = "VALID"
     self.BLOCKED = "BLOCKED"
     self.FAILURE = "FAILURE"
     self.SUCCESS = "SUCCESS"
     self.verdict_name = self.SUCCESS
     self.step_number = 0
     self.output_dict = {}
Пример #11
0
 def init(self, globalConf=None):
     self.__globalConf = globalConf
     self.__logger = LoggerModule.LoggerModule()
     self.__output = OutputModule.OutputModule()
     self.flash = None
     self.boot = None
     self.logs = None
     self.timeout = None
     self.download = None
     self.custom = None
     self.boot_vars = None
     self.workaround = None
     self.board = None
     self.branch = None
     self.test_description = None
     self.__configuration_list = dict()
     self.__branchList = None
     self.__boardList = None
     self.__current_config_name = ""
Пример #12
0
def main():

    t = Timer()

    input_object = InputReader.InputReader(CSVPath, TXTPath)

    t.tic()
    part_type, init_r, init_v, mass, init_rho, init_energy, body_forces, init_sigma, normals = input_object.read_csv(
    )
    params_list = input_object.read_txt()
    print()
    t.toc("Loading simulation data")

    t.tic()
    output_object = OutputModule.Output(outPath)
    solver_object = Solver.Solver(params_list, init_r, init_rho)
    solver_object.solve(part_type, init_r, init_v, init_rho, mass, init_sigma,
                        init_energy, body_forces, output_object)
    t.toc("Problem solution")
Пример #13
0
 def init(self, globalConf):
     self.__globalConf = globalConf
     self.__logger = LoggerModule.LoggerModule()
     self.__host = HostModule.HostModule()
     self.__output = OutputModule.OutputModule()
     self.__configuration = ConfigurationModule.ConfigurationModule()
Пример #14
0
def train():
	# read configuration file
	with open(FLAGS.config_json) as config_json:  
		config = json.load(config_json)

	"""Train the Vnet model"""
	with tf.Graph().as_default():
		global_step = tf.train.get_or_create_global_step()

		# patch_shape(batch_size, height, width, depth, channels)
		input_channel_num = len(config['TrainingSetting']['Data']['ImageFilenames'])

		input_batch_shape = (None, FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer, input_channel_num) 
		output_batch_shape = (None, FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer, 1) 
		
		if FLAGS.attention:
			images_placeholder, labels_placeholder, distmap_placeholder = placeholder_inputs(input_batch_shape,output_batch_shape, attention=True)
		else:
			images_placeholder, labels_placeholder = placeholder_inputs(input_batch_shape,output_batch_shape, attention=False)
		

		if FLAGS.image_log:
			for batch in range(FLAGS.batch_size):
				# plot images in tensorboard
				for image_channel in range(input_channel_num):
					images_log = tf.cast(images_placeholder[batch:batch+1,:,:,:,image_channel], dtype=tf.uint8)
					tf.summary.image(config['TrainingSetting']['Data']['ImageFilenames'][image_channel], tf.transpose(images_log,[3,1,2,0]),max_outputs=FLAGS.patch_layer)

				if FLAGS.attention:
					# dist map will be plot in color
					distmap_log = grayscale_to_rainbow(tf.transpose(distmap_placeholder[batch:batch+1,:,:,:,0],[3,1,2,0]))
					distmap_log = tf.cast(tf.scalar_mul(255,distmap_log), dtype=tf.uint8)
					tf.summary.image("distmap", distmap_log,max_outputs=FLAGS.patch_layer)

				# plot labels
				labels_log = tf.cast(tf.scalar_mul(255,labels_placeholder[batch:batch+1,:,:,:,0]), dtype=tf.uint8)
				tf.summary.image("label", tf.transpose(labels_log,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
				
		# Get images and labels
		train_data_dir = os.path.join(FLAGS.data_dir,'training')
		test_data_dir = os.path.join(FLAGS.data_dir,'testing')
		# support multiple image input, but here only use single channel, label file should be a single file with different classes

		# Force input pipepline to CPU:0 to avoid operations sometimes ended up at GPU and resulting a slow down
		with tf.device('/cpu:0'):
			# create transformations to image and labels
			trainTransforms = [
				NiftiDataset3D.ExtremumNormalization(0.1),
				# NiftiDataset.Normalization(),
				NiftiDataset3D.Resample((0.25,0.25,0.25)),
				NiftiDataset3D.Padding((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer)),
				NiftiDataset3D.RandomCrop((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),FLAGS.drop_ratio,FLAGS.min_pixel),
				# NiftiDataset.ConfidenceCrop((FLAGS.patch_size*3, FLAGS.patch_size*3, FLAGS.patch_layer*3),(0.0001,0.0001,0.0001)),
				# NiftiDataset.BSplineDeformation(randomness=2),
				# NiftiDataset.ConfidenceCrop((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),(0.5,0.5,0.5)),
				# NiftiDataset.ConfidenceCrop2((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),rand_range=32,probability=0.8),
				NiftiDataset3D.RandomFlip([True, False, False]),
				NiftiDataset3D.RandomNoise()
				]

			TrainDataset = NiftiDataset3D.NiftiDataset(
				data_dir=train_data_dir,
				image_filenames=config['TrainingSetting']['Data']['ImageFilenames'],
				label_filename=config['TrainingSetting']['Data']['LabelFilename'],
				transforms=trainTransforms,
				train=True,
				distmap=FLAGS.attention
				)
			
			trainDataset = TrainDataset.get_dataset()
			trainDataset = trainDataset.shuffle(buffer_size=5)
			trainDataset = trainDataset.batch(FLAGS.batch_size,drop_remainder=True)

			if FLAGS.testing:
				# use random crop for testing
				testTransforms = [
					NiftiDataset3D.ExtremumNormalization(0.1),
					# NiftiDataset.Normalization(),
					NiftiDataset3D.Resample((0.25,0.25,0.25)),
					NiftiDataset3D.Padding((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer)),
					NiftiDataset3D.RandomCrop((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),FLAGS.drop_ratio,FLAGS.min_pixel),
					# NiftiDataset.ConfidenceCrop((FLAGS.patch_size*2, FLAGS.patch_size*2, FLAGS.patch_layer*2),(0.0001,0.0001,0.0001)),
					# NiftiDataset.BSplineDeformation(),
					# NiftiDataset.ConfidenceCrop((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),(0.75,0.75,0.75)),
					# NiftiDataset.ConfidenceCrop2((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),rand_range=32,probability=0.8),
					NiftiDataset3D.RandomFlip([True, False, False]),
					]

				TestDataset = NiftiDataset3D.NiftiDataset(
					data_dir=test_data_dir,
					image_filenames=config['TrainingSetting']['Data']['ImageFilenames'],
					label_filename=config['TrainingSetting']['Data']['LabelFilename'],
					transforms=testTransforms,
					train=True,
					distmap=FLAGS.attention
				)

				testDataset = TestDataset.get_dataset()
				testDataset = testDataset.shuffle(buffer_size=5)
				testDataset = testDataset.batch(FLAGS.batch_size,drop_remainder=True)
			
		train_iterator = trainDataset.make_initializable_iterator()
		next_element_train = train_iterator.get_next()

		if FLAGS.testing:
			test_iterator = testDataset.make_initializable_iterator()
			next_element_test = test_iterator.get_next()

		# Initialize the model
		with tf.name_scope("vnet"):
			model = VNet.VNet(
				num_classes=2, # binary for 2
				keep_prob=1.0, # default 1
				num_channels=16, # default 16 
				num_levels=3,  # default 4
				num_convolutions=(1,2,2), # default (1,2,3,3), size should equal to num_levels
				bottom_convolutions=3, # default 3
				activation_fn="prelu") # default relu
			logits_vnet = model.network_fn(images_placeholder)

		if FLAGS.attention:
			with tf.name_scope("attention"):
				attentionModule = attention.AttentionModule(
					num_classes=2,
					is_training=True,
					activation_fn="relu",
					keep_prob=1.0)
				logits_attention = attentionModule.GetNetwork(logits_vnet)
				softmax_attention = tf.nn.softmax(logits_attention, name="softmax_attention")

			for batch in range(FLAGS.batch_size):
				softmax_att_log_0 = grayscale_to_rainbow(tf.transpose(softmax_attention[batch:batch+1,:,:,:,0],[3,1,2,0]))
				softmax_att_log_1 = grayscale_to_rainbow(tf.transpose(softmax_attention[batch:batch+1,:,:,:,1],[3,1,2,0]))
				softmax_att_log_0 = tf.cast(tf.scalar_mul(255,softmax_att_log_0), dtype=tf.uint8)
				softmax_att_log_1 = tf.cast(tf.scalar_mul(255,softmax_att_log_1), dtype=tf.uint8)
			   
				if FLAGS.image_log:
					tf.summary.image("softmax_attention_0", softmax_att_log_0,max_outputs=FLAGS.patch_layer)
					tf.summary.image("softmax_attention_1", softmax_att_log_1,max_outputs=FLAGS.patch_layer)

			with tf.name_scope("masked_vnet"):
				logits_masked = (1+softmax_attention)*logits_vnet

			with tf.name_scope("output"):
				outputModule = OutputModule.OutputModule(
					num_classes=2,
					is_training=True,
					activation_fn="relu",
					keep_prob=1.0)
				logits_output = outputModule.GetNetwork(logits_masked)
		else:
			logits_output = logits_vnet

		# for batch in range(FLAGS.batch_size):
		#     logits_max = tf.reduce_max(logits[batch:batch+1,:,:,:,:])
		#     logits_min = tf.reduce_min(logits[batch:batch+1,:,:,:,:])

		#     logits_log_0 = logits[batch:batch+1,:,:,:,0]
		#     logits_log_1 = logits[batch:batch+1,:,:,:,1]

		#     # normalize to 0-255 range
		#     logits_log_0 = tf.cast((logits_log_0-logits_min)*255./(logits_max-logits_min), dtype=tf.uint8)
		#     logits_log_1 = tf.cast((logits_log_1-logits_min)*255./(logits_max-logits_min), dtype=tf.uint8)

		#     tf.summary.image("logits_0", tf.transpose(logits_log_0,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
		#     tf.summary.image("logits_1", tf.transpose(logits_log_1,[3,1,2,0]),max_outputs=FLAGS.patch_layer)

		# learning rate
		with tf.name_scope("learning_rate"):
			learning_rate = tf.train.exponential_decay(FLAGS.init_learning_rate, global_step,
				FLAGS.decay_steps,FLAGS.decay_factor,staircase=False)
		tf.summary.scalar('learning_rate', learning_rate)

		# softmax op for masked logit layer
		with tf.name_scope("softmax"):
			softmax_op = tf.nn.softmax(logits_output,name="softmax")

		if FLAGS.image_log:
			for batch in range(FLAGS.batch_size):
				softmax_log_0 = grayscale_to_rainbow(tf.transpose(softmax_op[batch:batch+1,:,:,:,0],[3,1,2,0]))
				softmax_log_1 = grayscale_to_rainbow(tf.transpose(softmax_op[batch:batch+1,:,:,:,1],[3,1,2,0]))
				softmax_log_0 = tf.cast(tf.scalar_mul(255,softmax_log_0), dtype=tf.uint8)
				softmax_log_1 = tf.cast(tf.scalar_mul(255,softmax_log_1), dtype=tf.uint8)
			   
				tf.summary.image("softmax_0", softmax_log_0,max_outputs=FLAGS.patch_layer)
				tf.summary.image("softmax_1", softmax_log_1,max_outputs=FLAGS.patch_layer)

		# Op for calculating loss
		with tf.name_scope("loss"):
			if (FLAGS.loss_function == "xent"):
				loss_op = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(
					logits=logits_output,
					labels=tf.squeeze(labels_placeholder, 
					squeeze_dims=[4])))
			elif(FLAGS.loss_function == "weighted_cross_entropy"):
				class_weights = tf.constant([1.0, 1.0])

				# deduce weights for batch samples based on their true label
				onehot_labels = tf.one_hot(tf.squeeze(labels_placeholder,squeeze_dims=[4]),depth = 2)

				weights = tf.reduce_sum(class_weights * onehot_labels, axis=-1)
				# compute your (unweighted) softmax cross entropy loss
				unweighted_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
					logits=logits_masked,
					labels=tf.squeeze(labels_placeholder, 
					squeeze_dims=[4]))
				# apply the weights, relying on broadcasting of the multiplication
				weighted_loss = unweighted_loss * weights
				# reduce the result to get your final loss
				loss_op = tf.reduce_mean(weighted_loss)
			elif(FLAGS.loss_function == "sorensen"):
				# Dice Similarity, currently only for binary segmentation, here we provide two calculation methods, first one is closer to classical dice formula
				sorensen = dice_coe(tf.expand_dims(softmax_op[:,:,:,:,1],-1),tf.cast(labels_placeholder,dtype=tf.float32), loss_type='sorensen')
				# sorensen = dice_coe(softmax_op,tf.cast(tf.one_hot(labels_placeholder[:,:,:,:,0],depth=2),dtype=tf.float32), loss_type='sorensen', axis=[1,2,3])
				loss_op = 1. - sorensen
			elif(FLAGS.loss_function == "jaccard"):
				# Dice Similarity, currently only for binary segmentation, here we provide two calculation methods, first one is closer to classical dice formula
				# jaccard = dice_coe(tf.expand_dims(softmax_op[:,:,:,:,1],-1),tf.cast(labels_placeholder,dtype=tf.float32), loss_type='jaccard')
				jaccard = dice_coe(softmax_op,tf.cast(tf.one_hot(labels_placeholder[:,:,:,:,0],depth=2),dtype=tf.float32), loss_type='jaccard', axis=[1,2,3])
				loss_op = 1. - jaccard
		tf.summary.scalar('loss',loss_op)

		if FLAGS.attention:
			# attention loss function
			with tf.name_scope("attention_loss"):
				if (FLAGS.attention_loss_function == "l2"):
					distmap_0 = 1. - tf.squeeze(distmap_placeholder,axis=-1)
					distmap_1 = tf.squeeze(distmap_placeholder,axis=-1)
					# distmap = tf.stack([distmap_0,distmap_1],axis=-1)
					# att_loss_op_ = tf.square(softmax_attention-distmap)*100 # attention softmax and distmap are between 0 and 1, 100 is for regularization
					att_loss_op_  = tf.square(softmax_attention[:,:,:,:,1]-distmap_1)*100
					att_loss_op_ = tf.expand_dims(att_loss_op_,axis=-1)
					att_loss_op = tf.reduce_mean(att_loss_op_)
				elif (FLAGS.attention_loss_function == "abs"):
					distmap_0 = 1. - tf.squeeze(distmap_placeholder,axis=-1)
					distmap_1 = tf.squeeze(distmap_placeholder,axis=-1)
					distmap = tf.stack([distmap_0,distmap_1],axis=-1)
					att_loss_op_ = tf.abs(softmax_attention-distmap)
					att_loss_op = tf.reduce_mean(att_loss_op_)
				else:
					sys.exit("Invalid loss function");
			tf.summary.scalar('attention_loss',att_loss_op)

			if FLAGS.image_log:
				for batch in range(FLAGS.batch_size):
					att_loss_log_0 = grayscale_to_rainbow(tf.transpose(att_loss_op_[batch:batch+1,:,:,:,0],[3,1,2,0]))
					# att_loss_log_1 = grayscale_to_rainbow(tf.transpose(att_loss_op_[batch:batch+1,:,:,:,1],[3,1,2,0]))
					att_loss_log_0 = tf.cast(tf.scalar_mul(255,att_loss_log_0), dtype=tf.uint8)
					# att_loss_log_1 = tf.cast(tf.scalar_mul(255,att_loss_log_1), dtype=tf.uint8)
				   
					# this two values is the same for binary classification
					tf.summary.image("att_loss_0", att_loss_log_0,max_outputs=FLAGS.patch_layer)
					# tf.summary.image("att_loss_1", att_loss_log_1,max_outputs=FLAGS.patch_layer)

			# total loss
			with tf.name_scope("total_loss"):
				total_loss_op = att_loss_op + loss_op
			tf.summary.scalar('total_loss',total_loss_op)

		# Argmax Op to generate label from logits
		with tf.name_scope("predicted_label"):
			pred_op = tf.argmax(logits_output, axis=4 , name="prediction")

		if FLAGS.image_log:
			for batch in range(FLAGS.batch_size):
				pred_log = tf.cast(tf.scalar_mul(255,pred_op[batch:batch+1,:,:,:]), dtype=tf.uint8)
				tf.summary.image("pred", tf.transpose(pred_log,[3,1,2,0]),max_outputs=FLAGS.patch_layer)

		# Accuracy of model
		with tf.name_scope("accuracy"):
			correct_pred = tf.equal(tf.expand_dims(pred_op,-1), tf.cast(labels_placeholder,dtype=tf.int64))
			accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
		tf.summary.scalar('accuracy', accuracy)

		# metrics of the model
		with tf.name_scope("metrics"):
			tp, tp_op = tf.metrics.true_positives(labels_placeholder, pred_op, name="true_positives")
			tn, tn_op = tf.metrics.true_negatives(labels_placeholder, pred_op, name="true_negatives")
			fp, fp_op = tf.metrics.false_positives(labels_placeholder, pred_op, name="false_positives")
			fn, fn_op = tf.metrics.false_negatives(labels_placeholder, pred_op, name="false_negatives")
			sensitivity_op = tf.divide(tf.cast(tp_op,tf.float32),tf.cast(tf.add(tp_op,fn_op),tf.float32))
			specificity_op = tf.divide(tf.cast(tn_op,tf.float32),tf.cast(tf.add(tn_op,fp_op),tf.float32))
			dice_op = 2.*tp_op/(2.*tp_op+fp_op+fn_op)
		tf.summary.scalar('sensitivity', sensitivity_op)
		tf.summary.scalar('specificity', specificity_op)
		tf.summary.scalar('dice', dice_op)

		# Training Op
		with tf.name_scope("training"):
			# optimizer
			if FLAGS.optimizer == "sgd":
				optimizer = tf.train.GradientDescentOptimizer(learning_rate=FLAGS.init_learning_rate)
			elif FLAGS.optimizer == "adam":
				optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.init_learning_rate)
			elif FLAGS.optimizer == "momentum":
				optimizer = tf.train.MomentumOptimizer(learning_rate=FLAGS.init_learning_rate, momentum=FLAGS.momentum)
			elif FLAGS.optimizer == "nesterov_momentum":
				optimizer = tf.train.MomentumOptimizer(learning_rate=FLAGS.init_learning_rate, momentum=FLAGS.momentum, use_nesterov=True)
			else:
				sys.exit("Invalid optimizer");

			if FLAGS.attention:
				train_op = optimizer.minimize(
					loss = total_loss_op,
					global_step=global_step)
			else:
				train_op = optimizer.minimize(
					loss = loss_op,
					global_step=global_step)

			# the update op is required by batch norm layer: https://www.tensorflow.org/api_docs/python/tf/layers/batch_normalization
			update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
			train_op = tf.group([train_op, update_ops])

		# # epoch checkpoint manipulation
		start_epoch = tf.get_variable("start_epoch", shape=[1], initializer= tf.zeros_initializer,dtype=tf.int32)
		start_epoch_inc = start_epoch.assign(start_epoch+1)

		# saver
		summary_op = tf.summary.merge_all()
		checkpoint_prefix = os.path.join(FLAGS.checkpoint_dir ,"checkpoint")
		print("Setting up Saver...")
		saver = tf.train.Saver(keep_checkpoint_every_n_hours=5)

		config = tf.ConfigProto()
		config.gpu_options.allow_growth = True
		# config.gpu_options.per_process_gpu_memory_fraction = 0.4

		# training cycle
		with tf.Session(config=config) as sess:
			# Initialize all variables
			sess.run(tf.global_variables_initializer())
			print("{}: Start training...".format(datetime.datetime.now()))

			# summary writer for tensorboard
			train_summary_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph)
			if FLAGS.testing:
				test_summary_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test', sess.graph)

			# restore from checkpoint
			if FLAGS.restore_training:
				# check if checkpoint exists
				if os.path.exists(checkpoint_prefix+"-latest"):
					print("{}: Last checkpoint found at {}, loading...".format(datetime.datetime.now(),FLAGS.checkpoint_dir))
					latest_checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_dir,latest_filename="checkpoint-latest")
					saver.restore(sess, latest_checkpoint_path)
			
			print("{}: Last checkpoint epoch: {}".format(datetime.datetime.now(),start_epoch.eval()[0]))
			print("{}: Last checkpoint global step: {}".format(datetime.datetime.now(),tf.train.global_step(sess, global_step)))

			# loop over epochs
			for epoch in np.arange(start_epoch.eval(), FLAGS.epochs):
				# initialize iterator in each new epoch
				sess.run(train_iterator.initializer)
				if FLAGS.testing:
					sess.run(test_iterator.initializer)
				print("{}: Epoch {} starts".format(datetime.datetime.now(),epoch+1))

				# training phase
				while True:
					try:
						sess.run(tf.local_variables_initializer())
						if FLAGS.attention:
							[image, label, distMap] = sess.run(next_element_train)
							distMap = distMap[:,:,:,:,np.newaxis]
						else:
							[image, label] = sess.run(next_element_train)

						label = label[:,:,:,:,np.newaxis]
						
						model.is_training = True;
						if FLAGS.attention:
							train, summary, loss, att_loss = sess.run([train_op, summary_op, loss_op, att_loss_op], 
								feed_dict={
									images_placeholder: image, 
									labels_placeholder: label, 
									distmap_placeholder: distMap, 
									model.train_phase: True, 
									attentionModule.train_phase: False, 
									outputModule.train_phase: False})
							print('{}: Training dice loss: {}'.format(datetime.datetime.now(), str(loss)))
							print('{}: Training attention loss: {}'.format(datetime.datetime.now(), str(att_loss)))
						else:
							train, summary, loss = sess.run([train_op, summary_op, loss_op], feed_dict={images_placeholder: image, labels_placeholder: label, model.train_phase: True})
							print('{}: Training dice loss: {}'.format(datetime.datetime.now(), str(loss)))

						train_summary_writer.add_summary(summary, global_step=tf.train.global_step(sess, global_step))
						train_summary_writer.flush()

					except tf.errors.OutOfRangeError:
						start_epoch_inc.op.run()
						model.is_training = False;
						# print(start_epoch.eval())
						# save the model at end of each epoch training
						print("{}: Saving checkpoint of epoch {} at {}...".format(datetime.datetime.now(),epoch+1,FLAGS.checkpoint_dir))
						if not (os.path.exists(FLAGS.checkpoint_dir)):
							os.makedirs(FLAGS.checkpoint_dir,exist_ok=True)
						saver.save(sess, checkpoint_prefix, 
							global_step=tf.train.global_step(sess, global_step),
							latest_filename="checkpoint-latest")
						print("{}: Saving checkpoint succeed".format(datetime.datetime.now()))
						break
						
				if FLAGS.testing:
					# testing phase
					print("{}: Training of epoch {} finishes, testing start".format(datetime.datetime.now(),epoch+1))
					while True:
						try:
							sess.run(tf.local_variables_initializer())
							if FLAGS.attention:
								[image, label, distMap] = sess.run(next_element_test)
								distMap = distMap[:,:,:,:,np.newaxis]
							else:
								[image, label] = sess.run(next_element_test)

							label = label[:,:,:,:,np.newaxis]
							
							model.is_training = False;
							if FLAGS.attention:
								loss, summary, att_loss = sess.run([loss_op, summary_op, att_loss_op], 
									feed_dict={
										images_placeholder: image, 
										labels_placeholder: label, 
										distmap_placeholder: distMap, 
										model.train_phase: False, 
										attentionModule.train_phase: False, 
										outputModule.train_phase: False})
							else:
								loss, summary = sess.run([loss_op, summary_op], feed_dict={images_placeholder: image, labels_placeholder: label, model.train_phase: False})

							test_summary_writer.add_summary(summary, global_step=tf.train.global_step(sess, global_step))
							test_summary_writer.flush()

						except tf.errors.OutOfRangeError:
							break

		# close tensorboard summary writer
		train_summary_writer.close()
		if FLAGS.testing:
			test_summary_writer.close()
Пример #15
0
loop = None  # Embedded loop file
embeddedC = None  # Embedded loop connection object

inputList = []  # List of connected Input devices on the GPIO pins
outputList = []  # List of connected Output devices on the GPIO pins
pwmList = []  # List of connected PWM devices on the GPIO pins
adcList = []  # List of connected ADC devices on the Analog pins
i2cList = []  # List of connected IMU devices on the I2C ports
connectionState = ""  # Current connection state to be displayed
updateDuration = 0  # Cycle duration needed to update values
scanDuration = 0  # Cycle duration needed to scan for new devices

mux = MuxModule.GetMux()  # Initialize the SoftWEAR Mux Module
input = InputModule.Input()  # Initialize the SoftWEAR Input Module
output = OutputModule.Output()  # Initialize the SoftWEAR Output Module
pwm = PWMModule.PWM()  # Initialize the SoftWEAR PWM Module
adc = ADCModule.ADC()  # Initialize the SoftWEAR ADC Module
i2c = I2CModule.I2C()  # Initialize the SoftWEAR I2C Module

GPIO.setup(scanPin, GPIO.IN, GPIO.PUD_UP)  # Setup scan pin


def muxScan():
    """Scan for new mux devices."""
    global mux
    mux.scan()  # Scan devices on the mux pins


def inputScan():
    """Scan for new input devices."""
Пример #16
0
    def init(self, conf):
        # global conf
        self.__globalConf = conf

        # create all modules

        # create logger
        isExternal = self.__globalConf.get("EXTERNAL_LOGGER")
        LoggerModule.LoggerModule().init(self.__globalConf,
                                         externalLogger=isExternal)
        self.Logger = LoggerModule.LoggerModule()
        # create misc
        MiscModule.MiscModule().init(self.__globalConf)
        self.Misc = MiscModule.MiscModule()
        # create configs
        ConfigurationModule.ConfigurationModule().init(self.__globalConf)
        self.Configuration = ConfigurationModule.ConfigurationModule()
        # create workaround
        WorkaroundModule.WorkaroundModule().init(self.__globalConf)
        self.Workaround = WorkaroundModule.WorkaroundModule()
        # create host
        isExternal = self.__globalConf.get("EXTERNAL_LOCAL_EXEC", False)
        HostModule.HostModule().init(self.__globalConf,
                                     externalCmdExec=isExternal)
        self.Host = HostModule.HostModule()
        # create relay card
        RelayCardModule.RelayCardModule().init(
            self.__globalConf,
            externalRelayCard=self.__globalConf.get("EXTERNAL_RELAY_CARD"))
        self.RelayCard = RelayCardModule.RelayCardModule()
        # create report
        ReportModule.ReportModule().init(self.__globalConf)
        self.Report = ReportModule.ReportModule()
        # create output
        OutputModule.OutputModule().init(self.__globalConf)
        self.Output = OutputModule.OutputModule()
        # create device
        DeviceModule.DeviceModule().init(self.__globalConf)
        self.Device = DeviceModule.DeviceModule()
        # create campaign
        CampaignModule.CampaignModule().init(self.__globalConf)
        self.Campaign = CampaignModule.CampaignModule()
        # create OsManager
        OsManagerModule.OsManagerModule().init(self.__globalConf)
        self.OsManager = OsManagerModule.OsManagerModule()
        # create dediprog
        DediprogModule.DediprogModule().init(self.__globalConf)
        self.Dediprog = DediprogModule.DediprogModule()
        # create FlashFile
        FlashFileModule.FlashFileModule().init(self.__globalConf)
        self.FlashFile = FlashFileModule.FlashFileModule()
        # create logs
        LogsModule.LogsModule().init(self.__globalConf)
        self.Logs = LogsModule.LogsModule()
        # create efi var
        EfiVarModule.EfiVarModule().init(self.__globalConf)
        self.EfiVar = EfiVarModule.EfiVarModule()
        # create Flash
        FlashModule.FlashModule().init(self.__globalConf)
        self.Flash = FlashModule.FlashModule()
        # create reset irq
        ResetIrqModule.ResetIrqModule().init(self.__globalConf)
        self.ResetIrq = ResetIrqModule.ResetIrqModule()
        # create boot data
        BootDataModule.BootDataModule().init(self.__globalConf)
        self.BootData = BootDataModule.BootDataModule()
        # create watchdog
        WatchdogModule.WatchdogModule().init(self.__globalConf)
        self.Watchdog = WatchdogModule.WatchdogModule()
        # create download
        DownloadModule.DownloadModule().init(self.__globalConf)
        self.Download = DownloadModule.DownloadModule()
        # create testing
        TestingModule.TestingModule().init(self.__globalConf)
        self.Testing = TestingModule.TestingModule()
        # create DntEvents
        EventsModule.EventsModule().init(self.__globalConf)
        self.Events = EventsModule.EventsModule()

        # scripts
        scripts.COS_scripts.COS().init(self.__globalConf)
        self.COS_scripts = scripts.COS_scripts.COS()
        scripts.MOS_scripts.MOS().init(self.__globalConf)
        self.MOS_scripts = scripts.MOS_scripts.MOS()
        scripts.Watchdog_scripts.Watchdog().init(self.__globalConf)
        self.Watchdog_scripts = scripts.Watchdog_scripts.Watchdog()
        scripts.KeyPress_scripts.KeyPress().init(self.__globalConf)
        self.KeyPress_scripts = scripts.KeyPress_scripts.KeyPress()
        scripts.POS_scripts.POS().init(self.__globalConf)
        self.POS_scripts = scripts.POS_scripts.POS()
        scripts.RMA_scripts.RMA().init(self.__globalConf)
        self.RMA_scripts = scripts.RMA_scripts.RMA()
        scripts.ROS_scripts.ROS().init(self.__globalConf)
        self.ROS_scripts = scripts.ROS_scripts.ROS()
        scripts.DNX_scripts.DNX().init(self.__globalConf)
        self.DNX_scripts = scripts.DNX_scripts.DNX()
        scripts.Flash_scripts.Flash().init(self.__globalConf)
        self.Flash_scripts = scripts.Flash_scripts.Flash()
        scripts.Shell_scripts.Shell().init(self.__globalConf)
        self.Shell_scripts = scripts.Shell_scripts.Shell()
        scripts.Time_scripts.Time().init(self.__globalConf)
        self.Time_scripts = scripts.Time_scripts.Time()
        scripts.Debug_scripts.Debug().init(self.__globalConf)
        self.Debug_scripts = scripts.Debug_scripts.Debug()

        # print Library version
        self.Logger.printLog(
            "INFO", "BOOT/OTA LIBRARY Code Version: " + self.__version)
        self.__globalConf["BOOTOTA_LIBRARY_CODE_VERSION"] = self.__version
        if self.__framework_push:
            self.Logger.printLog(
                "INFO", "FRAMEWORK Code Version: " + self.__framework_push)
            self.__globalConf["FRAMEWORK_CODE_VERSION"] = self.__framework_push