def response_performed(attackDetails, responseTechnique, techniquePath): #Send a message to server letting it know that the response technique is performed client.sock.send( serialize('ack', '{}!.!{}'.format('response performed', responseTechnique))) #The client waits for the server to finish the effectiveness test attack print('Waiting for the server to perform the effectiveness attack') client.FINISH_TESTING_EFFECTIVENESS.wait() #If both are true, that means the response technique failed to stop the attack print('Server done with effectiveness attack') if UNDER_ATTACK.is_set(): isEffective = False elif not UNDER_ATTACK.is_set(): isEffective = True #Clear flags UNDER_ATTACK.clear() client.FINISH_TESTING_EFFECTIVENESS.clear() #Load the saved file print('Loading the log file') logFile = read_data('data/logs/{}/{}.json'.format(attackDetails, responseTechnique)) #Add effective status to the json file logFile['is_effective'] = isEffective #Send the log so we don't keep the server waiting for log file, in case this was the last technique to be assessed print('Sending the log file') client.sock.send( send_log( 'log', "{}!.!{}!.!{}!.!{}".format( specifications['software_version'], specifications['hardware_specifications'], attackDetails, responseTechnique), logFile)) #After sending the log, delete the log file try: os.remove('data/logs/{}/{}.json'.format(attackDetails, responseTechnique)) except: print("Error while deleting file: ", logFile) #Revert changes done when the response technique applied print('Running revert operation') subprocess.run(shlex.split("{} {}".format(techniquePath, "revert"))) print('Revert operation successful') #After getting back to the original state, change busy status to false specifications['is_busy'] = False #Save changes save_data('data/specifications.json', specifications) #Inform the server that the car is no longer busy print( 'Sending busy status to server. Setting it to false (the car is ready for more work)' ) client.sock.send( serialize('ack', '{}!.!{}'.format('client busy status', 'false')))
def running_response_technique(attackDetails, responseTechnique, path): #A place holder for log information. tempDict = {} tempDict['cpu_utilization'] = '100' print("Running {} from path {}".format(responseTechnique, path)) if specifications['is_collaborative']: os.makedirs(os.path.dirname('data/logs/{}/{}.json'.format( attackDetails, responseTechnique)), exist_ok=True) save_data( 'data/logs/{}/{}.json'.format(attackDetails, responseTechnique), tempDict) #Run response technique subprocess.run(shlex.split("{} {}".format(path, "apply")))
def log_received(data): print('A log has been received!') #Get contents between delimiters tempMessage = data.split(b'!:START:!')[-1].split(b'!:FINISH:!')[0] #Get the rest of file data tempFile = data.rsplit(b'!:FINISH:!', 1)[1] #Filter them out tempMessage = tempMessage.decode('utf-8') message = tempMessage.split('!.!') softwareVersion = message[0] hardwareSpecifications = message[1] attackDetails = message[2] responseTechnique = message[3] file = pickle.loads(tempFile) #Save the file filePath = 'data/logs/{}/{}/{}/{}.json'.format(softwareVersion, hardwareSpecifications, attackDetails, responseTechnique) save_data(filePath, file) #Update the log path in the response techniques data responseTechniquesData[softwareVersion][hardwareSpecifications][attackDetails][responseTechnique]['log_path'] = filePath
def evaluate_techniques(self, responseTechniques): print("Evaluating techniques...") #If one technique is not evaluated, then start the evaluation process isEvaluationRequired = False for technique in responseTechniques: if not responseTechniques[technique]['is_evaluated']: isEvaluationRequired = True break #Otherwise, no need for evaluation if not isEvaluationRequired : return #Update response techniques database import glob src = "data/logs/{}/{}/{}/".format(self.softwareVersion, self.hardwareSpecifications, self.attackDetails) files = glob.glob('{}/*'.format(src), recursive=False) # Loop through files for single_file in files: json_file = read_data(single_file) fileName = os.path.basename(single_file) techniqueName = fileName.replace('.json', '') isEffective = json_file['is_effective'] responseTechniquesData[self.softwareVersion][self.hardwareSpecifications][self.attackDetails][techniqueName]['is_effective'] = isEffective #Add every effective technique and its duration effectiveTechniques = {} for technique in responseTechniques: if responseTechniques[technique]['is_effective']: effectiveTechniques[technique] = responseTechniques[technique]['duration'] #Get the lowest duration technique bestResponseTechnique = min(effectiveTechniques, key=effectiveTechniques.get) print("Effective techniques are: ", effectiveTechniques) print("Best effective response technique is {}.".format(bestResponseTechnique)) #Modify as necessary responseTechniquesData[self.softwareVersion][self.hardwareSpecifications][self.attackDetails][bestResponseTechnique]['is_most_efficient'] = True responseTechniquesData[self.softwareVersion][self.hardwareSpecifications][self.attackDetails][bestResponseTechnique]['is_evaluated'] = True for technique in responseTechniques: if not technique == bestResponseTechnique: responseTechniquesData[self.softwareVersion][self.hardwareSpecifications][self.attackDetails][technique]['is_most_efficient'] = False responseTechniquesData[self.softwareVersion][self.hardwareSpecifications][self.attackDetails][technique]['is_evaluated'] = True #Save changes to the json file save_data('data/response_techniques_database.json', responseTechniquesData)
def response_system(attackDetails): #Under attack state is being processed, clear it UNDER_ATTACK.clear() if not specifications['is_collaborative']: #After detecting the attack -> request evaluation from the server evaluationThread = threading.Thread( target=request_evaluation(attackDetails)) evaluationThread.start() #Wait for the thread to finish evaluationThread.join() #Clear the response ack client.RESPONSE_ACK.clear() #Load the response system responseSystem = read_data('data/response_system.json') responseTechnique = responseSystem[attackDetails] techniquePath = responseTechniquesData[ responseTechnique] # The path for the technique script running_response_technique(attackDetails, responseTechnique, techniquePath) else: #Assign car to busy and save it locally specifications['is_busy'] = True save_data('data/specifications.json', specifications) #Load the response system responseSystem = read_data('data/response_system.json') responseTechnique = responseSystem[attackDetails] #Run the response technique applied by the response system print('Under {} attack! Running {}!'.format(attackDetails, responseTechnique)) techniquePath = responseTechniquesData[ responseTechnique] # The path for the technique script #Generate a new thread and run the response technique script responseThread = threading.Thread(target=running_response_technique( attackDetails, responseTechnique, techniquePath)) responseThread.start() #Wait for the thread to finish responseThread.join() response_performed(attackDetails, responseTechnique, techniquePath)
def evaluation_requested(self, data): message = pickle.loads(data).split('!.!') self.attackDetails = message[0] self.responseTechniqueApplied = message[1] print('Evaluation requested by ID: {} for {}. The applied response technique is: {}.'.format(str(self.id), self.attackDetails, self.responseTechniqueApplied)) #Start observer to wait for log arrivals #Monitor the directory and wait for the new file to arrive from the car logging.basicConfig(level=logging.ERROR) path = "data/logs/{}/{}/{}/".format(self.softwareVersion, self.hardwareSpecifications, self.attackDetails) os.makedirs(os.path.dirname(path), exist_ok=True) #Run observer to monitor log file changes observer = Observer() event_handler = FileEventHandler(observer) observer.schedule(event_handler, path, recursive=False) observer.start() #Load response techniques based on the software version, hardware specifications, and attack suitableTechniques = responseTechniquesData[self.softwareVersion][self.hardwareSpecifications][self.attackDetails] #Loop through the suitable techniques and assess un-assessed techniques assess_techniques(self, suitableTechniques) #Wait for the assessment to be done ASSESSMENT_FLAG.wait() #Clear the assessment flag ASSESSMENT_FLAG.clear() #Terminate the observer observer.stop() observer.join() #Save changes to the json file save_data('data/response_techniques_database.json', responseTechniquesData) #After assessing all un-assessed techniques, evaluate them evaluate_techniques(self, suitableTechniques) #After making sure that all techniques are assessed and evaluated, find the # best response technique for that metric and send it as update message final_update(self, self.attackDetails, suitableTechniques)
def apply_update(attackDetails, responseTechnique): print('Update received! Applying: {} for: {}'.format(responseTechnique, attackDetails)) #Read and update the response_system file responseSystem = read_data('data/response_system.json') responseSystem[attackDetails] = responseTechnique save_data('data/response_system.json', responseSystem)
def exp(args): #Get Image Data x_train, y_train, x_test, y_test, prior_test = load_dataset(args.dataset) #Get Sets Sizes args.set_sizes = get_set_sizes(args.sets, len(x_train), args.set_size_gen) #Randomly Generate Priors Pi args.Pi = get_Pi(args.sets, args.Pi_gen) #Sample Data According to Pi and Sets Sizes U_sets, priors_corr = get_U_sets(args.sets, y_train, args.set_sizes, args.Pi) print('Data prepared!') print("set_sizes: " + str(args.set_sizes)) print("test class prior: " + str(prior_test)) print("Pi: " + str(args.Pi)) # Get Model ExpModel = MultiLayerPerceptron(dataset=args.dataset, sets=args.sets, set_sizes=args.set_sizes, Pi=args.Pi, mode=args.mode, weight_decay=args.weightdecay) # Schedule Learning Rate if not specified if args.learningrate == -1: args.learningrate = lr_scheduler(args.dataset, args.sets, args.mode) # Get optimizer ExpModel.optimizer = Adam(args.optimizer, lr=args.learningrate, decay=args.lr_decay) # Build Model input_shape = x_train[0].shape ExpModel.build_model(priors_corr, prior_test, args.Pi, input_shape, mode=args.mode) #-----------------------------------------------------Start Training-----------------------------------------------------# history, loss_test = ExpModel.fit_model(U_sets=U_sets, x_train_total=x_train, batch_size=args.batchsize, epochs=args.epoch, x_test=x_test, y_test=y_test, Pi=args.Pi, priors_corr=priors_corr, prior_test=prior_test, mode=args.mode) np_loss_test = np.array(loss_test) np_loss_train = np.array(history['loss']) plot_curve(np_loss_test, args.epoch, label=args.mode, phase='test', dataset=args.dataset) plot_curve(np_loss_train, args.epoch, label=args.mode, phase='train', dataset=args.dataset) #---------------------------------------------Save files----------------------------------------------------------------# save_data(args, U_sets, priors_corr, prior_test, np_loss_train, np_loss_test)
step=0 for v in D: step+=1 u=v #u is actually v0 u=np.expand_dims(u,1) for t in range(k): h=sampleH(W,c,u) u=sampleU(W,b,h) v=np.expand_dims(v,axis=1) W += eta*(np.matmul(sigmoid(np.matmul(W,v)+c),v.T)-np.matmul(sigmoid(np.matmul(W,u)+c),u.T)) b += eta*(v.astype(np.int8) -u.astype(np.int8)) c += eta*(sigmoid(np.matmul(W,v)+c)-sigmoid(np.matmul(W,u)+c)) loss+=np.sum(np.abs(v.astype(np.int8) -u.astype(np.int8))) print 'epoch %d , loss %.4f'%(epoch,loss/len(D)) lost.append(loss/len(D)) return W,b,c # In[ ]: W,b,c=RBM(200,training_data) # In[ ]: save_data([W,b,c],'weights.bin')
def persistence(): '''Persists data every 10 seconds''' while True: save_data() sleep(10)
def train(self, sess, X, Y, label_names, label_colors, label_map_path, sampleOutputPath): # load model in case if (self.loadModel > -1): print("LOAD THE MODEL . . .") self.saver.restore(sess, './checkpoints/VAE_GAN-' + str(self.loadModel)) print("DONE!") # if new model set tensorboard summaryOp = tf.summary.merge_all() summaryWriter = tf.summary.FileWriter( "train/gan_{}".format(datetime.datetime.now().strftime("%s")), sess.graph) # get length of dataset nData = len(X) # sample noise Z to generate image - normal distributed sampleZ = np.random.normal(0, 1, size=self.zDim) # start training self.step = 0 self.losses = [] # train bool trainEncD = True trainD = True # epoch loop startEpoch = int(self.loadModel) + 1 for epoch in range(startEpoch, self.epochs): # print print("Starting epoch n {}/{} : ".format(epoch + 1, self.epochs)) # compute number of iteraton for an epoch nIter = nData # randomize indexes for batches batchAllIndexes = np.random.random_integers(0, nData - 1, size=nData) # iter loop for iter in range(nIter): # increment step counter self.step += 1 print("Starting iteration n {}/{} : ".format(iter + 1, nIter)) # get batch indexes batchIndexes = batchAllIndexes[iter] # get data and convert it to correct format thisX, thisY = h.processScene(X[batchIndexes], Y[batchIndexes], self.subVolumeShape) # apply some noise to the data noiseX = np.random.normal(loc=0, scale=0.001, size=self.subVolumeShape) noiseY = np.random.normal(loc=0, scale=0.001, size=self.subVolumeShape) thisX = np.add(thisX, noiseX) thisY = np.add(thisY, noiseY) # sample latent sample thisZ = np.random.normal(0, 1, size=self.zDim) # UPDATE E _ = sess.run(self.encTrainOpt, feed_dict={ self.x: thisX, self.y: thisY, self.z: thisZ }) # UPDATE G _, thisSummary = sess.run([self.genTrainOpt, summaryOp], feed_dict={ self.x: thisX, self.y: thisY, self.z: thisZ }) # UPDATE D if (trainD): # sample random noise for D _ = sess.run(self.discTrainOpt, feed_dict={ self.x: thisX, self.y: thisY, self.z: thisZ }) # UPDATE ENC D if (trainEncD): _ = sess.run(self.encDiscTrainOpt, feed_dict={ self.x: thisX, self.y: thisY, self.z: thisZ }) # write summary summaryWriter.add_summary(thisSummary, self.step) # eval losses and at the end of each iter, get the losses and print them out encDiscTrainLoss = self.encDLoss.eval({ self.x: thisX, self.y: thisY, self.z: thisZ }) discTrainLoss = self.dLoss.eval({ self.x: thisX, self.y: thisY, self.z: thisZ }) encTrainLoss = self.encLoss.eval({ self.x: thisX, self.y: thisY, self.z: thisZ }) genTrainLoss = self.gLoss.eval({ self.x: thisX, self.y: thisY, self.z: thisZ }) encDAccuracy = self.encDiscAccuracy.eval({ self.x: thisX, self.y: thisY, self.z: thisZ }) dAccuracy = self.discAccuracy.eval({ self.x: thisX, self.y: thisY, self.z: thisZ }) print("Encoder Discriminator Accuracy : {:.2f}...".format( encDAccuracy)) print("Discriminator Accuracy : {:.2f}...".format(dAccuracy)) # check training next iteration trainEncD = True trainD = True if (encDAccuracy >= 0.8): trainEncD = False if (dAccuracy >= 0.8): trainD = False # print losses print( "Encoder Discriminator Loss: {:.4f}...".format( encDiscTrainLoss), "Discriminator Loss: {:.4f}...".format(discTrainLoss), "Encoder Loss: {:.4f}".format(encTrainLoss), "Generator Loss: {:.4f}".format(genTrainLoss)) # get sample thisSample, thisSampleLogit = sess.run(self.generator( self.z, reuse=True, isTraining=False), feed_dict={self.z: sampleZ}) # visualize and save it sampleName = 'gen_' + str(epoch) h.save_data(thisSample[0], sampleName, './samples', label_names, label_colors, label_map_path, False) # save session print("SAVING SESSION! . . . ") self.saver.save(sess, './checkpoints/VAE_GAN', global_step=epoch) # return the losses return self.losses
def pv_data(coordinates=None, address=None, tilt=30, solar_size_kw_dc=4500, inverter_size_kw=4500, inverter_efficiency=96, system_losses_perc_of_dc_energy=14, mount="Fixed (open rack)", module_type="Default [Standard]"): """ :param coordinates: optional, set the coordinates of the location as (latitude, longitude) :param address: optional, set the address as a string :param tilt: integer, tilt angle default = 40 :param solar_size_kw_dc: solar size in KW :param inverter_size_kw: inverter size in KW :param inverter_efficiency: inverter efficiency, default = 96 (96%) :param system_losses_perc_of_dc_energy: system losses, default = 14 (14%) :param mount: mount type, default = 'Fixed (open rack)' :param module_type: module type, default= "Default [Standard]" :return: hourly data of expected solar energy """ # TODO: Figure why inverter_size_kw, export_limit_ac_k_w, export_limit_ac_k_w, dataset, # and annual_solar_degradation_perc_per_year are not implemented if coordinates: assert isinstance(coordinates, tuple), 'coordinates should be in tuple format' latitude = coordinates[0] longitude = coordinates[1] location = "&" + "lat=" + str(latitude) + "&" + "lon=" + str(longitude) tilt = latitude verbose_dic = {'Coordinates': coordinates, 'Tilt Angle': int(tilt)} elif address: location = "&" + "address=" + address verbose_dic = {'Address': address, 'Tilt Angle': int(tilt)} else: raise Exception('either coordinates or address must be input') verbose_dic.update({ 'Solar System Size (kW DC)': solar_size_kw_dc, 'Inverter Size (kW)': inverter_size_kw, 'Inverter Efficiency': inverter_efficiency, 'System Losses Percentage of DC Energy': system_losses_perc_of_dc_energy }) save_data(verbose_dic, 'solar_inputs', overwrite=True, description='Solar Inputs to View on Excel') # Solar Input Selection Setup and API_Key Setup pv_watts_api_key = "VTF48OxZfq7tlP4oriEDVK1qAnpOCPdzl0XGT2c0" mount_dict = { "Fixed (open rack)": 0, "Fixed (roof mount)": 1, "1-Axis Tracking": 2, "1-Axis Backtracking": 3, "2-Axis": 4, "Default [Fixed (open rack)]": 0 } module_type_dict = { "Standard": 0, "Premium": 1, "Thin film": 2, "Default [Standard]": 0 } array_azimuth_dict = {"135 (SE)": 135, "180 (S)": 180, "225 (SW)": 225} dataset = "nsrdb" array_azimuth = array_azimuth_dict["180 (S)"] mount = mount_dict[mount] module_type = module_type_dict[module_type] # Over a 25 year life annual_solar_degradation_perc_per_year = 0.5 * .01 # If no export limit then state "No Limit" export_limit_ac_k_w = 3000 variable = 1 get_link = ''.join([ "https://developer.nrel.gov/api/pvwatts/v6.json?" + "api_key=" + pv_watts_api_key, "&" + "system_capacity=" + str(solar_size_kw_dc), "&" + "module_type=" + str(module_type), "&" + "losses=" + str(system_losses_perc_of_dc_energy), "&" + "array_type=" + str(mount), "&" + "tilt=" + str(tilt), "&" + "azimuth=" + str(array_azimuth), location, "&" + "dataset=nsrdb", "&" + "timeframe=hourly", "&" + "inv_eff=" + str(inverter_efficiency), ]) result = requests.get(get_link) data = result.json() outs_w = data["outputs"]["ac"] outs_kw = [out / 1000 for out in outs_w] outs_poa = data["outputs"]["poa"] outs_dn = data["outputs"]["dn"] outs_tamb = data["outputs"]["tamb"] outs_poa = pd.Series(outs_poa) outs_kw = pd.Series(outs_kw) outs_dn = pd.Series(outs_dn) outs_tamb = pd.Series(outs_tamb) df = load_data('hours_in_year') df['solar_output_kw'] = outs_kw.values df['solar_irradiance'] = outs_poa.values df['direct_normal_irradiance'] = outs_dn.values df['ambient_temperature'] = outs_tamb.values * 1.8000 + 32 return df
return l_data, l_count, l_dictionary, l_reverse_dictionary data, count, dictionary, reverse_dictionary = build_dataset(arg_words=words) # print('l_data:', data) # print('l_count:', count) # print('l_dictionary:', dictionary) # print('l_reverse_dictionary:', reverse_dictionary) # 删除words节省内存 del words data_index = 0 # 保存字典和符号表文件,以便使用模型时使用 helper.save_data(token=token_dict, vocab_to_int=dictionary, int_to_vocab=reverse_dictionary) # 3.为skip-gram模型生产训练参数 def generate_batch(arg_batch_size, arg_num_skips, arg_skip_windows): global data_index l_batch = np.ndarray(shape=arg_batch_size, dtype=np.int32) # (1, arg_batch_size) l_labels = np.ndarray(shape=(arg_batch_size, 1), dtype=np.int32) # (arg_batch_size,1) span = 2 * arg_skip_windows + 1 # 示例[我 爱 祖国] buffer = collections.deque(maxlen=span) # 把词尽可能的随机分词组合
def suf_processing(user, mapping, data_to_save): index_name = mapping['identifier'] + "_" + user['username'] user = { "password": user['password'] } helper.save_data(user=user, datas=data_to_save, index_name=index_name)