def task2_linear(): gc.collect() result = [[],[],[]] for i in range(15): size = 10*pow(2, i) # Getting currante memory usage start_mem = memory_usage() for j in range(10): # Releasing unused memory gc.collect() # Getting current time start_time = time.clock() # Chain generation x = "".join([random.choice(z) for _ in range(size)]) y = "".join([random.choice(z) for _ in range(size)]) # Sequence Alignment v, cost = linear_sequence_alignment(x, y, g, a) out_x, out_y = get_sequence_linear(x, y, g, a) elapsed = (time.clock() - start_time)/60 used_men = memory_usage() - start_mem # Saves chain size result[0].append(size) # Saves time result[1].append(elapsed) # Saves consumed memory result[2].append(used_men) print "i = %d, j = %d\nElapsed Time: %.3f mins\nUsed memory: %.3f MB" % (i, j, elapsed, used_men) # Releasing unused memory del x, y, v, out_x, out_y gc.collect() if elapsed > 15: return result return result
def test_with(sess, solver, path, name, time_steps=26, batch_size=1): # Load test instances print("{timestamp}\t{memory}\tLoading test {name} instances ...".format( timestamp=timestamp(), memory=memory_usage(), name=name)) test_generator = instance_loader.InstanceLoader(path) test_loss = 0.0 test_accuracy = 0.0 test_avg_pred = 0.0 test_batches = 0 # Run with the test instances print("{timestamp}\t{memory}\t{name} TEST SET BEGIN".format( timestamp=timestamp(), memory=memory_usage(), name=name)) for b, batch in enumerate(test_generator.get_batches(batch_size)): l, a, p = run_and_log_batch(sess, solver, name, b, batch, time_steps, train=False) test_loss += l test_accuracy += a test_avg_pred += p test_batches += 1 #end for # Summarize results and print test summary test_loss /= test_batches test_accuracy /= test_batches test_avg_pred /= test_batches print( "{timestamp}\t{memory}\t{name} TEST SET END Mean loss: {loss:.4f} Mean Accuracy = {accuracy} Mean prediction {avg_pred:.4f}" .format(loss=test_loss, accuracy=test_accuracy, avg_pred=test_avg_pred, timestamp=timestamp(), memory=memory_usage(), name=name))
def run_and_log_batch(sess, solver, epoch, b, batch, time_steps, train=True): sat = list(1 if sat else 0 for sat in batch.sat) # Build feed_dict feed_dict = { solver["time_steps"]: time_steps, solver["M"]: batch.get_dense_matrix(), solver["instance_SAT"]: np.array(sat), solver["num_vars_on_instance"]: batch.n } # Run session if train: _, pred_SAT, loss_val, accuracy_val = sess.run([ solver["train_step"], solver["predicted_SAT"], solver["loss"], solver["accuracy"] ], feed_dict=feed_dict) else: pred_SAT, loss_val, accuracy_val = sess.run( [solver["predicted_SAT"], solver["loss"], solver["accuracy"]], feed_dict=feed_dict) #end if avg_pred = np.mean(np.round(sigmoid(pred_SAT))) # Print train step loss and accuracy, as well as predicted sat values compared with the normal ones print( "{timestamp}\t{memory}\tEpoch {epoch} Batch {batch} (n,m) ({n},{m}) Loss: {loss:.4f} Accuracy: {accuracy:.4f} Average Prediction: {avg_pred:.4f}" .format(timestamp=timestamp(), memory=memory_usage(), epoch=epoch, batch=b, loss=loss_val, accuracy=accuracy_val, avg_pred=avg_pred, n=batch.total_n, m=batch.total_m), flush=True) return loss_val, accuracy_val, avg_pred
def memory_usage(): payload = util.memory_usage() return json.dumps(payload)
GNN["nop"] = tf.no_op() return GNN #end build_network if __name__ == '__main__': d = 64 epochs = 100 batch_n_max = 4096 batches_per_epoch = 32 n_size_min = 16 n_loss_increase_threshold = 0.01 n_size_max = 512 edge_probability = 0.25 # Build model print( "{timestamp}\t{memory}\tBuilding model ...".format( timestamp = timestamp(), memory = memory_usage() ) ) GNN = build_network(d) # Disallow GPU use config = tf.ConfigProto( device_count = {"GPU":0}) with tf.Session(config=config) as sess: # Initialize global variables print( "{timestamp}\t{memory}\tInitializing global variables ... ".format( timestamp = timestamp(), memory = memory_usage() ) ) sess.run( tf.global_variables_initializer() ) # Run for a number of epochs print( "{timestamp}\t{memory}\tRunning for {} epochs".format( epochs, timestamp = timestamp(), memory = memory_usage() ) ) n_size = n_size_min for epoch in range( epochs ): # Run batches #instance_generator.reset()
def __call__(self, *args, **kwargs): timestamp = args[2] if not timestamp: """ sometimes we want to add something to the DNS map without using a timestamp; in this case we directly jump to dnsmap.add(..) """ return self.func.__call__(self.obj, *args, **kwargs) merged = False splitAndCleanedUp = False curTime = time.time() dnsmap = self.obj blocksMerged = set() numIPBlocksBeforeMerge = -1 if not self.tNextMerge: self.tNextMerge = timestamp + self.nextMergeInterval if not self.tNextSplitAndCleanup: self.tNextSplitAndCleanup = timestamp + self.nextSplitAndCleanupInterval if timestamp > self.tNextMerge: """ it's time to split and merge blocks """ merged = True """ remember how many IPBlocks we had before merging """ numIPBlocksBeforeMerge = dnsmap.getNumberOfIPBlocks() # FIXME, remove this: ensure that all IPBlocks are clustered, not just the ones # that are being merged/split below dnsmap.reclusterAll(config.clusteringThreshold, force=False) """ MERGE """ blocksMerged = dnsmap.mergeAllBlocks() numBlocksMerged = len(blocksMerged) """ Schedule next merge/split iteration """ self.tNextMerge += self.nextMergeInterval """ output some statistics """ msg = ('merged blocks: %u' % (numBlocksMerged)) logging.info(msg) if timestamp > self.tNextSplitAndCleanup: """ we do the cleanup AFTER the split/merge operation, as we need the domains set in each of IPBlocks not to be empty in order to cluster the domains for splitting/merging. The cleanup procedure *resets* the domains field, therefore it has to come after split/merge """ splitAndCleanedUp = True """ SPLIT we do this BEFORE calling dnsmap.cleanup(), as this call resets the active IP settings, and therefore affects the splitting. We want to split blocks only if the corresponding IPs were inactive for an entire cleanup time interval. NOTE: we do split blocks that were merged in the previous merge intervals though! """ numBlocksSplit = dnsmap.splitAllBlocks(blocksMerged) """ remove empty IPBlocks, remove unused domain names, uncollapse clusters, and reset all IPBlocks (i.e., reset the set of contained domains and set all IPs to not active) """ dnsmap.cleanup() """ After the first cleanup iteration we start to output suspicious activity """ dnsmap.doOutputSuspicious = True """ Schedule next cleanup iteration """ self.tNextSplitAndCleanup += self.nextSplitAndCleanupInterval """ output some statistics """ msg = ('split blocks: %u' % (numBlocksSplit)) logging.info(msg) """ dump current dnsmap to disk, omitting the domains """ dnsmap.dumpt(os.path.join(config.workingDir, 'dnsmap_' + str(timestamp) + '.txt'), withDomains=False) if merged or splitAndCleanedUp: """ output some statistics """ msg = ('t is now %u; merged: %s; splitAndCleanedUp: %s' % (timestamp, merged, splitAndCleanedUp)) logging.info(msg) logging.info('memory usage: ' + str(memory_usage())) msg = ('IPBlocks before merge: %u' % (numIPBlocksBeforeMerge)) logging.info(msg) msg = ('IPs/IPBlocks: %u/%u' % (dnsmap.getNumberOfIPs(), dnsmap.getNumberOfIPBlocks())) logging.info(msg) logging.info('domains: ' + str(dnsmap.getNumDomains())) logging.info('Clusters per IP: ' + str(dnsmap.getMeanStdClustersPerIP())) logging.info('Clusters per IPBlock: ' + str(dnsmap.getMeanStdClustersPerIPBlock())) logging.info('Collapsed clusters: ' + str(dnsmap.getNumCollapsedClusters())) logging.info('Blocks that reached cluster capacity: ' + str( len([ 1 for node in dnsmap.traverseTrees() if node.value.hasReachedClusterCapacity() ]))) logging.info('this took ' + str(time.time() - curTime) + ' seconds') return self.func.__call__(self.obj, *args, **kwargs)
GNN["train_step"] = train_step return GNN #end build_networks if __name__ == '__main__': d = 128 epochs = 100 batch_size = 32 batches_per_epoch = 128 time_steps = 200 # Build model print("{timestamp}\t{memory}\tBuilding model ...".format( timestamp=timestamp(), memory=memory_usage())) GNN = build_network(d) # Create train and test instance generators train_generator = InstanceLoader("train") test_generator = InstanceLoader("test") # Disallow GPU use config = tf.ConfigProto(device_count={"GPU": 0}) with tf.Session(config=config) as sess: # Initialize global variables print( "{timestamp}\t{memory}\tInitializing global variables ... ".format( timestamp=timestamp(), memory=memory_usage())) sess.run(tf.global_variables_initializer())
import tensorflow as tf import numpy as np import matplotlib.pyplot as plt from model import SAT_solver import instance_loader import itertools from util import test_with, timestamp, memory_usage if __name__ == "__main__": d = 128 batch_size = 64 test_time_steps = 28 # Use a much bigger number of time steps test_batch_size = batch_size # Build model print( "{timestamp}\t{memory}\tBuilding model ...".format( timestamp = timestamp(), memory = memory_usage() ) ) solver = SAT_solver( d ) # Create model saver saver = tf.train.Saver() with tf.Session() as sess: # Initialize global variables print( "{timestamp}\t{memory}\tInitializing global variables ... ".format( timestamp = timestamp(), memory = memory_usage() ) ) sess.run( tf.global_variables_initializer() ) # Restore saved weights print( "{timestamp}\t{memory}\tRestoring saved model ... ".format( timestamp = timestamp(), memory = memory_usage() ) ) saver.restore(sess, "./tmp/model.ckpt")
from model import build_neurosat import instance_loader import itertools from logutil import test_with from util import timestamp, memory_usage from cnf import ensure_datasets from sklearn.decomposition import PCA from sklearn.preprocessing import normalize from sklearn.cluster import KMeans from matplotlib import pyplot as plt from matplotlib.animation import FuncAnimation from numpy import linalg as LA if __name__ == "__main__": print("{timestamp}\t{memory}\tMaking sure ther datasets exits ...".format( timestamp=timestamp(), memory=memory_usage())) ensure_datasets(make_critical=True) if not os.path.isdir("tmp"): sys.exit(1) #end if d = 128 # Build model solver = build_neurosat(d) # Create model saver saver = tf.train.Saver() with tf.Session() as sess: # Initialize global variables
#import itertools from util import timestamp, memory_usage from dijkstra_util import baseline_error if __name__ == '__main__': d = 64 epochs = 100 batch_n_max = 4096 batches_per_epoch = 32 n_size_min = 16 n_loss_increase_threshold = 0.01 n_size_max = 512 edge_probability = 0.25 # Run for a number of epochs print( "{timestamp}\t{memory}\tRunning for {} epochs".format( epochs, timestamp = timestamp(), memory = memory_usage() ) ) n_size = n_size_min for epoch in range( epochs ): # Run batches #instance_generator.reset() epoch_loss = 0.0 epoch_err = 0 epoch_abserr = 0 epoch_n = 0 epoch_m = 0 for batch_i in range( batches_per_epoch ): # Create random graphs batch_n_size = np.random.randint( n_size_min, n_size+1 ) n_acc = 0 max_n = 0 instances = 0