def task2_linear():
gc.collect()
result = [[],[],[]]
for i in range(15):
size = 10*pow(2, i)
# Getting currante memory usage
start_mem = memory_usage()
for j in range(10):
# Releasing unused memory
gc.collect()
# Getting current time
start_time = time.clock()
# Chain generation
x = "".join([random.choice(z) for _ in range(size)])
y = "".join([random.choice(z) for _ in range(size)])
# Sequence Alignment
v, cost = linear_sequence_alignment(x, y, g, a)
out_x, out_y = get_sequence_linear(x, y, g, a)
elapsed = (time.clock() - start_time)/60
used_men = memory_usage() - start_mem
# Saves chain size
result[0].append(size)
# Saves time
result[1].append(elapsed)
# Saves consumed memory
result[2].append(used_men)
print "i = %d, j = %d\nElapsed Time: %.3f mins\nUsed memory: %.3f MB" % (i, j, elapsed, used_men)
# Releasing unused memory
del x, y, v, out_x, out_y
gc.collect()
if elapsed > 15:
return result
return result
def test_with(sess, solver, path, name, time_steps=26, batch_size=1):
# Load test instances
print("{timestamp}\t{memory}\tLoading test {name} instances ...".format(
timestamp=timestamp(), memory=memory_usage(), name=name))
test_generator = instance_loader.InstanceLoader(path)
test_loss = 0.0
test_accuracy = 0.0
test_avg_pred = 0.0
test_batches = 0
# Run with the test instances
print("{timestamp}\t{memory}\t{name} TEST SET BEGIN".format(
timestamp=timestamp(), memory=memory_usage(), name=name))
for b, batch in enumerate(test_generator.get_batches(batch_size)):
l, a, p = run_and_log_batch(sess,
solver,
name,
b,
batch,
time_steps,
train=False)
test_loss += l
test_accuracy += a
test_avg_pred += p
test_batches += 1
#end for
# Summarize results and print test summary
test_loss /= test_batches
test_accuracy /= test_batches
test_avg_pred /= test_batches
print(
"{timestamp}\t{memory}\t{name} TEST SET END Mean loss: {loss:.4f} Mean Accuracy = {accuracy} Mean prediction {avg_pred:.4f}"
.format(loss=test_loss,
accuracy=test_accuracy,
avg_pred=test_avg_pred,
timestamp=timestamp(),
memory=memory_usage(),
name=name))
def run_and_log_batch(sess, solver, epoch, b, batch, time_steps, train=True):
sat = list(1 if sat else 0 for sat in batch.sat)
# Build feed_dict
feed_dict = {
solver["time_steps"]: time_steps,
solver["M"]: batch.get_dense_matrix(),
solver["instance_SAT"]: np.array(sat),
solver["num_vars_on_instance"]: batch.n
}
# Run session
if train:
_, pred_SAT, loss_val, accuracy_val = sess.run([
solver["train_step"], solver["predicted_SAT"], solver["loss"],
solver["accuracy"]
],
feed_dict=feed_dict)
else:
pred_SAT, loss_val, accuracy_val = sess.run(
[solver["predicted_SAT"], solver["loss"], solver["accuracy"]],
feed_dict=feed_dict)
#end if
avg_pred = np.mean(np.round(sigmoid(pred_SAT)))
# Print train step loss and accuracy, as well as predicted sat values compared with the normal ones
print(
"{timestamp}\t{memory}\tEpoch {epoch} Batch {batch} (n,m) ({n},{m}) Loss: {loss:.4f} Accuracy: {accuracy:.4f} Average Prediction: {avg_pred:.4f}"
.format(timestamp=timestamp(),
memory=memory_usage(),
epoch=epoch,
batch=b,
loss=loss_val,
accuracy=accuracy_val,
avg_pred=avg_pred,
n=batch.total_n,
m=batch.total_m),
flush=True)
return loss_val, accuracy_val, avg_pred
def memory_usage():
payload = util.memory_usage()
return json.dumps(payload)
GNN["nop"] = tf.no_op()
return GNN
#end build_network
if __name__ == '__main__':
d = 64
epochs = 100
batch_n_max = 4096
batches_per_epoch = 32
n_size_min = 16
n_loss_increase_threshold = 0.01
n_size_max = 512
edge_probability = 0.25
# Build model
print( "{timestamp}\t{memory}\tBuilding model ...".format( timestamp = timestamp(), memory = memory_usage() ) )
GNN = build_network(d)
# Disallow GPU use
config = tf.ConfigProto( device_count = {"GPU":0})
with tf.Session(config=config) as sess:
# Initialize global variables
print( "{timestamp}\t{memory}\tInitializing global variables ... ".format( timestamp = timestamp(), memory = memory_usage() ) )
sess.run( tf.global_variables_initializer() )
# Run for a number of epochs
print( "{timestamp}\t{memory}\tRunning for {} epochs".format( epochs, timestamp = timestamp(), memory = memory_usage() ) )
n_size = n_size_min
for epoch in range( epochs ):
# Run batches
#instance_generator.reset()
def __call__(self, *args, **kwargs):
timestamp = args[2]
if not timestamp:
"""
sometimes we want to add something to the DNS map without using a
timestamp; in this case we directly jump to dnsmap.add(..)
"""
return self.func.__call__(self.obj, *args, **kwargs)
merged = False
splitAndCleanedUp = False
curTime = time.time()
dnsmap = self.obj
blocksMerged = set()
numIPBlocksBeforeMerge = -1
if not self.tNextMerge:
self.tNextMerge = timestamp + self.nextMergeInterval
if not self.tNextSplitAndCleanup:
self.tNextSplitAndCleanup = timestamp + self.nextSplitAndCleanupInterval
if timestamp > self.tNextMerge:
"""
it's time to split and merge blocks
"""
merged = True
"""
remember how many IPBlocks we had before merging
"""
numIPBlocksBeforeMerge = dnsmap.getNumberOfIPBlocks()
# FIXME, remove this: ensure that all IPBlocks are clustered, not just the ones
# that are being merged/split below
dnsmap.reclusterAll(config.clusteringThreshold, force=False)
"""
MERGE
"""
blocksMerged = dnsmap.mergeAllBlocks()
numBlocksMerged = len(blocksMerged)
"""
Schedule next merge/split iteration
"""
self.tNextMerge += self.nextMergeInterval
"""
output some statistics
"""
msg = ('merged blocks: %u' % (numBlocksMerged))
logging.info(msg)
if timestamp > self.tNextSplitAndCleanup:
"""
we do the cleanup AFTER the split/merge operation, as we need the
domains set in each of IPBlocks not to be empty in order to cluster
the domains for splitting/merging. The cleanup procedure *resets*
the domains field, therefore it has to come after split/merge
"""
splitAndCleanedUp = True
"""
SPLIT
we do this BEFORE calling dnsmap.cleanup(), as this call resets the
active IP settings, and therefore affects the splitting. We want to
split blocks only if the corresponding IPs were inactive for an
entire cleanup time interval.
NOTE: we do split blocks that were merged in the previous merge
intervals though!
"""
numBlocksSplit = dnsmap.splitAllBlocks(blocksMerged)
"""
remove empty IPBlocks, remove unused domain names, uncollapse
clusters, and reset all IPBlocks (i.e., reset the set of contained
domains and set all IPs to not active)
"""
dnsmap.cleanup()
"""
After the first cleanup iteration we start to output suspicious
activity
"""
dnsmap.doOutputSuspicious = True
"""
Schedule next cleanup iteration
"""
self.tNextSplitAndCleanup += self.nextSplitAndCleanupInterval
"""
output some statistics
"""
msg = ('split blocks: %u' % (numBlocksSplit))
logging.info(msg)
"""
dump current dnsmap to disk, omitting the domains
"""
dnsmap.dumpt(os.path.join(config.workingDir,
'dnsmap_' + str(timestamp) + '.txt'),
withDomains=False)
if merged or splitAndCleanedUp:
"""
output some statistics
"""
msg = ('t is now %u; merged: %s; splitAndCleanedUp: %s' %
(timestamp, merged, splitAndCleanedUp))
logging.info(msg)
logging.info('memory usage: ' + str(memory_usage()))
msg = ('IPBlocks before merge: %u' % (numIPBlocksBeforeMerge))
logging.info(msg)
msg = ('IPs/IPBlocks: %u/%u' %
(dnsmap.getNumberOfIPs(), dnsmap.getNumberOfIPBlocks()))
logging.info(msg)
logging.info('domains: ' + str(dnsmap.getNumDomains()))
logging.info('Clusters per IP: ' +
str(dnsmap.getMeanStdClustersPerIP()))
logging.info('Clusters per IPBlock: ' +
str(dnsmap.getMeanStdClustersPerIPBlock()))
logging.info('Collapsed clusters: ' +
str(dnsmap.getNumCollapsedClusters()))
logging.info('Blocks that reached cluster capacity: ' + str(
len([
1 for node in dnsmap.traverseTrees()
if node.value.hasReachedClusterCapacity()
])))
logging.info('this took ' + str(time.time() - curTime) +
' seconds')
return self.func.__call__(self.obj, *args, **kwargs)
GNN["train_step"] = train_step
return GNN
#end build_networks
if __name__ == '__main__':
d = 128
epochs = 100
batch_size = 32
batches_per_epoch = 128
time_steps = 200
# Build model
print("{timestamp}\t{memory}\tBuilding model ...".format(
timestamp=timestamp(), memory=memory_usage()))
GNN = build_network(d)
# Create train and test instance generators
train_generator = InstanceLoader("train")
test_generator = InstanceLoader("test")
# Disallow GPU use
config = tf.ConfigProto(device_count={"GPU": 0})
with tf.Session(config=config) as sess:
# Initialize global variables
print(
"{timestamp}\t{memory}\tInitializing global variables ... ".format(
timestamp=timestamp(), memory=memory_usage()))
sess.run(tf.global_variables_initializer())
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from model import SAT_solver
import instance_loader
import itertools
from util import test_with, timestamp, memory_usage
if __name__ == "__main__":
d = 128
batch_size = 64
test_time_steps = 28 # Use a much bigger number of time steps
test_batch_size = batch_size
# Build model
print( "{timestamp}\t{memory}\tBuilding model ...".format( timestamp = timestamp(), memory = memory_usage() ) )
solver = SAT_solver( d )
# Create model saver
saver = tf.train.Saver()
with tf.Session() as sess:
# Initialize global variables
print( "{timestamp}\t{memory}\tInitializing global variables ... ".format( timestamp = timestamp(), memory = memory_usage() ) )
sess.run( tf.global_variables_initializer() )
# Restore saved weights
print( "{timestamp}\t{memory}\tRestoring saved model ... ".format( timestamp = timestamp(), memory = memory_usage() ) )
saver.restore(sess, "./tmp/model.ckpt")
from model import build_neurosat
import instance_loader
import itertools
from logutil import test_with
from util import timestamp, memory_usage
from cnf import ensure_datasets
from sklearn.decomposition import PCA
from sklearn.preprocessing import normalize
from sklearn.cluster import KMeans
from matplotlib import pyplot as plt
from matplotlib.animation import FuncAnimation
from numpy import linalg as LA
if __name__ == "__main__":
print("{timestamp}\t{memory}\tMaking sure ther datasets exits ...".format(
timestamp=timestamp(), memory=memory_usage()))
ensure_datasets(make_critical=True)
if not os.path.isdir("tmp"):
sys.exit(1)
#end if
d = 128
# Build model
solver = build_neurosat(d)
# Create model saver
saver = tf.train.Saver()
with tf.Session() as sess:
# Initialize global variables
#import itertools
from util import timestamp, memory_usage
from dijkstra_util import baseline_error
if __name__ == '__main__':
d = 64
epochs = 100
batch_n_max = 4096
batches_per_epoch = 32
n_size_min = 16
n_loss_increase_threshold = 0.01
n_size_max = 512
edge_probability = 0.25
# Run for a number of epochs
print( "{timestamp}\t{memory}\tRunning for {} epochs".format( epochs, timestamp = timestamp(), memory = memory_usage() ) )
n_size = n_size_min
for epoch in range( epochs ):
# Run batches
#instance_generator.reset()
epoch_loss = 0.0
epoch_err = 0
epoch_abserr = 0
epoch_n = 0
epoch_m = 0
for batch_i in range( batches_per_epoch ):
# Create random graphs
batch_n_size = np.random.randint( n_size_min, n_size+1 )
n_acc = 0
max_n = 0
instances = 0
