def do_optimization(n,m,k,tau,lower_bounds, upper_bounds, r, rN, \
max_normal, sorted_index, max_processes, multi_event, get_values):
"""
Performs the optimization for the given parameters with max_proccesses
number of processes
Returns a list of the best C matrices and associated mu values
and likelihoods
"""
enum = Enumerator(n, m, k, tau, lower_bounds, upper_bounds, multi_event)
opt = Optimizer(r, rN, m, n, tau, upper_bound=max_normal)
MAX_QUEUE_SIZE = int(10E6)
try:
queue = Queue(MAX_QUEUE_SIZE) #Task queue for the processes
except OSError:
MAX_QUEUE_SIZE = 2**15 - 1
queue = Queue(MAX_QUEUE_SIZE)
returnQueue = Queue(
MAX_QUEUE_SIZE) #Shared queue for processes to return results
processes = start_processes(max_processes, queue, opt, returnQueue, \
sorted_index, get_values)
# fix problem with missing first matrix
#C = enum.generate_next_C()
C = enum._C_to_array()
count = 0
while C is not False:
count += 1
queue.put(C, True)
C = enum.generate_next_C()
if count == 0:
print "Error: No valid Copy Number Profiles exist for these intervals within the bounds specified. Exiting..."
sys.exit(1)
# Send STOP signal to all processes
for i in range(max_processes - 1):
queue.put(0)
best = []
for i in range(len(processes)):
item = returnQueue.get()
best.append(item)
for p in processes:
p.join()
best = find_mins(best)
return best
def do_optimization_single(n,m,k,tau,lower_bounds, upper_bounds, r, rN, \
max_normal, sorted_index, multi_event, get_values):
"""
Performs the optimization for the given parameters with a single process
Returns a list of the best C matrices and associated mu values
and likelihoods
"""
enum = Enumerator(n, m, k, tau, lower_bounds, upper_bounds, multi_event)
opt = Optimizer(r, rN, m, n, tau, upper_bound=max_normal)
min_likelihood = float("inf")
best = []
count = 0
#fix missing first matrix problem
C = enum._C_to_array()
#C = enum.generate_next_C()
if get_values: solns = []
while C is not False:
count += 1
soln = opt.solve(C)
if soln is not None:
(mu, likelihood, vals) = soln
if get_values: solns.append((C, mu, likelihood))
if isClose([likelihood], [min_likelihood]):
C_new = reverse_sort_C(C, sorted_index)
vals = reverse_sort_list(vals, sorted_index)
best.append((C_new, mu, likelihood, vals))
elif likelihood < min_likelihood:
C_new = reverse_sort_C(C, sorted_index)
vals = reverse_sort_list(vals, sorted_index)
best = [(C_new, mu, likelihood, vals)]
min_likelihood = likelihood
C = enum.generate_next_C()
if get_values:
with open(pre + "." + "likelihoods", 'w') as f:
for C, mu, likelihood in solns:
m, n = C.shape
stringC = "".join((str(int(C[i][1])) for i in range(m)))
f.write(stringC + "\t" + str(mu[0]) + "\t" + str(likelihood) +
"\n")
if count == 0:
print "Error: No valid Copy Number Profiles exist for these intervals within the bounds specified. Exiting..."
sys.exit(1)
return best
def do_optimization_single(n,m,k,tau,lower_bounds, upper_bounds, r, rN, \
max_normal, sorted_index, multi_event, get_values):
"""
Performs the optimization for the given parameters with a single process
Returns a list of the best C matrices and associated mu values
and likelihoods
"""
enum = Enumerator(n, m, k, tau, lower_bounds, upper_bounds, multi_event)
opt = Optimizer(r, rN, m, n,tau, upper_bound=max_normal)
min_likelihood = float("inf")
best = []
count = 0
#fix missing first matrix problem
C=enum._C_to_array()
#C = enum.generate_next_C()
if get_values: solns = []
while C is not False:
count += 1
soln = opt.solve(C)
if soln is not None:
(mu, likelihood,vals) = soln
if get_values: solns.append((C,mu,likelihood))
if isClose([likelihood],[min_likelihood]):
C_new = reverse_sort_C(C,sorted_index)
vals = reverse_sort_list(vals, sorted_index)
best.append((C_new, mu, likelihood, vals))
elif likelihood < min_likelihood:
C_new = reverse_sort_C(C,sorted_index)
vals = reverse_sort_list(vals, sorted_index)
best = [(C_new, mu, likelihood, vals)]
min_likelihood = likelihood
C = enum.generate_next_C()
if get_values:
with open(pre+"."+"likelihoods",'w') as f:
for C,mu,likelihood in solns:
m,n = C.shape
stringC = "".join((str(int(C[i][1])) for i in range(m)))
f.write(stringC+"\t"+str(mu[0])+"\t"+str(likelihood)+"\n")
if count == 0:
print "Error: No valid Copy Number Profiles exist for these intervals within the bounds specified. Exiting..."
sys.exit(1)
return best
def do_optimization(n,m,k,tau,lower_bounds, upper_bounds, r, rN, \ max_normal, sorted_index, max_processes, multi_event, get_values): """ Performs the optimization for the given parameters with max_proccesses number of processes Returns a list of the best C matrices and associated mu values and likelihoods """ enum = Enumerator(n, m, k, tau, lower_bounds, upper_bounds, multi_event) opt = Optimizer(r, rN, m, n,tau, upper_bound=max_normal) MAX_QUEUE_SIZE = int(10E6) try: queue = Queue(MAX_QUEUE_SIZE) #Task queue for the processes except OSError: MAX_QUEUE_SIZE = 2**15-1 queue = Queue(MAX_QUEUE_SIZE) returnQueue = Queue(MAX_QUEUE_SIZE) #Shared queue for processes to return results processes = start_processes(max_processes, queue, opt, returnQueue, \ sorted_index, get_values) # fix problem with missing first matrix #C = enum.generate_next_C() C=enum._C_to_array() count = 0 while C is not False: count += 1 queue.put(C, True) C = enum.generate_next_C() if count == 0: print "Error: No valid Copy Number Profiles exist for these intervals within the bounds specified. Exiting..." sys.exit(1) # Send STOP signal to all processes for i in range(max_processes-1): queue.put(0) best = [] for i in range(len(processes)): item = returnQueue.get() best.append(item) for p in processes: p.join() best = find_mins(best) return best
def run_tests():
global test_result
TEST_ENV = {
'stopped': False,
'test': True,
'send': False,
}
TEST_ENV['sender'] = TestSender()
TEST_ENV['enumerator'] = Enumerator(TEST_ENV)
TEST_ENV['parser'] = Parser(TEST_ENV)
TEST_ENV['screener'] = Screener(TEST_ENV)
TEST_ENV['logger'] = open('/dev/null', 'w')
TEST_ENV['processor'] = processor = Processor(TEST_ENV)
num_failures = 0
for (rcpttos, expected) in tests:
test_result = {}
processor.process_message({
'peer': None,
'mailfrom': 'mailfrom',
'rcpttos': rcpttos,
'data': 'message',
})
print 'testing sending to', rcpttos, '...'
for key in expected:
assert key in test_result
if type(test_result[key]) == type([]):
assert set(test_result[key]) == set(expected[key])
else:
assert test_result[key] == expected[key]
for key in test_result:
assert key in expected
def lookup_lists(self, key, cred):
request = self.get_sendlist_link(key)
with NamedTemporaryFile(prefix="siab_ccache_") as ccache:
# Write the credentials into a krb5 ccache.
ccache.write(make_ccache(cred))
ccache.flush()
# Use Enumerator to get mailing list members
enumerator = Enumerator({'test': True})
env = dict(os.environ)
env['KRB5CCNAME'] = ccache.name
valid = []
for mailing_list in request['lists']:
result = enumerator.list_emails(mailing_list, env)
if 'error' not in result and len(result['warnings']) == 0:
self.ENV['db'].list_members.update(
{'list': mailing_list},
{'$set': {
'members': '\n'.join(result['result'])
}},
upsert=True)
valid.append(mailing_list)
# Check whether all lists have known members
invalid = []
for mailing_list in request['lists']:
if not self.ENV['db'].list_members.find_one(
{'list': mailing_list}):
invalid.append(mailing_list)
sent = len(invalid) == 0
if sent:
self.ENV['db'].email_pool.update(
{
'_id': request['message_id'],
'status': 'FAILED'
}, {'$set': {
'status': 'VERIFIED'
}})
return {
'valid': ', '.join(valid),
'invalid': ', '.join(invalid),
'sent': sent
}
def run_tests():
TEST_ENV = {
'test': True,
}
enumerator = Enumerator(TEST_ENV)
# hard-coded list of sample query, expected result
tests = [('michaelx', ['*****@*****.**']),
('*****@*****.**', ['*****@*****.**']),
('next-code-admin', ['*****@*****.**', '*****@*****.**']),
('Next-Code-Admin', ['*****@*****.**', '*****@*****.**'])]
num_failures = 0
for query, expected in tests:
result = enumerator.list_emails(query)['result']
print 'testing ' + query + '...'
if set(result) != set(expected):
num_failures += 1
print 'ERROR: got', result, 'but expected', expected
print 'Ran', len(tests), 'tests, got', num_failures, 'failures'
def time_estimate(n,m,k,tau,lower_bounds, upper_bounds, r, rN, \ max_normal, sorted_index, num_processes, multi_event, force): """ Estimates the runtime with the specified bounds and parameters """ print "Estimating time..." if n is 3 and m > 30 and not force: print "\tWARNING: With n=3 and", m, "intervals, the runtime would likely be excessive. Try reducing the number of intervals below 25. Run with --FORCE to continue." exit(1) enum = Enumerator(n, m, k, tau, lower_bounds, upper_bounds, multi_event) opt = Optimizer(r, rN, m, n,tau, upper_bound=max_normal) if n == 2: TEST_NUM=100 count = count_number_matrices_2(m,upper_bounds, lower_bounds) else: TEST_NUM=20 count = count_number_matrices_3(m,upper_bounds, lower_bounds, enum) C = enum.generate_next_C() if C is False: print "ERROR: No valid Copy Number Profiles exist for these intervals within the bounds specified. Exiting..." sys.exit(1) start = time.clock() for i in range(TEST_NUM): try: soln = opt.solve(C) C = enum.generate_next_C() except: break end = time.clock() avgVal = float(end-start)/TEST_NUM seconds = count * (float(end-start)/TEST_NUM) / num_processes print "\tEstimated Total Time:", if seconds < 60: print int(seconds + .5), "second(s)" elif seconds < 3600: print int((seconds/60)+.5) , "minute(s)" else:
def time_estimate(n,m,k,tau,lower_bounds, upper_bounds, r, rN, \
max_normal, sorted_index, num_processes, multi_event, force):
"""
Estimates the runtime with the specified bounds and parameters
"""
print "Estimating time..."
if n is 3 and m > 30 and not force:
print "\tWARNING: With n=3 and", m, "intervals, the runtime would likely be excessive. Try reducing the number of intervals below 25. Run with --FORCE to continue."
exit(1)
enum = Enumerator(n, m, k, tau, lower_bounds, upper_bounds, multi_event)
opt = Optimizer(r, rN, m, n, tau, upper_bound=max_normal)
if n == 2:
TEST_NUM = 100
count = count_number_matrices_2(m, upper_bounds, lower_bounds)
else:
TEST_NUM = 20
count = count_number_matrices_3(m, upper_bounds, lower_bounds, enum)
C = enum.generate_next_C()
if C is False:
print "ERROR: No valid Copy Number Profiles exist for these intervals within the bounds specified. Exiting..."
exit(1)
start = time.clock()
for i in range(TEST_NUM):
try:
soln = opt.solve(C)
C = enum.generate_next_C()
except:
break
end = time.clock()
avgVal = float(end - start) / TEST_NUM
seconds = count * (float(end - start) / TEST_NUM) / num_processes
print "\tEstimated Total Time:",
if seconds < 60: print int(seconds + .5), "second(s)"
elif seconds < 3600: print int((seconds / 60) + .5), "minute(s)"
else:
def __init__(self, editor):
GObject.__init__(self)
self.__init_attributes(editor)
from Feedback import Feedback
Feedback(self, editor)
from URIOpener import Opener
Opener(self, editor)
from MatchFilterer import Filterer
Filterer(self, editor)
from FileFormatter import Formatter
Formatter(self, editor)
from FileinfoFilterer import Filterer
Filterer(self, editor)
from Enumerator import Enumerator
Enumerator(self, editor)
from FilesAggregator import Aggregator
Aggregator(self, editor)
from FolderPathUpdater import Updater
Updater(self, editor)
from GUI.Manager import Manager
Manager(self, editor)
def process_message(self, peer, mailfrom, rcpttos, data):
GLOBAL_ENV['db'].email_pool.insert({
'peer': peer,
'mailfrom': mailfrom,
'rcpttos': rcpttos,
'data': data,
'status': 'UNVERIFIED',
})
if __name__ == '__main__':
# Initialize components
GLOBAL_ENV['db'] = MongoClient().emails
sender = Sender(GLOBAL_ENV)
enumerator = Enumerator(GLOBAL_ENV)
parser = Parser(GLOBAL_ENV)
screener = Screener(GLOBAL_ENV)
processor = Processor(GLOBAL_ENV)
sendlist_server = SendlistServer(GLOBAL_ENV, IP_ADDRESS)
# Initialize global environment
GLOBAL_ENV['sender'] = sender
GLOBAL_ENV['enumerator'] = enumerator
GLOBAL_ENV['parser'] = parser
GLOBAL_ENV['screener'] = screener
GLOBAL_ENV['logger'] = open(LOG_FILE, 'a', 0)
GLOBAL_ENV['processor'] = processor
GLOBAL_ENV['sendlist_server'] = sendlist_server
server = SimpleSMTPServer((IP_ADDRESS, SMTP_PORT), None)
valid_input = False
input_extract = None
while (not valid_input):
input_extract = getQueryAndEvidenceVariables(input_problem)
print('Processing:' + str(input_extract))
if len(input_extract['query']) > 0 and len(input_extract['evidence']) > 0:
valid_input = True
if not valid_input:
input_problem = get_input(user_input_warning + user_input_note)
print('Performing enumeration....')
alarmNetwork = constructAlarmNetwork()
enumerated_result = Enumerator().askQueries(input_extract['query'],
input_extract['evidence'],
alarmNetwork)
def addArrays(array1, array2):
return [srcV + targV for srcV, targV in zip(array1, array2)]
def averageQueries(queries, samplesize):
denominator = [samplesize for x in range(len(queries[0]))]
returnArray = []
#print(queries)
for query in queries:
returnArray.append(
[probV / den for probV, den in zip(query, denominator)])
return returnArray
