def __init__(self, problem, shared, verbose=False, memory_limit=0):
"""
Constructor.
"""
multiprocessing.Process.__init__(self)
self.problem = problem
self.shared = shared
self.verbose = verbose
self.memory_limit = memory_limit
if memory_limit > 0:
from memorymonitor import MemoryMonitor
self.memmon = MemoryMonitor()
def __init__(self, problem, shared, verbose = False, memory_limit = 0):
"""
Constructor.
"""
multiprocessing.Process.__init__(self)
self.problem = problem
self.shared = shared
self.verbose = verbose
self.memory_limit = memory_limit
if memory_limit > 0:
from memorymonitor import MemoryMonitor
self.memmon = MemoryMonitor()
#!/usr/bin/env python
#http://www-rohan.sdsu.edu/~gawron/mt_plus/mt/course_core/lectures/assignment_five.pdf
#http://www.cs.jhu.edu/~alopez/papers/model1-note.pdf
#http://www.inf.ed.ac.uk/teaching/courses/mt/assignments/assignment2.pdf
#http://www.mt-archive.info/MTMarathon-2010-Lambert-ppt.pdf
import optparse
import sys
from collections import defaultdict
from memorymonitor import MemoryMonitor
memory_mon = MemoryMonitor('madmaze')
startMemory = memory_mon.usage()
def trainfw(itr):
x=0
print "Memory increased by", int(memory_mon.usage()) - startMemory
for (n, (f, e)) in enumerate(bitext):
if n%1000==0:
print "loading...",n
for f_i in set(f):
f_total[f_i] = 0.0
for e_j in set(e):
ef_count[(e_j,f_i)] = 0.0
t_ef[(e_j,f_i)]=1.0
x=0
print "Memory increased by", int(memory_mon.usage()) - startMemory
class ProcessedSolver(multiprocessing.Process):
def __init__(self, problem, shared, verbose = False, memory_limit = 0):
"""
Constructor.
"""
multiprocessing.Process.__init__(self)
self.problem = problem
self.shared = shared
self.verbose = verbose
self.memory_limit = memory_limit
if memory_limit > 0:
from memorymonitor import MemoryMonitor
self.memmon = MemoryMonitor()
@threadsafe_function
def run(self):
if self.verbose:
print self.name,'is working! at',time.time()
while True:
try:
UB, node = self.shared.inbox.get(True,1)
except:
continue
# stopping criterion
if isinstance(node,str) and node == 'stop_computation':
if self.verbose:
print 'stop computation recieved by',self.name,'at',time.time()
break
if self.verbose:
print self.name,'found node with UB',UB,'at',time.time()
# stopping criterion moved to master
#if UB - self.shared.get_LB() < self.problem.tol:
# a tolerable solution has been found!
# put current node back in the partition
#if self.verbose: print 'Stopping criterion reached by %s' % (self.name)
#self.shared.outbox.put((UB,node))
#break
# memory check
if self.memory_limit > 0:
usage = self.memmon.usage() / 1000 # in MB
if usage > self.memory_limit:
# put node back so partition stays complete
self.shared.outbox.put((UB,node))
if self.verbose:
print 'killing process',self.name,\
'because memory usage',usage,\
'exceeded',self.memory_limit
self.shared.killed.value = self.shared.killed.value + 1
break
# keep going
for child in node.split(self.problem):
if child.LB > self.shared.get_LB():
if self.verbose:
print 'Node with best lower bound %.4f found by %s' \
% (child.LB, self.name)
self.shared.put(child)
self.shared.outbox.put((child.UB,child))
### script to generate time series data from Will's computer ###
import os, timeit, sys, getpass
from time import sleep, time
from datetime import datetime
sys.path.append(os.getcwd() + '/include/')
from memorymonitor import MemoryMonitor
import commands
small_ratio = 48
## how many hours of recording
# ----- Initialize -----
path = os.getcwd() + '/../../track/'
file = 'track_' + datetime.now().isoformat() + '.txt'
# --- Memory monitor ---
mem_mon = MemoryMonitor(getpass.getuser())
T = 0
# ----- Name & create files -----
tl = ['cpu_', 'mem_']
tfile = []
for t in tl:
temp = path + t + file
os.system('touch ' + temp)
tf = open(temp, 'a')
tfile.append(tf)
tf.write(datetime.now().isoformat() + '\n')
tf.write(' record time: ' + str(small_ratio) + ' hours')
# ----- Start tracking -----
while (T < small_ratio * 3600):
### script to generate time series data from Will's computer ###
import os, timeit, sys, getpass
from time import sleep,time
from datetime import datetime
sys.path.append(os.getcwd()+'/include/');
from memorymonitor import MemoryMonitor
import commands
small_ratio = 48; ## how many hours of recording
# ----- Initialize -----
path = os.getcwd()+'/../../track/'; file = 'track_'+datetime.now().isoformat()+'.txt';
# --- Memory monitor ---
mem_mon = MemoryMonitor(getpass.getuser());
T = 0;
# ----- Name & create files -----
tl=['cpu_','mem_']; tfile=[];
for t in tl:
temp=path+t+file;os.system('touch '+temp);tf=open(temp,'a');tfile.append(tf);tf.write(datetime.now().isoformat()+'\n');tf.write(' record time: '+str(small_ratio)+' hours');
# ----- Start tracking -----
while(T<small_ratio*3600):
upt = commands.getoutput('uptime');
totalcpu = float(upt.split('averages: ')[1].split(' ')[0]);
memuse = mem_mon.usage();
tfile[0].write(str(time())+', '+str(totalcpu)+'\n');
tfile[1].write(str(time())+', '+str(memuse)+'\n');
T = T + 1;
if T%300==1:
print('\n'+'writing at time '+str(T)+'/'+str(3600*small_ratio)+'\n');
class ProcessedSolver(multiprocessing.Process):
def __init__(self, problem, shared, verbose=False, memory_limit=0):
"""
Constructor.
"""
multiprocessing.Process.__init__(self)
self.problem = problem
self.shared = shared
self.verbose = verbose
self.memory_limit = memory_limit
if memory_limit > 0:
from memorymonitor import MemoryMonitor
self.memmon = MemoryMonitor()
@threadsafe_function
def run(self):
if self.verbose:
print self.name, 'is working! at', time.time()
while True:
try:
UB, node = self.shared.inbox.get(True, 1)
except:
continue
# stopping criterion
if isinstance(node, str) and node == 'stop_computation':
if self.verbose:
print 'stop computation recieved by', self.name, 'at', time.time(
)
break
if self.verbose:
print self.name, 'found node with UB', UB, 'at', time.time()
# stopping criterion moved to master
#if UB - self.shared.get_LB() < self.problem.tol:
# a tolerable solution has been found!
# put current node back in the partition
#if self.verbose: print 'Stopping criterion reached by %s' % (self.name)
#self.shared.outbox.put((UB,node))
#break
# memory check
if self.memory_limit > 0:
usage = self.memmon.usage() / 1000 # in MB
if usage > self.memory_limit:
# put node back so partition stays complete
self.shared.outbox.put((UB, node))
if self.verbose:
print 'killing process',self.name,\
'because memory usage',usage,\
'exceeded',self.memory_limit
self.shared.killed.value = self.shared.killed.value + 1
break
# keep going
for child in node.split(self.problem):
if child.LB > self.shared.get_LB():
if self.verbose:
print 'Node with best lower bound %.4f found by %s' \
% (child.LB, self.name)
self.shared.put(child)
self.shared.outbox.put((child.UB, child))