def requestTask(self):
"""
Request task from master
"""
if self.currentTask is not None:
# Already got task
return
if not self.uuid:
# Nowhere to request from
return
proxy = slave.Slave().getProxy()
try:
options = proxy.job.requestTask(self.uuid)
if options:
Logger.log('Got new task {0} for job {1}' .
format(options['task'], options['jobUUID']))
self.currentTask = spawnNewTask(options)
except socket.error as strerror:
Logger.log('Error requesting task: {0}'. format(strerror))
except:
Logger.log('Unexpected error: {0}' . format(sys.exc_info()[0]))
raise
def touch(self):
"""
Touch master to tell we're still alive
"""
# Ensure we're registered at serevr
if not self.check():
if self.uuid is not None:
Logger.log('Connection to server lost, registering again...')
self.uuid = None
self.register()
if self.uuid is None:
# Not registered at server
return
proxy = slave.Slave().getProxy()
try:
proxy.node.touch(self.uuid)
except socket.error as strerror:
Logger.log('Error touching master: {0}'. format(strerror))
except:
Logger.log('Unexpected error: {0}' . format(sys.exc_info()[0]))
raise
def sendMessages(self):
"""
Send specified task to master
"""
proxy = slave.Slave().getProxy()
nodeUUID = self.node.getUUID()
if self.buffer and nodeUUID is not None:
if proxy.node.logMessage(nodeUUID, self.buffer):
self.buffer = ''
def isChecksumOk(self):
"""
Compare MD5 checksum of received file and file at serevr
"""
proxy = slave.Slave().getProxy()
self_checksum = md5_for_file(self.getBlend())
master_checksum = proxy.job.getBlendChecksum(self.jobUUID)
return self_checksum == master_checksum
def __init__(self, options):
"""
Initialize environment
"""
node = slave.Slave().getRenderNode()
self.jobUUID = options['jobUUID']
self.task_nr = options['task']
self.storage = os.path.join(Config.slave['storage_path'],
'node-' + node.uuid, 'job-' + options['jobUUID'])
def restoreTask(self):
"""
Restart assigned task on master
"""
jobUUID = self.currentTask.getJobUUID()
task_nr = self.currentTask.getTaskNum()
Logger.log('Error occured while rendering task {0} of job {1}' .
format(task_nr, jobUUID))
proxy = slave.Slave().getProxy()
proxy.job.restartTask(self.uuid, jobUUID, task_nr)
def check(self):
"""
Check if node still walid
"""
if self.uuid is None:
return False
proxy = slave.Slave().getProxy()
try:
return proxy.node.check(self.uuid)
except socket.error as strerror:
Logger.log('Error checking self: {0}'. format(strerror))
except:
Logger.log('Unexpected error: {0}' . format(sys.exc_info()[0]))
raise
return False
def register(self):
"""
Register node at master
"""
if self.uuid is not None:
return
proxy = slave.Slave().getProxy()
try:
self.uuid = proxy.node.register(self.host_info)
Logger.log('Registered at master under uuid {0}' .
format(self.uuid))
except socket.error as strerror:
Logger.log('Error registering self: {0}'. format(strerror))
except:
Logger.log('Unexpected error: {0}' . format(sys.exc_info()[0]))
raise
def unregister(self):
"""
Unregister node from renderfarm master
"""
if not self.uuid:
# Not registered at server -- no need in unregistering
return
proxy = slave.Slave().getProxy()
try:
proxy.node.unregister(self.uuid)
self.uuid = None
Logger.log('Node unregisered')
except socket.error as strerror:
Logger.log('Error registering self: {0}'. format(strerror))
except:
Logger.log('Unexpected error: {0}' . format(sys.exc_info()[0]))
raise
# IMPORT CUSTOM FILES
import slave
import master
import mqtt
from sensorlist import SensorList
# TEST SENSORLIST ENUM
print(repr(SensorList.IR))
# INIT SLAVE
s = slave.Slave(0)
# INIT MASTER
m = master.Master()
# PRINT TEMPERATURE
print(s.getTemperature())
s.sendTemperature()
s.getValue(SensorList.IR)
# CALLBACK FUNCTIONS
def on_message(client, userdata, msg):
print("on_message function")
print(msg.topic)
print(msg.payload)
if (int(msg.payload) == 2):
print("Equal to 2")
else:
print("Not equal to 2")
first = dict()
data.append(first)
for i in range(size - 1):
e = dict()
e["section"] = sec2habs[i]
partHabNeighMap = dict()
partHabNeighSFDMap = dict()
for habid in sec2habs[i]:
partHabNeighMap[habid] = habitatNeighMap[habid]
partHabNeighSFDMap[habid] = hab2sfd[habid]
for n in habitatNeighMap[habid]:
partHabNeighSFDMap[n] = hab2sfd[n]
e["neighbhors"] = (partHabNeighMap, partHabNeighSFDMap)
data.append(e)
data = comm.scatter(data, root=0)
master = master.Master(sects, habitats)
else:
data = None
data = comm.scatter(data, root=0)
sec2habsk = data["section"]
if data["neighbhors"] is not None:
partHabNeighMap = data["neighbhors"][0]
partHabNeighSFDMap = data["neighbhors"][1]
print "sec --", sec2habsk
print "parthabneighMAp", partHabNeighMap
slave = slave.Slave(rank, sec2habsk, partHabNeighMap,
partHabNeighSFDMap)
#sim = Simulation()
#sim.startSimulation(logWriter)
def receiveFile(self):
"""
Receive .blend file from master
"""
fname = self.getBlend()
proxy = slave.Slave().getProxy()
node = slave.Slave().getRenderNode()
nodeUUID = node.getUUID()
if os.path.isfile(fname):
if self.isChecksumOk():
# checksum matched -- nothing to do here
return True
else:
Logger.log(
'Checksum mistmatch, ' +
're-receiving file {0} from master'.format(self.fname))
else:
# XXX: need better handling
Logger.log('Receiving file {0} from master'.format(self.fname))
with open(fname, 'wb') as handle:
chunk_nr = 0
while True:
try:
chunk = proxy.job.getBlendChunk(nodeUUID, self.jobUUID,
self.task_nr, chunk_nr)
ok = True
except socket.error as strerror:
Logger.log(
'Error receiving .blend file from master: {0}'.format(
strerror))
time.sleep(0.2)
except:
err = sys.exc_info()[0]
Logger.log('Unexpected error: {0}'.format(err))
raise
if ok:
if type(chunk) is dict:
if 'FINISHED' in chunk:
return True
elif 'CANCELLED' in chunk:
# Transmission was cancelled by master
# Happens after job reassigning, cancelling
# jobs and so on
Logger.log('File transmission was cancelled ' +
'by master')
return False
else:
return False
handle.write(chunk.data)
chunk_nr += 1
return False
def doSendTask(self, task):
"""
Send specified task to master
"""
proxy = slave.Slave().getProxy()
chunk_size = Config.slave['chunk_size']
jobUUID = task.getJobUUID()
nodeUUID = self.node.getUUID()
task_nr = task.getTaskNum()
output_fname = task.getOutputPath()
for root, dirs, files in os.walk(output_fname):
for f in files:
Logger.log(
'Sending file {0} as result of job {1} task {2}'.format(
f, task.getJobUUID(), task.getTaskNum()))
fname_path = os.path.join(root, f)
with open(fname_path, 'rb') as handle:
chunk = handle.read(chunk_size)
chunk_nr = 0
enc_chunk = None
while len(chunk) > 0:
ok = False
if enc_chunk is None:
enc_chunk = xmlrpc.client.Binary(chunk)
try:
res = proxy.job.putRenderChunk(
nodeUUID, jobUUID, task_nr, f, enc_chunk,
chunk_nr)
ok = True
except socket.error as strerror:
Logger.log(
'Error sending image to master: {0}'.format(
strerror))
time.sleep(0.2)
except:
Logger.log('Unexpected error: {0}'.format(
sys.exc_info()[0]))
raise
if ok:
if res == False:
# Chunk was rejected. Happens after network
# lags and jobs re-assignment
return
chunk = handle.read(chunk_size)
chunk_nr += 1
enc_chunk = None
proxy.job.putRenderChunk(nodeUUID, jobUUID, task_nr, f,
False, -1)
Logger.log('File {0} sent to job {1} task {2}'.format(
f, jobUUID, task_nr))
proxy.job.taskComplete(nodeUUID, jobUUID, task_nr)
def getBlenderArguments(self):
"""
Get arguments to pass to blender binary
"""
# Get absolute path to blender setup script
program_startup = os.path.abspath(os.path.dirname(sys.argv[0]))
blender_setup = os.path.join(program_startup, 'lib',
'blender_setup.py')
args = []
sl = slave.Slave()
proxy_addr = sl.getProxyAddress()
node = sl.getRenderNode()
# File to be rendered
args.append('--background')
args.append(self.fpath)
# Configuration script
args.append('--python')
args.append(blender_setup)
# No more blender-related arguments
args.append('--')
args.append('--node-id')
args.append(node.getUUID())
args.append('--task-id')
args.append(str(self.task))
args.append('--output-path')
args.append(self.output_fpath)
if 'use_stamp' in self.options and self.options['use_stamp']:
# Useful for debugging
args.append('--use-stamp')
if 'file_format' in self.options:
args.append('--file-format')
args.append(self.options['file_format'])
if 'resol_x' in self.options:
args.append('--resolution-x')
args.append(str(self.options['resol_x']))
if 'resol_y' in self.options:
args.append('--resolution-y')
args.append(str(self.options['resol_y']))
if 'percentage' in self.options:
args.append('--percentage')
args.append(str(self.options['percentage']))
if 'color_mode' in self.options:
args.append('--color-mode')
args.append(str(self.options['color_mode']))
# For correct node IP detection (for stamp)
args.append('--master-addr')
args.append(proxy_addr[0] + ':' + str(proxy_addr[1]))
return args
# Defining the population size.
pop_size = (sol_per_pop,num_weights) # The population will have sol_per_pop chromosome where each chromosome has num_weights genes.
offspring_size= (sol_per_pop-num_parents_mating,num_weights)
#Creating the initial population.
new_population = numpy.random.uniform(low=-4.0, high=4.0, size=pop_size)
num_generations = 5
the_master=master.Master(sol_per_pop,new_population,num_generations,pop_size,offspring_size)
slave1=slave.Slave("slave1")
slave2=slave.Slave("slave2")
#slave3=slave.Slave("slave3")
print("Slave 1 Process")
slave1_process=multiprocessing.Process(target=slave1.run(),args=(1,))
slave1_process.start()
print("Slave 2 second Process")
slave2_process=multiprocessing.Process(target=slave2.run(),args=(1,))
slave2_process.start()
#print("Slave 3 Process")
#slave3_process=multiprocessing.Process(target=slave3.run(),args=(1,))
print("Master Process")
master_process=multiprocessing.Process(target=the_master.run(), args=(1,))
master_process.start()
while (the_master.not_done_yet):
continue
def runProcess(self, input, wait_time):
s = slave.Slave("./stub_subprocess.py", str(wait_time))
s.run(input)
return s