def work(master_id, worker_id):
"""
Connect to mongodb at the given location, grab a work item from it, invoke
the mb calculation for the given region, and push the results back.
The path element of the master_id (which is considered to be a URL) is used
as collection ID, the client ID will specify the worker document.
"""
[host, port, dbname, cname, _, _, _] = ru.split_dburl(master_id)
[host, port, dbname, cname, _, _, _] = ru.split_dburl(master_id)
print " %s:%d" % (host, port)
db_client = pymongo.MongoClient(host=host, port=port)
database = db_client[dbname]
collection = database[cname]
docs = collection.find()
# for i in db_client.database_names () : print " %s" % i
# for i in database.collection_names () : print " %s" % i
# print " %d docs" % docs.count()
print 'master id : %s' % master_id
print 'worker id : %s' % worker_id
print 'host : %s' % host
print 'port : %s' % port
print 'dbname : %s' % dbname
print 'cname : %s' % cname
print 'worker_id : %s' % worker_id
print 'works : %s' % collection.find().count()
attempts = 0
while attempts < ATTEMPTS:
work = collection.find_one({
'worker_id': int(worker_id),
'type': 'work'
})
if work:
break
else:
print 'waiting for work'
time.sleep(1)
if not work:
print "cannot find work for %s : %s" % (master_id, worker_id)
sys.exit(0) # silent exit
print 'work : %s' % str(work)
result = mandelbrot_calc(work)
print 'publish result'
collection.insert(result)
db_client.disconnect()
def work (master_id, worker_id) :
"""
Connect to mongodb at the given location, grab a work item from it, invoke
the mb calculation for the given region, and push the results back.
The path element of the master_id (which is considered to be a URL) is used
as collection ID, the client ID will specify the worker document.
"""
[host, port, dbname, cname, _, _, _] = ru.split_dburl (master_id)
[host, port, dbname, cname, _, _, _] = ru.split_dburl (master_id)
print " %s:%d" % (host, port)
db_client = pymongo.MongoClient (host=host, port=port)
database = db_client[dbname]
collection = database[cname]
docs = collection.find ()
# for i in db_client.database_names () : print " %s" % i
# for i in database.collection_names () : print " %s" % i
# print " %d docs" % docs.count()
print 'master id : %s' % master_id
print 'worker id : %s' % worker_id
print 'host : %s' % host
print 'port : %s' % port
print 'dbname : %s' % dbname
print 'cname : %s' % cname
print 'worker_id : %s' % worker_id
print 'works : %s' % collection.find ().count()
attempts = 0
while attempts < ATTEMPTS :
work = collection.find_one ({'worker_id': int(worker_id),
'type' : 'work'})
if work :
break
else :
print 'waiting for work'
time.sleep (1)
if not work :
print "cannot find work for %s : %s" % (master_id, worker_id)
sys.exit (0) # silent exit
print 'work : %s' % str(work)
result = mandelbrot_calc (work)
print 'publish result'
collection.insert (result)
db_client.disconnect ()
def timed_store (self, url) :
# get mongodb database details, and connect to it
host, port, dbname, _, _ = ru.split_dburl (url)
mongodb = pymongo.MongoClient (host=host, port=port)
database = mongodb[dbname]
collection = database[self.timed_id]
# first store the session itself
# build up an index of related components
components = list()
for ct in self.timed_components :
components.append ({'type' : ct,
'ids' : self.timed_components[ct].keys ()})
# store the timing and component info
collection.save ({'_id' : self.timed_id,
'components' : components,
'events' : self.timed_events,
'durations' : self.timed_durations})
# then store all known components
for component_type in self.timed_components :
for component_id in self.timed_components[component_type] :
component = self.timed_components[component_type][component_id]()
# build up an index of related components
components = list()
for ct in component.timed_components :
components.append ({'type' : ct,
'ids' : component.timed_components[ct].keys ()})
# store the timing and component info
collection.save ({'_id' : component.timed_id,
'type' : component.timed_type,
'components' : components,
'events' : component.timed_events,
'durations' : component.timed_durations})
def get_profiles (command, tags=None, url=None, mode=None) :
print command
if not url:
url = os.environ.get ('RADICAL_SYNAPSE_DBURL')
if not url:
print "warning: need dburl to retrieve profiles"
return None
url = ru.Url(url)
if mode and not isinstance (mode, list):
mode = [mode]
if not tags:
tags = dict()
elems = filter (None, os.environ.get('RADICAL_SYNAPSE_TAGS', '').split(','))
for elem in elems:
if ':' in elem:
key, val = elem.split(':', 1)
tags[key] = val
else:
tags[elem] = None
command_idx = index_command (command, tags)
if url.schema == 'mongodb':
[dbhost, port, dbname, _, _, _, _] = ru.split_dburl (url)
db_client = pymongo.MongoClient (host=dbhost, port=port)
database = db_client[dbname]
collection = database['profiles']
# FIXME: eval partial tags
if mode:
results = collection.find ({'type' : 'synapse_profile',
'tags' : tags,
'mode' : {'$in': mode}, # FIXME: check
'command_idx' : command_idx})
else:
results = collection.find ({'type' : 'synapse_profile',
'tags' : tags,
'command_idx' : command_idx})
if not results.count() :
raise RuntimeError ("Could not get profile for %s at %s/profiles"
% (command, url))
ret = list(results)
elif url.schema == 'file':
path = url.path
if not os.path.isdir (path):
raise ValueError ("dburl (%s) must point to an existing dir" % url)
name = command_idx.split()[0]
# for key, val in tags.iteritems():
# if val != None: name += "_%s:%s" % (key, val)
# else : name += "_%s" % (key)
for tag in sorted(tags.keys()):
if tags[tag] != None: name += "_%s" % tags[tag]
else : name += "_%s" % tag
# print "checking profiles %s/synapse_profile_%s_*.json" % (path, name)
base = "%s/synapse_profile_%s_*.json" % (path, name)
fnames = glob.glob (base)
ret = list()
for fname in fnames:
# print 'reading profile %s' % fname
doc = ru.read_json_str (fname)
use = False
docmode = doc['mode'][0:3]
doc['fname'] = fname
if doc['command'] == command:
if not mode :
use = True
elif docmode in mode:
use = True
else:
print "skip: mode %s not in %s" % (docmode, mode)
else:
print "skip command %s" % command
print " ! command %s" % doc['command']
if use:
ret.append (doc)
if not len(ret):
raise LookupError ("No matching profile at %s" % base)
# print 'retrieved %d profiles from %s' % (len(ret), url)
# pprint.pprint (ret)
return ret
def store_profile (profile, tags=None, url=None, mode=None) :
if not url:
url = os.environ.get ('RADICAL_SYNAPSE_DBURL')
if not url:
# print "warning: need dburl to store profiles"
return None
if not mode:
raise ValueError ("document needs mode (emulated | eecuted | profiled)")
url = ru.Url (url)
if not tags:
tags = dict()
elems = filter (None, os.environ.get('RADICAL_SYNAPSE_TAGS', '').split(','))
for elem in elems:
if ':' in elem:
key, val = elem.split(':', 1)
tags[key] = val
else:
tags[elem] = None
command_idx = index_command (profile['cmd'], tags)
print "index %s (%s) to %s" % (profile['cmd'], tags, command_idx)
host = profile['sys'].get ('hostname')
if not host:
host = os.environ.get ('RADICAL_SYNAPSE_HOSTNAME', socket.gethostname())
profile['sys']['hostname'] = host
doc = {'type' : 'synapse_profile',
'mode' : mode,
'command_idx' : command_idx,
'command' : profile['cmd'],
'tags' : tags,
'profile' : profile}
if url.schema == 'mongodb':
print 'store profile in db %s' % url
[dbhost, port, dbname, _, _, _, _] = ru.split_dburl (url)
db_client = pymongo.MongoClient (host=dbhost, port=port)
database = db_client[dbname]
collection = database['profiles']
collection.insert (doc)
elif url.schema == 'file':
path = url.path
if not os.path.isdir (path):
os.system ('mkdir -p "%s"' % path)
name = command_idx.split()[0]
# for key, val in tags.iteritems():
# if val != None: name += "_%s:%s" % (key, val)
# else : name += "_%s" % (key)
for tag in sorted(tags.keys()):
if tags[tag] != None: name += "_%s" % tags[tag]
else : name += "_%s" % tag
idx = 0
while True:
fname = "%s/synapse_profile_%s_%s_%s_%03d.json" % (path, name, host, mode[0:3], idx)
if not os.path.exists (fname):
break
idx += 1
print 'store profile in file %s' % fname
os.system ('mkdir -p "%s/"' % path)
ru.write_json (doc, fname)
def main(master_id, num_workers, mb_size, mb_depth):
[host, port, dbname, cname, _, _, _] = ru.split_dburl(master_id)
db_client = pymongo.MongoClient(host=host, port=port)
database = db_client[dbname]
collection = database[cname]
# collection.remove ()
subsx = int(math.sqrt(num_workers))
subsy = int(math.sqrt(num_workers))
minx = -2.0
maxx = 1.0
miny = -1.5
maxy = 1.5
pixx = mb_size # divisible by subsx
pixy = mb_size # divisible by subsy
iters = mb_depth
image = Image.new("RGB", (pixx, pixy))
stepx = (maxx - minx) / subsx
stepy = (maxy - miny) / subsy
spixx = pixx / subsx
spixy = pixy / subsy
# initialize white pic
for x in range(0, pixx):
for y in range(0, pixy):
image.putpixel((x, y), (255, 255, 255))
image.save("%s/mandel.png" % os.environ['HOME'], "PNG")
workers = dict()
# subdivide and publish work items
for subx in range(0, subsx):
for suby in range(0, subsy):
worker_idx = subx * subsx + suby
worker_doc = {
'worker_id': int(worker_idx),
'type': 'work',
'minx': float(minx + subx * stepx),
'miny': float(miny + suby * stepy),
'maxx': float(minx + subx * stepx + stepx),
'maxy': float(miny + suby * stepy + stepy),
'pixx': spixx,
'pixy': spixy,
'subx': subx,
'suby': suby,
'iters': int(iters)
}
workers[worker_idx] = worker_doc
collection.insert(workers[worker_idx])
# wait for and evaluate results
done = list()
while len(done) != len(workers):
print "waiting (%s/%s)" % (len(done), len(workers))
results = collection.find({'type': 'result'})
active = False
for result in results:
worker_id = result['worker_id']
pixx = result['pixx']
pixy = result['pixy']
subx = result['subx']
suby = result['suby']
if worker_id not in done:
active = True
data = result['data']
for x in range(0, pixx):
for y in range(0, pixy):
i = data[x][y]
image.putpixel((spixx * subx + y, spixy * suby + x),
(i / 4, i / 4, i))
done.append(worker_id)
image.save("%s/mandel.png" % os.environ['HOME'], "PNG")
if not active:
time.sleep(1)
print "done (%s/%s)" % (len(done), len(workers))
db_client.disconnect()
def get_profiles(command, tags=None, url=None, mode=None):
print command
if not url:
url = os.environ.get('RADICAL_SYNAPSE_DBURL')
if not url:
print "warning: need dburl to retrieve profiles"
return None
url = ru.Url(url)
if mode and not isinstance(mode, list):
mode = [mode]
if not tags:
tags = dict()
elems = filter(None,
os.environ.get('RADICAL_SYNAPSE_TAGS', '').split(','))
for elem in elems:
if ':' in elem:
key, val = elem.split(':', 1)
tags[key] = val
else:
tags[elem] = None
command_idx = index_command(command, tags)
if url.schema == 'mongodb':
[dbhost, port, dbname, _, _, _, _] = ru.split_dburl(url)
db_client = pymongo.MongoClient(host=dbhost, port=port)
database = db_client[dbname]
collection = database['profiles']
# FIXME: eval partial tags
if mode:
results = collection.find({
'type': 'synapse_profile',
'tags': tags,
'mode': {
'$in': mode
}, # FIXME: check
'command_idx': command_idx
})
else:
results = collection.find({
'type': 'synapse_profile',
'tags': tags,
'command_idx': command_idx
})
if not results.count():
raise RuntimeError("Could not get profile for %s at %s/profiles" %
(command, url))
ret = list(results)
elif url.schema == 'file':
path = url.path
if not os.path.isdir(path):
raise ValueError("dburl (%s) must point to an existing dir" % url)
name = command_idx.split()[0]
# for key, val in tags.iteritems():
# if val != None: name += "_%s:%s" % (key, val)
# else : name += "_%s" % (key)
for tag in sorted(tags.keys()):
if tags[tag] != None: name += "_%s" % tags[tag]
else: name += "_%s" % tag
# print "checking profiles %s/synapse_profile_%s_*.json" % (path, name)
base = "%s/synapse_profile_%s_*.json" % (path, name)
fnames = glob.glob(base)
ret = list()
for fname in fnames:
# print 'reading profile %s' % fname
doc = ru.read_json_str(fname)
use = False
docmode = doc['mode'][0:3]
doc['fname'] = fname
if doc['command'] == command:
if not mode:
use = True
elif docmode in mode:
use = True
else:
print "skip: mode %s not in %s" % (docmode, mode)
else:
print "skip command %s" % command
print " ! command %s" % doc['command']
if use:
ret.append(doc)
if not len(ret):
raise LookupError("No matching profile at %s" % base)
# print 'retrieved %d profiles from %s' % (len(ret), url)
# pprint.pprint (ret)
return ret
def store_profile(profile, tags=None, url=None, mode=None):
if not url:
url = os.environ.get('RADICAL_SYNAPSE_DBURL')
if not url:
# print "warning: need dburl to store profiles"
return None
if not mode:
raise ValueError("document needs mode (emulated | eecuted | profiled)")
url = ru.Url(url)
if not tags:
tags = dict()
elems = filter(None,
os.environ.get('RADICAL_SYNAPSE_TAGS', '').split(','))
for elem in elems:
if ':' in elem:
key, val = elem.split(':', 1)
tags[key] = val
else:
tags[elem] = None
command_idx = index_command(profile['cmd'], tags)
print "index %s (%s) to %s" % (profile['cmd'], tags, command_idx)
host = profile['sys'].get('hostname')
if not host:
host = os.environ.get('RADICAL_SYNAPSE_HOSTNAME', socket.gethostname())
profile['sys']['hostname'] = host
doc = {
'type': 'synapse_profile',
'mode': mode,
'command_idx': command_idx,
'command': profile['cmd'],
'tags': tags,
'profile': profile
}
if url.schema == 'mongodb':
print 'store profile in db %s' % url
[dbhost, port, dbname, _, _, _, _] = ru.split_dburl(url)
db_client = pymongo.MongoClient(host=dbhost, port=port)
database = db_client[dbname]
collection = database['profiles']
collection.insert(doc)
elif url.schema == 'file':
path = url.path
if not os.path.isdir(path):
os.system('mkdir -p "%s"' % path)
name = command_idx.split()[0]
# for key, val in tags.iteritems():
# if val != None: name += "_%s:%s" % (key, val)
# else : name += "_%s" % (key)
for tag in sorted(tags.keys()):
if tags[tag] != None: name += "_%s" % tags[tag]
else: name += "_%s" % tag
idx = 0
while True:
fname = "%s/synapse_profile_%s_%s_%s_%03d.json" % (
path, name, host, mode[0:3], idx)
if not os.path.exists(fname):
break
idx += 1
print 'store profile in file %s' % fname
os.system('mkdir -p "%s/"' % path)
ru.write_json(doc, fname)
def main (master_id, num_workers, mb_size, mb_depth) :
[host, port, dbname, cname, _, _, _] = ru.split_dburl (master_id)
db_client = pymongo.MongoClient (host=host, port=port)
database = db_client[dbname]
collection = database[cname]
# collection.remove ()
subsx = int(math.sqrt(num_workers))
subsy = int(math.sqrt(num_workers))
minx = -2.0
maxx = 1.0
miny = -1.5
maxy = 1.5
pixx = mb_size # divisible by subsx
pixy = mb_size # divisible by subsy
iters = mb_depth
image = Image.new ("RGB", (pixx, pixy))
stepx = (maxx-minx) / subsx
stepy = (maxy-miny) / subsy
spixx = pixx / subsx
spixy = pixy / subsy
# initialize white pic
for x in range (0, pixx) :
for y in range (0, pixy) :
image.putpixel ((x, y), (255, 255, 255))
image.save ("%s/mandel.png" % os.environ['HOME'], "PNG")
workers = dict()
# subdivide and publish work items
for subx in range (0, subsx) :
for suby in range (0, subsy) :
worker_idx = subx * subsx + suby
worker_doc = {'worker_id' : int(worker_idx),
'type' : 'work',
'minx' : float(minx + subx * stepx),
'miny' : float(miny + suby * stepy),
'maxx' : float(minx + subx * stepx + stepx),
'maxy' : float(miny + suby * stepy + stepy),
'pixx' : spixx,
'pixy' : spixy,
'subx' : subx,
'suby' : suby,
'iters' : int(iters)
}
workers[worker_idx] = worker_doc
collection.insert (workers[worker_idx])
# wait for and evaluate results
done = list()
while len(done) != len(workers) :
print "waiting (%s/%s)" % (len(done), len(workers))
results = collection.find ({'type' : 'result'})
active = False
for result in results :
worker_id = result['worker_id']
pixx = result['pixx']
pixy = result['pixy']
subx = result['subx']
suby = result['suby']
if worker_id not in done :
active = True
data = result['data']
for x in range (0, pixx) :
for y in range (0, pixy) :
i = data[x][y]
image.putpixel ((spixx*subx+y, spixy*suby+x), (i/4, i/4, i))
done.append (worker_id)
image.save ("%s/mandel.png" % os.environ['HOME'], "PNG")
if not active:
time.sleep (1)
print "done (%s/%s)" % (len(done), len(workers))
db_client.disconnect ()