def work(master_id, worker_id):
    """
    Connect to mongodb at the given location, grab a work item from it, invoke
    the mb calculation for the given region, and push the results back.

    The path element of the master_id (which is considered to be a URL) is used
    as collection ID, the client ID will specify the worker document.
    """

    [host, port, dbname, cname, _, _, _] = ru.split_dburl(master_id)
    [host, port, dbname, cname, _, _, _] = ru.split_dburl(master_id)

    print "  %s:%d" % (host, port)
    db_client = pymongo.MongoClient(host=host, port=port)
    database = db_client[dbname]
    collection = database[cname]
    docs = collection.find()

    # for i in db_client.database_names  () : print "  %s" % i
    # for i in database.collection_names () : print "     %s" % i
    # print "      %d docs" % docs.count()

    print 'master id : %s' % master_id
    print 'worker id : %s' % worker_id
    print 'host      : %s' % host
    print 'port      : %s' % port
    print 'dbname    : %s' % dbname
    print 'cname     : %s' % cname
    print 'worker_id : %s' % worker_id
    print 'works     : %s' % collection.find().count()

    attempts = 0

    while attempts < ATTEMPTS:
        work = collection.find_one({
            'worker_id': int(worker_id),
            'type': 'work'
        })
        if work:
            break
        else:
            print 'waiting for work'
            time.sleep(1)

    if not work:
        print "cannot find work for %s : %s" % (master_id, worker_id)
        sys.exit(0)  # silent exit

    print 'work      : %s' % str(work)

    result = mandelbrot_calc(work)

    print 'publish result'
    collection.insert(result)

    db_client.disconnect()
def work (master_id, worker_id) :
    """
    Connect to mongodb at the given location, grab a work item from it, invoke
    the mb calculation for the given region, and push the results back.

    The path element of the master_id (which is considered to be a URL) is used
    as collection ID, the client ID will specify the worker document.
    """


    [host, port, dbname, cname, _, _, _] = ru.split_dburl (master_id)
    [host, port, dbname, cname, _, _, _] = ru.split_dburl (master_id)

    print "  %s:%d" % (host, port)
    db_client  = pymongo.MongoClient (host=host, port=port)
    database   = db_client[dbname]
    collection = database[cname]
    docs = collection.find ()

  # for i in db_client.database_names  () : print "  %s" % i
  # for i in database.collection_names () : print "     %s" % i
  # print "      %d docs" % docs.count()

    print 'master id : %s' % master_id
    print 'worker id : %s' % worker_id
    print 'host      : %s' % host
    print 'port      : %s' % port
    print 'dbname    : %s' % dbname
    print 'cname     : %s' % cname
    print 'worker_id : %s' % worker_id 
    print 'works     : %s' % collection.find ().count()

    attempts = 0

    while attempts < ATTEMPTS :
        work = collection.find_one ({'worker_id': int(worker_id),
                                     'type'     : 'work'})
        if  work :
            break
        else :
            print 'waiting for work'
            time.sleep (1)

    if  not work :
        print "cannot find work for %s : %s" % (master_id, worker_id)
        sys.exit (0) # silent exit


    print 'work      : %s' % str(work)

    result = mandelbrot_calc (work)

    print 'publish result'
    collection.insert (result)

    db_client.disconnect ()
예제 #3
0
    def timed_store (self, url) :

        # get mongodb database details, and connect to it
        host, port, dbname, _, _ = ru.split_dburl (url)

        mongodb    = pymongo.MongoClient (host=host, port=port)
        database   = mongodb[dbname]

        collection = database[self.timed_id]

        # first store the session itself
        # build up an index of related components
        components = list()
        for ct in self.timed_components :
            components.append ({'type' : ct, 
                                'ids'  : self.timed_components[ct].keys ()})
        # store the timing and component info
        collection.save ({'_id'        : self.timed_id, 
                          'components' : components,
                          'events'     : self.timed_events, 
                          'durations'  : self.timed_durations})

        # then store all known components
        for component_type in self.timed_components :
            for component_id in self.timed_components[component_type] :

                component  = self.timed_components[component_type][component_id]()
                # build up an index of related components
                components = list()
                for ct in component.timed_components :
                    components.append ({'type' : ct, 
                                        'ids'  : component.timed_components[ct].keys ()})

                # store the timing and component info
                collection.save ({'_id'        : component.timed_id, 
                                  'type'       : component.timed_type, 
                                  'components' : components,
                                  'events'     : component.timed_events, 
                                  'durations'  : component.timed_durations})
예제 #4
0
def get_profiles (command, tags=None, url=None, mode=None) :

    print command

    if not url:
        url = os.environ.get ('RADICAL_SYNAPSE_DBURL')

    if not url:
        print "warning: need dburl to retrieve profiles"
        return None

    url = ru.Url(url)

    if mode and not isinstance (mode, list):
        mode = [mode]

    if not tags:
        tags  = dict()
        elems = filter (None, os.environ.get('RADICAL_SYNAPSE_TAGS', '').split(','))
        for elem in elems:
            if ':' in elem:
                key, val  = elem.split(':', 1)
                tags[key] = val
            else:
                tags[elem] = None


    command_idx = index_command (command, tags)

    if url.schema == 'mongodb':

        [dbhost, port, dbname, _, _, _, _] = ru.split_dburl (url)

        db_client  = pymongo.MongoClient (host=dbhost, port=port)
        database   = db_client[dbname]
        collection = database['profiles']

        # FIXME: eval partial tags

        if mode:
            results = collection.find ({'type'        : 'synapse_profile',
                                        'tags'        : tags,
                                        'mode'        : {'$in': mode},  # FIXME: check
                                        'command_idx' : command_idx})
        else:
            results = collection.find ({'type'        : 'synapse_profile',
                                        'tags'        : tags,
                                        'command_idx' : command_idx})

        if  not results.count() :
            raise RuntimeError ("Could not get profile for %s at %s/profiles"
                    % (command, url))

        ret = list(results)


    elif url.schema == 'file':

        path = url.path

        if not os.path.isdir (path):
            raise ValueError ("dburl (%s) must point to an existing dir" % url)

        name = command_idx.split()[0]
      # for key, val in tags.iteritems():
      #     if val != None: name += "_%s:%s" % (key, val)
      #     else          : name += "_%s"    % (key)
        for tag in sorted(tags.keys()):
            if tags[tag] != None: name += "_%s" % tags[tag]
            else                : name += "_%s" % tag

      # print    "checking profiles %s/synapse_profile_%s_*.json" % (path, name)
        base   = "%s/synapse_profile_%s_*.json" % (path, name)
        fnames = glob.glob (base)
        ret    = list()
        for fname in fnames:

          # print 'reading profile %s' % fname

            doc     = ru.read_json_str (fname)
            use     = False
            docmode = doc['mode'][0:3]

            doc['fname'] = fname

            if doc['command'] == command:
                if not mode :
                    use = True
                elif docmode in mode:
                    use = True
                else:
                    print "skip: mode %s not in %s" % (docmode, mode)
            else:
                print "skip command %s" % command
                print "   ! command %s" % doc['command']

            if use:
                ret.append (doc)

        if not len(ret):
            raise LookupError ("No matching profile at %s" % base)

  # print 'retrieved %d profiles from %s' % (len(ret), url)
  # pprint.pprint (ret)

    return ret
예제 #5
0
def store_profile (profile, tags=None, url=None, mode=None) :

    if not url:
        url = os.environ.get ('RADICAL_SYNAPSE_DBURL')

    if not url:
      # print "warning: need dburl to store profiles"
        return None

    if not mode:
        raise ValueError ("document needs mode (emulated | eecuted | profiled)")

    url = ru.Url (url)

    if not tags:
        tags  = dict()
        elems = filter (None, os.environ.get('RADICAL_SYNAPSE_TAGS', '').split(','))
        for elem in elems:
            if ':' in elem:
                key, val  = elem.split(':', 1)
                tags[key] = val
            else:
                tags[elem] = None


    command_idx = index_command (profile['cmd'], tags)
    print "index %s (%s) to %s" % (profile['cmd'], tags, command_idx)

    host = profile['sys'].get ('hostname')
    if not host:
        host = os.environ.get ('RADICAL_SYNAPSE_HOSTNAME', socket.gethostname())
        profile['sys']['hostname'] = host

    doc  = {'type'        : 'synapse_profile',
            'mode'        : mode,
            'command_idx' : command_idx,
            'command'     : profile['cmd'],
            'tags'        : tags,
            'profile'     : profile}



    if url.schema == 'mongodb':

        print 'store profile in db %s' % url

        [dbhost, port, dbname, _, _, _, _] = ru.split_dburl (url)

        db_client  = pymongo.MongoClient (host=dbhost, port=port)
        database   = db_client[dbname]
        collection = database['profiles']

        collection.insert (doc)


    elif url.schema == 'file':

        path = url.path

        if not os.path.isdir (path):
            os.system ('mkdir -p "%s"' % path)

        name = command_idx.split()[0]
      # for key, val in tags.iteritems():
      #     if val != None: name += "_%s:%s" % (key, val)
      #     else          : name += "_%s"    % (key)
        for tag in sorted(tags.keys()):
            if tags[tag] != None: name += "_%s" % tags[tag]
            else                : name += "_%s" % tag

        idx  = 0
        while True:
            fname = "%s/synapse_profile_%s_%s_%s_%03d.json" % (path, name, host, mode[0:3], idx)
            if not os.path.exists (fname):
                break
            idx += 1

        print 'store profile in file %s' % fname
        os.system ('mkdir -p "%s/"' % path)
        ru.write_json (doc, fname)
예제 #6
0
def main(master_id, num_workers, mb_size, mb_depth):

    [host, port, dbname, cname, _, _, _] = ru.split_dburl(master_id)

    db_client = pymongo.MongoClient(host=host, port=port)
    database = db_client[dbname]
    collection = database[cname]

    # collection.remove ()

    subsx = int(math.sqrt(num_workers))
    subsy = int(math.sqrt(num_workers))

    minx = -2.0
    maxx = 1.0
    miny = -1.5
    maxy = 1.5
    pixx = mb_size  # divisible by subsx
    pixy = mb_size  # divisible by subsy
    iters = mb_depth
    image = Image.new("RGB", (pixx, pixy))

    stepx = (maxx - minx) / subsx
    stepy = (maxy - miny) / subsy

    spixx = pixx / subsx
    spixy = pixy / subsy

    # initialize white pic
    for x in range(0, pixx):
        for y in range(0, pixy):
            image.putpixel((x, y), (255, 255, 255))
    image.save("%s/mandel.png" % os.environ['HOME'], "PNG")

    workers = dict()

    # subdivide and publish work items
    for subx in range(0, subsx):
        for suby in range(0, subsy):

            worker_idx = subx * subsx + suby
            worker_doc = {
                'worker_id': int(worker_idx),
                'type': 'work',
                'minx': float(minx + subx * stepx),
                'miny': float(miny + suby * stepy),
                'maxx': float(minx + subx * stepx + stepx),
                'maxy': float(miny + suby * stepy + stepy),
                'pixx': spixx,
                'pixy': spixy,
                'subx': subx,
                'suby': suby,
                'iters': int(iters)
            }

            workers[worker_idx] = worker_doc

            collection.insert(workers[worker_idx])

    # wait for and evaluate results
    done = list()

    while len(done) != len(workers):

        print "waiting (%s/%s)" % (len(done), len(workers))

        results = collection.find({'type': 'result'})
        active = False

        for result in results:

            worker_id = result['worker_id']
            pixx = result['pixx']
            pixy = result['pixy']
            subx = result['subx']
            suby = result['suby']

            if worker_id not in done:

                active = True
                data = result['data']

                for x in range(0, pixx):
                    for y in range(0, pixy):
                        i = data[x][y]
                        image.putpixel((spixx * subx + y, spixy * suby + x),
                                       (i / 4, i / 4, i))

                done.append(worker_id)
                image.save("%s/mandel.png" % os.environ['HOME'], "PNG")

        if not active:
            time.sleep(1)

    print "done    (%s/%s)" % (len(done), len(workers))

    db_client.disconnect()
예제 #7
0
def get_profiles(command, tags=None, url=None, mode=None):

    print command

    if not url:
        url = os.environ.get('RADICAL_SYNAPSE_DBURL')

    if not url:
        print "warning: need dburl to retrieve profiles"
        return None

    url = ru.Url(url)

    if mode and not isinstance(mode, list):
        mode = [mode]

    if not tags:
        tags = dict()
        elems = filter(None,
                       os.environ.get('RADICAL_SYNAPSE_TAGS', '').split(','))
        for elem in elems:
            if ':' in elem:
                key, val = elem.split(':', 1)
                tags[key] = val
            else:
                tags[elem] = None

    command_idx = index_command(command, tags)

    if url.schema == 'mongodb':

        [dbhost, port, dbname, _, _, _, _] = ru.split_dburl(url)

        db_client = pymongo.MongoClient(host=dbhost, port=port)
        database = db_client[dbname]
        collection = database['profiles']

        # FIXME: eval partial tags

        if mode:
            results = collection.find({
                'type': 'synapse_profile',
                'tags': tags,
                'mode': {
                    '$in': mode
                },  # FIXME: check
                'command_idx': command_idx
            })
        else:
            results = collection.find({
                'type': 'synapse_profile',
                'tags': tags,
                'command_idx': command_idx
            })

        if not results.count():
            raise RuntimeError("Could not get profile for %s at %s/profiles" %
                               (command, url))

        ret = list(results)

    elif url.schema == 'file':

        path = url.path

        if not os.path.isdir(path):
            raise ValueError("dburl (%s) must point to an existing dir" % url)

        name = command_idx.split()[0]
        # for key, val in tags.iteritems():
        #     if val != None: name += "_%s:%s" % (key, val)
        #     else          : name += "_%s"    % (key)
        for tag in sorted(tags.keys()):
            if tags[tag] != None: name += "_%s" % tags[tag]
            else: name += "_%s" % tag

    # print    "checking profiles %s/synapse_profile_%s_*.json" % (path, name)
        base = "%s/synapse_profile_%s_*.json" % (path, name)
        fnames = glob.glob(base)
        ret = list()
        for fname in fnames:

            # print 'reading profile %s' % fname

            doc = ru.read_json_str(fname)
            use = False
            docmode = doc['mode'][0:3]

            doc['fname'] = fname

            if doc['command'] == command:
                if not mode:
                    use = True
                elif docmode in mode:
                    use = True
                else:
                    print "skip: mode %s not in %s" % (docmode, mode)
            else:
                print "skip command %s" % command
                print "   ! command %s" % doc['command']

            if use:
                ret.append(doc)

        if not len(ret):
            raise LookupError("No matching profile at %s" % base)

# print 'retrieved %d profiles from %s' % (len(ret), url)
# pprint.pprint (ret)

    return ret
예제 #8
0
def store_profile(profile, tags=None, url=None, mode=None):

    if not url:
        url = os.environ.get('RADICAL_SYNAPSE_DBURL')

    if not url:
        # print "warning: need dburl to store profiles"
        return None

    if not mode:
        raise ValueError("document needs mode (emulated | eecuted | profiled)")

    url = ru.Url(url)

    if not tags:
        tags = dict()
        elems = filter(None,
                       os.environ.get('RADICAL_SYNAPSE_TAGS', '').split(','))
        for elem in elems:
            if ':' in elem:
                key, val = elem.split(':', 1)
                tags[key] = val
            else:
                tags[elem] = None

    command_idx = index_command(profile['cmd'], tags)
    print "index %s (%s) to %s" % (profile['cmd'], tags, command_idx)

    host = profile['sys'].get('hostname')
    if not host:
        host = os.environ.get('RADICAL_SYNAPSE_HOSTNAME', socket.gethostname())
        profile['sys']['hostname'] = host

    doc = {
        'type': 'synapse_profile',
        'mode': mode,
        'command_idx': command_idx,
        'command': profile['cmd'],
        'tags': tags,
        'profile': profile
    }

    if url.schema == 'mongodb':

        print 'store profile in db %s' % url

        [dbhost, port, dbname, _, _, _, _] = ru.split_dburl(url)

        db_client = pymongo.MongoClient(host=dbhost, port=port)
        database = db_client[dbname]
        collection = database['profiles']

        collection.insert(doc)

    elif url.schema == 'file':

        path = url.path

        if not os.path.isdir(path):
            os.system('mkdir -p "%s"' % path)

        name = command_idx.split()[0]
        # for key, val in tags.iteritems():
        #     if val != None: name += "_%s:%s" % (key, val)
        #     else          : name += "_%s"    % (key)
        for tag in sorted(tags.keys()):
            if tags[tag] != None: name += "_%s" % tags[tag]
            else: name += "_%s" % tag

        idx = 0
        while True:
            fname = "%s/synapse_profile_%s_%s_%s_%03d.json" % (
                path, name, host, mode[0:3], idx)
            if not os.path.exists(fname):
                break
            idx += 1

        print 'store profile in file %s' % fname
        os.system('mkdir -p "%s/"' % path)
        ru.write_json(doc, fname)
def main (master_id, num_workers, mb_size, mb_depth) :

    [host, port, dbname, cname, _, _, _] = ru.split_dburl (master_id)

    db_client  = pymongo.MongoClient (host=host, port=port)
    database   = db_client[dbname]
    collection = database[cname]

  # collection.remove ()
  
    subsx   = int(math.sqrt(num_workers))
    subsy   = int(math.sqrt(num_workers))
    
    minx    = -2.0
    maxx    =  1.0
    miny    = -1.5
    maxy    =  1.5
    pixx    =  mb_size  # divisible by subsx
    pixy    =  mb_size  # divisible by subsy
    iters   =  mb_depth
    image   =  Image.new ("RGB", (pixx, pixy))
    
    stepx   = (maxx-minx) / subsx
    stepy   = (maxy-miny) / subsy
    
    spixx   =  pixx / subsx
    spixy   =  pixy / subsy

    # initialize white pic
    for     x in range (0, pixx) :
        for y in range (0, pixy) :
            image.putpixel ((x, y), (255, 255, 255))
    image.save  ("%s/mandel.png" % os.environ['HOME'], "PNG")


    workers = dict()
    
    # subdivide and publish work items
    for   subx in range (0, subsx) :
      for suby in range (0, subsy) :
    
          worker_idx = subx * subsx + suby
          worker_doc = {'worker_id' : int(worker_idx), 
                        'type'      : 'work',
                        'minx'      : float(minx + subx * stepx),
                        'miny'      : float(miny + suby * stepy),
                        'maxx'      : float(minx + subx * stepx + stepx),
                        'maxy'      : float(miny + suby * stepy + stepy),
                        'pixx'      : spixx,
                        'pixy'      : spixy,
                        'subx'      : subx,
                        'suby'      : suby,
                        'iters'     : int(iters)
                        }
  
          workers[worker_idx] = worker_doc
  
          collection.insert (workers[worker_idx])


    # wait for and evaluate results
    done    = list()

    while len(done) != len(workers) :

        print "waiting (%s/%s)" % (len(done), len(workers))

        results = collection.find ({'type' : 'result'})
        active  = False

        for result in results :

            worker_id = result['worker_id']
            pixx      = result['pixx']
            pixy      = result['pixy']
            subx      = result['subx']
            suby      = result['suby']

            if  worker_id not in done :

                active = True
                data   = result['data']

                for     x in range (0, pixx) :
                    for y in range (0, pixy) :
                        i = data[x][y]
                        image.putpixel ((spixx*subx+y, spixy*suby+x), (i/4, i/4, i))

                done.append (worker_id)
                image.save  ("%s/mandel.png" % os.environ['HOME'], "PNG")

        if  not active:
            time.sleep (1)

    print "done    (%s/%s)" % (len(done), len(workers))

    db_client.disconnect ()