def main(conf): """Execute manage or monitoring operation""" psbase.init_logging(conf, logging) logging.debug("starting") for msg in psbase.extract_warnings(conf): logging.warn(msg) if not sys.argv[1:]: usage() sys.exit(1) cmd = sys.argv[1] cmd_args = sys.argv[2:] ready = psbase.wait_for_pastset(conf, 10) if not ready: logging.error("gave up locating active PastSet") sys.exit(1) pset = pastset.PastSet() if cmd in specs_cmds: specs = client_specs(conf, pset, cmd_args) show_clients(specs) elif cmd in tasks_cmds: tasks = client_tasks(conf, pset, cmd_args) show_tasks(tasks) else: logging.error("unknown command: %s" % cmd) usage() sys.exit(1)
def client(conf): """Run client side of conversion""" pset = pastset.PastSet() jobs = pset.enter(('convert-jobs', str)) results = pset.enter(('convert-results', bool, str)) while True: path = jobs.observe()[0] # Job is a tuple (str,) convert = True src_path = os.path.join(conf['source'], path) if not os.path.isfile(src_path): msg = 'no such source: %s' % src_path results.move((False, msg)) print msg continue src_dir = os.path.dirname(src_path) dst_path = os.path.join(conf['destination'], path.replace('flac', 'mp3')) dst_dir = os.path.dirname(dst_path) try: os.makedirs(dst_dir) except Exception, exc: pass if not os.path.isdir(dst_dir): msg = 'could not create destination dir %s: %s' % (dst_dir, exc) results.move((False, msg)) print msg continue if conf.get('sync-time-stamps', False): source_stamp = os.stat(src_dir) os.utime(dst_dir, (source_stamp.st_atime, source_stamp.st_mtime)) helper = {'src_path': src_path, 'dst_path': dst_path} tags_cmd = [conf['tags_path']] + \ [i % helper for i in conf['tags_opts']] readtags = subprocess.Popen(tags_cmd, stdout=subprocess.PIPE) readtags.wait() tags = dict([(i, '') for i in conf['tag_map'].keys()]) tags.update(parse_tags(readtags.stdout)) helper.update(tags) decode_cmd = [conf['decode_path']] + \ [i % helper for i in conf['decode_opts']] encode_cmd = [conf['encode_path']] + \ [i % helper for i in conf['encode_opts']] if not os.path.isfile(dst_path) or conf['overwrite']: decode = subprocess.Popen(decode_cmd, stdout=subprocess.PIPE) encode = subprocess.Popen(encode_cmd, stdin=decode.stdout) decode.wait() encode.wait() convert += decode.returncode convert += encode.returncode if conf.get('sync-time-stamps', False): source_stamp = os.stat(src_path) os.utime(dst_path, (source_stamp.st_atime, source_stamp.st_mtime)) results.move((convert, ''))
def server(conf): """Run server side of conversion""" total_time = -time.time() pset = pastset.PastSet() jobs = pset.enter(('convert-jobs', str)) results = pset.enter(('convert-results', bool, str)) tasks = 0 if os.path.isfile(conf['source']): print 'enqueue single file %s' % conf['source'] jobs.move((conf['source'], )) tasks += 1 elif os.path.isdir(conf['source']) and conf['recursive']: print 'enqueue files recusively in %s' % conf['source'] for (root, _, files) in os.walk(conf['source']): for filename in files: if not filename.endswith('.flac'): continue real_path = os.path.join(root, filename) relative_path = real_path.replace(conf['source'] + os.sep, '') print 'enqueue nested file %s' % relative_path jobs.move((relative_path, )) tasks += 1 else: print 'invalid source: %s' % conf["source"] print 'source must be an existing file or directory.' print 'if it is a directory recursive must be set.' print 'spawning %(workers)d workers' % conf for _ in xrange(conf['workers']): pset.spawn(sys.argv[0], sys.argv[1:]+['--client=True']) convert_val = True convert_err = [] print 'waiting for %d results' % tasks for _ in xrange(tasks): res = results.observe() convert_val &= res[0] # result is a tuple(bool,) if res[1]: convert_err.append(res[1]) print 'Convert result is %s with %d worker(s), %d task(s)' \ % (convert_val, conf['workers'], tasks) if convert_err: print "Convert errors:\n%s" % '\n'.join(convert_err) pset.halt() total_time += time.time() if tasks: avg_time = total_time / tasks else: avg_time = total_time print 'Convert finished in %.2fs (avg %.2fs)' % (total_time, avg_time) return True
def main(conf): """Run client""" psbase.init_logging(conf, logging) logging.debug("starting") for msg in psbase.extract_warnings(conf): logging.warn(msg) ps_ready = psbase.wait_for_pastset(conf) if not ps_ready: logging.error("gave up locating pastset - server not running?") sys.exit(1) pset = pastset.PastSet() logging.debug("got pastset handle") psid = str(random.random()) # Register clients with psid, host, system, cores, mem, disk, bench clients = pset.enter(('__clients', str, str, str, int, int, int, float)) token = pset.enter(('__jobreq', str)) jobs = pset.enter((psid, str, str)) (host, system, arch, cores, mem, disk, bench) = client_specs() logging.info("register with ID %s (%s %s %s %d %d %d %f)" % \ (psid, host, system, arch, cores, mem, disk, bench)) clients.move((psid, host, system, arch, cores, mem, disk, bench)) app = None while True: token.move(psid) result = jobs.observe() (cmd, args) = result if cmd == '__HALT': logging.debug("%s received halt" % psid) break try: app_path = os.path.realpath(os.path.join(conf['psbin'], cmd)) if not os.path.exists(app_path): raise OSError('No such file: %s' % app_path) except Exception, err: logging.error("%s unable to locate application: %s : %s" % \ (psid, cmd, err)) usage() continue logging.debug("%s launching %s" % (psid, app_path)) app = subprocess.Popen(conf['pspython'].split() + [app_path] + args) # Dummy poll to make sure app starts __ = app.poll()
"""Simple example PastSet application inserting and reading tuples with python objects""" import pastset import pickle class ObjectTest: """Simple class for object test""" def __init__(self, data): self.data = data def show(self): """Display object data""" print self.data pset = pastset.PastSet() data = ObjectTest('Hello world...') test = pset.enter(('Test', str)) test.move((pickle.dumps(data), )) print test.first(), test.last() b = pickle.loads(test.observe()[0]) print test.first(), test.last() b.show() pset.halt()