def main(conf):
"""Execute manage or monitoring operation"""
psbase.init_logging(conf, logging)
logging.debug("starting")
for msg in psbase.extract_warnings(conf):
logging.warn(msg)
if not sys.argv[1:]:
usage()
sys.exit(1)
cmd = sys.argv[1]
cmd_args = sys.argv[2:]
ready = psbase.wait_for_pastset(conf, 10)
if not ready:
logging.error("gave up locating active PastSet")
sys.exit(1)
pset = pastset.PastSet()
if cmd in specs_cmds:
specs = client_specs(conf, pset, cmd_args)
show_clients(specs)
elif cmd in tasks_cmds:
tasks = client_tasks(conf, pset, cmd_args)
show_tasks(tasks)
else:
logging.error("unknown command: %s" % cmd)
usage()
sys.exit(1)
def client(conf):
"""Run client side of conversion"""
pset = pastset.PastSet()
jobs = pset.enter(('convert-jobs', str))
results = pset.enter(('convert-results', bool, str))
while True:
path = jobs.observe()[0] # Job is a tuple (str,)
convert = True
src_path = os.path.join(conf['source'], path)
if not os.path.isfile(src_path):
msg = 'no such source: %s' % src_path
results.move((False, msg))
print msg
continue
src_dir = os.path.dirname(src_path)
dst_path = os.path.join(conf['destination'], path.replace('flac',
'mp3'))
dst_dir = os.path.dirname(dst_path)
try:
os.makedirs(dst_dir)
except Exception, exc:
pass
if not os.path.isdir(dst_dir):
msg = 'could not create destination dir %s: %s' % (dst_dir, exc)
results.move((False, msg))
print msg
continue
if conf.get('sync-time-stamps', False):
source_stamp = os.stat(src_dir)
os.utime(dst_dir, (source_stamp.st_atime, source_stamp.st_mtime))
helper = {'src_path': src_path, 'dst_path': dst_path}
tags_cmd = [conf['tags_path']] + \
[i % helper for i in conf['tags_opts']]
readtags = subprocess.Popen(tags_cmd, stdout=subprocess.PIPE)
readtags.wait()
tags = dict([(i, '') for i in conf['tag_map'].keys()])
tags.update(parse_tags(readtags.stdout))
helper.update(tags)
decode_cmd = [conf['decode_path']] + \
[i % helper for i in conf['decode_opts']]
encode_cmd = [conf['encode_path']] + \
[i % helper for i in conf['encode_opts']]
if not os.path.isfile(dst_path) or conf['overwrite']:
decode = subprocess.Popen(decode_cmd, stdout=subprocess.PIPE)
encode = subprocess.Popen(encode_cmd, stdin=decode.stdout)
decode.wait()
encode.wait()
convert += decode.returncode
convert += encode.returncode
if conf.get('sync-time-stamps', False):
source_stamp = os.stat(src_path)
os.utime(dst_path, (source_stamp.st_atime, source_stamp.st_mtime))
results.move((convert, ''))
def server(conf):
"""Run server side of conversion"""
total_time = -time.time()
pset = pastset.PastSet()
jobs = pset.enter(('convert-jobs', str))
results = pset.enter(('convert-results', bool, str))
tasks = 0
if os.path.isfile(conf['source']):
print 'enqueue single file %s' % conf['source']
jobs.move((conf['source'], ))
tasks += 1
elif os.path.isdir(conf['source']) and conf['recursive']:
print 'enqueue files recusively in %s' % conf['source']
for (root, _, files) in os.walk(conf['source']):
for filename in files:
if not filename.endswith('.flac'):
continue
real_path = os.path.join(root, filename)
relative_path = real_path.replace(conf['source'] + os.sep, '')
print 'enqueue nested file %s' % relative_path
jobs.move((relative_path, ))
tasks += 1
else:
print 'invalid source: %s' % conf["source"]
print 'source must be an existing file or directory.'
print 'if it is a directory recursive must be set.'
print 'spawning %(workers)d workers' % conf
for _ in xrange(conf['workers']):
pset.spawn(sys.argv[0], sys.argv[1:]+['--client=True'])
convert_val = True
convert_err = []
print 'waiting for %d results' % tasks
for _ in xrange(tasks):
res = results.observe()
convert_val &= res[0] # result is a tuple(bool,)
if res[1]:
convert_err.append(res[1])
print 'Convert result is %s with %d worker(s), %d task(s)' \
% (convert_val, conf['workers'], tasks)
if convert_err:
print "Convert errors:\n%s" % '\n'.join(convert_err)
pset.halt()
total_time += time.time()
if tasks:
avg_time = total_time / tasks
else:
avg_time = total_time
print 'Convert finished in %.2fs (avg %.2fs)' % (total_time, avg_time)
return True
def main(conf):
"""Run client"""
psbase.init_logging(conf, logging)
logging.debug("starting")
for msg in psbase.extract_warnings(conf):
logging.warn(msg)
ps_ready = psbase.wait_for_pastset(conf)
if not ps_ready:
logging.error("gave up locating pastset - server not running?")
sys.exit(1)
pset = pastset.PastSet()
logging.debug("got pastset handle")
psid = str(random.random())
# Register clients with psid, host, system, cores, mem, disk, bench
clients = pset.enter(('__clients', str, str, str, int, int, int, float))
token = pset.enter(('__jobreq', str))
jobs = pset.enter((psid, str, str))
(host, system, arch, cores, mem, disk, bench) = client_specs()
logging.info("register with ID %s (%s %s %s %d %d %d %f)" % \
(psid, host, system, arch, cores, mem, disk, bench))
clients.move((psid, host, system, arch, cores, mem, disk, bench))
app = None
while True:
token.move(psid)
result = jobs.observe()
(cmd, args) = result
if cmd == '__HALT':
logging.debug("%s received halt" % psid)
break
try:
app_path = os.path.realpath(os.path.join(conf['psbin'], cmd))
if not os.path.exists(app_path):
raise OSError('No such file: %s' % app_path)
except Exception, err:
logging.error("%s unable to locate application: %s : %s" % \
(psid, cmd, err))
usage()
continue
logging.debug("%s launching %s" % (psid, app_path))
app = subprocess.Popen(conf['pspython'].split() + [app_path] + args)
# Dummy poll to make sure app starts
__ = app.poll()
"""Simple example PastSet application inserting and reading tuples with
python objects"""
import pastset
import pickle
class ObjectTest:
"""Simple class for object test"""
def __init__(self, data):
self.data = data
def show(self):
"""Display object data"""
print self.data
pset = pastset.PastSet()
data = ObjectTest('Hello world...')
test = pset.enter(('Test', str))
test.move((pickle.dumps(data), ))
print test.first(), test.last()
b = pickle.loads(test.observe()[0])
print test.first(), test.last()
b.show()
pset.halt()
