def _spawnvef(mode, file, args, env, func):
pid = fork()
if not pid:
try:
if env is None:
func(file, args)
else:
func(file, args, env)
except:
_exit(127)
else:
if mode == P_NOWAIT:
return pid
while 1:
wpid, sts = waitpid(pid, 0)
if WIFSTOPPED(sts):
continue
else:
if WIFSIGNALED(sts):
return -WTERMSIG(sts)
if WIFEXITED(sts):
return WEXITSTATUS(sts)
raise error, 'Not stopped, signaled or exited???'
return
def load_source_tokens(dataset,
export_dataset=None,
quit_after_exporting=True):
if dataset is None:
tf.logging.info("Generating random fake tokens")
tokens = [(_ + 0) % n_vocab for _ in range(0, 100000)]
elif dataset.endswith('.tok16'):
tf.logging.info("Reading tokens from %s...", dataset)
with tf.io.gfile.GFile(dataset, 'rb') as f:
data = f.read()
tf.logging.info(
"len(data)=%s; np.frombuffer(%s, dtype=np.uint16)...",
len(data), repr(dataset))
tokens = np.frombuffer(data, dtype=np.uint16)
else:
tf.logging.info("Loading tokens from %s...", dataset)
tokens = []
npz = np.load(dataset)
for item in npz.files:
tokens.extend(npz[item])
tf.logging.info("Finished reading tokens.")
if export_dataset:
export_source_tokens(export_dataset, tokens)
if quit_after_exporting:
tf.logging.info("Tokens exported; quitting.")
import posix
posix._exit(0)
return tokens
def _spawnvef(mode, file, args, env, func):
if not isinstance(args, (tuple, list)):
raise TypeError('argv must be a tuple or a list')
if not args or not args[0]:
raise ValueError('argv first element cannot be empty')
pid = fork()
if not pid:
try:
if env is None:
func(file, args)
else:
func(file, args, env)
except:
_exit(127)
else:
if mode == P_NOWAIT:
return pid
while 1:
wpid, sts = waitpid(pid, 0)
if WIFSTOPPED(sts):
continue
elif WIFSIGNALED(sts):
return -WTERMSIG(sts)
elif WIFEXITED(sts):
return WEXITSTATUS(sts)
else:
raise OSError('Not stopped, signaled or exited???')
def _spawnvef(mode, file, args, env, func):
pid = fork()
if not pid:
try:
if env is None:
func(file, args)
else:
func(file, args, env)
except:
_exit(127)
else:
if mode == P_NOWAIT:
return pid
while 1:
wpid, sts = waitpid(pid, 0)
if WIFSTOPPED(sts):
continue
else:
if WIFSIGNALED(sts):
return -WTERMSIG(sts)
if WIFEXITED(sts):
return WEXITSTATUS(sts)
raise error, 'Not stopped, signaled or exited???'
return
def _spawnvef(mode, file, args, env, func):
# Internal helper; func is the exec*() function to use
pid = fork()
if not pid:
# Child
try:
if env is None:
func(file, args)
else:
func(file, args, env)
except:
_exit(127)
else:
# Parent
if mode == P_NOWAIT:
return pid # Caller is responsible for waiting!
while 1:
wpid, sts = waitpid(pid, 0)
if WIFSTOPPED(sts):
continue
elif WIFSIGNALED(sts):
return -WTERMSIG(sts)
elif WIFEXITED(sts):
return WEXITSTATUS(sts)
else:
raise error("Not stopped, signaled or exited???")
def _spawnvef(mode, file, args, env, func):
# Internal helper; func is the exec*() function to use
if not isinstance(args, (tuple, list)):
raise TypeError('argv must be a tuple or a list')
if not args or not args[0]:
raise ValueError('argv first element cannot be empty')
pid = fork()
if not pid:
# Child
try:
if env is None:
func(file, args)
else:
func(file, args, env)
except:
_exit(127)
else:
# Parent
if mode == P_NOWAIT:
return pid # Caller is responsible for waiting!
while 1:
wpid, sts = waitpid(pid, 0)
if WIFSTOPPED(sts):
continue
elif WIFSIGNALED(sts):
return -WTERMSIG(sts)
elif WIFEXITED(sts):
return WEXITSTATUS(sts)
else:
raise OSError("Not stopped, signaled or exited???")
def _spawnvef(mode, file, args, env, func):
# Internal helper; func is the exec*() function to use
pid = fork()
if not pid:
# Child
try:
if env is None:
func(file, args)
else:
func(file, args, env)
except:
_exit(127)
else:
# Parent
if mode == P_NOWAIT:
return pid # Caller is responsible for waiting!
while 1:
wpid, sts = waitpid(pid, 0)
if WIFSTOPPED(sts):
continue
elif WIFSIGNALED(sts):
return -WTERMSIG(sts)
elif WIFEXITED(sts):
return WEXITSTATUS(sts)
else:
raise error, "Not stopped, signaled or exited???"
def _spawnvef(mode, file, args, env, func):
# Internal helper; func is the exec*() function to use
if not isinstance(args, (tuple, list)):
raise TypeError('argv must be a tuple or a list')
if not args or not args[0]:
raise ValueError('argv first element cannot be empty')
pid = fork()
if not pid:
# Child
try:
if env is None:
func(file, args)
else:
func(file, args, env)
except:
_exit(127)
else:
# Parent
if mode == P_NOWAIT:
return pid # Caller is responsible for waiting!
while 1:
wpid, sts = waitpid(pid, 0)
if WIFSTOPPED(sts):
continue
elif WIFSIGNALED(sts):
return -WTERMSIG(sts)
elif WIFEXITED(sts):
return WEXITSTATUS(sts)
else:
raise OSError("Not stopped, signaled or exited???")
def ioloop1(s, otheraddr):
#
# Watch out! data is in bytes, but the port counts in samples,
# which are two bytes each (for 16-bit samples).
# Luckily, we use mono, else it would be worse (2 samples/frame...)
#
SAMPSPERBUF = 500
BYTESPERSAMP = 2 # AL.SAMPLE_16
BUFSIZE = BYTESPERSAMP * SAMPSPERBUF
QSIZE = 4 * SAMPSPERBUF
#
config = al.newconfig()
config.setqueuesize(QSIZE)
config.setwidth(AL.SAMPLE_16)
config.setchannels(AL.MONO)
#
pid = posix.fork()
if pid:
# Parent -- speaker/headphones handler
log('parent started')
spkr = al.openport('spkr', 'w', config)
while 1:
data = s.recv(BUFSIZE)
if len(data) == 0:
# EOF packet
log('parent got empty packet; killing child')
posix.kill(pid, 15)
return
# Discard whole packet if we are too much behind
if spkr.getfillable() > len(data) / BYTESPERSAMP:
if len(debug) >= 2:
log('parent Q full; dropping packet')
spkr.writesamps(data)
else:
# Child -- microphone handler
log('child started')
try:
try:
mike = al.openport('mike', 'r', config)
# Sleep a while to let the other side get started
time.sleep(1)
# Drain the queue before starting to read
data = mike.readsamps(mike.getfilled())
# Loop, sending packets from the mike to the net
while 1:
data = mike.readsamps(SAMPSPERBUF)
s.sendto(data, otheraddr)
except KeyboardInterrupt:
log('child got interrupt; exiting')
posix._exit(0)
except error:
log('child got error; exiting')
posix._exit(1)
finally:
log('child got unexpected error; leaving w/ traceback')
def ioloop1(s, otheraddr):
#
# Watch out! data is in bytes, but the port counts in samples,
# which are two bytes each (for 16-bit samples).
# Luckily, we use mono, else it would be worse (2 samples/frame...)
#
SAMPSPERBUF = 500
BYTESPERSAMP = 2 # AL.SAMPLE_16
BUFSIZE = BYTESPERSAMP*SAMPSPERBUF
QSIZE = 4*SAMPSPERBUF
#
config = al.newconfig()
config.setqueuesize(QSIZE)
config.setwidth(AL.SAMPLE_16)
config.setchannels(AL.MONO)
#
pid = posix.fork()
if pid:
# Parent -- speaker/headphones handler
log('parent started')
spkr = al.openport('spkr', 'w', config)
while 1:
data = s.recv(BUFSIZE)
if len(data) == 0:
# EOF packet
log('parent got empty packet; killing child')
posix.kill(pid, 15)
return
# Discard whole packet if we are too much behind
if spkr.getfillable() > len(data) / BYTESPERSAMP:
if len(debug) >= 2:
log('parent Q full; dropping packet')
spkr.writesamps(data)
else:
# Child -- microphone handler
log('child started')
try:
try:
mike = al.openport('mike', 'r', config)
# Sleep a while to let the other side get started
time.sleep(1)
# Drain the queue before starting to read
data = mike.readsamps(mike.getfilled())
# Loop, sending packets from the mike to the net
while 1:
data = mike.readsamps(SAMPSPERBUF)
s.sendto(data, otheraddr)
except KeyboardInterrupt:
log('child got interrupt; exiting')
posix._exit(0)
except error:
log('child got error; exiting')
posix._exit(1)
finally:
log('child got unexpected error; leaving w/ traceback')
def __call__(self, unused_word, state):
"""Return a single match."""
try:
return self._GetNextCompletion(state)
except Exception as e: # ESSENTIAL because readline swallows exceptions.
import traceback
traceback.print_exc()
log('Unhandled exception while completing: %s', e)
self.debug_f.log('Unhandled exception while completing: %s', e)
except SystemExit as e:
# Because readline ignores SystemExit!
posix._exit(e.code)
def __call__(self, unused_word, state):
"""Return a single match."""
# NOTE: The readline library tokenizes words. We bypass that and use
# get_line_buffer(). So we get 'for x in l' instead of just 'l'.
#self.debug_f.log(0, 'word %r state %s', unused_word, state)
try:
return self._GetNextCompletion(state)
except Exception as e:
#traceback.print_exc()
self.debug_f.log('Unhandled exception while completing: %s', e)
except SystemExit as e:
# Because readline ignores SystemExit!
posix._exit(e.code)
def main():
hash(MAGIC + 1)
s = socket(AF_INET, SOCK_STREAM)
s.connect((b'127.0.0.1', 8001))
data = b''
while b'\r\n\r\n' not in data:
data += s.recv(8192)
heading, content = data.split(b'\r\n\r\n', 1)
lines = heading.splitlines()
version, status, text = lines[0].split()
headers = dict(line.split(b': ', 1) for line in lines[1:])
hash(MAGIC + 2)
posix._exit(42)
print(headers)
def __call__(self, unused_word, state):
"""Return a single match."""
try:
return self._GetNextCompletion(state)
except util.FatalRuntimeError as e:
# From -W. TODO: -F is swallowed now.
# We should have a nicer UI for displaying errors. Maybe they shouldn't
# print it to stderr. That messes up the completion display. We could
# print what WOULD have been COMPREPLY here.
log('Runtime error while completing: %s', e)
self.debug_f.log('Runtime error while completing: %s', e)
except Exception as e: # ESSENTIAL because readline swallows exceptions.
import traceback
traceback.print_exc()
log('Unhandled exception while completing: %s', e)
self.debug_f.log('Unhandled exception while completing: %s', e)
except SystemExit as e:
# Because readline ignores SystemExit!
posix._exit(e.code)
def gethostbyaddr(ip, timeout=5, default="<???>"):
try:
return Cache[ip]
except LookupError:
pass
host = default
(pin, pout) = os.pipe()
pid = os.fork()
if not pid:
# Child
os.close(pin)
try:
host = socket.gethostbyaddr(ip)[0]
except socket.herror:
pass
os.write(pout, host)
posix._exit(127)
#Parent
os.close(pout)
signal.signal(signal.SIGALRM,
lambda sig, frame: os.kill(pid, signal.SIGKILL))
signal.alarm(timeout)
try:
os.waitpid(pid, 0)
host = os.read(pin, 8192)
except OSError:
pass
signal.alarm(0)
os.close(pin)
Cache[ip] = host
return host
def gethostbyaddr( ip, timeout = 5, default = "<???>" ):
try:
return Cache[ip]
except LookupError:
pass
host = default
( pin, pout ) = os.pipe()
pid = os.fork()
if not pid:
# Child
os.close( pin )
try:
host = socket.gethostbyaddr( ip )[0]
except socket.herror:
pass
os.write( pout, host )
posix._exit(127)
#Parent
os.close( pout )
signal.signal( signal.SIGALRM, lambda sig, frame: os.kill( pid, signal.SIGKILL ) )
signal.alarm( timeout )
try:
os.waitpid( pid, 0 )
host = os.read( pin, 8192 )
except OSError:
pass
signal.alarm( 0 )
os.close( pin )
Cache[ip] = host;
return host
async def on_ready():
if lock.locked() or 'logged_in' in state:
print('Tried to log in again; terminating.')
import posix
posix._exit(1)
assert False
state['logged_in'] = True
print('Logged on as {0}!'.format(client.user))
try:
channel = None if args.channel is None else [
x for x in list(client.get_all_channels())
if args.channel == x.name
]
if channel is not None:
assert len(channel) == 1
channel = channel[0]
print(channel)
warnevent = 0.0
while True:
results = list(
sorted([(index, event)
for index, event in get_image_events(event_acc)]))
emas = dict([(event.step, event)
for index, event in get_image_events(
event_acc, 'fake_images_ema_image_0')])
lastevent = utc()
start_time = 0.0
with lock:
for index, event in results:
lastevent = event.wall_time
if index == 0:
start_time = event.wall_time
text = "```#{} step {} elapsed {:.2f}m```".format(
index, event.step,
(event.wall_time - start_time) / 60.0)
print(text)
if index >= args.start and (args.end is None
or index <= args.end):
desc = get_description(event_acc, step=event.step)
if len(desc.strip()) > 0:
desc = '\n' + desc.strip()
text = "```#{} step {} elapsed {:.2f}m\n{}\n{}{}```".format(
index, event.step,
(event.wall_time - start_time) / 60.0,
timestamp(event.wall_time), args.logdir, desc)
print(text)
args.start = index + 1
try:
ema = emas.get(event.step)
if ema is not None:
ema_image = get_image_from_event(ema)
await send_picture(channel,
ema_image,
'jpg',
text=text)
text = None
image = get_image_from_event(event)
await send_picture(channel,
image,
'jpg',
text=text)
if args.logstart is not None:
with open(args.logstart, "w") as f:
f.write(str(args.start))
except:
import traceback
traceback.print_exc()
now = utc()
if args.warnsec is not None and args.warnsec > 0 and now - lastevent > args.warnsec and now - warnevent > args.warnsec:
await send_message(
channel,
text=
"I've fallen and I can't get up. Please send help for logdir {}. Last update was {:.2f}m ago."
.format(args.logdir, (now - lastevent) / 60.0))
warnevent = now
if args.waitsec is not None and args.waitsec > 0:
print("Sleeping for {} secs".format(args.waitsec))
await asyncio.sleep(args.waitsec)
else:
print("Done. Bye!")
print("--start {}".format(args.start))
import posix
posix._exit(args.start)
assert False
break
print('Reloading events for {}'.format(args.logdir))
event_acc.Reload()
print('Reloaded events for {}'.format(args.logdir))
except:
import traceback
traceback.print_exc()
finally:
await client.logout()
def ripread(track, offset=0):
"rip one track from an image file."
data = {}
start_time = time.time()
pid, master_fd = pty.fork() # this could also be done with a pipe, anyone?
if pid == CHILD:
# debug:
# so=open("/tmp/stdout", "w")
# sys.stdout = so
# se=open("/tmp/stderr", "w+")
# sys.stderr = se
default_signals()
# FIXME: all this offset stuff has to go, track 0 support has to come.
print ":fAE: waiting for status report..."
sys.stdout.flush()
hdr = sndhdr.whathdr(cf['_image_file'])
my_swap_byteorder = cf['_swap_byteorder']
my_offset = offset
if hdr:
# I guess most people use cdparanoia 1- (instead of 0- if applicable)
# for image creation, so for a wav file use:
image_offset = -offset
else:
if string.upper(cf['_image_file'])[-4:] == ".CDR":
hdr = ('cdr', 44100, 2, -1, 16) # Unknown header, assuming cdr
# assume old cdrdao which started at track 1, not at block 0
image_offset = -offset
elif string.upper(cf['_image_file'])[-4:] == ".BIN":
hdr = ('bin', 44100, 2, -1, 16) # Unknown header, assuming bin
# assume new cdrdao which starts at block 0, byteorder is reversed.
my_swap_byteorder = not my_swap_byteorder
image_offset = 0
elif string.upper(cf['_image_file'])[-4:] == ".RAW":
hdr = ('bin', 44100, 2, -1, 16) # Unknown header, assuming raw
image_offset = 0
else:
debug("unsupported image file " + cf['_image_file'])
posix._exit(4)
expected_filesize = jack.functions.tracksize(
jack.ripstuff.all_tracks)[CDR] + CDDA_BLOCKSIZE * offset
# WAVE header is 44 Bytes for normal PCM files...
if hdr[0] == 'wav':
expected_filesize = expected_filesize + 44
if abs(jack.utils.filesize(cf['_image_file']) -
expected_filesize) > CDDA_BLOCKSIZE:
# we *do* allow a difference of one frame
debug("image file size mismatch, aborted. %d != %d" %
(jack.utils.filesize(cf['_image_file']), expected_filesize))
posix._exit(1)
elif hdr[0] == 'wav' and (hdr[1], hdr[2], hdr[4]) != (44100, 2, 16):
debug("unsupported WAV, need CDDA_fmt, aborted.")
posix._exit(2)
elif hdr[0] not in ('wav', 'cdr', 'bin'):
debug("unsupported: " + hdr[0] + ", aborted.")
posix._exit(3)
else:
f = open(cf['_image_file'], 'r')
# set up output wav file:
wav = wave.open(track[NAME] + ".wav", 'w')
wav.setnchannels(2)
wav.setsampwidth(2)
wav.setframerate(44100)
wav.setnframes(0)
wav.setcomptype('NONE', 'not compressed')
# calculate (and seek to) position in image file
track_start = (track[START] + image_offset) * CDDA_BLOCKSIZE
if hdr[0] == 'wav':
track_start = track_start + 44
f.seek(track_start)
# copy / convert the stuff
for i in range(0, track[LEN]):
buf = array.array("h")
buf.fromfile(f, 1176) # CDDA_BLOCKSIZE / 2
if not my_swap_byteorder: # this is inverted as WAVE swabs them anyway.
buf.byteswap()
wav.writeframesraw(buf.tostring())
if i % 1000 == 0:
print ":fAE: Block " + ` i ` + "/" + ` track[LEN] ` + (
" (%2i%%)" % (i * 100 / track[LEN]))
sys.stdout.flush()
wav.close()
f.close()
stop_time = time.time()
read_speed = track[LEN] / CDDA_BLOCKS_PER_SECOND / (stop_time -
start_time)
if read_speed < 100:
print "[%2.0fx]" % read_speed,
else:
print "[99x]",
if hdr[0] in ('bin', 'wav'):
print "[ - read from image - ]"
else:
print "[cdr-WARNING, check byteorder !]"
sys.stdout.flush()
posix._exit(0)
else:
# we are not the child
data['start_time'] = start_time
data['pid'] = pid
data['fd'] = master_fd
data['file'] = os.fdopen(master_fd)
data['cmd'] = ""
data['buf'] = ""
data['type'] = "image_reader"
data['prog'] = "builtin"
data['track'] = track
data['percent'] = 0
data['otf'] = 0
data['elapsed'] = 0
return data
# intercom -- use mike and headset to *talk* to a person on another host.
def parse_command_line():
action_keys = {"install": True, "start": True, "stop": True, "uninstall": True}
argv0Dir = os.path.dirname(sys.argv[0])
defaultConfig = os.path.join(argv0Dir, "sample_setup.cfg")
defaultConfig = os.path.abspath(defaultConfig)
defaultSrcDir = os.path.join(argv0Dir, "../..")
defaultSrcDir = os.path.abspath(defaultSrcDir)
defaultRelDir = os.path.join(argv0Dir, "../../build/release")
defaultRelDir = os.path.abspath(defaultRelDir)
if not os.path.exists(defaultRelDir):
defaultRelDir = os.path.join(argv0Dir, "../..")
defaultRelDir = os.path.abspath(defaultRelDir)
formatter = IndentedHelpFormatter(max_help_position=50, width=120)
usage = "usage: ./%prog [options] -a <ACTION>"
parser = OptionParser(usage, formatter=formatter, add_help_option=False)
parser.add_option(
"-c", "--config-file", action="store", default=defaultConfig, metavar="FILE", help="Setup config file."
)
parser.add_option(
"-a", "--action", action="store", default=None, metavar="ACTION", help="One of install, uninstall, or stop."
)
parser.add_option(
"-r", "--release-dir", action="store", default=defaultRelDir, metavar="DIR", help="QFS release directory."
)
parser.add_option(
"-s", "--source-dir", action="store", default=defaultSrcDir, metavar="DIR", help="QFS source directory."
)
parser.add_option("-h", "--help", action="store_true", help="Print this help message and exit.")
actions = """
Actions:
install = setup meta and chunk server directories, restarting/starting them
start = start meta and chunk servers
stop = stop meta and chunk servers
uninstall = remove meta and chunk server directories after stopping them"""
sampleSession = """
Hello World example of a client session:
# Install sample server setup, only needed once.
./examples/sampleservers/sample_setup.py -a install
PATH=<bin-tools-path>:${PATH}
# Make temp directory.
qfsshell -s localhost -p 20000 -q -- mkdir /qfs/tmp
# Create file containing Hello World, Reed-Solomon encoded, replication 1.
echo 'Hello World' \
| cptoqfs -s localhost -p 20000 -S -r 1 -k /qfs/tmp/helloworld -d -
# Cat file content.
qfscat -s localhost -p 20000 /qfs/tmp/helloworld
# Stat file to see encoding (RS or not), replication level, mtime.
qfsshell -s localhost -p 20000 -q -- stat /qfs/tmp/helloworld
# Copy file locally to current directory.
cpfromqfs -s localhost -p 20000 -k /qfs/tmp/helloworld -d ./helloworld
# Remove file from QFS.
qfsshell -s localhost -p 20000 -q -- rm /qfs/tmp/helloworld
# Stop the server and remove the custom install.
./examples/sampleservers/sample_setup.py -a stop
./examples/sampleservers/sample_setup.py -a uninstall
Use qfs to manipulate files the same way you would use 'hadoop fs':
# Set qfs command alias.
alias qfs='<QFS_INSTALL_PATH>/bin/tools/qfs \
-cfg ./examples/sampleservers/sample_qfs_tool.cfg'
qfs -h
qfs -stat /
qfs -mkdir /some-dir
qfs -ls /
Did you notice how fast it is? :)
"""
# An install sets up all config files and (re)starts the servers.
# An uninstall stops the servers and removes the config files.
# A stop stops the servers.
opts, args = parser.parse_args()
if opts.help:
parser.print_help()
print actions
print sampleSession
print
posix._exit(0)
e = []
if not os.path.isfile(opts.config_file):
e.append("specified 'config-file' does not exist: %s" % opts.config_file)
if not opts.action:
e.append("'action' must be specified")
elif not action_keys.has_key(opts.action):
e.append("invalid 'action' specified: %s" % opts.action)
if not os.path.isdir(opts.release_dir):
e.append("specified 'release-dir' does not exist: %s" % opts.release_dir)
if not os.path.isdir(opts.source_dir):
e.append("specified 'source-dir' does not exist: %s" % opts.source_dir)
if len(e) > 0:
parser.print_help()
print actions
print sampleSession
print
for error in e:
print "*** %s" % error
print
posix._exit(1)
return opts
def main():
QSIZE = 16
TIME = 5
audio = 0
opts, args = getopt.getopt(sys.argv[1:], 'aq:t:')
for opt, arg in opts:
if opt == '-a':
audio = 1
elif opt == '-q':
QSIZE = string.atoi(arg)
elif opt == '-t':
TIME = string.atoi(arg)
if args:
filename = args[0]
else:
filename = 'film.video'
if audio:
if args[1:]:
audiofilename = args[1]
else:
audiofilename = 'film.aiff'
gl.foreground()
x, y = SV.PAL_XMAX / 4, SV.PAL_YMAX / 4
print x, 'x', y
gl.minsize(40, 30)
gl.stepunit(8, 6)
gl.maxsize(SV.PAL_XMAX, SV.PAL_YMAX)
gl.keepaspect(SV.PAL_XMAX, SV.PAL_YMAX)
win = gl.winopen(filename)
x, y = gl.getsize()
print x, 'x', y
v = sv.OpenVideo()
v.BindGLWindow(win, SV.IN_REPLACE)
v.SetSize(x, y)
v.BindGLWindow(win, SV.IN_REPLACE)
v.SetCaptureFormat(SV.RGB_FRAMES)
v.SetCaptureMode(SV.BLOCKING_CAPTURE)
v.SetQueueSize(QSIZE)
v.InitCapture()
if v.GetQueueSize() != QSIZE:
QSIZE = v.GetQueueSize()
print 'Warning: QSIZE reduced to', QSIZE
gl.qdevice(DEVICE.LEFTMOUSE)
gl.qdevice(DEVICE.WINQUIT)
gl.qdevice(DEVICE.WINSHUT)
gl.qdevice(DEVICE.ESCKEY)
print 'Click left mouse to start recording', TIME, 'seconds'
ofile = None
afile = None
# Mouse down opens the file & freezes window
# Mouse up starts recording frames
while 1:
dev, val = gl.qread()
if dev == DEVICE.LEFTMOUSE:
# Start recording
if val == 1:
# Mouse down -- preparations
if ofile == None:
ofile = VFile.VoutFile().init(filename)
ofile.format = 'rgb8'
ofile.width = x
ofile.height = y
ofile.writeheader()
# XXX other format bits?
# The window can't be resized from now
gl.prefsize(x, y)
gl.winconstraints()
gl.wintitle('* ' + filename)
if audio:
afile = initaudio(audiofilename)
continue
# Mouse up -- start actual recording
global recording, stop_recording
if audio:
stop_recording = 0
recording.release()
t0 = time.millitimer()
v.StartCapture()
while 1:
t = time.millitimer() - t0
if t >= TIME*1000:
break
if v.GetCaptured() > 2:
doframe(v, ofile, x, y, t)
v.StopCapture()
stop_recording = 1
while v.GetCaptured() > 0:
doframe(v, ofile, x, y, t)
t = time.millitimer() - t0
gl.wintitle(filename)
elif dev == DEVICE.REDRAW:
# Window resize (or move)
x, y = gl.getsize()
print x, 'x', y
v.SetSize(x, y)
v.BindGLWindow(win, SV.IN_REPLACE)
elif dev in (DEVICE.ESCKEY, DEVICE.WINQUIT, DEVICE.WINSHUT):
# Quit
if ofile:
ofile.close()
if afile:
afile.destroy()
posix._exit(0)
# EndCapture dumps core...
v.EndCapture()
v.CloseVideo()
gl.winclose(win)
def parse_command_line():
action_keys = {
'install': True,
'start': True,
'stop': True,
'uninstall': True
}
argv0Dir = os.path.dirname(sys.argv[0])
defaultConfig = os.path.join(argv0Dir, 'sample_setup.cfg')
defaultConfig = os.path.abspath(defaultConfig)
defaultSrcDir = os.path.join(argv0Dir, '../..')
defaultSrcDir = os.path.abspath(defaultSrcDir)
defaultRelDir = os.path.join(argv0Dir, '../../build/release')
defaultRelDir = os.path.abspath(defaultRelDir)
if not os.path.exists(defaultRelDir):
defaultRelDir = os.path.join(argv0Dir, '../..')
defaultRelDir = os.path.abspath(defaultRelDir)
formatter = IndentedHelpFormatter(max_help_position=50, width=120)
usage = "usage: ./%prog [options] -a <ACTION>"
parser = OptionParser(usage, formatter=formatter, add_help_option=False)
parser.add_option('-c',
'--config-file',
action='store',
default=defaultConfig,
metavar='FILE',
help='Setup config file.')
parser.add_option('-a',
'--action',
action='store',
default=None,
metavar='ACTION',
help='One of install, uninstall, or stop.')
parser.add_option('-r',
'--release-dir',
action='store',
default=defaultRelDir,
metavar='DIR',
help='QFS release directory.')
parser.add_option('-s',
'--source-dir',
action='store',
default=defaultSrcDir,
metavar='DIR',
help='QFS source directory.')
parser.add_option('-u',
'--auth',
action='store_true',
help="Configure QFS authentication.")
parser.add_option('-o',
'--os',
action='store_true',
help="Configure object store (S3) support.")
parser.add_option('-h',
'--help',
action='store_true',
help="Print this help message and exit.")
actions = """
Actions:
install = setup meta and chunk server directories, restarting/starting them
start = start meta and chunk servers
stop = stop meta and chunk servers
uninstall = remove meta and chunk server directories after stopping them"""
sampleSession = """
Hello World example of a client session:
# Install sample server setup, only needed once.
%s/examples/sampleservers/sample_setup.py -a install
PATH="%s:${PATH}"
# Make temp directory.
qfsshell -s localhost -p 20000 -q -- mkdir /qfs/tmp
# Create file containing Hello World, Reed-Solomon encoded, replication 1.
echo 'Hello World' \
| cptoqfs -s localhost -p 20000 -S -r 1 -k /qfs/tmp/helloworld -d -
# Cat file content.
qfscat -s localhost -p 20000 /qfs/tmp/helloworld
# Stat file to see encoding (RS or not), replication level, mtime.
qfsshell -s localhost -p 20000 -q -- stat /qfs/tmp/helloworld
# Copy file locally to current directory.
cpfromqfs -s localhost -p 20000 -k /qfs/tmp/helloworld -d ./helloworld
# Remove file from QFS.
qfsshell -s localhost -p 20000 -q -- rm /qfs/tmp/helloworld
# Stop the server and remove the custom install.
%s/examples/sampleservers/sample_setup.py -a stop
%s/examples/sampleservers/sample_setup.py -a uninstall
Use qfs to manipulate files the same way you would use 'hadoop fs':
# Set qfs command alias.
alias qfs='%s/bin/tools/qfs -cfg ~/qfsbase/client/clidefault.prp'
qfs -h
qfs -stat /
qfs -mkdir /some-dir
qfs -ls /
Did you notice how fast it is? :)
Run the following to test with hadoop:
%s/src/test-scripts/qfshadoop.sh
"""
# An install sets up all config files and (re)starts the servers.
# An uninstall stops the servers and removes the config files.
# A stop stops the servers.
opts, args = parser.parse_args()
sampleSession = sampleSession % (opts.source_dir, opts.release_dir,
opts.source_dir, opts.source_dir,
opts.release_dir, opts.source_dir)
if opts.help:
parser.print_help()
print actions
print sampleSession
print
posix._exit(0)
e = []
if not os.path.isfile(opts.config_file):
e.append("specified 'config-file' does not exist: %s" %
opts.config_file)
if not opts.action:
e.append("'action' must be specified")
elif not action_keys.has_key(opts.action):
e.append("invalid 'action' specified: %s" % opts.action)
if not os.path.isdir(opts.release_dir):
e.append("specified 'release-dir' does not exist: %s" %
opts.release_dir)
if not os.path.isdir(opts.source_dir):
e.append("specified 'source-dir' does not exist: %s" % opts.source_dir)
if len(e) > 0:
parser.print_help()
print actions
print sampleSession
print
for error in e:
print "*** %s" % error
print
posix._exit(1)
return opts
for section in config.sections():
dir = config.get(section, 'rundir')
config.set(section, 'rundir', os.path.expanduser(dir))
if section.startswith('chunkserver'):
dir = config.get(section, 'chunkdirs')
dirstowrite = []
dirs = dir.split(' ')
for d in dirs:
dirstowrite.append(os.path.expanduser(d))
config.set(section, 'chunkdirs', ' '.join(dirstowrite))
return config
if __name__ == '__main__':
opts = parse_command_line()
config = parse_config(opts.config_file)
if opts.action in ('uninstall', 'stop'):
do_cleanup(config, opts.action == 'uninstall')
posix._exit(0)
check_binaries(opts.release_dir, opts.source_dir, opts.auth)
check_ports(config)
if opts.action == 'install':
setup_directories(config, opts.auth)
setup_config_files(config, opts.auth, opts.os)
copy_files(config, opts.source_dir)
elif opts.action == 'start':
check_directories(config)
start_servers(config, 'all', opts.action == 'install', opts.auth)
def parse_command_line():
action_keys = { 'install' : True,
'start' : True,
'stop' : True,
'uninstall' : True }
argv0Dir = os.path.dirname(sys.argv[0])
defaultConfig = os.path.join(argv0Dir, 'sample_setup.cfg')
defaultConfig = os.path.abspath(defaultConfig)
defaultSrcDir = os.path.join(argv0Dir, '../..')
defaultSrcDir = os.path.abspath(defaultSrcDir)
defaultRelDir = os.path.join(argv0Dir, '../../build/release')
defaultRelDir = os.path.abspath(defaultRelDir)
if not os.path.exists(defaultRelDir):
defaultRelDir = os.path.join(argv0Dir, '../..')
defaultRelDir = os.path.abspath(defaultRelDir)
formatter = IndentedHelpFormatter(max_help_position=50, width=120)
usage = "usage: ./%prog [options] -a <ACTION>"
parser = OptionParser(usage, formatter=formatter, add_help_option=False)
parser.add_option('-c', '--config-file', action='store',
default=defaultConfig, metavar='FILE', help='Setup config file.')
parser.add_option('-a', '--action', action='store', default=None,
metavar='ACTION', help='One of install, uninstall, or stop.')
parser.add_option('-r', '--release-dir', action='store',
default=defaultRelDir, metavar='DIR', help='QFS release directory.')
parser.add_option('-s', '--source-dir', action='store',
default=defaultSrcDir, metavar='DIR', help='QFS source directory.')
parser.add_option('-h', '--help', action='store_true',
help="Print this help message and exit.")
actions = """
Actions:
install = setup meta and chunk server directories, restarting/starting them
start = start meta and chunk servers
stop = stop meta and chunk servers
uninstall = remove meta and chunk server directories after stopping them"""
sampleSession = """
Hello World example of a client session:
# Install sample server setup, only needed once.
./examples/sampleservers/sample_setup.py -a install
PATH=<bin-tools-path>:${PATH}
# Make temp directory.
qfsshell -s localhost -p 20000 -q -- mkdir /qfs/tmp
# Create file containing Hello World, Reed-Solomon encoded, replication 1.
echo 'Hello World' \
| cptoqfs -s localhost -p 20000 -S -r 1 -k /qfs/tmp/helloworld -d -
# Cat file content.
qfscat -s localhost -p 20000 /qfs/tmp/helloworld
# Stat file to see encoding (RS or not), replication level, mtime.
qfsshell -s localhost -p 20000 -q -- stat /qfs/tmp/helloworld
# Copy file locally to current directory.
cpfromqfs -s localhost -p 20000 -k /qfs/tmp/helloworld -d ./helloworld
# Remove file from QFS.
qfsshell -s localhost -p 20000 -q -- rm /qfs/tmp/helloworld
# Stop the server and remove the custom install.
./examples/sampleservers/sample_setup.py -a stop
./examples/sampleservers/sample_setup.py -a uninstall
"""
# An install sets up all config files and (re)starts the servers.
# An uninstall stops the servers and removes the config files.
# A stop stops the servers.
opts, args = parser.parse_args()
if opts.help:
parser.print_help()
print actions
print sampleSession
print
posix._exit(0)
e = []
if not os.path.isfile(opts.config_file):
e.append("specified 'config-file' does not exist: %s"
% opts.config_file)
if not opts.action:
e.append("'action' must be specified")
elif not action_keys.has_key(opts.action):
e.append("invalid 'action' specified: %s" % opts.action)
if not os.path.isdir(opts.release_dir):
e.append("specified 'release-dir' does not exist: %s"
% opts.release_dir)
if not os.path.isdir(opts.source_dir):
e.append("specified 'source-dir' does not exist: %s" % opts.source_dir)
if len(e) > 0:
parser.print_help()
print actions
print sampleSession
print
for error in e:
print "*** %s" % error
print
posix._exit(1)
return opts
for section in config.sections():
dir = config.get(section, 'rundir')
config.set(section, 'rundir', os.path.expanduser(dir))
if section.startswith('chunkserver'):
dir = config.get(section, 'chunkdirs')
dirstowrite = []
dirs = dir.split(' ')
for d in dirs:
dirstowrite.append(os.path.expanduser(d))
config.set(section, 'chunkdirs', ' '.join(dirstowrite))
return config
if __name__ == '__main__':
opts = parse_command_line()
config = parse_config(opts.config_file)
check_binaries(opts.release_dir, opts.source_dir)
if opts.action in ('uninstall', 'stop'):
do_cleanup(config, opts.action == 'uninstall')
posix._exit(0)
check_ports(config)
if opts.action == 'install':
setup_directories(config)
setup_config_files(config)
copy_files(config, opts.source_dir)
elif opts.action == 'start':
check_directories(config)
start_servers(config)
r"""OS routines for Mac, DOS, NT, or Posix depending on what system we're on.
