def find_matching_samples(options):
"""Generator which returns all samples at target sever which
match the requested predicate expression."""
all_samples = {}
def req_error(c, url, errmsg, errno):
print >> sys.stderr, "%s: failed to retrieve samples: %s (%d)" \
% (options.server, errmsg, errno)
sys.exit(1)
def req_done(c):
all_samples['result'] = cjson.decode(c.buffer.getvalue())
reqman = RequestManager(ssl_opts = ssl_opts,
user_agent = ident,
request_respond = req_done,
request_error = req_error)
print options.server + "/samples"
reqman.put((options.server + "/samples",))
reqman.process()
if not all_samples:
print >> sys.stderr, "%s: no samples" % options.server
sys.exit(1)
for sample_type in all_samples['result']['samples']:
for sample in sample_type['items']:
if should_process_sample(sample, options.sample_expr):
yield sample
def fetch_tstreamerinfo(options, dataset):
topdir = {}
def req_error(c, url, errmsg, errno):
print >> sys.stderr, "%s: failed to retrieve TStreamerInfo: %s (%d)" \
% (options.server, errmsg, errno)
sys.exit(1)
def req_done(c):
topdir["contents"] = cjson.decode(c.buffer.getvalue())['contents']
reqman = RequestManager(ssl_opts = ssl_opts,
user_agent = ident,
request_respond = req_done,
request_error = req_error)
reqman.put((options.server + "/archive/" + dataset + "?rootcontent=1",))
reqman.process()
return topdir["contents"][0]["streamerinfo"]
def find_matching_samples(options):
"""Generator which returns all samples at target sever which
match the requested predicate expression."""
all_samples = {}
def req_error(c, url, errmsg, errno):
print >> sys.stderr, "%s: failed to retrieve samples: %s (%d)" \
% (options.server, errmsg, errno)
sys.exit(1)
def req_done(c):
all_samples['result'] = cjson.decode(c.buffer.getvalue())
reqman = RequestManager(ssl_opts=ssl_opts,
user_agent=ident,
request_respond=req_done,
request_error=req_error)
print options.server + "/samples"
reqman.put((options.server + "/samples", ))
reqman.process()
if not all_samples:
print >> sys.stderr, "%s: no samples" % options.server
sys.exit(1)
for sample_type in all_samples['result']['samples']:
for sample in sample_type['items']:
if should_process_sample(sample, options.sample_expr):
yield sample
def fetch_tstreamerinfo(options, dataset):
topdir = {}
def req_error(c, url, errmsg, errno):
print >> sys.stderr, "%s: failed to retrieve TStreamerInfo: %s (%d)" \
% (options.server, errmsg, errno)
sys.exit(1)
def req_done(c):
topdir["contents"] = cjson.decode(c.buffer.getvalue())['contents']
reqman = RequestManager(ssl_opts=ssl_opts,
user_agent=ident,
request_respond=req_done,
request_error=req_error)
reqman.put((options.server + "/archive/" + dataset + "?rootcontent=1", ))
reqman.process()
return topdir["contents"][0]["streamerinfo"]
if args:
print >> sys.stderr, "Too many arguments"
sys.exit(1)
if not options.server:
print >> sys.stderr, "Server contact string required"
sys.exit(1)
UMASK = myumask()
# Get SSL X509 parametres.
ssl_opts = SSLOptions()
if options.verbose:
print "Using SSL cert dir", ssl_opts.ca_path
print "Using SSL private key", ssl_opts.key_file
print "Using SSL public key", ssl_opts.cert_file
# Start a request manager for contents.
reqman = RequestManager(num_connections = options.connections,
ssl_opts = ssl_opts,
user_agent = ident,
request_init = request_init,
request_respond = parse_request_and_tasks
)
# Process from root directory.
start = time()
reqman.put((options, options.server, options.request, ''))
reqman.process()
end = time()
#proxy=True
#ssl_opts = SSLOptions(proxy)
ssl_opts = SSLOptions()
if options.verbose:
print "Using SSL cert dir", ssl_opts.ca_path
print "Using SSL private key", ssl_opts.key_file
print "Using SSL public key", ssl_opts.cert_file
# Convert each glob pattern into a filter expression.
filter_sets = map(lambda glob: pattern_to_filter(glob), options.glob)
# Start a request manager.
reqman = RequestManager(num_connections = options.connections,
ssl_opts = ssl_opts,
user_agent = ident,
request_init = request_init,
request_respond = process,
request_error = report_error)
# Process all samples matching the predicate.
ntotreq = 0
nfound = 0
start = time()
for sample in find_matching_samples(options):
tstreamerinfo = None
if options.write:
tstreamerinfo = fetch_tstreamerinfo(options, "%(run)d%(dataset)s" % sample)
literal2root(tstreamerinfo, "TStreamerInfo", options.debug_streamers)
nreq = 0
found = []
#proxy=True
#ssl_opts = SSLOptions(proxy)
ssl_opts = SSLOptions()
if options.verbose:
print "Using SSL cert dir", ssl_opts.ca_path
print "Using SSL private key", ssl_opts.key_file
print "Using SSL public key", ssl_opts.cert_file
# Convert each glob pattern into a filter expression.
filter_sets = map(lambda glob: pattern_to_filter(glob), options.glob)
# Start a request manager.
reqman = RequestManager(num_connections=options.connections,
ssl_opts=ssl_opts,
user_agent=ident,
request_init=request_init,
request_respond=process,
request_error=report_error)
# Process all samples matching the predicate.
ntotreq = 0
nfound = 0
start = time()
for sample in find_matching_samples(options):
tstreamerinfo = None
if options.write:
tstreamerinfo = fetch_tstreamerinfo(options,
"%(run)d%(dataset)s" % sample)
literal2root(tstreamerinfo, "TStreamerInfo", options.debug_streamers)
nreq = 0
