def find_matching_samples(options): """Generator which returns all samples at target sever which match the requested predicate expression.""" all_samples = {} def req_error(c, url, errmsg, errno): print >> sys.stderr, "%s: failed to retrieve samples: %s (%d)" \ % (options.server, errmsg, errno) sys.exit(1) def req_done(c): all_samples['result'] = cjson.decode(c.buffer.getvalue()) reqman = RequestManager(ssl_opts = ssl_opts, user_agent = ident, request_respond = req_done, request_error = req_error) print options.server + "/samples" reqman.put((options.server + "/samples",)) reqman.process() if not all_samples: print >> sys.stderr, "%s: no samples" % options.server sys.exit(1) for sample_type in all_samples['result']['samples']: for sample in sample_type['items']: if should_process_sample(sample, options.sample_expr): yield sample
def fetch_tstreamerinfo(options, dataset): topdir = {} def req_error(c, url, errmsg, errno): print >> sys.stderr, "%s: failed to retrieve TStreamerInfo: %s (%d)" \ % (options.server, errmsg, errno) sys.exit(1) def req_done(c): topdir["contents"] = cjson.decode(c.buffer.getvalue())['contents'] reqman = RequestManager(ssl_opts = ssl_opts, user_agent = ident, request_respond = req_done, request_error = req_error) reqman.put((options.server + "/archive/" + dataset + "?rootcontent=1",)) reqman.process() return topdir["contents"][0]["streamerinfo"]
def find_matching_samples(options): """Generator which returns all samples at target sever which match the requested predicate expression.""" all_samples = {} def req_error(c, url, errmsg, errno): print >> sys.stderr, "%s: failed to retrieve samples: %s (%d)" \ % (options.server, errmsg, errno) sys.exit(1) def req_done(c): all_samples['result'] = cjson.decode(c.buffer.getvalue()) reqman = RequestManager(ssl_opts=ssl_opts, user_agent=ident, request_respond=req_done, request_error=req_error) print options.server + "/samples" reqman.put((options.server + "/samples", )) reqman.process() if not all_samples: print >> sys.stderr, "%s: no samples" % options.server sys.exit(1) for sample_type in all_samples['result']['samples']: for sample in sample_type['items']: if should_process_sample(sample, options.sample_expr): yield sample
def fetch_tstreamerinfo(options, dataset): topdir = {} def req_error(c, url, errmsg, errno): print >> sys.stderr, "%s: failed to retrieve TStreamerInfo: %s (%d)" \ % (options.server, errmsg, errno) sys.exit(1) def req_done(c): topdir["contents"] = cjson.decode(c.buffer.getvalue())['contents'] reqman = RequestManager(ssl_opts=ssl_opts, user_agent=ident, request_respond=req_done, request_error=req_error) reqman.put((options.server + "/archive/" + dataset + "?rootcontent=1", )) reqman.process() return topdir["contents"][0]["streamerinfo"]
if args: print >> sys.stderr, "Too many arguments" sys.exit(1) if not options.server: print >> sys.stderr, "Server contact string required" sys.exit(1) UMASK = myumask() # Get SSL X509 parametres. ssl_opts = SSLOptions() if options.verbose: print "Using SSL cert dir", ssl_opts.ca_path print "Using SSL private key", ssl_opts.key_file print "Using SSL public key", ssl_opts.cert_file # Start a request manager for contents. reqman = RequestManager(num_connections = options.connections, ssl_opts = ssl_opts, user_agent = ident, request_init = request_init, request_respond = parse_request_and_tasks ) # Process from root directory. start = time() reqman.put((options, options.server, options.request, '')) reqman.process() end = time()
#proxy=True #ssl_opts = SSLOptions(proxy) ssl_opts = SSLOptions() if options.verbose: print "Using SSL cert dir", ssl_opts.ca_path print "Using SSL private key", ssl_opts.key_file print "Using SSL public key", ssl_opts.cert_file # Convert each glob pattern into a filter expression. filter_sets = map(lambda glob: pattern_to_filter(glob), options.glob) # Start a request manager. reqman = RequestManager(num_connections = options.connections, ssl_opts = ssl_opts, user_agent = ident, request_init = request_init, request_respond = process, request_error = report_error) # Process all samples matching the predicate. ntotreq = 0 nfound = 0 start = time() for sample in find_matching_samples(options): tstreamerinfo = None if options.write: tstreamerinfo = fetch_tstreamerinfo(options, "%(run)d%(dataset)s" % sample) literal2root(tstreamerinfo, "TStreamerInfo", options.debug_streamers) nreq = 0 found = []
#proxy=True #ssl_opts = SSLOptions(proxy) ssl_opts = SSLOptions() if options.verbose: print "Using SSL cert dir", ssl_opts.ca_path print "Using SSL private key", ssl_opts.key_file print "Using SSL public key", ssl_opts.cert_file # Convert each glob pattern into a filter expression. filter_sets = map(lambda glob: pattern_to_filter(glob), options.glob) # Start a request manager. reqman = RequestManager(num_connections=options.connections, ssl_opts=ssl_opts, user_agent=ident, request_init=request_init, request_respond=process, request_error=report_error) # Process all samples matching the predicate. ntotreq = 0 nfound = 0 start = time() for sample in find_matching_samples(options): tstreamerinfo = None if options.write: tstreamerinfo = fetch_tstreamerinfo(options, "%(run)d%(dataset)s" % sample) literal2root(tstreamerinfo, "TStreamerInfo", options.debug_streamers) nreq = 0