def main(): global options logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO) parser = OptionParser() parser.add_option("-I", "--identity", dest="identity", help="Use IDENTITY as identity", metavar="IDENTITY") parser.add_option("-H", "--hostname", dest="hostname", default='localhost', help="Send table to HOSTNAME qasino server", metavar="HOSTNAME") parser.add_option("-p", "--port", dest="port", default=constants.ZMQ_RPC_PORT, type=int, help="Use PORT for qasino server", metavar="PORT") parser.add_option("-u", "--username", dest="username", help="HTTPS auth username") parser.add_option("-w", "--password", dest="password", help="HTTPS auth password") parser.add_option("-P", "--pubsub-port", dest="pubsub_port", default=constants.ZMQ_PUBSUB_PORT, type=int, help="Use PORT for qasino pubsub connection", metavar="PORT") parser.add_option("-i", "--index", dest="indexes", action="append", help="Path to a index file to process" ) parser.add_option("-f", "--index-list", dest="index_list", help="Path to a file with a list of index files to process in it" ) parser.add_option("-T", "--table", dest="tables", action="append", help="Tables to limit publishing to" ) parser.add_option("-t", "--table-list", dest="table_list", help="Path to a file with a list of tables to limit publishing to" ) parser.add_option("-d", "--send-delay-max", dest="send_delay_max", default=15, help="Max delay to add when its time to send tables." ) parser.add_option("-x", "--interval", dest="interval", default=None, type=int, help="Interval to send updates (This will turn off subscribing)." ) parser.add_option("-s", "--use-https", dest="use_https", default=False, action="store_true", help="Use HTTP over SSL/TLS protocol to publish table.") parser.add_option("-k", "--skip-ssl-verify", dest="skip_ssl_verify", default=False, action="store_true", help="Don't verify SSL certificates.") parser.add_option("-g", "--gen-signal-timeout", dest="gen_signal_timeout", default=120, type=int, help="Timeout after which we restart the generation signal subscription.") (options, args) = parser.parse_args() logging.info("Qasino csv publisher starting") if options.identity != None: Identity.set_identity(options.identity) logging.info("Identity is %s", Identity.get_identity()) if options.hostname == None: logging.info("Please specify a hostname to connect to.") exit(1) zmq_factory = ZmqFactory() # Create a request object (either ZMQ or HTTPS). if options.use_https: import http_requestor # Change the default port if we're https if options.port == constants.ZMQ_RPC_PORT: options.port = constants.HTTPS_PORT logging.info("Connecting to {}:{} with HTTPS to send tables.".format(options.hostname, options.port)) # Disable extraneous logging in requests. requests_log = logging.getLogger("requests") requests_log.setLevel(logging.WARNING) requestor = http_requestor.HttpRequestor(options.hostname, options.port, username = options.username, password = options.password, skip_ssl_verify = options.skip_ssl_verify ) else: # Use zmq requestor import zmq_requestor logging.info("Connecting to {}:{} with ZeroMQ to send tables.".format(options.hostname, options.port)) requestor = zmq_requestor.ZmqRequestor(options.hostname, options.port, zmq_factory) # Determine the update trigger (interval or signal). if options.interval is None or options.interval < 10: logging.info("Connecting to {}:{} on pubsub ZeroMQ channel to listen for generation signals.".format(options.hostname, options.pubsub_port)) # Create a zeromq pub sub subscriber. import zmq_subscriber subscriber = zmq_subscriber.ZmqSubscriber(options.hostname, options.pubsub_port, zmq_factory) # Read and send the table when a generation signal comes in. global last_gen_signal_time last_gen_signal_time = time.time() subscriber.subscribe_generation_signal(initiate_read_and_send_tables, requestor, options) # Set a timeout so we can restart the subscribe if we haven't heard from the server in a while. reactor.callLater(5, check_for_gen_signal_timeout, options, subscriber, requestor, zmq_factory) else: # Read and send the table at a fixed interval. logging.info("Sending data on fixed interval ({} seconds).".format(options.interval)) request_metadata_task = task.LoopingCall(read_and_send_tables, requestor, options) request_metadata_task.start(int(options.interval)) # Read and send immediately, uncomment. # read_and_send_tables(requestor, options) # Run the event loop reactor.run() logging.info("Qasino csv publisher exiting")
def main(): global options logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO) parser = OptionParser() parser.add_option("-I", "--identity", dest="identity", help="Use IDENTITY as identity", metavar="IDENTITY") parser.add_option("-H", "--hostname", dest="hostname", default='localhost', help="Send table to HOSTNAME qasino server", metavar="HOSTNAME") parser.add_option("-p", "--port", dest="port", default=constants.ZMQ_RPC_PORT, type=int, help="Use PORT for qasino server", metavar="PORT") parser.add_option("-u", "--username", dest="username", help="HTTPS auth username") parser.add_option("-w", "--password", dest="password", help="HTTPS auth password") parser.add_option("-P", "--pubsub-port", dest="pubsub_port", default=constants.ZMQ_PUBSUB_PORT, type=int, help="Use PORT for qasino pubsub connection", metavar="PORT") parser.add_option("-i", "--index", dest="indexes", action="append", help="Path to a index file to process") parser.add_option( "-f", "--index-list", dest="index_list", help="Path to a file with a list of index files to process in it") parser.add_option("-T", "--table", dest="tables", action="append", help="Tables to limit publishing to") parser.add_option( "-t", "--table-list", dest="table_list", help="Path to a file with a list of tables to limit publishing to") parser.add_option("-d", "--send-delay-max", dest="send_delay_max", default=15, help="Max delay to add when its time to send tables.") parser.add_option( "-x", "--interval", dest="interval", default=None, type=int, help="Interval to send updates (This will turn off subscribing).") parser.add_option("-s", "--use-https", dest="use_https", default=False, action="store_true", help="Use HTTP over SSL/TLS protocol to publish table.") parser.add_option("-k", "--skip-ssl-verify", dest="skip_ssl_verify", default=False, action="store_true", help="Don't verify SSL certificates.") parser.add_option( "-g", "--gen-signal-timeout", dest="gen_signal_timeout", default=120, type=int, help= "Timeout after which we restart the generation signal subscription.") (options, args) = parser.parse_args() logging.info("Qasino csv publisher starting") if options.identity != None: Identity.set_identity(options.identity) logging.info("Identity is %s", Identity.get_identity()) if options.hostname == None: logging.info("Please specify a hostname to connect to.") exit(1) zmq_factory = ZmqFactory() # Create a request object (either ZMQ or HTTPS). if options.use_https: import http_requestor # Change the default port if we're https if options.port == constants.ZMQ_RPC_PORT: options.port = constants.HTTPS_PORT logging.info("Connecting to {}:{} with HTTPS to send tables.".format( options.hostname, options.port)) # Disable extraneous logging in requests. requests_log = logging.getLogger("requests") requests_log.setLevel(logging.WARNING) requestor = http_requestor.HttpRequestor( options.hostname, options.port, username=options.username, password=options.password, skip_ssl_verify=options.skip_ssl_verify) else: # Use zmq requestor import zmq_requestor logging.info("Connecting to {}:{} with ZeroMQ to send tables.".format( options.hostname, options.port)) requestor = zmq_requestor.ZmqRequestor(options.hostname, options.port, zmq_factory) # Determine the update trigger (interval or signal). if options.interval is None or options.interval < 10: logging.info( "Connecting to {}:{} on pubsub ZeroMQ channel to listen for generation signals." .format(options.hostname, options.pubsub_port)) # Create a zeromq pub sub subscriber. import zmq_subscriber subscriber = zmq_subscriber.ZmqSubscriber(options.hostname, options.pubsub_port, zmq_factory) # Read and send the table when a generation signal comes in. global last_gen_signal_time last_gen_signal_time = time.time() subscriber.subscribe_generation_signal(initiate_read_and_send_tables, requestor, options) # Set a timeout so we can restart the subscribe if we haven't heard from the server in a while. reactor.callLater(5, check_for_gen_signal_timeout, options, subscriber, requestor, zmq_factory) else: # Read and send the table at a fixed interval. logging.info("Sending data on fixed interval ({} seconds).".format( options.interval)) request_metadata_task = task.LoopingCall(read_and_send_tables, requestor, options) request_metadata_task.start(int(options.interval)) # Read and send immediately, uncomment. # read_and_send_tables(requestor, options) # Run the event loop reactor.run() logging.info("Qasino csv publisher exiting")
help="Use HOSTNAME to connect to", metavar="HOSTNAME") parser.add_option("-p", "--persist", dest="persist", default=False, action="store_true", help="Re-apply table every generation automatically on the server") parser.add_option("-S", "--static", dest="static", default=False, action="store_true", help="Send table as a 'static' table on the server") parser.add_option("-t", "--tablename", dest="tablename", default="dummy", help="Use TABLENAME as the table name", metavar="TABLENAME") #parser.add_option("-q", "--quiet", # action="store_false", dest="verbose", default=True, # help="don't print status messages to stdout") (options, args) = parser.parse_args() if options.identity != None: Identity.set_identity(options.identity) logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO) logging.info("Sending dummy table on port %d", constants.ZMQ_RPC_PORT) zmq_factory = ZmqFactory() # Create a zeromq requestor object. zmq_requestor = zmq_requestor.ZmqRequestor(options.hostname, constants.ZMQ_RPC_PORT, zmq_factory) # Send the table at fixed intervals
"--data-format", dest="data_format", default='json', help="Read data in as either 'json' or 'csv'.") parser.add_option("-f", "--filename", dest="filename", help="Read data from FILENAME.") (options, args) = parser.parse_args() print "Qasino publish starting" if options.identity != None: Identity.set_identity(options.identity) print "Identity is {}".format(Identity.get_identity()) if options.hostname == None: print "Please specify a hostname to connect to." exit(1) # invalid combos if (options.use_zmq and options.use_https) or ( options.use_http and options.use_https) or (options.use_http and options.use_zmq): print "Pick one of --use-https, --use-https or --use-zmq" exit(1) # default to https