help="the momentum.", type=float, default=0.9) parser.add_argument("-nl", "--num_layers", help="Number of LSTM hidden layers.", type=int, default=2) parser.add_argument("-hu", "--hidden_units", help="Number of units in LSTM hidden layer.", type=int, default=128) args = parser.parse_args() redis_logger_handler.logging_setup(args.redis) logging.info("===== Start") images, labels = parseFile(args.images, args.labels, args.format) dataRDD = images.zip(labels) args.train_size = labels.count() - args.test_size logging.info(args) cluster = TFCluster.run(sc, lstm_ctc_ocr_dist.map_fun, args, args.cluster_size, num_ps, args.tensorboard, TFCluster.InputMode.SPARK) if args.mode == "train": cluster.train(dataRDD, args.epochs) else: labelRDD = cluster.inference(dataRDD) labelRDD.saveAsTextFile(args.output)
def print_log(args, ctx): import logging import redis_logger_handler redis_logger_handler.logging_setup(args.redis) logging.info('print log..............................')