def main(argv):
if len(argv) != 1:
sys.exit(__doc__)
config = entconfig.EntConfig(argv[0])
if not config.Load(): sys.exit(__doc__)
# Collect syslogs only if active or serve
state = install_utilities.install_state(config.var('VERSION'))
if not state in ['ACTIVE', 'SERVE']:
sys.exit(0)
# Collect syslogs only from master node.
if not isMaster(config):
logging.fatal('Not a oneway or cluster master node. Return!')
pywrapbase.InitGoogleScript('', [
'foo',
'--gfs_aliases=%s' % config.var("GFS_ALIASES"),
'--bnsresolver_use_svelte=false', '--logtostderr'
], 0)
gfile.Init()
first_date, last_date, printable_date, file_date = \
liblog.ParseDateRange('all',[])
apache_main_dir = liblog.get_apache_dir(config)
checkpoint_dir = liblog.get_syslog_checkpoint_dir(config)
liblog.MakeGoogleDir(config, checkpoint_dir)
if (config.var('SYSLOG_SERVER') == None
or config.var('ENT_SYSLOG_GWS_FAC') == None):
logging.fatal('SYSLOG logging is disabled')
lockfile = '%s/syslog_lock' % config.var('LOGDIR')
lock = E.acquire_lock(lockfile, 1, breakLockAfterGracePeriod=0)
if not lock:
return
try:
logger = syslog_client.SyslogClient(config.var('SYSLOG_SERVER'),
config.var('EXTERNAL_WEB_IP'))
logging.info("syslog-server = %s" % config.var('SYSLOG_SERVER'))
for collection in os.listdir(apache_main_dir):
apache_dir = '%s/%s' % (apache_main_dir, collection)
checkpoint_file = "%s/%s" % (checkpoint_dir, collection)
apache_logs = liblog.FindLogFiles(apache_dir, first_date,
last_date)
logging.info('syslog handles collection %s' % collection)
if not SyslogLogs(apache_logs, apache_dir, checkpoint_file, logger,
config):
sys.exit('Updating failed')
logger.close()
finally:
lock.close()
os.unlink(lockfile)
sys.exit(0)
def SetupSwigFlags():
"""Initialize the C++-level commandline flags (swig flags) with the
python flags. We use a module-level variable to ensure that
InitGoogleScript is only called once.
"""
global setup_swig_flags_called
if not setup_swig_flags_called:
cpp_args = _GenerateArgsForSwigFlags()
pywrapbase.InitGoogleScript(cpp_args[0], cpp_args, 1)
setup_swig_flags_called = True
def main(argv):
argc = len(argv)
if argc < 10:
sys.exit(__doc__)
config = entconfig.EntConfig(argv[0])
if not config.Load(): sys.exit(__doc__)
pywrapbase.InitGoogleScript('', [
'foo',
'--gfs_aliases=%s' % config.var("GFS_ALIASES"),
'--bnsresolver_use_svelte=false', '--logtostderr'
], 0)
gfile.Init()
client = argv[1]
date_fields = string.split(argv[2], '_')
date_range = liblog.ParseDateRange(date_fields[0], date_fields[1:])
withResults = argv[3]
topCount = argv[4]
diagnosticTerms = argv[5]
html_file = argv[6]
valid_file = argv[7]
new_html_file = argv[8]
new_valid_file = argv[9]
if not date_range:
sys.exit(__doc__)
first_date, last_date, printable_date, file_date = date_range
if last_date.as_int() < first_date.as_int():
logging.fatal('invalid date range')
gws_log_dir = liblog.get_gws_log_dir(config)
collect_dir = liblog.get_collect_dir(config)
partition_dir = liblog.get_partition_dir(config)
directory_map_file = liblog.get_directory_map_file(config)
# Collect logs first from all gws nodes and preprocess
# logs to make sure logs are up to date.
all_machines = config.var('MACHINES')
collect_logs.CollectLogs(all_machines, gws_log_dir, collect_dir)
preprocess_logs.PartitionLogs(config)
gws_logs = liblog.FindClientLogFiles(partition_dir, directory_map_file,
client, first_date, last_date)
# note that collection (client) has been factored into gwslog_dir.
result = CreateLogReport(config, printable_date, gws_logs,
config.var('MAIN_GOOGLE3_DIR'), withResults,
topCount, diagnosticTerms, html_file, valid_file,
new_html_file, new_valid_file)
if result == liblog.FAILURE:
logging.error('CreateLogReport Failed')
sys.exit(result)
if not FLAGS.enthome:
sys.exit('Must specify --enthome')
if not FLAGS.xml_path:
sys.exit('Must specify --xml_path')
if not (FLAGS.do_import or FLAGS.do_export):
sys.exit('Must specify --do_import or --do_export')
ent_config = entconfig.EntConfig(FLAGS.enthome)
if not ent_config.Load():
sys.exit('Cannot load configuration.')
pywrapbase.InitGoogleScript('', [
'foo',
'--gfs_aliases=%s' % ent_config.var('GFS_ALIASES'),
'--bnsresolver_use_svelte=false', '--logtostderr', '--minloglevel=1'
], 0)
gfile.Init()
begin = time.time()
cfg = configurator.configurator(FLAGS.enthome)
if FLAGS.do_import:
logging.info("initializing global default files")
if not cfg.globalParams.InitGlobalDefaultFiles(overwrite=0):
logging.fatal("failed to initialize global default files")
config_xml_serialization.ImportConfiguration(cfg, open(FLAGS.xml_path))
if FLAGS.do_export:
def main(argv):
argc = len(argv)
if argc < 4:
sys.exit(__doc__)
config = entconfig.EntConfig(argv[0])
if not config.Load():
sys.exit(__doc__)
pywrapbase.InitGoogleScript('', [
'foo',
'--gfs_aliases=%s' % config.var("GFS_ALIASES"),
'--bnsresolver_use_svelte=false', '--logtostderr'
], 0)
gfile.Init()
cmd = argv[1]
if cmd != 'crawlqueue':
sys.exit(__doc__)
server_str = argv[2]
result_file = argv[3]
index_file = argv[4]
queue_name = argv[5]
caption_time = argv[6]
num_urls_requested = int(argv[7])
next_hours = int(argv[8])
if argc == 10:
crawl_host = argv[9]
else:
crawl_host = ''
command = 'B= V=20 P'
cr_request = crawlreport_pb.CrawlReportRequest()
cr_request.set_commandtype(cr_request.CRAWL_QUEUE_COMMAND)
cr_params = cr_request.mutable_crawlqueueparams()
if crawl_host != '':
cr_params.set_host(crawl_host)
cr_params.set_numurls(num_urls_requested)
cr_params.set_nexthours(next_hours)
servers = server_str.split(',')
cq_mixer = CrawlQueueResponseMixer(num_urls_requested, len(servers),
result_file, index_file)
threads = []
for server in servers:
(server_host, server_port) = server.split(':')
t = threading.Thread(target=GetCrawlqueueFromOneShard,
name=server_host,
args=(server_host, int(server_port), cr_request,
cq_mixer))
threads.append(t)
t.start()
for t in threads:
t.join()
sys.exit(cq_mixer.finalStatus)
def main(argv):
argc = len(argv)
if argc < 6:
sys.exit(__doc__)
config = entconfig.EntConfig(argv[0])
if not config.Load():
sys.exit(__doc__)
pywrapbase.InitGoogleScript('', [
'foo',
'--gfs_aliases=%s' % config.var("GFS_ALIASES"),
'--bnsresolver_use_svelte=false', '--logtostderr'
], 0)
gfile.Init()
client = argv[1]
date_arg = argv[2]
html_file = argv[3]
valid_file = argv[4]
new_valid_file = argv[5]
# extract tag and date_range from command line args
date_fields = string.split(date_arg, '_')
date_range = liblog.ParseDateRange(date_fields[0], date_fields[1:])
if not date_range:
sys.exit(__doc__)
first_date, last_date, printable_date, file_date = date_range
if last_date.as_int() < first_date.as_int():
sys.exit(__doc__)
gws_log_dir = liblog.get_gws_log_dir(config)
click_dir = liblog.get_click_dir(config)
collect_dir = liblog.get_collect_dir(config)
apache_dir = liblog.get_apache_dir(config)
directory_map_file = liblog.get_directory_map_file(config)
# we need to collect logs first from all gws nodes and preprocess
# logs first to make sure logs are up to date.
all_machines = config.var('MACHINES')
collect_logs.CollectLogs(all_machines, gws_log_dir, collect_dir)
preprocess_logs.PartitionLogs(config)
# make a vector of Log objects for all apache_logs and click_logs matching
# the given date range and client.
apache_logs = liblog.FindClientLogFiles(apache_dir, directory_map_file,
client, first_date, last_date)
click_logs = liblog.FindClientLogFiles(click_dir, directory_map_file,
client, first_date, last_date)
# If we have valid file and report file, we check to see if the data in
# apache_dir has been changed and if the report is still valid.
if (gfile.Exists(html_file) and gfile.Exists(valid_file)
and liblog.checkValid(html_file, valid_file, apache_logs)):
logging.info('%s still valid.' % html_file)
sys.exit(liblog.STILL_VALID)
# if there is no valid report, we create a new one
DumpApacheAndClickLogs(apache_logs, click_logs)
if not liblog.makeValid(new_valid_file, apache_logs):
logging.error('Error validating %s' % html_file)
sys.exit(liblog.FAILURE)
logging.info('done apache_log, new_valid_file: %s' % new_valid_file)
sys.exit(liblog.SUCCESS)