def GetEnterpriseConfiguration():
"""Get the enterprise configuration object."""
export_hda = '/export/hda3'
c = entconfig.EntConfig(None)
c.set_allow_config_mgr_requests(0)
c.set_var('DEFAULTS', ['enterprise'])
c.AddDefaults()
version = c.var('VERSION')
ent_home = '%s/%s' % (export_hda, version)
ec = entconfig.EntConfig(ent_home)
ec.Load()
return ec
def main(argv):
""" runs PeriodicScript() function in a loop"""
# global MACHINES
# global MAIN_GOOGLE3_DIR
if len(argv) != 1:
sys.exit(__doc__)
try:
logging.info("Running periodic_script (pid = %d)..." % os.getpid())
config = entconfig.EntConfig(argv[0]) # user config
if not config.Load():
sys.exit("Cannot load the config file %s" % argv[0])
PeriodicScript(config)
svs_utilities.CheckSvsAlive(["localhost"])
monitorSnmp()
EnableGFS(config)
EnableNamed()
DnsConfig(config)
admin_runner_utils.SyncOneboxLog(config)
WarmIndex(config)
logging.info("Finished periodic_script.")
except:
# collect the exception traceback so we know what went wrong
(t, v, tb) = sys.exc_info()
logging.error("\nPeriodic script: Fatal Error:\n" +
"=======================\n" +
string.join(traceback.format_exception(t, v, tb)))
def main(argv):
if len(argv) != 1:
sys.exit(__doc__)
config = entconfig.EntConfig(argv[0])
if not config.Load(): sys.exit(__doc__)
# Collect syslogs only if active or serve
state = install_utilities.install_state(config.var('VERSION'))
if not state in ['ACTIVE', 'SERVE']:
sys.exit(0)
# Collect syslogs only from master node.
if not isMaster(config):
logging.fatal('Not a oneway or cluster master node. Return!')
pywrapbase.InitGoogleScript('', [
'foo',
'--gfs_aliases=%s' % config.var("GFS_ALIASES"),
'--bnsresolver_use_svelte=false', '--logtostderr'
], 0)
gfile.Init()
first_date, last_date, printable_date, file_date = \
liblog.ParseDateRange('all',[])
apache_main_dir = liblog.get_apache_dir(config)
checkpoint_dir = liblog.get_syslog_checkpoint_dir(config)
liblog.MakeGoogleDir(config, checkpoint_dir)
if (config.var('SYSLOG_SERVER') == None
or config.var('ENT_SYSLOG_GWS_FAC') == None):
logging.fatal('SYSLOG logging is disabled')
lockfile = '%s/syslog_lock' % config.var('LOGDIR')
lock = E.acquire_lock(lockfile, 1, breakLockAfterGracePeriod=0)
if not lock:
return
try:
logger = syslog_client.SyslogClient(config.var('SYSLOG_SERVER'),
config.var('EXTERNAL_WEB_IP'))
logging.info("syslog-server = %s" % config.var('SYSLOG_SERVER'))
for collection in os.listdir(apache_main_dir):
apache_dir = '%s/%s' % (apache_main_dir, collection)
checkpoint_file = "%s/%s" % (checkpoint_dir, collection)
apache_logs = liblog.FindLogFiles(apache_dir, first_date,
last_date)
logging.info('syslog handles collection %s' % collection)
if not SyslogLogs(apache_logs, apache_dir, checkpoint_file, logger,
config):
sys.exit('Updating failed')
logger.close()
finally:
lock.close()
os.unlink(lockfile)
sys.exit(0)
def __load_default_params__():
global __default_params__
if __default_params__ == None:
from google3.enterprise.legacy.adminrunner import entconfig
# read some default parameter
__default_params__ = entconfig.EntConfig(None)
__default_params__.set_var('DEFAULTS', ['enterprise'])
__default_params__.AddDefaults()
__default_params__.update_derived_info()
return __default_params__
def main():
latest_version = install_utilities.get_latest_version()
ent_home = '/export/hda3/%s' % latest_version
backup_dir = '/export/hda3/%s/data/googleconfig.bak' % latest_version
log_file_name = '/export/hda3/tmp/google_config_backup.log'
log_file = open(log_file_name, 'a')
logging.set_logfile(log_file)
config = entconfig.EntConfig(ent_home)
number_of_backups = 21
return BackupGoogleConfigFile(config, backup_dir, number_of_backups)
def ClearResetIndexStatus(ent_home, unittestdir=None):
""" Clear Reset index status
Parameters:
ent_home: '/export/hda3/4.6.0.G.35/'
unittestdir: dir to put the RESET_STATE file. -- for unittest only.
"""
cfg = entconfig.EntConfig(ent_home)
cfg.Load()
version = cfg.var('VERSION')
_ClearResetStatus(version, unittestdir=unittestdir)
def main():
# on the GSA-Lite, there is no SVS, so just return
if core_utils.GetEntConfigVar('ENT_CONFIG_TYPE') == 'LITE':
return 0
enthome = sys.argv[1]
if len(sys.argv) > 2:
machine = sys.argv[2]
else:
machine = E.getCrtHostName()
cfg = entconfig.EntConfig(enthome)
if not cfg.Load():
return 1
restart_svs(machine)
return 0
def main(enthome):
config = entconfig.EntConfig(enthome)
if not config.Load(): sys.exit(__doc__)
user = pwd.getpwuid(os.getuid())
if (user[0] != config.var('ENTERPRISE_USER')):
sys.exit("user %s cannot run this script; expected user is %s" %
(user[0], config.var('ENTERPRISE_USER')))
ENTID_TAG = "ENT_ID=%s_crawl" % config.var('VERSION')
tabs = []
logging.info("starting to fill tabs..")
# Setup standard jobs
every_ten_minutes = "*/10 * * * *"
conf_dir = config.var('ENTERPRISE_HOME') + "/local/conf"
source_env = ". /etc/profile; . %s/ent_bashrc " % conf_dir
logging.info("preparing collect_logs.py entry..")
log_collect_cmd = "cd ../logs;"\
"%s ./collect_logs.py %s" % (ENTID_TAG, enthome)
log_collect_log_file = "/%s/cronLogCollectOut_%s" % (config.var('LOGDIR'),
config.var('VERSION'))
tabs.append(
CronTabEntry(config, every_ten_minutes, (source_env, log_collect_cmd),
log_collect_log_file, source_env))
logging.info("preparing syslog_logs.py entry..")
syslog_log_cmd = "cd ../logs;"\
"%s ./syslog_logs.py %s" % (ENTID_TAG, enthome)
syslog_log_logfile = "/%s/cronSyslogLogOut_%s" % (config.var('LOGDIR'),
config.var('VERSION'))
tabs.append(
CronTabEntry(config, every_ten_minutes, (source_env, syslog_log_cmd),
syslog_log_logfile, source_env))
# write out a temp cron tab
tmp_cron_file = "%s/tmp/crawl_cron_%s" % (config.var('ENTERPRISE_HOME'),
config.var('VERSION'))
fout = open(tmp_cron_file, "w")
fout.write(string.join(tabs))
fout.close()
os.system("crontab -l|grep -v %s| cat - %s > %s_" %
(ENTID_TAG, tmp_cron_file, tmp_cron_file))
#user_arg = "-u %s " % crontab_user
os.system("crontab %s_" % tmp_cron_file)
def CreateConfig(self, config_file, config_dir=None):
"""
Given a config file name this returns a Config or an EntConfig object
"""
# Get here only for strings
ent_home = self.get_enterprise_home_(config_file)
if ent_home != None:
from google3.enterprise.legacy.adminrunner import entconfig
return entconfig.EntConfig(ent_home)
# This is not an enterprise config. How can it not be? Die horribly and
# noisily. This code would/should never be up integrated. There are no
# such plans anyway
raise Exception("%s is not a valid enterprise config file!" % config_file)
die = 5 / 0 # Just in case...
sys.exit(1) # yes, I really want you dead
def main(argv):
argc = len(argv)
if argc != 1:
sys.exit(__doc__)
config = entconfig.EntConfig(argv[0])
if not config.Load():
sys.exit(__doc__)
success = GenerateStatusReport(config)
if not success:
logging.error("Failed")
sys.exit(__doc__)
logging.info("Done")
sys.exit(0)
def init_service(self, ent_home):
"""
Does the actual initialization. Reads the config file in the
cp (EntConfig) member and initializes some members for easy access
to usual parameters
"""
self.cp = entconfig.EntConfig(ent_home)
if not self.cp.Load():
sys.exit("Cannot load the config file %s" % self.cp.GetConfigFileName())
# Get some params for easy access
self.configfile = self.cp.GetConfigFileName()
self.version = str(self.cp.var("VERSION"))
self.entid_tag = "ENT_ID=%s_%s" % (self.version, self.service_name)
self.ent_user = self.cp.var("ENTERPRISE_USER")
self.ent_group = self.cp.var("ENTERPRISE_GROUP")
self.ent_bashrc = self.cp.var("ENTERPRISE_BASHRC")
self.ent_home = self.cp.var("ENTERPRISE_HOME")
self.googlebot_dir = self.cp.var("GOOGLEBOT_DIR")
self.version_tmpdir= "%s/tmp" % self.cp.var("ENTERPRISE_HOME")
self.tmpdir = self.cp.var("TMPDIR")
self.logdir = self.cp.var("LOGDIR")
self.datadir = self.cp.var("DATADIR")
self.scripts_dir = ("%s/local/google3/enterprise/legacy/scripts" %
self.ent_home)
self.util_dir = ("%s/local/google3/enterprise/legacy/util" %
self.ent_home)
self.machines = self.cp.var("MACHINES")
# The master depends on the install state : for active / test / install
# we have the adminrunner on the master, else we get it from MASTER
# parameter
self.install_state = install_utilities.install_state(
self.version, rootdir = self.cp.var('ENT_DISK_ROOT'))
self.local_machine = E.getCrtHostName()
testver = install_utilities.is_test(self.version)
if self.install_state in ["ACTIVE", "TEST", "INSTALL"]:
try:
self.master_machine = find_master.FindMasterUsingChubby(self.cp.var('VERSION'))
except core_utils.EntMasterError, e:
# Something is seriously wrong.
logging.error("ERROR: Couldn't determine master")
# Assume we aren't master, so we can at least do inactivate
self.master_machine = None
def IsResetIndexInProgress(ent_home, unittestdir=None):
""" If reset index is in progress
Parameters:
ent_home: '/export/hda3/4.6.0.G.35/'
unittestdir: dir to put the RESET_STATE file. -- for unittest only.
Return:
1 - reset index is in progress. 0 - otherwise.
"""
cfg = entconfig.EntConfig(ent_home)
cfg.Load()
version = cfg.var('VERSION')
status = _GetResetStatus(version, unittestdir=unittestdir)
if status == 'RESET_IN_PROGRESS':
return 1
else:
return 0
def main(argv):
config = entconfig.EntConfig(argv[0])
if not config.Load(): sys.exit(__doc__)
try:
crawl = argv[1]
port = string.atoi(argv[2])
except:
sys.exit(__doc__)
machines = config.var('MACHINES')
master = find_master.FindMaster(port, machines)
# The name of this machine
crt_machine = E.getCrtHostName()
if len(master) != 1 or master[0] != crt_machine:
logging.info("I am not the master")
sys.exit(0) # not a problem
# find out the date 24 hours ago
year, month, day = time.localtime(time.time() - 60 * 60 * 24)[0:3]
date1 = "date_%d_%d_%d" % (month, day, year)
date2 = "date %d %d %d" % (month, day, year)
logging.info("Running %s for %s" % (sys.argv[0], date2))
# 1. run log_report.py for each crawl
ar = adminrunner_client.AdminRunnerClient("localhost", port)
if not ar.LogReportStart(crawl, date1):
sys.exit("Error starting crawl for " + crawl)
logging.info("Started log_report for %s/%s" % (crawl, date1))
# 2. run apache_log.py for each crawl
apache_cmd = "./apache_log.py %s update %s %s" % (argv[0], crawl, date2)
(status, output) = commands.getstatusoutput(apache_cmd)
if status != 0:
logging.fatal("apache_log failed: %s" % output)
logging.info("Finished apache_log: %s" % output)
logging.info("Done")
def __init__(self, enthome, installstate, port, box_keys_dir,
license_keys_dir):
ent_config = entconfig.EntConfig(enthome) # read-only entconfig object
if not ent_config.Load():
sys.exit('Cannot load configuration.')
flags.FLAGS.gfs_aliases = ent_config.var('GFS_ALIASES')
flags.FLAGS.bnsresolver_use_svelte = False
flags.FLAGS.logtostderr = True
gfile.Init()
self.ss = pywrapselectserver.SelectServer()
# Initialize the configurator.
self.cfg = configurator.configurator(enthome,
box_keys_dir=box_keys_dir,
license_keys_dir=license_keys_dir,
install_state=installstate,
startup_mode=1)
self.cfg.lm.setInWorking()
self.thread_pool = pywrapthreadpool.ThreadPool(1, 5, 5, 0)
self.thread_pool.StartWorkers()
self.server = pywraphttpserver.HTTPServer(self.ss, port, '1.0')
self.server.SetTrustedClients(
string.join(
self.cfg.globalParams.var('HTTPSERVER_TRUSTED_CLIENTS'), ','))
self.handlers = []
self.RegisterSimpleGetHandler('/', self.RootHandler)
self.RegisterSimpleGetHandler('/healthz', self.HealthzHandler)
self.RegisterSimpleGetHandler('/servers', self.ServersHandler)
self.RegisterSimpleGetHandler('/configversion',
self.ConfigVersionHandler)
self.RegisterSimpleGetHandler('/flushlog', self.FlushLogHandler)
self.RegisterGetOrPostHandler('/legacy', self.LegacyHandler)
self.RegisterSimplePostHandler('/legacyCommand',
self.LegacyCommandHandler)
license_handler.NewLicenseHandler(self)
def main(argv):
if FLAGS.enthome is None:
raise SystemExit('--enthome must be set')
ec = entconfig.EntConfig(FLAGS.enthome)
ec.Load()
# ensure that the new serving directory exists and is empty
idx_config_name = 'ENT_SPELL_SERVING_ID'
current_idx = ec.var(idx_config_name)
next_idx = current_idx + 1
spell_path_pattern_prefix = '%s/spell-' % ec.var('ENT_SPELL_ROOT_DIR')
current_dir = spell_path_pattern_prefix + str(current_idx)
next_dir = spell_path_pattern_prefix + str(next_idx)
RunCommand('rm -rf %s' % next_dir)
RunCommand('mkdir -p -m 777 %s' % next_dir) # TODO(mgp): check permissions
# generate all spelling data, placing it in the new serving directory
for lang in FLAGS.langs:
sdg = SpellingDataGenerator(FLAGS.enthome, ec, current_dir, next_dir,
lang)
sdg.GenerateAll()
# tell adminrunner about new serving directory and restart spellmixer
ar = adminrunner_client.AdminRunnerClient(ec.var('MASTER'), 2100)
ar.SetParam(idx_config_name, next_idx)
ar.RestartServer('spellmixer')
# remove older, unused spelling data directories, if they exist
prefix_len = len(spell_path_pattern_prefix)
for path in glob.glob(spell_path_pattern_prefix + '*'):
# get integer suffix
suffix = path[prefix_len:len(path)]
# note: don't delete currently used data
if int(suffix) < current_idx:
RunCommand('rm -rf %s' % path)
def main(argv):
argc = len(argv)
if argc < 4:
sys.exit(__doc__)
config = entconfig.EntConfig(argv[0])
if not config.Load():
sys.exit(__doc__)
pywrapbase.InitGoogleScript('', [
'foo',
'--gfs_aliases=%s' % config.var("GFS_ALIASES"),
'--bnsresolver_use_svelte=false', '--logtostderr'
], 0)
gfile.Init()
cmd = argv[1]
if cmd != 'crawlqueue':
sys.exit(__doc__)
server_str = argv[2]
result_file = argv[3]
index_file = argv[4]
queue_name = argv[5]
caption_time = argv[6]
num_urls_requested = int(argv[7])
next_hours = int(argv[8])
if argc == 10:
crawl_host = argv[9]
else:
crawl_host = ''
command = 'B= V=20 P'
cr_request = crawlreport_pb.CrawlReportRequest()
cr_request.set_commandtype(cr_request.CRAWL_QUEUE_COMMAND)
cr_params = cr_request.mutable_crawlqueueparams()
if crawl_host != '':
cr_params.set_host(crawl_host)
cr_params.set_numurls(num_urls_requested)
cr_params.set_nexthours(next_hours)
servers = server_str.split(',')
cq_mixer = CrawlQueueResponseMixer(num_urls_requested, len(servers),
result_file, index_file)
threads = []
for server in servers:
(server_host, server_port) = server.split(':')
t = threading.Thread(target=GetCrawlqueueFromOneShard,
name=server_host,
args=(server_host, int(server_port), cr_request,
cq_mixer))
threads.append(t)
t.start()
for t in threads:
t.join()
sys.exit(cq_mixer.finalStatus)
class InstallManager:
def __init__(self):
self.cp_ = None
self.machines_ = None
self.target_state_ = None
self.version_ = None
self.enthome_ = None
self.core_only_ = None
self.nonecore_only_ = None
self.retry_ = None
self.core_op_ = None
self.reinitok_ = None
def parse_cmd_line_args(self, argv):
try:
argv = FLAGS(argv) # parse flags
except flags.FlagsError, e:
sys.exit("%s\nUsage: %s ARGS\n%s" % (e, argv[0], FLAGS))
if ((FLAGS.enthome and FLAGS.config)
or (not FLAGS.enthome and not FLAGS.config)):
sys.exit("Please specify only one of --enthome and --config")
if FLAGS.makeactive:
self.target_state_ = "ACTIVE"
elif FLAGS.makeserve:
self.target_state_ = "SERVE"
elif FLAGS.makeinactive:
self.target_state_ = "INACTIVE"
elif FLAGS.maketesting:
self.target_state_ = "TEST"
elif FLAGS.makeinstall:
self.target_state_ = "INSTALL"
elif FLAGS.list:
self.target_state_ = "LIST"
elif FLAGS.docoreop:
self.core_op_ = FLAGS.docoreop
if FLAGS.coreonly:
self.core_only_ = 1
if FLAGS.nonecoreonly:
self.nonecore_only_ = 1
if FLAGS.retry:
self.retry = 1
if FLAGS.reinitok:
self.reinitok_ = 1
if not self.target_state_ and not FLAGS.docoreop:
sys.exit("Please specify one and only one of "\
"--makeactive --makeserve --makeinactive "\
"--maketesting --makeinstall --list or --docoreop")
if self.core_only_ and self.nonecore_only_:
sys.exit(
"Please specify only one of --coreonly and --nonecoreonly")
if FLAGS.enthome:
self.cp_ = entconfig.EntConfig(FLAGS.enthome)
else:
cf = config_factory.ConfigFactory()
self.cp_ = cf.CreateConfig(FLAGS.config)
if not self.cp_.Load():
sys.exit("ERROR: Cannot load the config file")
if FLAGS.machines:
self.machines_ = map(string.strip,
string.split(FLAGS.machines, ","))
else:
self.machines_ = self.cp_.var("MACHINES")
self.version_ = self.cp_.var('VERSION')
self.enthome_ = self.cp_.GetEntHome()
E.ENTERPRISE_HOME = self.enthome_
def main(argv):
try:
argv = FLAGS(argv) # parse flags
except flags.FlagsError, e:
sys.exit('%s\nUsage: %s ARGS\n%s' % (e, argv[0], FLAGS))
if not FLAGS.enthome:
sys.exit('Must specify --enthome')
if not FLAGS.xml_path:
sys.exit('Must specify --xml_path')
if not (FLAGS.do_import or FLAGS.do_export):
sys.exit('Must specify --do_import or --do_export')
ent_config = entconfig.EntConfig(FLAGS.enthome)
if not ent_config.Load():
sys.exit('Cannot load configuration.')
pywrapbase.InitGoogleScript('', [
'foo',
'--gfs_aliases=%s' % ent_config.var('GFS_ALIASES'),
'--bnsresolver_use_svelte=false', '--logtostderr', '--minloglevel=1'
], 0)
gfile.Init()
begin = time.time()
cfg = configurator.configurator(FLAGS.enthome)
if FLAGS.do_import:
logging.info("initializing global default files")
if cmd_idx > op_index:
err = 0
s.send(cmd + "\n")
cmd_idx = cmd_idx + 1
except Exception, e:
logging.error(e)
pass
s.close()
return err
if __name__ == "__main__":
config = entconfig.EntConfig(sys.argv[1])
if not config.Load(): sys.exit(__doc__)
try:
victim = sys.argv[2]
apcval = APC_VALUES[sys.argv[3]]
except:
sys.exit(__doc__)
if not config.var('AUTO_REBOOT'):
sys.exit(0)
if (not config.var('APC_MAP')
or not config.var('APC_MAP').has_key(victim)):
## just return with status value 1
sys.exit(1)
def execute(flag_enthome, flag_restrict, flag_value, flag_op, flag_force,
flag_no_extra_req):
if not flag_restrict: sys.exit("Must specify a paramter")
if not flag_enthome: sys.exit("Config file must be specified")
if not flag_op or not flag_op in ("add", "del"):
sys.exit("Invalid value")
value = None
try:
exec("value = %s" % flag_value)
except:
sys.exit("Invalid value %s" % flag_value)
cp = entconfig.EntConfig(flag_enthome)
if not cp.Load():
sys.exit("Unable to load config file %s" % (cp.GetConfigFileName()))
do_update = true
if not flag_force:
try:
oldval = open(cp.GetRestrictParam(flag_restrict), "r").read()
do_update = oldval != value
except IOError:
pass # we update
if not do_update:
logging.info("New value for restrict [%s] unchanged." % flag_restrict)
return (1, None)
# Perform the actual request. On error we return
rt_op = None
if flag_op == "del":
rt_op = 1
cp.DeleteRestrict(flag_restrict)
elif flag_op == "add":
rt_op = 0
err = cp.SetRestrict(flag_restrict, value)
if err not in validatorlib.VALID_CODES:
logging.error("New value for file restrict [%s] did "\
"not validate correctly [%s]" % (flag_restrict, err))
return (2, None)
else:
sys.exit("Invalid restrict operation %s" % flag_op)
# Distribute config file
cp.DistributeAll()
# Register requests per machine
# Prepare adummy request to send server command
req = server_requests.SendServerCommandRequest()
req.Set(
"dummy", 1,
"GET /update?RestrictUpdate=b&IsDeleteOrAddRestrict=%s&RestrictFileName=%s&RestrictUpdate=e HTTP/1.0\r\n\r\n"
% (rt_op, cp.GetRestrictParam(flag_restrict)), "HTTP/1.0 200 OK",
cp.GetConfigFileName())
# The actual request that registers req for each machine:port in base_indexer
the_req = configmgr_request.MultiRequestCreateRequest()
the_req.Set(req.datadict, "base_indexer", server_requests.MACHINE,
server_requests.PORT, cp.GetConfigFileName())
if not flag_no_extra_req and not cp.WriteConfigManagerRequest(the_req):
sys.exit("Cannot save subsequent request to update param on machine")
cp.DistributeAll()
return (0, the_req)
def __init__(self,
enthome,
box_keys_dir="",
license_keys_dir="",
install_state='ACTIVE',
startup_mode=0):
self.entHome = enthome
self.install_state = install_state
self.globalLock = threading.Lock()
# Init the global params
self.mach_param_cache = machine_param_cache.MachineParamCache()
self.globalParams = entconfig.EntConfig(self.entHome,
1) # Allow writes
if not self.globalParams.Load():
logging.error(
"Cannot load the global parameters - think about reinstall")
if install_state != 'INSTALL':
if not self.globalParams.ValidateAll():
logging.error("Global parameters were not validated correctly")
# The operator log file
oplogname = "%s/AdminRunner.OPERATOR.%s" % (
self.globalParams.var("LOGDIR"),
time.strftime("%Y_%m_%d_%H_%M_%S", E.getLocaltime(time.time())))
self.oplog = open(oplogname, "w")
self.lm = license_manager.LicenseManager(self, box_keys_dir,
license_keys_dir)
self.um = user_manager.UserManager(self)
self.logmanager = log_manager.LogManager(self)
self.crawlqueuemanager = crawlqueue_manager.CrawlQueueManager(self)
self.startup_mode = startup_mode # statup in startup mode
self.startup_time = int(time.time())
self.graphs = {
'SUM_URLS_TOTAL':
self.graph_data(file_name="sumurlstotalgraph.png",
vars=[
"num_urls_in_index_total[24h]",
"num_urls_available_total[24h]"
],
last_modified=-1),
'SUM_URLS_CRAWLED':
self.graph_data(file_name="sumurlscrawledgraph.png",
vars=["num_urls_in_index_total[24h]"],
last_modified=-1),
'SUM_URLS_AVAILABLE':
self.graph_data(file_name="sumurlsavailablegraph.png",
vars=["num_urls_available_total[24h]"],
last_modified=-1),
'QUERIES_PER_MINUTE':
self.graph_data(file_name="queriesperminutegraph.png",
vars=["gws_searches_per_minute[24h]"],
last_modified=-1),
'SUM_URLS_TOTAL_THUMBNAIL':
self.graph_data(file_name="sumurlstotalthumb.png",
vars=[
"num_urls_in_index_total[24h]",
"num_urls_available_total[24h]"
],
last_modified=-1),
'QUERIES_PER_MINUTE_THUMBNAIL':
self.graph_data(file_name="queriesperminutethumb.png",
vars=["gws_searches_per_minute[24h]"],
last_modified=-1),
'TMP_GRAPH':
self.graph_data(file_name="tmp_graph.png",
vars=[],
last_modified=-1),
'ERROR_TEXT':
self.graph_data(file_name="errortext.png",
vars=[],
last_modified=-1),
}
self.graphs_lock_ = threading.Lock()
def main(argv):
argc = len(argv)
if argc < 10:
sys.exit(__doc__)
config = entconfig.EntConfig(argv[0])
if not config.Load(): sys.exit(__doc__)
pywrapbase.InitGoogleScript('', [
'foo',
'--gfs_aliases=%s' % config.var("GFS_ALIASES"),
'--bnsresolver_use_svelte=false', '--logtostderr'
], 0)
gfile.Init()
client = argv[1]
date_fields = string.split(argv[2], '_')
date_range = liblog.ParseDateRange(date_fields[0], date_fields[1:])
withResults = argv[3]
topCount = argv[4]
diagnosticTerms = argv[5]
html_file = argv[6]
valid_file = argv[7]
new_html_file = argv[8]
new_valid_file = argv[9]
if not date_range:
sys.exit(__doc__)
first_date, last_date, printable_date, file_date = date_range
if last_date.as_int() < first_date.as_int():
logging.fatal('invalid date range')
gws_log_dir = liblog.get_gws_log_dir(config)
collect_dir = liblog.get_collect_dir(config)
partition_dir = liblog.get_partition_dir(config)
directory_map_file = liblog.get_directory_map_file(config)
# Collect logs first from all gws nodes and preprocess
# logs to make sure logs are up to date.
all_machines = config.var('MACHINES')
collect_logs.CollectLogs(all_machines, gws_log_dir, collect_dir)
preprocess_logs.PartitionLogs(config)
gws_logs = liblog.FindClientLogFiles(partition_dir, directory_map_file,
client, first_date, last_date)
# note that collection (client) has been factored into gwslog_dir.
result = CreateLogReport(config, printable_date, gws_logs,
config.var('MAIN_GOOGLE3_DIR'), withResults,
topCount, diagnosticTerms, html_file, valid_file,
new_html_file, new_valid_file)
if result == liblog.FAILURE:
logging.error('CreateLogReport Failed')
sys.exit(result)
def main(argv):
argc = len(argv)
if argc < 6:
sys.exit(__doc__)
config = entconfig.EntConfig(argv[0])
if not config.Load():
sys.exit(__doc__)
pywrapbase.InitGoogleScript('', [
'foo',
'--gfs_aliases=%s' % config.var("GFS_ALIASES"),
'--bnsresolver_use_svelte=false', '--logtostderr'
], 0)
gfile.Init()
client = argv[1]
date_arg = argv[2]
html_file = argv[3]
valid_file = argv[4]
new_valid_file = argv[5]
# extract tag and date_range from command line args
date_fields = string.split(date_arg, '_')
date_range = liblog.ParseDateRange(date_fields[0], date_fields[1:])
if not date_range:
sys.exit(__doc__)
first_date, last_date, printable_date, file_date = date_range
if last_date.as_int() < first_date.as_int():
sys.exit(__doc__)
gws_log_dir = liblog.get_gws_log_dir(config)
click_dir = liblog.get_click_dir(config)
collect_dir = liblog.get_collect_dir(config)
apache_dir = liblog.get_apache_dir(config)
directory_map_file = liblog.get_directory_map_file(config)
# we need to collect logs first from all gws nodes and preprocess
# logs first to make sure logs are up to date.
all_machines = config.var('MACHINES')
collect_logs.CollectLogs(all_machines, gws_log_dir, collect_dir)
preprocess_logs.PartitionLogs(config)
# make a vector of Log objects for all apache_logs and click_logs matching
# the given date range and client.
apache_logs = liblog.FindClientLogFiles(apache_dir, directory_map_file,
client, first_date, last_date)
click_logs = liblog.FindClientLogFiles(click_dir, directory_map_file,
client, first_date, last_date)
# If we have valid file and report file, we check to see if the data in
# apache_dir has been changed and if the report is still valid.
if (gfile.Exists(html_file) and gfile.Exists(valid_file)
and liblog.checkValid(html_file, valid_file, apache_logs)):
logging.info('%s still valid.' % html_file)
sys.exit(liblog.STILL_VALID)
# if there is no valid report, we create a new one
DumpApacheAndClickLogs(apache_logs, click_logs)
if not liblog.makeValid(new_valid_file, apache_logs):
logging.error('Error validating %s' % html_file)
sys.exit(liblog.FAILURE)
logging.info('done apache_log, new_valid_file: %s' % new_valid_file)
sys.exit(liblog.SUCCESS)