def GetEnterpriseConfiguration(): """Get the enterprise configuration object.""" export_hda = '/export/hda3' c = entconfig.EntConfig(None) c.set_allow_config_mgr_requests(0) c.set_var('DEFAULTS', ['enterprise']) c.AddDefaults() version = c.var('VERSION') ent_home = '%s/%s' % (export_hda, version) ec = entconfig.EntConfig(ent_home) ec.Load() return ec
def main(argv): """ runs PeriodicScript() function in a loop""" # global MACHINES # global MAIN_GOOGLE3_DIR if len(argv) != 1: sys.exit(__doc__) try: logging.info("Running periodic_script (pid = %d)..." % os.getpid()) config = entconfig.EntConfig(argv[0]) # user config if not config.Load(): sys.exit("Cannot load the config file %s" % argv[0]) PeriodicScript(config) svs_utilities.CheckSvsAlive(["localhost"]) monitorSnmp() EnableGFS(config) EnableNamed() DnsConfig(config) admin_runner_utils.SyncOneboxLog(config) WarmIndex(config) logging.info("Finished periodic_script.") except: # collect the exception traceback so we know what went wrong (t, v, tb) = sys.exc_info() logging.error("\nPeriodic script: Fatal Error:\n" + "=======================\n" + string.join(traceback.format_exception(t, v, tb)))
def main(argv): if len(argv) != 1: sys.exit(__doc__) config = entconfig.EntConfig(argv[0]) if not config.Load(): sys.exit(__doc__) # Collect syslogs only if active or serve state = install_utilities.install_state(config.var('VERSION')) if not state in ['ACTIVE', 'SERVE']: sys.exit(0) # Collect syslogs only from master node. if not isMaster(config): logging.fatal('Not a oneway or cluster master node. Return!') pywrapbase.InitGoogleScript('', [ 'foo', '--gfs_aliases=%s' % config.var("GFS_ALIASES"), '--bnsresolver_use_svelte=false', '--logtostderr' ], 0) gfile.Init() first_date, last_date, printable_date, file_date = \ liblog.ParseDateRange('all',[]) apache_main_dir = liblog.get_apache_dir(config) checkpoint_dir = liblog.get_syslog_checkpoint_dir(config) liblog.MakeGoogleDir(config, checkpoint_dir) if (config.var('SYSLOG_SERVER') == None or config.var('ENT_SYSLOG_GWS_FAC') == None): logging.fatal('SYSLOG logging is disabled') lockfile = '%s/syslog_lock' % config.var('LOGDIR') lock = E.acquire_lock(lockfile, 1, breakLockAfterGracePeriod=0) if not lock: return try: logger = syslog_client.SyslogClient(config.var('SYSLOG_SERVER'), config.var('EXTERNAL_WEB_IP')) logging.info("syslog-server = %s" % config.var('SYSLOG_SERVER')) for collection in os.listdir(apache_main_dir): apache_dir = '%s/%s' % (apache_main_dir, collection) checkpoint_file = "%s/%s" % (checkpoint_dir, collection) apache_logs = liblog.FindLogFiles(apache_dir, first_date, last_date) logging.info('syslog handles collection %s' % collection) if not SyslogLogs(apache_logs, apache_dir, checkpoint_file, logger, config): sys.exit('Updating failed') logger.close() finally: lock.close() os.unlink(lockfile) sys.exit(0)
def __load_default_params__(): global __default_params__ if __default_params__ == None: from google3.enterprise.legacy.adminrunner import entconfig # read some default parameter __default_params__ = entconfig.EntConfig(None) __default_params__.set_var('DEFAULTS', ['enterprise']) __default_params__.AddDefaults() __default_params__.update_derived_info() return __default_params__
def main(): latest_version = install_utilities.get_latest_version() ent_home = '/export/hda3/%s' % latest_version backup_dir = '/export/hda3/%s/data/googleconfig.bak' % latest_version log_file_name = '/export/hda3/tmp/google_config_backup.log' log_file = open(log_file_name, 'a') logging.set_logfile(log_file) config = entconfig.EntConfig(ent_home) number_of_backups = 21 return BackupGoogleConfigFile(config, backup_dir, number_of_backups)
def ClearResetIndexStatus(ent_home, unittestdir=None): """ Clear Reset index status Parameters: ent_home: '/export/hda3/4.6.0.G.35/' unittestdir: dir to put the RESET_STATE file. -- for unittest only. """ cfg = entconfig.EntConfig(ent_home) cfg.Load() version = cfg.var('VERSION') _ClearResetStatus(version, unittestdir=unittestdir)
def main(): # on the GSA-Lite, there is no SVS, so just return if core_utils.GetEntConfigVar('ENT_CONFIG_TYPE') == 'LITE': return 0 enthome = sys.argv[1] if len(sys.argv) > 2: machine = sys.argv[2] else: machine = E.getCrtHostName() cfg = entconfig.EntConfig(enthome) if not cfg.Load(): return 1 restart_svs(machine) return 0
def main(enthome): config = entconfig.EntConfig(enthome) if not config.Load(): sys.exit(__doc__) user = pwd.getpwuid(os.getuid()) if (user[0] != config.var('ENTERPRISE_USER')): sys.exit("user %s cannot run this script; expected user is %s" % (user[0], config.var('ENTERPRISE_USER'))) ENTID_TAG = "ENT_ID=%s_crawl" % config.var('VERSION') tabs = [] logging.info("starting to fill tabs..") # Setup standard jobs every_ten_minutes = "*/10 * * * *" conf_dir = config.var('ENTERPRISE_HOME') + "/local/conf" source_env = ". /etc/profile; . %s/ent_bashrc " % conf_dir logging.info("preparing collect_logs.py entry..") log_collect_cmd = "cd ../logs;"\ "%s ./collect_logs.py %s" % (ENTID_TAG, enthome) log_collect_log_file = "/%s/cronLogCollectOut_%s" % (config.var('LOGDIR'), config.var('VERSION')) tabs.append( CronTabEntry(config, every_ten_minutes, (source_env, log_collect_cmd), log_collect_log_file, source_env)) logging.info("preparing syslog_logs.py entry..") syslog_log_cmd = "cd ../logs;"\ "%s ./syslog_logs.py %s" % (ENTID_TAG, enthome) syslog_log_logfile = "/%s/cronSyslogLogOut_%s" % (config.var('LOGDIR'), config.var('VERSION')) tabs.append( CronTabEntry(config, every_ten_minutes, (source_env, syslog_log_cmd), syslog_log_logfile, source_env)) # write out a temp cron tab tmp_cron_file = "%s/tmp/crawl_cron_%s" % (config.var('ENTERPRISE_HOME'), config.var('VERSION')) fout = open(tmp_cron_file, "w") fout.write(string.join(tabs)) fout.close() os.system("crontab -l|grep -v %s| cat - %s > %s_" % (ENTID_TAG, tmp_cron_file, tmp_cron_file)) #user_arg = "-u %s " % crontab_user os.system("crontab %s_" % tmp_cron_file)
def CreateConfig(self, config_file, config_dir=None): """ Given a config file name this returns a Config or an EntConfig object """ # Get here only for strings ent_home = self.get_enterprise_home_(config_file) if ent_home != None: from google3.enterprise.legacy.adminrunner import entconfig return entconfig.EntConfig(ent_home) # This is not an enterprise config. How can it not be? Die horribly and # noisily. This code would/should never be up integrated. There are no # such plans anyway raise Exception("%s is not a valid enterprise config file!" % config_file) die = 5 / 0 # Just in case... sys.exit(1) # yes, I really want you dead
def main(argv): argc = len(argv) if argc != 1: sys.exit(__doc__) config = entconfig.EntConfig(argv[0]) if not config.Load(): sys.exit(__doc__) success = GenerateStatusReport(config) if not success: logging.error("Failed") sys.exit(__doc__) logging.info("Done") sys.exit(0)
def init_service(self, ent_home): """ Does the actual initialization. Reads the config file in the cp (EntConfig) member and initializes some members for easy access to usual parameters """ self.cp = entconfig.EntConfig(ent_home) if not self.cp.Load(): sys.exit("Cannot load the config file %s" % self.cp.GetConfigFileName()) # Get some params for easy access self.configfile = self.cp.GetConfigFileName() self.version = str(self.cp.var("VERSION")) self.entid_tag = "ENT_ID=%s_%s" % (self.version, self.service_name) self.ent_user = self.cp.var("ENTERPRISE_USER") self.ent_group = self.cp.var("ENTERPRISE_GROUP") self.ent_bashrc = self.cp.var("ENTERPRISE_BASHRC") self.ent_home = self.cp.var("ENTERPRISE_HOME") self.googlebot_dir = self.cp.var("GOOGLEBOT_DIR") self.version_tmpdir= "%s/tmp" % self.cp.var("ENTERPRISE_HOME") self.tmpdir = self.cp.var("TMPDIR") self.logdir = self.cp.var("LOGDIR") self.datadir = self.cp.var("DATADIR") self.scripts_dir = ("%s/local/google3/enterprise/legacy/scripts" % self.ent_home) self.util_dir = ("%s/local/google3/enterprise/legacy/util" % self.ent_home) self.machines = self.cp.var("MACHINES") # The master depends on the install state : for active / test / install # we have the adminrunner on the master, else we get it from MASTER # parameter self.install_state = install_utilities.install_state( self.version, rootdir = self.cp.var('ENT_DISK_ROOT')) self.local_machine = E.getCrtHostName() testver = install_utilities.is_test(self.version) if self.install_state in ["ACTIVE", "TEST", "INSTALL"]: try: self.master_machine = find_master.FindMasterUsingChubby(self.cp.var('VERSION')) except core_utils.EntMasterError, e: # Something is seriously wrong. logging.error("ERROR: Couldn't determine master") # Assume we aren't master, so we can at least do inactivate self.master_machine = None
def IsResetIndexInProgress(ent_home, unittestdir=None): """ If reset index is in progress Parameters: ent_home: '/export/hda3/4.6.0.G.35/' unittestdir: dir to put the RESET_STATE file. -- for unittest only. Return: 1 - reset index is in progress. 0 - otherwise. """ cfg = entconfig.EntConfig(ent_home) cfg.Load() version = cfg.var('VERSION') status = _GetResetStatus(version, unittestdir=unittestdir) if status == 'RESET_IN_PROGRESS': return 1 else: return 0
def main(argv): config = entconfig.EntConfig(argv[0]) if not config.Load(): sys.exit(__doc__) try: crawl = argv[1] port = string.atoi(argv[2]) except: sys.exit(__doc__) machines = config.var('MACHINES') master = find_master.FindMaster(port, machines) # The name of this machine crt_machine = E.getCrtHostName() if len(master) != 1 or master[0] != crt_machine: logging.info("I am not the master") sys.exit(0) # not a problem # find out the date 24 hours ago year, month, day = time.localtime(time.time() - 60 * 60 * 24)[0:3] date1 = "date_%d_%d_%d" % (month, day, year) date2 = "date %d %d %d" % (month, day, year) logging.info("Running %s for %s" % (sys.argv[0], date2)) # 1. run log_report.py for each crawl ar = adminrunner_client.AdminRunnerClient("localhost", port) if not ar.LogReportStart(crawl, date1): sys.exit("Error starting crawl for " + crawl) logging.info("Started log_report for %s/%s" % (crawl, date1)) # 2. run apache_log.py for each crawl apache_cmd = "./apache_log.py %s update %s %s" % (argv[0], crawl, date2) (status, output) = commands.getstatusoutput(apache_cmd) if status != 0: logging.fatal("apache_log failed: %s" % output) logging.info("Finished apache_log: %s" % output) logging.info("Done")
def __init__(self, enthome, installstate, port, box_keys_dir, license_keys_dir): ent_config = entconfig.EntConfig(enthome) # read-only entconfig object if not ent_config.Load(): sys.exit('Cannot load configuration.') flags.FLAGS.gfs_aliases = ent_config.var('GFS_ALIASES') flags.FLAGS.bnsresolver_use_svelte = False flags.FLAGS.logtostderr = True gfile.Init() self.ss = pywrapselectserver.SelectServer() # Initialize the configurator. self.cfg = configurator.configurator(enthome, box_keys_dir=box_keys_dir, license_keys_dir=license_keys_dir, install_state=installstate, startup_mode=1) self.cfg.lm.setInWorking() self.thread_pool = pywrapthreadpool.ThreadPool(1, 5, 5, 0) self.thread_pool.StartWorkers() self.server = pywraphttpserver.HTTPServer(self.ss, port, '1.0') self.server.SetTrustedClients( string.join( self.cfg.globalParams.var('HTTPSERVER_TRUSTED_CLIENTS'), ',')) self.handlers = [] self.RegisterSimpleGetHandler('/', self.RootHandler) self.RegisterSimpleGetHandler('/healthz', self.HealthzHandler) self.RegisterSimpleGetHandler('/servers', self.ServersHandler) self.RegisterSimpleGetHandler('/configversion', self.ConfigVersionHandler) self.RegisterSimpleGetHandler('/flushlog', self.FlushLogHandler) self.RegisterGetOrPostHandler('/legacy', self.LegacyHandler) self.RegisterSimplePostHandler('/legacyCommand', self.LegacyCommandHandler) license_handler.NewLicenseHandler(self)
def main(argv): if FLAGS.enthome is None: raise SystemExit('--enthome must be set') ec = entconfig.EntConfig(FLAGS.enthome) ec.Load() # ensure that the new serving directory exists and is empty idx_config_name = 'ENT_SPELL_SERVING_ID' current_idx = ec.var(idx_config_name) next_idx = current_idx + 1 spell_path_pattern_prefix = '%s/spell-' % ec.var('ENT_SPELL_ROOT_DIR') current_dir = spell_path_pattern_prefix + str(current_idx) next_dir = spell_path_pattern_prefix + str(next_idx) RunCommand('rm -rf %s' % next_dir) RunCommand('mkdir -p -m 777 %s' % next_dir) # TODO(mgp): check permissions # generate all spelling data, placing it in the new serving directory for lang in FLAGS.langs: sdg = SpellingDataGenerator(FLAGS.enthome, ec, current_dir, next_dir, lang) sdg.GenerateAll() # tell adminrunner about new serving directory and restart spellmixer ar = adminrunner_client.AdminRunnerClient(ec.var('MASTER'), 2100) ar.SetParam(idx_config_name, next_idx) ar.RestartServer('spellmixer') # remove older, unused spelling data directories, if they exist prefix_len = len(spell_path_pattern_prefix) for path in glob.glob(spell_path_pattern_prefix + '*'): # get integer suffix suffix = path[prefix_len:len(path)] # note: don't delete currently used data if int(suffix) < current_idx: RunCommand('rm -rf %s' % path)
def main(argv): argc = len(argv) if argc < 4: sys.exit(__doc__) config = entconfig.EntConfig(argv[0]) if not config.Load(): sys.exit(__doc__) pywrapbase.InitGoogleScript('', [ 'foo', '--gfs_aliases=%s' % config.var("GFS_ALIASES"), '--bnsresolver_use_svelte=false', '--logtostderr' ], 0) gfile.Init() cmd = argv[1] if cmd != 'crawlqueue': sys.exit(__doc__) server_str = argv[2] result_file = argv[3] index_file = argv[4] queue_name = argv[5] caption_time = argv[6] num_urls_requested = int(argv[7]) next_hours = int(argv[8]) if argc == 10: crawl_host = argv[9] else: crawl_host = '' command = 'B= V=20 P' cr_request = crawlreport_pb.CrawlReportRequest() cr_request.set_commandtype(cr_request.CRAWL_QUEUE_COMMAND) cr_params = cr_request.mutable_crawlqueueparams() if crawl_host != '': cr_params.set_host(crawl_host) cr_params.set_numurls(num_urls_requested) cr_params.set_nexthours(next_hours) servers = server_str.split(',') cq_mixer = CrawlQueueResponseMixer(num_urls_requested, len(servers), result_file, index_file) threads = [] for server in servers: (server_host, server_port) = server.split(':') t = threading.Thread(target=GetCrawlqueueFromOneShard, name=server_host, args=(server_host, int(server_port), cr_request, cq_mixer)) threads.append(t) t.start() for t in threads: t.join() sys.exit(cq_mixer.finalStatus)
class InstallManager: def __init__(self): self.cp_ = None self.machines_ = None self.target_state_ = None self.version_ = None self.enthome_ = None self.core_only_ = None self.nonecore_only_ = None self.retry_ = None self.core_op_ = None self.reinitok_ = None def parse_cmd_line_args(self, argv): try: argv = FLAGS(argv) # parse flags except flags.FlagsError, e: sys.exit("%s\nUsage: %s ARGS\n%s" % (e, argv[0], FLAGS)) if ((FLAGS.enthome and FLAGS.config) or (not FLAGS.enthome and not FLAGS.config)): sys.exit("Please specify only one of --enthome and --config") if FLAGS.makeactive: self.target_state_ = "ACTIVE" elif FLAGS.makeserve: self.target_state_ = "SERVE" elif FLAGS.makeinactive: self.target_state_ = "INACTIVE" elif FLAGS.maketesting: self.target_state_ = "TEST" elif FLAGS.makeinstall: self.target_state_ = "INSTALL" elif FLAGS.list: self.target_state_ = "LIST" elif FLAGS.docoreop: self.core_op_ = FLAGS.docoreop if FLAGS.coreonly: self.core_only_ = 1 if FLAGS.nonecoreonly: self.nonecore_only_ = 1 if FLAGS.retry: self.retry = 1 if FLAGS.reinitok: self.reinitok_ = 1 if not self.target_state_ and not FLAGS.docoreop: sys.exit("Please specify one and only one of "\ "--makeactive --makeserve --makeinactive "\ "--maketesting --makeinstall --list or --docoreop") if self.core_only_ and self.nonecore_only_: sys.exit( "Please specify only one of --coreonly and --nonecoreonly") if FLAGS.enthome: self.cp_ = entconfig.EntConfig(FLAGS.enthome) else: cf = config_factory.ConfigFactory() self.cp_ = cf.CreateConfig(FLAGS.config) if not self.cp_.Load(): sys.exit("ERROR: Cannot load the config file") if FLAGS.machines: self.machines_ = map(string.strip, string.split(FLAGS.machines, ",")) else: self.machines_ = self.cp_.var("MACHINES") self.version_ = self.cp_.var('VERSION') self.enthome_ = self.cp_.GetEntHome() E.ENTERPRISE_HOME = self.enthome_
def main(argv): try: argv = FLAGS(argv) # parse flags except flags.FlagsError, e: sys.exit('%s\nUsage: %s ARGS\n%s' % (e, argv[0], FLAGS)) if not FLAGS.enthome: sys.exit('Must specify --enthome') if not FLAGS.xml_path: sys.exit('Must specify --xml_path') if not (FLAGS.do_import or FLAGS.do_export): sys.exit('Must specify --do_import or --do_export') ent_config = entconfig.EntConfig(FLAGS.enthome) if not ent_config.Load(): sys.exit('Cannot load configuration.') pywrapbase.InitGoogleScript('', [ 'foo', '--gfs_aliases=%s' % ent_config.var('GFS_ALIASES'), '--bnsresolver_use_svelte=false', '--logtostderr', '--minloglevel=1' ], 0) gfile.Init() begin = time.time() cfg = configurator.configurator(FLAGS.enthome) if FLAGS.do_import: logging.info("initializing global default files")
if cmd_idx > op_index: err = 0 s.send(cmd + "\n") cmd_idx = cmd_idx + 1 except Exception, e: logging.error(e) pass s.close() return err if __name__ == "__main__": config = entconfig.EntConfig(sys.argv[1]) if not config.Load(): sys.exit(__doc__) try: victim = sys.argv[2] apcval = APC_VALUES[sys.argv[3]] except: sys.exit(__doc__) if not config.var('AUTO_REBOOT'): sys.exit(0) if (not config.var('APC_MAP') or not config.var('APC_MAP').has_key(victim)): ## just return with status value 1 sys.exit(1)
def execute(flag_enthome, flag_restrict, flag_value, flag_op, flag_force, flag_no_extra_req): if not flag_restrict: sys.exit("Must specify a paramter") if not flag_enthome: sys.exit("Config file must be specified") if not flag_op or not flag_op in ("add", "del"): sys.exit("Invalid value") value = None try: exec("value = %s" % flag_value) except: sys.exit("Invalid value %s" % flag_value) cp = entconfig.EntConfig(flag_enthome) if not cp.Load(): sys.exit("Unable to load config file %s" % (cp.GetConfigFileName())) do_update = true if not flag_force: try: oldval = open(cp.GetRestrictParam(flag_restrict), "r").read() do_update = oldval != value except IOError: pass # we update if not do_update: logging.info("New value for restrict [%s] unchanged." % flag_restrict) return (1, None) # Perform the actual request. On error we return rt_op = None if flag_op == "del": rt_op = 1 cp.DeleteRestrict(flag_restrict) elif flag_op == "add": rt_op = 0 err = cp.SetRestrict(flag_restrict, value) if err not in validatorlib.VALID_CODES: logging.error("New value for file restrict [%s] did "\ "not validate correctly [%s]" % (flag_restrict, err)) return (2, None) else: sys.exit("Invalid restrict operation %s" % flag_op) # Distribute config file cp.DistributeAll() # Register requests per machine # Prepare adummy request to send server command req = server_requests.SendServerCommandRequest() req.Set( "dummy", 1, "GET /update?RestrictUpdate=b&IsDeleteOrAddRestrict=%s&RestrictFileName=%s&RestrictUpdate=e HTTP/1.0\r\n\r\n" % (rt_op, cp.GetRestrictParam(flag_restrict)), "HTTP/1.0 200 OK", cp.GetConfigFileName()) # The actual request that registers req for each machine:port in base_indexer the_req = configmgr_request.MultiRequestCreateRequest() the_req.Set(req.datadict, "base_indexer", server_requests.MACHINE, server_requests.PORT, cp.GetConfigFileName()) if not flag_no_extra_req and not cp.WriteConfigManagerRequest(the_req): sys.exit("Cannot save subsequent request to update param on machine") cp.DistributeAll() return (0, the_req)
def __init__(self, enthome, box_keys_dir="", license_keys_dir="", install_state='ACTIVE', startup_mode=0): self.entHome = enthome self.install_state = install_state self.globalLock = threading.Lock() # Init the global params self.mach_param_cache = machine_param_cache.MachineParamCache() self.globalParams = entconfig.EntConfig(self.entHome, 1) # Allow writes if not self.globalParams.Load(): logging.error( "Cannot load the global parameters - think about reinstall") if install_state != 'INSTALL': if not self.globalParams.ValidateAll(): logging.error("Global parameters were not validated correctly") # The operator log file oplogname = "%s/AdminRunner.OPERATOR.%s" % ( self.globalParams.var("LOGDIR"), time.strftime("%Y_%m_%d_%H_%M_%S", E.getLocaltime(time.time()))) self.oplog = open(oplogname, "w") self.lm = license_manager.LicenseManager(self, box_keys_dir, license_keys_dir) self.um = user_manager.UserManager(self) self.logmanager = log_manager.LogManager(self) self.crawlqueuemanager = crawlqueue_manager.CrawlQueueManager(self) self.startup_mode = startup_mode # statup in startup mode self.startup_time = int(time.time()) self.graphs = { 'SUM_URLS_TOTAL': self.graph_data(file_name="sumurlstotalgraph.png", vars=[ "num_urls_in_index_total[24h]", "num_urls_available_total[24h]" ], last_modified=-1), 'SUM_URLS_CRAWLED': self.graph_data(file_name="sumurlscrawledgraph.png", vars=["num_urls_in_index_total[24h]"], last_modified=-1), 'SUM_URLS_AVAILABLE': self.graph_data(file_name="sumurlsavailablegraph.png", vars=["num_urls_available_total[24h]"], last_modified=-1), 'QUERIES_PER_MINUTE': self.graph_data(file_name="queriesperminutegraph.png", vars=["gws_searches_per_minute[24h]"], last_modified=-1), 'SUM_URLS_TOTAL_THUMBNAIL': self.graph_data(file_name="sumurlstotalthumb.png", vars=[ "num_urls_in_index_total[24h]", "num_urls_available_total[24h]" ], last_modified=-1), 'QUERIES_PER_MINUTE_THUMBNAIL': self.graph_data(file_name="queriesperminutethumb.png", vars=["gws_searches_per_minute[24h]"], last_modified=-1), 'TMP_GRAPH': self.graph_data(file_name="tmp_graph.png", vars=[], last_modified=-1), 'ERROR_TEXT': self.graph_data(file_name="errortext.png", vars=[], last_modified=-1), } self.graphs_lock_ = threading.Lock()
def main(argv): argc = len(argv) if argc < 10: sys.exit(__doc__) config = entconfig.EntConfig(argv[0]) if not config.Load(): sys.exit(__doc__) pywrapbase.InitGoogleScript('', [ 'foo', '--gfs_aliases=%s' % config.var("GFS_ALIASES"), '--bnsresolver_use_svelte=false', '--logtostderr' ], 0) gfile.Init() client = argv[1] date_fields = string.split(argv[2], '_') date_range = liblog.ParseDateRange(date_fields[0], date_fields[1:]) withResults = argv[3] topCount = argv[4] diagnosticTerms = argv[5] html_file = argv[6] valid_file = argv[7] new_html_file = argv[8] new_valid_file = argv[9] if not date_range: sys.exit(__doc__) first_date, last_date, printable_date, file_date = date_range if last_date.as_int() < first_date.as_int(): logging.fatal('invalid date range') gws_log_dir = liblog.get_gws_log_dir(config) collect_dir = liblog.get_collect_dir(config) partition_dir = liblog.get_partition_dir(config) directory_map_file = liblog.get_directory_map_file(config) # Collect logs first from all gws nodes and preprocess # logs to make sure logs are up to date. all_machines = config.var('MACHINES') collect_logs.CollectLogs(all_machines, gws_log_dir, collect_dir) preprocess_logs.PartitionLogs(config) gws_logs = liblog.FindClientLogFiles(partition_dir, directory_map_file, client, first_date, last_date) # note that collection (client) has been factored into gwslog_dir. result = CreateLogReport(config, printable_date, gws_logs, config.var('MAIN_GOOGLE3_DIR'), withResults, topCount, diagnosticTerms, html_file, valid_file, new_html_file, new_valid_file) if result == liblog.FAILURE: logging.error('CreateLogReport Failed') sys.exit(result)
def main(argv): argc = len(argv) if argc < 6: sys.exit(__doc__) config = entconfig.EntConfig(argv[0]) if not config.Load(): sys.exit(__doc__) pywrapbase.InitGoogleScript('', [ 'foo', '--gfs_aliases=%s' % config.var("GFS_ALIASES"), '--bnsresolver_use_svelte=false', '--logtostderr' ], 0) gfile.Init() client = argv[1] date_arg = argv[2] html_file = argv[3] valid_file = argv[4] new_valid_file = argv[5] # extract tag and date_range from command line args date_fields = string.split(date_arg, '_') date_range = liblog.ParseDateRange(date_fields[0], date_fields[1:]) if not date_range: sys.exit(__doc__) first_date, last_date, printable_date, file_date = date_range if last_date.as_int() < first_date.as_int(): sys.exit(__doc__) gws_log_dir = liblog.get_gws_log_dir(config) click_dir = liblog.get_click_dir(config) collect_dir = liblog.get_collect_dir(config) apache_dir = liblog.get_apache_dir(config) directory_map_file = liblog.get_directory_map_file(config) # we need to collect logs first from all gws nodes and preprocess # logs first to make sure logs are up to date. all_machines = config.var('MACHINES') collect_logs.CollectLogs(all_machines, gws_log_dir, collect_dir) preprocess_logs.PartitionLogs(config) # make a vector of Log objects for all apache_logs and click_logs matching # the given date range and client. apache_logs = liblog.FindClientLogFiles(apache_dir, directory_map_file, client, first_date, last_date) click_logs = liblog.FindClientLogFiles(click_dir, directory_map_file, client, first_date, last_date) # If we have valid file and report file, we check to see if the data in # apache_dir has been changed and if the report is still valid. if (gfile.Exists(html_file) and gfile.Exists(valid_file) and liblog.checkValid(html_file, valid_file, apache_logs)): logging.info('%s still valid.' % html_file) sys.exit(liblog.STILL_VALID) # if there is no valid report, we create a new one DumpApacheAndClickLogs(apache_logs, click_logs) if not liblog.makeValid(new_valid_file, apache_logs): logging.error('Error validating %s' % html_file) sys.exit(liblog.FAILURE) logging.info('done apache_log, new_valid_file: %s' % new_valid_file) sys.exit(liblog.SUCCESS)