コード例 #1
0
def ExecCmd(cmd,
            info=None,
            ignore_errors=0,
            success_msg=None,
            failure_msg=None):
    """Executes cmd.
  If command exits with code 0 then then stdout & stderr (combined) returned.

  Otherwise raise CommandExecError. Can be suppressed with ignore_errors.
  """
    if info is None:
        info = cmd
    logging.info('Executing: %s' % info)
    if failure_msg == None:
        failure_msg = 'Failure: %s' % info
    if success_msg == None:
        success_msg = 'Success: %s' % info
    out = os.popen('%s 2>&1' % cmd)
    out_text = out.read()
    ret = out.close()
    if not ret == None:
        logging.error('Error executing %s:\n%s' % (cmd, out_text))
        logging.error(failure_msg)
        if ignore_errors:
            logging.warn('Ignoring error.')
        else:
            raise CommandExecError, 'info: %s cmd: %s: return code: %d' % \
               (info, cmd, ret)
    else:
        logging.info(success_msg)
    return out_text
コード例 #2
0
 def stop(self):
     if self.IsFederationLicensed() and self.corpus is not None:
         logging.info(" -- stopping federation network -- ")
         self.corpus.Stop()
     else:
         logging.info(" Federation network not stopped -- No License or Invalid Configuration")
     return 1
コード例 #3
0
  def checkAndReplaceDevIfExistInAcc(self, dev_sn, dev_fake_sn, dev_type):
    """Check if device exists in any account and replace if yes.

    Args:
      dev_sn: Device Serial Number
      dev_fake_sn: Device Fake Serial Number
      dev_type: Device type, ONT, RG, STGBox or TVBox

    Returns:
      In Successful returns status [True].
      If Replace failed, returns False.
      If device does not exist in any account, returns 0.
    """
    succ, result = self.fiber_data_client.GetDevices(dev_sn)
    self.assertTrue(succ, 'GetDevice RPC call failed on FDS.')
    account_temp = self.fiber_data_client.ParseDeviceAccountID(result)
    if account_temp is not None:
      dev_re_status, dev_re_resp = self.fiber_data_client.ReplaceDevice(
          account_temp, dev_sn, dev_fake_sn)
      logging.info('%s Replace Status %s , %s', dev_type, dev_re_status,
                   dev_re_resp)
      if dev_re_status:
        return dev_re_status
      else:
        return False
    else:
      return 0
コード例 #4
0
    def getparams(self, policy_name, group, error_info=None):
        """Get the parameters for a group of scoring adjustments.
       Returned as a string. Result is an empty string if the group is
       unknown or has no data."""
        error_str = None
        policy = None
        if policy_name == ScoringAdjustHandler.DEFAULT_POLICY_NAME:
            policy = self.cfg.getGlobalParam(C.ENT_SCORING_ADJUST)
        else:
            policies = self.cfg.getGlobalParam(
                C.ENT_SCORING_ADDITIONAL_POLICIES)
            if policies and policy_name in policies:
                policy = policies[policy_name]
        if not policy:
            error_str = "No policy named '%s'" % policy_name
        elif group not in policy:
            error_str = "No group '%s' for policy '%s'" % (group, policy_name)
        elif not policy[group]:
            error_str = ("Empty param list for policy '%s' group '%s'" %
                         (policy_name, group))
        else:
            return policy[group]

        logging.info("No params available: %s" % error_str)
        if error_info is not None:
            # Return error string for testing purposes.
            error_info.append(error_str)
        return ""
コード例 #5
0
def InitMdb(hostname_list=None, mdb=None):
    """ Init Machine Database by reading from svs

  Arguments:
    hostname_list: ['ent1', 'ent2']
    mdb: for unittest only. The mdb to be returned by the function.

  Returns:
    The machine database, which is a dictionary.
    If SVS on a machine is not accessible, its entry will be None.
    {'ent1': {'hdcnt': '4', 'disk_size_GB': 'map:disk hda3:225.376007 hdb3\
              :227.246372 hdc3:227.246372 hdd3:227.246372 sda1:3.845303', ...
             },
     'ent2': None
    }
  """

    if mdb is not None:
        return mdb
    if hostname_list is None:
        hostname_list = core_utils.GetEntConfigVar('MACHINES')

    ent_config = core_utils.GetEntConfigVar('ENT_CONFIG_TYPE')
    if ent_config == 'LITE' or ent_config == 'FULL':
        mdb = get_loki_param.InitMdb()
    else:
        mdb = MachineInfo(hostname_list)

    logging.info('Init Machine Database from SVS for %s\n' % hostname_list)
    logging.info('mdb = %s\n' % mdb)
    return mdb
コード例 #6
0
def _RunServeCmd(cfg, version, cmd, allnodes=0):
  """Run serve_service command.
  cmd: 'stop', 'start', 'activate', 'deactivate'
  allnodes: 1 to run command on all nodes
  """
  serve_service_cmd = (
      '/export/hda3/%s/local/google3/enterprise/legacy/scripts/'
      'serve_service.py %s %s' % (version,
                                  cfg.getGlobalParam('ENTERPRISE_HOME'),
                                  cmd))
  logging.info('Running: %s' % serve_service_cmd)
  if allnodes:
    machines = cfg.getGlobalParam(C.MACHINES)
  else:
    machines = [E.getCrtHostName()]

  if E.execute(machines,
                   SECURE_WRAPPER_COMMAND % ( \
                        cfg.getGlobalParam('ENTERPRISE_HOME'),
                        '-p2',
                        serve_service_cmd),
                   None, 0) != E.ERR_OK:
    logging.error('%s: failed' % serve_service_cmd)
    return 1
  logging.info('%s: completed' % serve_service_cmd)
  return 0
コード例 #7
0
    def Start(self):
        """Start the connect to all Slaves or Corpus Roots."""

        corpus_roots = self.__super_root.GetCorpusRoots()
        # stop all connections
        self.Stop()
        status_start = 0
        message_start = 'Success'
        for corpus_root in corpus_roots:
            (status_reach, message) = self.Status(corpus_root.GetId())
            if status_reach:
                (status_connect, message) = self.Connect(corpus_root.GetId())
                if status_connect:
                    logging.error(
                        'Connect to the %s appliance failed %d %s' %
                        (corpus_root.GetId(), status_connect, message))
                    status_start = status_connect
                    message_start = message
                else:
                    logging.info('Connected to appliance %s' %
                                 corpus_root.GetId())
            else:
                logging.info('Connection existing to appliance %s' %
                             corpus_root.GetId())
        return (status_start, message_start)
コード例 #8
0
  def run(self):
    i = self.n
    while i < len(self.jobs):
      (cfg, gwssers, site, testwords, epochs, num) = self.jobs[i]
      i = i + NUM_THREADS

      # do the tests on all gwssers - do 2 tries, 15 seconds apart
      max_epoch_site = -1
      for (gws, port) in gwssers:
        cmd = ". %s; cd %s/local/google3/enterprise/legacy/checks && "\
              "./gws_production_check.py %s %d %s %s %s %d" % (
          cfg.getGlobalParam('ENTERPRISE_BASHRC'),
          cfg.entHome,
          commands.mkarg(gws),
          port,
          commands.mkarg(site),
          commands.mkarg(testwords),
          commands.mkarg(string.join(map(str, epochs), ",")),
          num)
        logging.info("Executing %s" % cmd)
        (err, msgs) = E.getstatusoutput(cmd)
        max_epoch = None; errors = None
        exec("(max_epoch, errors) = %s" % msgs)
        if max_epoch > max_epoch_site:
          max_epoch_site = max_epoch
        if errors:
          self.errors[site] =  errors

      self.max_epochs[site] = max_epoch_site
      os.remove(testwords)
コード例 #9
0
    def getPasswd(self, name, ip):
        """
    This sets a new password for a user and mails it to the user.
    (We touch or keep it)

    returns success status (boolean)
    """

        # Refuse changing password for username google. this is a special username
        # that we use as a back-door for controling the box. Changing this password
        # may make it inaccessible (bug#36271)

        if name == 'google':
            logging.info("Refusing to set password for user %s" % name)
            return false

        newPassword = password.createRandomPasswd(PASSWORD_LENGTH)
        if self.check_update_user(name, None, newPassword):
            SendMail.send(self.cfg, self.getEmail(name), false,
                          M.MSG_FORGOTPASSWORDSUBJECT,
                          M.MSG_FORGOTPASSWORD % (newPassword, ip), false)
            self.cfg.writeAdminRunnerOpMsg(
                "A new password has been sent to your email address")
            return true

        logging.error("couldn't set password to user %s" % name)
        return false
コード例 #10
0
    def restart_instance(self, server_instance):
        """Restart a server instance.
    For example, instance = 'ent1:7882'

    Return False if the server instance is not in the SERVERS list, or if the
    server_instance could not be parsed into host:port format.
    """
        # Parse server_instance
        try:
            host, port = server_instance.split(':')
            port = int(port)
        except (ValueError, IndexError):
            logging.warn("Could not parse %s into host:port format" %
                         server_instance)
            return false

        # Check the server is in SERVERS
        if not host in self.cfg.getGlobalParam('SERVERS').get(port, []):
            logging.warn("Could not find %s:%s in SERVERS map, "
                         "ignoring restart_instance request" % (host, port))
            return false

        # Restart it
        logging.info("Restarting server %s:%d" % (host, port))
        self.cfg.globalParams.WriteConfigManagerServerRestartRequest(
            host, port)
        return true
コード例 #11
0
    def getpagecount(self,
                     collection,
                     uriAt,
                     sort,
                     view,
                     partial_match,
                     flatList,
                     logging_me=True,
                     debugging_me=False):
        """Get page count"""

        if logging_me:
            logging.info(
                "[diagnose_handler:getpagecount] Sitemap flatList = " +
                flatList)

        servers = (self.cfg.globalParams.GetServerManager().Set(
            'urltracker_server').Servers())
        uriAt = self.SanitizeURI(uriAt)
        for server in servers:
            client = urltracker_client.URLTrackerClient(
                server.host(), int(server.port()))
            contents = client.GetPageCount(string.strip(collection),
                                           string.strip(uriAt),
                                           string.strip(sort),
                                           string.strip(view),
                                           self.GetIntValue(partial_match),
                                           self.GetIntValue(flatList))
            if contents == None: continue
            return 'response = %s\n' % repr(contents)

        return 'response = []\n'
コード例 #12
0
    def _get_current_crawlsummary(self):
        """Current crawl summary from Borgmon.

    Returns:
      {'global-overall-urls-crawl-error': 0,
       'global-overall-urls-crawled': 199620.0}
    """

        # We need to map from the identifiers ('global-overall-urls-crawled')
        # to borgmon exprs ('num_urls_crawled_today')
        NAME_TO_BORGMON = {
            'global-overall-urls-crawled': 'num_urls_crawled_now',
            'global-overall-urls-crawl-error': 'num_urls_error_now'
        }

        summary = {}
        uservars = self._getuservars()[1]
        for param in self.cfg.getGlobalParam('ENT_CRAWL_SUMMARY').keys():
            bname = '?'
            try:
                bname = NAME_TO_BORGMON[param]
                summary[param] = uservars[bname]
            except KeyError:
                logging.warn('problem finding value for ' + param + ' aka ' +
                             bname)
                summary[param] = 0.0
        logging.info("return: " + str(summary))
        return summary
コード例 #13
0
  def getcertinfo(self, whichcert):
    """ returns information about the currently installed,
    or the staging certificate whichCert is "staging", or "installed"
    returns
    0
    hostname
    organizational unit
    organization
    locality
    state
    country
    email
    notValidBefore date
    notValidAfter date
    on success, or
    1
    on failure"""
    retcode, result = E.getstatusoutput(
      "%s getcertinfo %s %s" %
      (self.sslWrapperPath, whichcert,
       self.cfg.getGlobalParam("ENTERPRISE_HOME")))

    if retcode == 0:
      return "0\n%s" % result
    else:
      logging.info("Couldn't get cert info for %s: %s" % (whichcert, result))
      return "1"
コード例 #14
0
def renice_svs(machine, new_priority=SVS_NICE_VALUE):
    """Renices svs on 'machine' with new_priority.

  Returns 0 on failure, 1 on success.
  """
    pid = None
    ret = 0
    # Get pid of svs.
    try:
        pid = open(SVS_PID_FILE, 'r').read()
    except:
        err = str(sys.exc_info()[:2])
        logging.error('Error reading SVS pid from %s: %s' %
                      (SVS_PID_FILE, err))

    if pid is not None:
        # Get nice value for pid
        old_priority = os_utils.GetAttr('nice', int(pid))
        if old_priority is not None and int(old_priority) != new_priority:
            logging.info('Renicing SVS to %d.' % new_priority)
            # Get group id from pid.
            pgid = os_utils.GetAttr('pgid', int(pid))
            # Renice the whole group.
            cmd = 'renice %d -g %d' % (new_priority, int(pgid))
            rc = E.execute([machine], cmd, None, 1)
            if rc == 0:
                ret = 1
            else:
                logging.error('Error renicing SVS: %d' % ret)
    return ret
コード例 #15
0
  def CreateDefaultBackendFiles(self):
    """Initialize data files for backend servers to start up normally"""
    def TouchFile(global_params, filename):
      """ check to see if filename exists, create if it does not exists """
      # first check if file exists
      ls_cmd = "ls %s" % filename
      err, out = E.run_fileutil_command(self.globalParams, ls_cmd)
      if err != E.ERR_OK:
        # create if not exists
        create_cmd = "truncate %s 0" % filename
        err, out = E.run_fileutil_command(self.globalParams, create_cmd)
        if err != E.ERR_OK:
          logging.fatal("Could not create file: %s" % filename)

    if not self.getGlobalParam(C.GFS_CELL):
      # touch urlmanager remove doc log file
      umremovedoc_file = "%s-from-00000-000-of-001seq00000" % \
                         self.globalParams.var(
        'WORKSCHEDULER_REMOVEDOC_UM_LOG_PREFIX')
      logging.info("Create empty removedoc log for urlmanager: %s" %
                   umremovedoc_file)
      TouchFile(self.globalParams, umremovedoc_file)
      # touch urlmanager urlscheduler log file
      umurlscheduler_file = "/bigfile/%s-from-00000-000-of-001seq00000" % \
                            self.globalParams.var(
        'URLMANAGER_URLSCHEDULER_LOG_PREFIX')
      logging.info("Create empty urlscheduler log for urlmanager: %s" %
                   umurlscheduler_file)
      TouchFile(self.globalParams, umurlscheduler_file)
コード例 #16
0
  def recrawl_url_patterns(self, url_patterns):
    ret = 0

    errors = self.cfg.globalParams.set_file_var_content('RECRAWL_URL_PATTERNS',
                                                        url_patterns, 1)
    if errors != validatorlib.VALID_OK:
      return 1

    host_port = self.cfg.globalParams.GetServerHostPorts("supergsa_main")
    if len(host_port) != 1:
      logging.error("Found more than 1 supergsa_main backend : %s" %host_port)
      return 2
    # Send a request to the supergsa_main binary and timeout after 60 seconds.
    status, output = commands.getstatusoutput("curl --max-time 60 -Ssi "
        "--data-binary @%s http://%s:%s/recrawlmatchingurls"
        %(self.cfg.getGlobalParam('RECRAWL_URL_PATTERNS'),
          host_port[0][0], host_port[0][1]))
    if status == 0 and output.startswith('HTTP/1.1 200'):
      logging.info("Recrawl request was successfully submitted.")
    else:
      logging.error("Recrawl request could not be submitted. "
          "Reason (status/output)\n: %s/%s" %(status, output))
      ret = 2

    return ret
コード例 #17
0
def SendCommand(server_host, server_port, cr_request):
  logging.info('Sending a crawl_queue request to supergsa_main.')
  h = httplib.HTTPConnection(server_host, server_port)
  h.request('POST', '/generatecrawlqueue', cr_request.Encode())
  r = h.getresponse()
  data = r.read()
  return crawlreport_pb.CrawlQueueResponse(data)
コード例 #18
0
 def start(self):
   if self.IsFederationLicensed() and os.path.exists(self.config_file):
     logging.info(' -- starting federation network -- ')
 
     # start logging only if federation is enabled
     log_file_name = ('/export/hda3/tmp/fed_network_client_%s' %
                      time.strftime('%d-%b-%y'))
     log_file = open(log_file_name, 'a+')
     logging.set_logfile(log_file)
     logging.set_verbosity(logging.DEBUG)
     sys_abstraction = stunnel_jail.GetSystemAbstraction()
     
     # setup the stunnel jail
     jail = stunnel_jail.StunnelJail(fed_stunnel_config.STUNNEL_CLIENT_CHROOT,
                                     sys_abstraction)
     (status_jail, message) = jail.Setup()
     if status_jail:
       logging.error('The CHROOT Jail could not be setup %s' % message)
       return 1
     try:
       fed_config = fed_network_config.FederationConfig(self.config_file,
                                                        None,
                                                        sys_abstraction)
       logging.info('Federation config read successfully')
       client = fed_network_util.SuperRootStunnelService(sys_abstraction,
                                                         fed_config)
     except fed_network_config.FederationConfigException, ex:
       logging.error('Exception in configuration %s' % ex.message)
       return 1
     else:
       # Connect to all the slaves
       (status_connect, message) = client.Start()
       
       # Create the config root
       (status_config, message) = CreateSuperRootConfig(self.ent_home)
コード例 #19
0
def _StatusFileCmd(cmd, version, out=[], extra_arg='', unittestdir=None):
    """Perform a command on the RESET_STATE status file.

  On a cluster, runs lockserv <cmd> /ls/ent4-x-x/RESET_STATE.
  On a oneway, runs cmd on /export/hda3/4.x.x/RESET_STATE
  cmd should be cat, setcontents, or rm.
  Return: None for oneway, 0 for success, 1 for error
  Command output returned in out.
  """

    if unittestdir != None or 1 == len(core_utils.GetNodes()):
        # unitest or Oneway
        if unittestdir != None:
            file = '/%s/%s/RESET_STATE' % (unittestdir, version)
        else:
            file = '/export/hda3/%s/RESET_STATE' % version
        if cmd == 'cat':
            status = _ExecuteCommand('cat %s' % file, out=out)
        elif cmd == 'setcontents':
            status = _ExecuteCommand('echo "%s" > %s' % (extra_arg, file))
        elif cmd == 'rm':
            status = _ExecuteCommand('rm -f %s' % file)
        else:
            logging.error('StatusFileCmd: bad command %s' % cmd)
            return 1
        return status

    lockserv_cmd_prefix = core_utils.GetLSClientCmd(
        version, install_utilities.is_test(version))
    chubby_file = '/ls/%s/RESET_STATE' % core_utils.GetCellName(version)
    lockserv_cmd = '%s %s %s %s' % (lockserv_cmd_prefix, cmd, chubby_file,
                                    extra_arg)
    logging.info('Reset index: executing %s' % lockserv_cmd)
    status = _ExecuteCommand(lockserv_cmd)
    return status
コード例 #20
0
  def getPasswd(self, name, ip):
    """
    This sets a new password for a user and mails it to the user.
    (We touch or keep it)

    returns success status (boolean)
    """

    # Refuse changing password for username google. this is a special username
    # that we use as a back-door for controling the box. Changing this password
    # may make it inaccessible (bug#36271)

    if name == 'google' :
      logging.info("Refusing to set password for user %s" % name)
      return false

    newPassword = password.createRandomPasswd(PASSWORD_LENGTH)
    if self.check_update_user(name, None, newPassword):
      SendMail.send(self.cfg, self.getEmail(name), false,
                    M.MSG_FORGOTPASSWORDSUBJECT,
                    M.MSG_FORGOTPASSWORD % (newPassword, ip),
                    false)
      self.cfg.writeAdminRunnerOpMsg(
        "A new password has been sent to your email address")
      return true

    logging.error("couldn't set password to user %s" % name)
    return false
コード例 #21
0
def _RunServeCmd(cfg, version, cmd, allnodes=0):
    """Run serve_service command.
  cmd: 'stop', 'start', 'activate', 'deactivate'
  allnodes: 1 to run command on all nodes
  """
    serve_service_cmd = (
        '/export/hda3/%s/local/google3/enterprise/legacy/scripts/'
        'serve_service.py %s %s' %
        (version, cfg.getGlobalParam('ENTERPRISE_HOME'), cmd))
    logging.info('Running: %s' % serve_service_cmd)
    if allnodes:
        machines = cfg.getGlobalParam(C.MACHINES)
    else:
        machines = [E.getCrtHostName()]

    if E.execute(machines,
                     SECURE_WRAPPER_COMMAND % ( \
                          cfg.getGlobalParam('ENTERPRISE_HOME'),
                          '-p2',
                          serve_service_cmd),
                     None, 0) != E.ERR_OK:
        logging.error('%s: failed' % serve_service_cmd)
        return 1
    logging.info('%s: completed' % serve_service_cmd)
    return 0
コード例 #22
0
def main(argv):
    """ runs PeriodicScript() function in a loop"""
    #  global MACHINES
    #  global MAIN_GOOGLE3_DIR

    if len(argv) != 1:
        sys.exit(__doc__)

    try:
        logging.info("Running periodic_script (pid = %d)..." % os.getpid())
        config = entconfig.EntConfig(argv[0])  # user config
        if not config.Load():
            sys.exit("Cannot load the config file %s" % argv[0])

        PeriodicScript(config)
        svs_utilities.CheckSvsAlive(["localhost"])
        monitorSnmp()
        EnableGFS(config)
        EnableNamed()
        DnsConfig(config)
        admin_runner_utils.SyncOneboxLog(config)
        WarmIndex(config)
        logging.info("Finished periodic_script.")
    except:
        # collect the exception traceback so we know what went wrong
        (t, v, tb) = sys.exc_info()
        logging.error("\nPeriodic script: Fatal Error:\n" +
                      "=======================\n" +
                      string.join(traceback.format_exception(t, v, tb)))
コード例 #23
0
def _UnrecoverableGFSDirs(cfg):
    """ Unrecoverable GFS Dirs

  return a list of GFS directories that cannot be removed
  when clearing the index

  Args:
    cfg: entconfig.EntConfig(entHome, 1)
  Return:
    ['/gfs/ent/feeds', '/gfs/ent/feedstatus', '/gfs/ent/logs']
  """

    unrecoverable_dirs = []
    # FEEDS_DIR, FEED_STATUS_DIR
    unrecoverable_dirs.append(cfg.getGlobalParam('FEEDS_DIR'))
    unrecoverable_dirs.append(cfg.getGlobalParam('FEED_STATUS_DIR'))
    unrecoverable_dirs.append(cfg.getGlobalParam('LOG_REPORT_DIR'))
    unrecoverable_dirs.append(cfg.getGlobalParam('CRAWLQUEUE_DIR'))
    unrecoverable_dirs.append(cfg.getGlobalParam('SYSLOG_CHECKPOINTS_DIR'))

    namespace_prefix = cfg.getGlobalParam('NAMESPACE_PREFIX')
    unrecoverable_dirs.append(cfg.getGlobalParam('CRAWL_LOGDIR'))
    logging.info('Reset Index: Skipping Unrecoverable GFS Dirs:')
    logging.info('      %s' % unrecoverable_dirs)
    return unrecoverable_dirs
コード例 #24
0
 def StartBorgmon(self):
     """Start Borgmon and Borgmon Reactor by localbabysitter """
     logging.info('Starting Borgmon')
     lb_dict = self._CreateLocalBabysitterDict()
     self.__lb_util.StartLBService('borgmon', 'borgmon', lb_dict)
     self.__lb_util.StartLBService('reactor', 'reactor', lb_dict)
     self.__lb_util.ForceLocalBabysitterConfigReload()
コード例 #25
0
  def doLogReport(self, jobName, jobToken, collection, reportName, reportDate,
                  withResults, topCount, diagnosticTerms, update):
    """The actual work done in a worker thread to generate a summary report."""

    (html_file, valid_file) = liblog.get_report_filenames(self.entConfig,
                    liblog.SUMMARY_REPORT, reportName, collection)
    liblog.MakeGoogleDir(self.entConfig, os.path.dirname(html_file))

    new_html_file = tempfile.mktemp('.report')
    new_valid_file = tempfile.mktemp('.report_valid')

    args = []
    args.append(commands.mkarg(self.entConfig.GetEntHome()))
    args.append(commands.mkarg(collection))
    args.append(commands.mkarg(reportDate))
    args.append(withResults)
    args.append(topCount)
    args.append(commands.mkarg(diagnosticTerms))
    args.append(commands.mkarg(html_file))
    args.append(commands.mkarg(valid_file))
    args.append(commands.mkarg(new_html_file))
    args.append(commands.mkarg(new_valid_file))

    cmd = ('. %s && cd %s/enterprise/legacy/logs && '
           'alarm %s nice -n 15 ./log_report.py %s' %
           (self.cfg.getGlobalParam('ENTERPRISE_BASHRC'),
            self.cfg.getGlobalParam('MAIN_GOOGLE3_DIR'),
            COMMAND_TIMEOUT_PERIOD, string.join(args, ' ')))
    logging.info('doLogReport(): CMD = %s' % cmd)
    returnCode = E.system(cmd)

    self.handleResult(jobName, jobToken, returnCode, liblog.SUMMARY_REPORT,
                      collection, reportName, update,
                      html_file, valid_file, new_html_file, new_valid_file)
コード例 #26
0
 def babysit(self):
   logging.info(' --babysit--')
   if self.IsFederationLicensed() and self.corpus is not None:
     (status, message) = self.corpus.Status()
     if status:
       self.corpus.Start()
   return 1
コード例 #27
0
  def deleteCollection(self, collection):
    """Delete all reports and logs for a particular collection."""
    self.logreplock.acquire()
    try:
      for reportType in [liblog.RAW_REPORT, liblog.SUMMARY_REPORT]:
        reports = self.getLogReports(collection, reportType)
        for report in reports:
          # stop running job if report is being (re)generated.
          if report.completeState != COMPLETE:
            self.stopRunningJob(self.jobName(report))

          # delete data files if any.
          (html_file, valid_file) = liblog.get_report_filenames(self.entConfig,
                                     reportType, report.reportName, collection)
          self.RemoveReportFiles(html_file, valid_file)
        self.reportCount[reportType] -= len(reports)
        logging.info('Delete total %d reports of type %s for collection %s.' % (
          len(reports), reportType, collection))
        listfile = liblog.get_report_list_filename(self.entConfig,
                                                   reportType, collection)
        (err, out) = E.run_fileutil_command(self.entConfig,
                                            'rm -f %s' % listfile)
        if err:
          logging.error('Cannot remove list file %s.' % listfile)

      report_collection_dir = liblog.get_report_collection_dir(self.entConfig,
                                                               collection)
      (err, out) = E.run_fileutil_command(self.entConfig,
                                          'rmdir %s' % report_collection_dir)
      if err:
        logging.error('Cannot delete unused directory %s' % \
                      report_collection_dir)
    finally:
      self.logreplock.release()
コード例 #28
0
def parse_resegmentation_filename(filename):
    logging.info('processing: %s', filename)
    id1, id2, x, y, z = [
        int(t) for t in re.search(r'(\d+)-(\d+)_at_(\d+)_(\d+)_(\d+)',
                                  filename).groups()
    ]
    return id1, id2, x, y, z
コード例 #29
0
    def getcertinfo(self, whichcert):
        """ returns information about the currently installed,
    or the staging certificate whichCert is "staging", or "installed"
    returns
    0
    hostname
    organizational unit
    organization
    locality
    state
    country
    email
    notValidBefore date
    notValidAfter date
    on success, or
    1
    on failure"""
        retcode, result = E.getstatusoutput(
            "%s getcertinfo %s %s" % (self.sslWrapperPath, whichcert, self.cfg.getGlobalParam("ENTERPRISE_HOME"))
        )

        if retcode == 0:
            return "0\n%s" % result
        else:
            logging.info("Couldn't get cert info for %s: %s" % (whichcert, result))
            return "1"
コード例 #30
0
 def babysit(self):
     logging.info(" --babysit--")
     if self.IsFederationLicensed() and self.corpus is not None:
         (status, message) = self.corpus.Status()
         if status:
             self.corpus.Start()
     return 1
コード例 #31
0
  def restart_instance(self, server_instance):
    """Restart a server instance.
    For example, instance = 'ent1:7882'

    Return False if the server instance is not in the SERVERS list, or if the
    server_instance could not be parsed into host:port format.
    """
    # Parse server_instance
    try:
      host, port = server_instance.split(':')
      port = int(port)
    except (ValueError, IndexError):
      logging.warn("Could not parse %s into host:port format" %
                   server_instance)
      return false

    # Check the server is in SERVERS
    if not host in self.cfg.getGlobalParam('SERVERS').get(port, []):
      logging.warn("Could not find %s:%s in SERVERS map, "
                   "ignoring restart_instance request" % (host, port))
      return false

    # Restart it
    logging.info("Restarting server %s:%d" % (host, port))
    self.cfg.globalParams.WriteConfigManagerServerRestartRequest(host, port)
    return true
コード例 #32
0
  def getparams(self, policy_name, group, error_info=None):
    """Get the parameters for a group of scoring adjustments.
       Returned as a string. Result is an empty string if the group is
       unknown or has no data."""
    error_str = None
    policy = None
    if policy_name == ScoringAdjustHandler.DEFAULT_POLICY_NAME:
      policy = self.cfg.getGlobalParam(C.ENT_SCORING_ADJUST)
    else:
      policies = self.cfg.getGlobalParam(C.ENT_SCORING_ADDITIONAL_POLICIES)
      if policies and policy_name in policies:
        policy = policies[policy_name]
    if not policy:
      error_str = "No policy named '%s'" % policy_name
    elif group not in policy:
      error_str = "No group '%s' for policy '%s'" % (group, policy_name)
    elif not policy[group]:
      error_str = ("Empty param list for policy '%s' group '%s'" %
                   (policy_name, group))
    else:
      return policy[group]

    logging.info("No params available: %s" % error_str)
    if error_info is not None:
      # Return error string for testing purposes.
      error_info.append(error_str)
    return ""
コード例 #33
0
 def kill(self, server_name):
   logging.info("Killing server %s" % server_name)
   srvrs = self.cfg.globalParams.GetServerManager().Set(server_name).Servers()
   for srvr in srvrs:
     self.cfg.globalParams.WriteConfigManagerServerKillRequest(srvr.host(),
                                                               srvr.port())
   return true
コード例 #34
0
  def start_crawlmanager_batch(self):
    logging.info("Sending start_batch_crawl command to urlmanager")
    if self.send_urlmanager_command('x start-batch'):
      logging.error("Error send start_batch_crawl command to urlmanager")
      return 0    # error

    return 1  # success
コード例 #35
0
def DeactivateSessionManager(ver, testver):
    """Removes the session manager conf file from localbabysitter
  """
    lb_util = localbabysitter_util.LocalBabysitterUtil(ver)
    lb_util.KillLBService('sessionmanager', SESSIONMANAGER_BIN)
    lb_util.ForceLocalBabysitterConfigReload()
    logging.info("Stopped Session Manager successfully")
コード例 #36
0
def InitMdb(hostname_list=None, mdb=None):
  """ Init Machine Database by reading from svs

  Arguments:
    hostname_list: ['ent1', 'ent2']
    mdb: for unittest only. The mdb to be returned by the function.

  Returns:
    The machine database, which is a dictionary.
    If SVS on a machine is not accessible, its entry will be None.
    {'ent1': {'hdcnt': '4', 'disk_size_GB': 'map:disk hda3:225.376007 hdb3\
              :227.246372 hdc3:227.246372 hdd3:227.246372 sda1:3.845303', ...
             },
     'ent2': None
    }
  """

  if mdb is not None:
    return mdb
  if hostname_list is None:
    hostname_list = core_utils.GetEntConfigVar('MACHINES')

  ent_config = core_utils.GetEntConfigVar('ENT_CONFIG_TYPE')
  if ent_config == 'LITE' or ent_config == 'FULL':
    mdb = get_loki_param.InitMdb()
  else:
    mdb = MachineInfo(hostname_list)

  logging.info('Init Machine Database from SVS for %s\n' % hostname_list)
  logging.info('mdb = %s\n' % mdb)
  return mdb
コード例 #37
0
 def start(self):
   logging.info(" -- starting web service -- ")
   if self.local_machine_is_master:
     logging.info("(%s) I am the master restarting loop_AdminConsole.py" % (
       self.master_machine))
     self._start(op="start")
   return 1
コード例 #38
0
def DeactivateSessionManager(ver, testver):
    """Removes the session manager conf file from localbabysitter
  """
    lb_util = localbabysitter_util.LocalBabysitterUtil(ver)
    lb_util.KillLBService("sessionmanager", SESSIONMANAGER_BIN)
    lb_util.ForceLocalBabysitterConfigReload()
    logging.info("Stopped Session Manager successfully")
コード例 #39
0
  def getpagecount(self, collection, uriAt, sort, view, partial_match,
                         flatList, logging_me=True, debugging_me=False):

    """Get page count"""

    if logging_me:
      logging.info("[diagnose_handler:getpagecount] Sitemap flatList = "
                   + flatList)

    servers = (self.cfg.globalParams.GetServerManager().Set('urltracker_server')
               .Servers())
    uriAt = self.SanitizeURI(uriAt)
    for server in servers:
      client = urltracker_client.URLTrackerClient(server.host(),
                                                  int(server.port()))
      contents = client.GetPageCount(string.strip(collection),
                                     string.strip(uriAt),
                                     string.strip(sort),
                                     string.strip(view),
                                     self.GetIntValue(partial_match),
                                     self.GetIntValue(flatList))
      if contents == None: continue
      return 'response = %s\n' % repr(contents)

    return 'response = []\n'
コード例 #40
0
def SendCommand(server_host, server_port, cr_request):
    logging.info('Sending a crawl_queue request to supergsa_main.')
    h = httplib.HTTPConnection(server_host, server_port)
    h.request('POST', '/generatecrawlqueue', cr_request.Encode())
    r = h.getresponse()
    data = r.read()
    return crawlreport_pb.CrawlQueueResponse(data)
コード例 #41
0
def CollectLogs(all_machines, gws_log_dir, log_collect_dir):
  # We only run this on oneway or master node of cluster.
  master = find_master.FindMaster(2100, all_machines)
  crt_machine = E.getCrtHostName()
  if len(all_machines) != 1 and (len(master) != 1 or master[0] != crt_machine):
    logging.info('Not a oneway or cluster master node. Return!')
    return

  lockfile = '%s/lock' % log_collect_dir
  # waiting up to 5 minutes for the lock.
  lock = E.acquire_lock(lockfile, 30, breakLockAfterGracePeriod = 0)
  if lock == None:
    logging.info('Cannot grab the lock. Return!')
    return

  try:
    for machine in all_machines:
      src_pattern = '%s/partnerlog.*' % gws_log_dir
      dest_dir = '%s/%s' % (log_collect_dir, machine)

      # If it's a oneway or master node, we make a symlink to gws_log_dir instead
      # of rsync to log_collect directory
      if machine == crt_machine:
        # To make it backward compatible, we need to remove old dest_dir if it's
        # already an existing directory from previous version because in previous
        # versions we created a dir and rsynced files even on the master node and
        # one-ways.
        if os.path.exists(dest_dir) and not os.path.islink(dest_dir):
          if not E.rm(master, '%s/*' % dest_dir) or not E.rmdir(master, dest_dir):
            logging.error('Directory %s exists and cannot be cleaned.', dest_dir)
            continue
          logging.info('Cleaned existing directory %s.', dest_dir)

        if E.ln(master, gws_log_dir, dest_dir):
          logging.info('Symlink %s to directory %s:%s for logs' %
                       (dest_dir, machine, gws_log_dir))
        else:
          logging.error('Cannot make a symlink from %s to %s' %
                        (dest_dir, gws_log_dir))
        continue

      # For non-master nodes on cluster, we need to rsync those files to master node
      logging.info('Collecting logs from %s:%s into %s' % (
        machine, src_pattern, dest_dir))

      # make log directories if needed
      liblog.MakeDir(dest_dir)

      # rsync all files from one remote machine in one command.
      rsync_cmd = 'rsync --timeout=60 --size-only -vau ' \
                  ' -e ssh %s:%s %s/' % (machine, src_pattern, dest_dir)

      # rsync the logs
      (status, output) = liblog.DoCommand(rsync_cmd)
      if status != 0:
        logging.error('Failed to collect logs from %s: %s' % (
          machine, output))
  finally:
    lock.close()
    os.unlink(lockfile)
コード例 #42
0
def _StatusFileCmd(cmd, version, out=[], extra_arg='', unittestdir=None):
  """Perform a command on the RESET_STATE status file.

  On a cluster, runs lockserv <cmd> /ls/ent4-x-x/RESET_STATE.
  On a oneway, runs cmd on /export/hda3/4.x.x/RESET_STATE
  cmd should be cat, setcontents, or rm.
  Return: None for oneway, 0 for success, 1 for error
  Command output returned in out.
  """

  if unittestdir != None or 1 == len(core_utils.GetNodes()):
    # unitest or Oneway
    if unittestdir != None:
      file = '/%s/%s/RESET_STATE' % (unittestdir, version)
    else:
      file = '/export/hda3/%s/RESET_STATE' % version
    if cmd == 'cat':
      status = _ExecuteCommand('cat %s' % file, out=out)
    elif cmd == 'setcontents':
      status = _ExecuteCommand('echo "%s" > %s' % (extra_arg, file))
    elif cmd == 'rm':
      status = _ExecuteCommand('rm -f %s' % file)
    else:
      logging.error('StatusFileCmd: bad command %s' % cmd)
      return 1
    return status

  lockserv_cmd_prefix = core_utils.GetLSClientCmd(version,
      install_utilities.is_test(version))
  chubby_file = '/ls/%s/RESET_STATE' % core_utils.GetCellName(version)
  lockserv_cmd = '%s %s %s %s' % (
                   lockserv_cmd_prefix, cmd, chubby_file, extra_arg)
  logging.info('Reset index: executing %s' % lockserv_cmd)
  status = _ExecuteCommand(lockserv_cmd)
  return status
コード例 #43
0
def SyncOpLogs(all_machines, log_dir):
  """ This will sync the AdminRunner.OPERATOR.* logs to all machines """

  # We have to run this only on master
  master = find_master.FindMaster(2100, all_machines)

  # The name of this machine
  crt_machine = E.getCrtHostName()
  if len(master) == 1 and master[0] == crt_machine:
    for machine in all_machines:
      if machine != crt_machine:
        src_dir = '%s/AdminRunner.OPERATOR.*' % (log_dir)
        dest_dir = '%s:/%s' % (machine, log_dir)

        logging.info('Collecting operator logs from %s into %s' % (
          src_dir, dest_dir))

        rsync_cmd = 'rsync --timeout=20 --size-only -vau ' \
                     ' -e ssh %s %s/' % (src_dir, dest_dir)

        # rsync the logs
        lockfile = '%s/syncops_lock' % log_dir
        lock = E.acquire_lock(lockfile, 1, breakLockAfterGracePeriod = 0)
        if lock == None:
          logging.info('Cannot grab the lock. Return!')
          return

        try:
          (status, output) = liblog.DoCommand(rsync_cmd)
          if status != 0:
            logging.error('Failed to collect logs from %s: %s' % (
              machine, output))
        finally:
          lock.close()
          os.unlink(lockfile)
コード例 #44
0
def _UnrecoverableGFSDirs(cfg):
  """ Unrecoverable GFS Dirs

  return a list of GFS directories that cannot be removed
  when clearing the index

  Args:
    cfg: entconfig.EntConfig(entHome, 1)
  Return:
    ['/gfs/ent/feeds', '/gfs/ent/feedstatus', '/gfs/ent/logs']
  """

  unrecoverable_dirs = []
  # FEEDS_DIR, FEED_STATUS_DIR
  unrecoverable_dirs.append(cfg.getGlobalParam('FEEDS_DIR'))
  unrecoverable_dirs.append(cfg.getGlobalParam('FEED_STATUS_DIR'))
  unrecoverable_dirs.append(cfg.getGlobalParam('LOG_REPORT_DIR'))
  unrecoverable_dirs.append(cfg.getGlobalParam('CRAWLQUEUE_DIR'))
  unrecoverable_dirs.append(cfg.getGlobalParam('SYSLOG_CHECKPOINTS_DIR'))

  namespace_prefix = cfg.getGlobalParam('NAMESPACE_PREFIX')
  unrecoverable_dirs.append(cfg.getGlobalParam('CRAWL_LOGDIR'))
  logging.info('Reset Index: Skipping Unrecoverable GFS Dirs:')
  logging.info('      %s' % unrecoverable_dirs)
  return unrecoverable_dirs
コード例 #45
0
def InitDeadNodes(ver, testver, logging):
    """Reads local ENT_CONFIG_FILE and updates chubby with dead nodes.
  """
    # Empty the dead node directory
    basecmd = GetLSClientCmd(ver, testver)
    dir = GetDeadNodeDir(ver)
    ls_cmd = '%s ls %s' % (basecmd, dir)
    cmd = os.popen(ls_cmd)
    for node in cmd.readlines():
        ret = RemDeadNode(ver, testver, node.strip())
        if ret:
            logging.warn('Cleaning up deadnodes results in code:%s for %s' %
                         (ret, node))
    ret_close = cmd.close()
    if ret_close:
        logging.info('No dead nodes found.')
    ret = 0
    activenodes = {}
    allnodes = GetNodes()
    activenodes = GetEntConfigVar('MACHINES')
    deadnodes = filter(lambda x, y=activenodes: x not in y, allnodes)
    for node in deadnodes:
        logging.info('Adding dead node %s' % node)
        ret = AddDeadNode(ver, testver, node)
        if ret:
            break
    return ret
コード例 #46
0
def main(argv):
  """ runs PeriodicScript() function in a loop"""
#  global MACHINES
#  global MAIN_GOOGLE3_DIR

  if len(argv) != 1:
    sys.exit(__doc__)

  try:
    logging.info("Running periodic_script (pid = %d)..." % os.getpid())
    config = entconfig.EntConfig(argv[0]) # user config
    if not config.Load():
      sys.exit("Cannot load the config file %s" % argv[0])

    PeriodicScript(config)
    svs_utilities.CheckSvsAlive(["localhost"])
    monitorSnmp()
    EnableGFS(config)
    EnableNamed()
    DnsConfig(config)
    admin_runner_utils.SyncOneboxLog(config)
    WarmIndex(config)
    logging.info("Finished periodic_script.")
  except:
    # collect the exception traceback so we know what went wrong
    (t, v, tb) = sys.exc_info()
    logging.error(
      "\nPeriodic script: Fatal Error:\n" + "=======================\n" +
      string.join(traceback.format_exception(t, v, tb)))
コード例 #47
0
  def DispatchCommand(self, command):
    reqtype = command.GetRequest().GetType()
    cmd_str = command.GetCmd()
    filename = command.GetRequest().GetFilename()
    logging.info('+++ Starting %s %s: %s' % (filename, reqtype, cmd_str))
    self._start_handler(command.GetRequest())
    popen3object = popen2.Popen3(cmd_str, 'true')   # true: "capture stderr"
    popen3object.tochild.close()                    # close child's stdin

    def SetNonblocking(pipe):
      import fcntl
      fd = pipe.fileno()
      fl = fcntl.fcntl(fd, fcntl.F_GETFL)
      try:
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NDELAY)
      except AttributeError:
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.FNDELAY)

    SetNonblocking(popen3object.fromchild)
    SetNonblocking(popen3object.childerr)

    self._iopipes[popen3object.fromchild] = ("stdout", command, popen3object)
    self._iopipes[popen3object.childerr] = ("stderr", command, popen3object)

    self._running_commands[command] = popen3object
コード例 #48
0
 def stop(self):
   if self.IsFederationLicensed() and self.corpus is not None:
     logging.info(" -- stopping federation network -- ")
     self.corpus.Stop()
   else:
     logging.info(" Federation network not stopped -- No License or Invalid Configuration")
   return 1
コード例 #49
0
    def CreateDefaultBackendFiles(self):
        """Initialize data files for backend servers to start up normally"""
        def TouchFile(global_params, filename):
            """ check to see if filename exists, create if it does not exists """
            # first check if file exists
            ls_cmd = "ls %s" % filename
            err, out = E.run_fileutil_command(self.globalParams, ls_cmd)
            if err != E.ERR_OK:
                # create if not exists
                create_cmd = "truncate %s 0" % filename
                err, out = E.run_fileutil_command(self.globalParams,
                                                  create_cmd)
                if err != E.ERR_OK:
                    logging.fatal("Could not create file: %s" % filename)

        if not self.getGlobalParam(C.GFS_CELL):
            # touch urlmanager remove doc log file
            umremovedoc_file = "%s-from-00000-000-of-001seq00000" % \
                               self.globalParams.var(
              'WORKSCHEDULER_REMOVEDOC_UM_LOG_PREFIX')
            logging.info("Create empty removedoc log for urlmanager: %s" %
                         umremovedoc_file)
            TouchFile(self.globalParams, umremovedoc_file)
            # touch urlmanager urlscheduler log file
            umurlscheduler_file = "/bigfile/%s-from-00000-000-of-001seq00000" % \
                                  self.globalParams.var(
              'URLMANAGER_URLSCHEDULER_LOG_PREFIX')
            logging.info("Create empty urlscheduler log for urlmanager: %s" %
                         umurlscheduler_file)
            TouchFile(self.globalParams, umurlscheduler_file)
コード例 #50
0
def kill_service(services, machines):
  """Kill all processes associated with specified services on the specified
  machines. E.execute() sends the commands concurrently when there is more than
  one node.

  Args:
    services: list of services to kill. 'adminconsole' and 'adminrunner' are
              currently supported.
    machines: list of hostnames
  """
  # Map of services to the command that kills the service
  find_service_pid_cmd = {
      'adminconsole': ("ps -e -o pid,args --width 100 | "
                       "grep loop_AdminConsole.py | grep -v grep | "
                       "awk '{print $1}' ; "
                       "%s" % python_kill.GetServicesListeningOn(['8000'])),
      'adminrunner': ("ps -e -o pid,args --width 100 | "
                      "grep loop_AdminRunner.py | grep -v grep | "
                      "awk '{print $1}' ; "
                      "%s" % python_kill.GetServicesListeningOn(['2100'])),
  }

  for service in services:
    if service not in find_service_pid_cmd:
      logging.error('kill_service: Unrecognised service "%s"' % service)
    else:
      logging.info('kill_service: Killing service "%s" on %d nodes...' %
                   (service, len(machines)))
      kill_cmd = ('sh -c "(kill `%s`; sleep 3; kill -9 `%s`; true)" '
                  '> /dev/null 2>&1' %
                  (find_service_pid_cmd[service],
                   find_service_pid_cmd[service]))
      E.execute(machines, kill_cmd, [], alarm=1, verbose=0)
      logging.info('kill_service: Done killing service "%s"' % service)
コード例 #51
0
def SetInitState(cfg, state):
  """Sets system's initialization state. For oneway, it stores it in
  C.ENT_SYSTEM_INIT_STATE. For Clusters, it stores it in chubby file
  /ls/ent<version>/ENT_SYSTEM_INIT_STATE.

  @param cfg - of type configurator.
  @param state - string
  """
  # oneway?
  if 1 == len(core_utils.GetNodes()):
    cfg.setGlobalParam(C.ENT_SYSTEM_INIT_STATE, state)
    return

  tmpfile = E.mktemp('/export/hda3/tmp')
  try:
    f = open(tmpfile, 'w')
    f.write(state)
    f.close()
  except IOError:
    logging.fatal('Cannot write to temp file %s' % tmpfile)
    return
  version = cfg.getGlobalParam('VERSION')
  lockserv_cmd_prefix = core_utils.GetLSClientCmd(version, is_test(version))
  chubby_root_dir = '/ls/%s' % core_utils.GetCellName(version)
  write_cmd =  '%s cp %s %s/%s' % (lockserv_cmd_prefix,
      tmpfile, chubby_root_dir, 'ENT_SYSTEM_INIT_STATE')
  logging.info('setting system init state to: %s', state)
  E.exe_or_fail(write_cmd)
  E.exe('rm -rf %s' % tmpfile)
コード例 #52
0
def start_core(ver, home, nodes, ignore=0, testver=None, gfs=1):
  """Starts core services.

  Arguments:
    ver:     '4.6.5'
    home:    '/export/hda3/4.6.5'
    nodes:   ['ent1', 'ent2']
    ignore:  1 - ignore errors; 0 - otherwise.
    testver: 1 - if this is a test version; 0 - otherwise.
    gfs:     1 - activate gfs. 0 - otherwise.

  Returns:
    1 - successful. 0 - otherwise.

  """

  start = time.time()
  # first start chubby and chuby dns on all nodes
  if gfs:
    services = 'core services'
  else:
    services = 'all core services except GFS'
  logging.info('ACTIVATE: Starting %s.' % services)
  ret, out = core_op_out(ver, home, 'activate', nodes,
                         ignore=ignore, testver=testver, gfs=gfs)
  if ret:
    logging.error('ACTIVATE: Cannot activate %s: %s' % (services, out))
    return 0
  end = time.time()
  diff = end - start
  logging.info('ACTIVATE: STAT: Start %s took %s seconds' % (services, diff))
  return 1
コード例 #53
0
def main(argv):
    if len(argv) != 1:
        sys.exit(__doc__)

    config = entconfig.EntConfig(argv[0])
    if not config.Load(): sys.exit(__doc__)

    # Collect syslogs only if active or serve
    state = install_utilities.install_state(config.var('VERSION'))
    if not state in ['ACTIVE', 'SERVE']:
        sys.exit(0)

    # Collect syslogs only from master node.
    if not isMaster(config):
        logging.fatal('Not a oneway or cluster master node. Return!')

    pywrapbase.InitGoogleScript('', [
        'foo',
        '--gfs_aliases=%s' % config.var("GFS_ALIASES"),
        '--bnsresolver_use_svelte=false', '--logtostderr'
    ], 0)
    gfile.Init()

    first_date, last_date, printable_date, file_date = \
                liblog.ParseDateRange('all',[])

    apache_main_dir = liblog.get_apache_dir(config)
    checkpoint_dir = liblog.get_syslog_checkpoint_dir(config)
    liblog.MakeGoogleDir(config, checkpoint_dir)

    if (config.var('SYSLOG_SERVER') == None
            or config.var('ENT_SYSLOG_GWS_FAC') == None):
        logging.fatal('SYSLOG logging is disabled')

    lockfile = '%s/syslog_lock' % config.var('LOGDIR')
    lock = E.acquire_lock(lockfile, 1, breakLockAfterGracePeriod=0)
    if not lock:
        return

    try:
        logger = syslog_client.SyslogClient(config.var('SYSLOG_SERVER'),
                                            config.var('EXTERNAL_WEB_IP'))
        logging.info("syslog-server = %s" % config.var('SYSLOG_SERVER'))
        for collection in os.listdir(apache_main_dir):
            apache_dir = '%s/%s' % (apache_main_dir, collection)
            checkpoint_file = "%s/%s" % (checkpoint_dir, collection)
            apache_logs = liblog.FindLogFiles(apache_dir, first_date,
                                              last_date)
            logging.info('syslog handles collection %s' % collection)
            if not SyslogLogs(apache_logs, apache_dir, checkpoint_file, logger,
                              config):
                sys.exit('Updating failed')

        logger.close()
    finally:
        lock.close()
        os.unlink(lockfile)

    sys.exit(0)