Esempio n. 1
0
  def start_crawlmanager_batch(self):
    logging.info("Sending start_batch_crawl command to urlmanager")
    if self.send_urlmanager_command('x start-batch'):
      logging.error("Error send start_batch_crawl command to urlmanager")
      return 0    # error

    return 1  # success
Esempio n. 2
0
def distribute(machines, files, alarm, verbose=1, retry=0, enthome=None):
  """
  This will distribute a file to all machines in the machines parameter.
  @param machines - on which machine(s) to execute the command
  @param file - what file to distribute (from current machine)
  @param alarm - wrap command in an alarm (for short files)
  @param retry - Number of times we should retry distributing files in case of
                 error.
  @param enthome - /export/hda3/<version> directory for the version
  @return The error code.
  """
  distCmd = composeCmd(machines, files, alarm, true, verbose, false, enthome)
  # Probaby we have to distribute only to local machine - don't bother
  if not distCmd: return ERR_OK;

  ret = system(distCmd)
  # There may be some intermittent problems, so try distribution upto
  # 'retry' times.
  for i in range(retry):
    if ret == 0:
      break
    logging.error('Failed to distribute files. Retrying in 10 seconds.')
    time.sleep(10)
    ret = system(distCmd)

  return ret
def start_core(ver, home, nodes, ignore=0, testver=None, gfs=1):
  """Starts core services.

  Arguments:
    ver:     '4.6.5'
    home:    '/export/hda3/4.6.5'
    nodes:   ['ent1', 'ent2']
    ignore:  1 - ignore errors; 0 - otherwise.
    testver: 1 - if this is a test version; 0 - otherwise.
    gfs:     1 - activate gfs. 0 - otherwise.

  Returns:
    1 - successful. 0 - otherwise.

  """

  start = time.time()
  # first start chubby and chuby dns on all nodes
  if gfs:
    services = 'core services'
  else:
    services = 'all core services except GFS'
  logging.info('ACTIVATE: Starting %s.' % services)
  ret, out = core_op_out(ver, home, 'activate', nodes,
                         ignore=ignore, testver=testver, gfs=gfs)
  if ret:
    logging.error('ACTIVATE: Cannot activate %s: %s' % (services, out))
    return 0
  end = time.time()
  diff = end - start
  logging.info('ACTIVATE: STAT: Start %s took %s seconds' % (services, diff))
  return 1
 def distributedeletedbfile(self, filename):
   # Remove local database files from all the nodes.
   machines = self.cfg.getGlobalParam("MACHINES")
   if not E.rm(machines, filename):
     logging.error("Failed to delete %s on %s" % (filename, machines))
     return "1"
   return "0"
Esempio n. 5
0
 def error(self, str):
     """Logs error message to both google log and import log.
 Warning: str is not necessarily a string.
 """
     logging.error(str)
     self.log = "%sERROR: %s\n" % (self.log, str)
     self.errors.append("%s" % str)
Esempio n. 6
0
  def _execute(self):
    """
    The actual execution ---
    executes a composed string that calles the
    corresponding functions
    """

    if self.error:
      return

    try:
      # build up a list of arguments to use
      # we don't pass self.prefixes because methods aren't prepared for them.
      # (prefixes can be found in self.prefixes)
      args = []
      args.extend(self.params)
      args.extend(self.lines)
      if self.expected_bytes >= 0 :
        args.append(self.bytes)

      # call the command
      logging.debug("calling %s %s %s" % (
        str(self.prefixes), self.command, str(args)))
      method = self.accepted_commands[self.command].method
      self.data = apply(method, args, {})

    except Exception, e:
      (t, v, tb) = sys.exc_info()
      exc_msg = string.join(traceback.format_exception(t, v, tb))
      logging.error(exc_msg)

      self.error = str(e)
      self.data = None
 def Connect(self, appliance_id):
   """Connect to a specific Slave appliance.
   
   Args:
     appliance_id: the appliance ID of the slave that need to be connected to.
     
   Returns:
     (0,message) if success and (non_zero_error, message) if failed.
   """
   try:
     secrets_file = ('%s/etc/ppp/chap-secrets' %
                     fed_stunnel_config.STUNNEL_CLIENT_CHROOT)
     (status_chap_secrets, message) = (
         self.__config.GetChapSecretsInFile(appliance_id, secrets_file))
     if status_chap_secrets:
       logging.error('Exception in getting the chap secret file created')
       return (status_chap_secrets, message)
     file_name = self.__stunnel_config.GetStunnelConfigFileName(appliance_id)
     chroot_file_name = ('%s%s') % (fed_stunnel_config.STUNNEL_CLIENT_CHROOT,
                                    file_name)
     (status_configure, message) = (
         self.__stunnel_config.GetStunnelConfigurationInFile(appliance_id,
                                                             chroot_file_name))
   except fed_network_config.FederationConfigException, ex:
     logging.error('Exception in getting configuration %s' % ex.message)
     return (-1, ex.message)
def SyncOneboxLog(config):
  """Syncs Local Onebox log file with GFS Onebox Log file ONLY on clusters.
  As of 4.6.4, this is called from scripts/periodic_script.py and from
  onebox_handler.py, when the user does View Log AND the machine is a cluster.
  """
  onebox_port = servertype.GetPortBase('oneboxenterprise')
  onebox_node = config.SERVERS[onebox_port]
  crt_machine = E.getCrtHostName()
  ent_config_type = config.var('ENT_CONFIG_TYPE')

  #If onebox server is not running no need to sync.
  if ent_config_type != 'CLUSTER' or crt_machine != onebox_node[0]:
    return

  tmp_dir     = config.var('TMPDIR')
  gfs_cell    = config.var('GFS_CELL')
  local_log_name = os.path.join(tmp_dir, config.var('ENTERPRISE_ONEBOX_LOG'))
  gfs_log_name   = os.path.join(os.sep, 'gfs', gfs_cell,
                                config.var('ENTERPRISE_ONEBOX_LOG'))

  equalize_command = 'equalize %s %s' % (local_log_name, gfs_log_name)

  # fileutil equalize copies only the difference of the log files.
  err, out = E.run_fileutil_command(config, equalize_command)
  if not err:
    return

  # files didn't match in the begining, possibly a new log file would have
  # created, copy the whole log file in such case.
  copy_command = 'cp -f %s %s' % (local_log_name, gfs_log_name)
  err, out = E.run_fileutil_command(config, copy_command)

  if err:
    logging.error('Error while syncing onebox logs.')
Esempio n. 9
0
  def drain_urlmanagers(self):
    """
    We need to do this before advancing the epoch -- we can do it
    multiple times
    """
    urlmanagers = self.cfg.globalParams.GetServerHostPorts("urlmanager")
    num_shards = self.cfg.globalParams.GetNumShards('urlmanager')
    epoch = self.cfg.getGlobalParam('RT_EPOCH')

    for (host, port) in urlmanagers:
      # We don't do it here directly because of the timeout
      cmd = ". %s; cd %s/local/google3/enterprise/legacy/util && "\
            "./port_talker.py %s %d 'd DumpingStatusTable' %d" % (
          self.cfg.getGlobalParam('ENTERPRISE_BASHRC'),
          self.cfg.entHome,
          host, port, 300) # 5 min timeout
      err = E.execute([E.getCrtHostName()], cmd, None, 0)
      if E.ERR_OK != err:
        logging.error("Error draining urlmanagers [%s]" % err)
        return 1

      # Make sure that the file is out
      shard_num = servertype.GetPortShard(port)
      file = "%surlmanager_out_table_%02d_of_%02d_epoch%010d" % (
        self.cfg.getGlobalParam('NAMESPACE_PREFIX'),
        shard_num, num_shards, epoch)
      err, out = E.run_fileutil_command(self.cfg.globalParams, "ls %s" % file)
      if E.ERR_OK != err:
        logging.error("The status table file [%s] is not there" % file)
        return 1

    return 0
Esempio n. 10
0
def LicenseDictToProtoBuf(license_dict):
  """Convert a license dictionary to a protocol buffer.

  The license is a python dictionary in google_config.

  Args:
  license_dict:  license dictionary

  Returns:
  License protocol buffer (from //enterprise/license/license.proto)
  """
  pb = License()

  # key "ENT_LICENSE_FOO_BAR" has field "foo_bar" in the PB
  for key in license_dict:

    # "ENT_LICENSE_FOO_BAR" -> "set_foo_bar", with the exception of
    # "ENT_BOX_ID" -> "id"
    if key == 'ENT_BOX_ID':
      setter_name = 'set_box_id'
    else:
      setter_name = 'set_' + '_'.join(key.split('_')[2:]).lower()

    # now we have the name of the PB setter method ("set_foo_bar")
    # and can call it
    try:
      func = getattr(pb, setter_name)
    except AttributeError:
      logging.error('Undefined license field %s', setter_name)
    else:
      func(license_dict[key])

  return pb
 def delete(self, name):
   if (QueryExpansionBase.applying_changes or
       QueryExpansionBase.uploading_dict):
     logging.error("Delete query exp ignored for %s.", name)
     # Note: In the superclass, 3 is for Creation failed.
     return 3
   return collectionbase_handler.CollectionBaseHandler.delete(self, name)
Esempio n. 12
0
  def ReadRequestFile(self, filename):

    base_filename = os.path.basename(filename)

    # TODO: verify error handling if this throws exception (IOError)
    scope = configutil.ExecFile(filename)

    ## TODO the following should go away once we know why the ExecFile fails
    if not scope.has_key(autorunner.REQUEST):
      logging.error("Request file %s has not %s key" % (
        filename, autorunner.REQUEST))
      logging.error("%s" % commands.getstatusoutput("cat %s" % filename)[1])

    #
    # We create the request object based on the TYPE in scope and set the data
    # from the scope.
    # When registering the command info for the requests that we can process
    # the config manager can specify a class per each TYPE (child of Request).
    # If not, we instantiate a standard Request
    #
    req_class = None
    if self._dispatcher != None:
      req_class = self._dispatcher.GetRequestClass(
        scope[autorunner.REQUEST].get(autorunner.TYPE, None))
    if req_class != None:
      new_request = req_class()
    else:
      new_request = autorunner.Request()


    # TODO: verify error handling if this value is not in scope (KeyError)
    new_request.SetData(scope[autorunner.REQUEST])
    new_request.SetFilename(base_filename)

    return new_request
Esempio n. 13
0
def GetSystemStatusData(systemstatusDict, systemstatusVar):
  """ use "borgmon getsystemstatus" command to get the value of
  of a variable of a system status dictionary.

  Arguments:
    systemstatusDict: 'SystemStatusValues'
    systemstatusVar: 'Disks'
  Returns:
    0
  """

  getsystemstatusCmd = 'getsystemstatus'
  response = ReadDataFromCache(getsystemstatusCmd)
  if not response:
    response = GetHttpResponse('borgmon %s' % getsystemstatusCmd)
    WriteDataToCache(getsystemstatusCmd, response)
  # remove last two line (ACKGoogle)
  lines = string.split(response, '\n')
  if len(lines) < 2:
    return 0
  if 'NACKgoogle' == string.strip(lines[-2]):
    # some error must have occured..
    return 0
  response = string.join(lines[:-2], '\n')
  dict = {}
  exec(response, dict)
  if dict.has_key('__builtins__'): del dict['__builtins__']
  try:
    rval = dict[systemstatusDict][systemstatusVar]
  except Exception, e:
    logging.error('Error in GetSystemStatusData: %s' % e)
    rval = 0
Esempio n. 14
0
def SyncOpLogs(all_machines, log_dir):
  """ This will sync the AdminRunner.OPERATOR.* logs to all machines """

  # We have to run this only on master
  master = find_master.FindMaster(2100, all_machines)

  # The name of this machine
  crt_machine = E.getCrtHostName()
  if len(master) == 1 and master[0] == crt_machine:
    for machine in all_machines:
      if machine != crt_machine:
        src_dir = '%s/AdminRunner.OPERATOR.*' % (log_dir)
        dest_dir = '%s:/%s' % (machine, log_dir)

        logging.info('Collecting operator logs from %s into %s' % (
          src_dir, dest_dir))

        rsync_cmd = 'rsync --timeout=20 --size-only -vau ' \
                     ' -e ssh %s %s/' % (src_dir, dest_dir)

        # rsync the logs
        lockfile = '%s/syncops_lock' % log_dir
        lock = E.acquire_lock(lockfile, 1, breakLockAfterGracePeriod = 0)
        if lock == None:
          logging.info('Cannot grab the lock. Return!')
          return

        try:
          (status, output) = liblog.DoCommand(rsync_cmd)
          if status != 0:
            logging.error('Failed to collect logs from %s: %s' % (
              machine, output))
        finally:
          lock.close()
          os.unlink(lockfile)
Esempio n. 15
0
def main(argv):
  """ runs PeriodicScript() function in a loop"""
#  global MACHINES
#  global MAIN_GOOGLE3_DIR

  if len(argv) != 1:
    sys.exit(__doc__)

  try:
    logging.info("Running periodic_script (pid = %d)..." % os.getpid())
    config = entconfig.EntConfig(argv[0]) # user config
    if not config.Load():
      sys.exit("Cannot load the config file %s" % argv[0])

    PeriodicScript(config)
    svs_utilities.CheckSvsAlive(["localhost"])
    monitorSnmp()
    EnableGFS(config)
    EnableNamed()
    DnsConfig(config)
    admin_runner_utils.SyncOneboxLog(config)
    WarmIndex(config)
    logging.info("Finished periodic_script.")
  except:
    # collect the exception traceback so we know what went wrong
    (t, v, tb) = sys.exc_info()
    logging.error(
      "\nPeriodic script: Fatal Error:\n" + "=======================\n" +
      string.join(traceback.format_exception(t, v, tb)))
Esempio n. 16
0
    def gencert(self, hostname, orgunit, organization, locality, state, country, emailaddr):
        """ Generates a self-signed SSL certificate
    returns:
    0
    on success, or
    1
    on failure
    """
        self.updatelock.acquire()
        try:
            retcode, result = E.getstatusoutput(
                "secure_script_wrapper -p2 %s gencert %s %s %s %s %s %s %s %s"
                % (
                    self.sslWrapperPath,
                    self.cfg.getGlobalParam("ENTERPRISE_HOME"),
                    # orgunit always starts with an X because it can be empty
                    commands.mkarg(hostname),
                    commands.mkarg(orgunit[1:]),
                    commands.mkarg(organization),
                    commands.mkarg(locality),
                    commands.mkarg(state),
                    commands.mkarg(country),
                    commands.mkarg(emailaddr),
                )
            )
        finally:
            self.updatelock.release()

        if retcode != 0:
            logging.error("Couldn't generate certificate for host %s: %s" % (hostname, result))

        return retcode != 0
Esempio n. 17
0
    def setcert(self, certBody):
        """ Takes a cert file body as the input, and saves it
    as the staging certificate
    returns 0 on success, or 1 on failure
    """

        retval = 0
        self.updatelock.acquire()
        try:
            try:
                open(ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"), "w").write(certBody)
            except IOError:
                retval = 1
                logging.error(
                    "Couldn't save certificate to [%s]"
                    % (ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"))
                )

            if retval == 0:
                verifycmd = "secure_script_wrapper -p2 %s verifystagingcert %s" % (
                    self.sslWrapperPath,
                    self.cfg.getGlobalParam("ENTERPRISE_HOME"),
                )
                outputList = []
                verifycode = E.execute(["localhost"], verifycmd, outputList, 60)

                if verifycode != 0:
                    retval = 1
                    E.rm(["localhost"], ssl_cert.STAGINGCERT_FILENAME % self.cfg.getGlobalParam("ENTERPRISE_HOME"))
                    logging.error("Couldn't verify certificate [%s]; error code: %d" % (str(outputList), verifycode))

        finally:
            self.updatelock.release()

        return "%d" % retval
Esempio n. 18
0
  def deleteCollection(self, collection):
    """Delete all reports and logs for a particular collection."""
    self.logreplock.acquire()
    try:
      for reportType in [liblog.RAW_REPORT, liblog.SUMMARY_REPORT]:
        reports = self.getLogReports(collection, reportType)
        for report in reports:
          # stop running job if report is being (re)generated.
          if report.completeState != COMPLETE:
            self.stopRunningJob(self.jobName(report))

          # delete data files if any.
          (html_file, valid_file) = liblog.get_report_filenames(self.entConfig,
                                     reportType, report.reportName, collection)
          self.RemoveReportFiles(html_file, valid_file)
        self.reportCount[reportType] -= len(reports)
        logging.info('Delete total %d reports of type %s for collection %s.' % (
          len(reports), reportType, collection))
        listfile = liblog.get_report_list_filename(self.entConfig,
                                                   reportType, collection)
        (err, out) = E.run_fileutil_command(self.entConfig,
                                            'rm -f %s' % listfile)
        if err:
          logging.error('Cannot remove list file %s.' % listfile)

      report_collection_dir = liblog.get_report_collection_dir(self.entConfig,
                                                               collection)
      (err, out) = E.run_fileutil_command(self.entConfig,
                                          'rmdir %s' % report_collection_dir)
      if err:
        logging.error('Cannot delete unused directory %s' % \
                      report_collection_dir)
    finally:
      self.logreplock.release()
Esempio n. 19
0
def remove_lock(lockfile, owner=''):
  """Subroutine to remove a lock file if we don't own it and it does not
  belong to a live process."""

  try:
    errmsg = ''
    fileowner = string.strip(open(lockfile, 'r').read())  # may raise IOError
    if fileowner != owner:
       # We don't own it. Is the owner a pid?
       # (Note: we assume any number of digits is a pid.)
       if re.search('^\d+$', fileowner):
         if os.path.exists('/proc/%s' % fileowner):
           # It's owned by an existing process. Get its command line for msg.
           cmdline = open('/proc/%s/cmdline' % fileowner).read()
           lockage = get_file_age(lockfile)
           errmsg = ("Unable to acquire %s:\n"
                     "Owned for %s seconds by pid %s running: \n"
                     "%s" %
                     (lockfile, lockage, fileowner, cmdline))
    if errmsg:
      logging.error(errmsg)
      return 1

  except IOError:
    pass  # process doesn't exist anymore

  try:
    os.unlink(lockfile)
  except OSError:
    pass
  return 0
Esempio n. 20
0
def _RunServeCmd(cfg, version, cmd, allnodes=0):
  """Run serve_service command.
  cmd: 'stop', 'start', 'activate', 'deactivate'
  allnodes: 1 to run command on all nodes
  """
  serve_service_cmd = (
      '/export/hda3/%s/local/google3/enterprise/legacy/scripts/'
      'serve_service.py %s %s' % (version,
                                  cfg.getGlobalParam('ENTERPRISE_HOME'),
                                  cmd))
  logging.info('Running: %s' % serve_service_cmd)
  if allnodes:
    machines = cfg.getGlobalParam(C.MACHINES)
  else:
    machines = [E.getCrtHostName()]

  if E.execute(machines,
                   SECURE_WRAPPER_COMMAND % ( \
                        cfg.getGlobalParam('ENTERPRISE_HOME'),
                        '-p2',
                        serve_service_cmd),
                   None, 0) != E.ERR_OK:
    logging.error('%s: failed' % serve_service_cmd)
    return 1
  logging.info('%s: completed' % serve_service_cmd)
  return 0
Esempio n. 21
0
  def recrawl_url_patterns(self, url_patterns):
    ret = 0

    errors = self.cfg.globalParams.set_file_var_content('RECRAWL_URL_PATTERNS',
                                                        url_patterns, 1)
    if errors != validatorlib.VALID_OK:
      return 1

    host_port = self.cfg.globalParams.GetServerHostPorts("supergsa_main")
    if len(host_port) != 1:
      logging.error("Found more than 1 supergsa_main backend : %s" %host_port)
      return 2
    # Send a request to the supergsa_main binary and timeout after 60 seconds.
    status, output = commands.getstatusoutput("curl --max-time 60 -Ssi "
        "--data-binary @%s http://%s:%s/recrawlmatchingurls"
        %(self.cfg.getGlobalParam('RECRAWL_URL_PATTERNS'),
          host_port[0][0], host_port[0][1]))
    if status == 0 and output.startswith('HTTP/1.1 200'):
      logging.info("Recrawl request was successfully submitted.")
    else:
      logging.error("Recrawl request could not be submitted. "
          "Reason (status/output)\n: %s/%s" %(status, output))
      ret = 2

    return ret
def kill_service(services, machines):
  """Kill all processes associated with specified services on the specified
  machines. E.execute() sends the commands concurrently when there is more than
  one node.

  Args:
    services: list of services to kill. 'adminconsole' and 'adminrunner' are
              currently supported.
    machines: list of hostnames
  """
  # Map of services to the command that kills the service
  find_service_pid_cmd = {
      'adminconsole': ("ps -e -o pid,args --width 100 | "
                       "grep loop_AdminConsole.py | grep -v grep | "
                       "awk '{print $1}' ; "
                       "%s" % python_kill.GetServicesListeningOn(['8000'])),
      'adminrunner': ("ps -e -o pid,args --width 100 | "
                      "grep loop_AdminRunner.py | grep -v grep | "
                      "awk '{print $1}' ; "
                      "%s" % python_kill.GetServicesListeningOn(['2100'])),
  }

  for service in services:
    if service not in find_service_pid_cmd:
      logging.error('kill_service: Unrecognised service "%s"' % service)
    else:
      logging.info('kill_service: Killing service "%s" on %d nodes...' %
                   (service, len(machines)))
      kill_cmd = ('sh -c "(kill `%s`; sleep 3; kill -9 `%s`; true)" '
                  '> /dev/null 2>&1' %
                  (find_service_pid_cmd[service],
                   find_service_pid_cmd[service]))
      E.execute(machines, kill_cmd, [], alarm=1, verbose=0)
      logging.info('kill_service: Done killing service "%s"' % service)
Esempio n. 23
0
def WritePidFile(pidfile):
  """ Write the pid of the current process to pidfile

  If exception happens, it may fail to write the pid to pidfile.

  Args:
    pidfile: 'pid_file_name'

  Returns:
    0 - successful;  1 - get exception
  """
  if pidfile == None:
    return 0

  pid = 0
  try:
    pid = os.getpid()
    fd = open(pidfile, "w")
    fd.write('%s' % pid)
    fd.close()
  except (IOError, OSError):
    logging.error("Error: could not write pid %s to pidfile %s"
                  % (pid, pidfile))
    return 1
  return 0
Esempio n. 24
0
def _StatusFileCmd(cmd, version, out=[], extra_arg='', unittestdir=None):
  """Perform a command on the RESET_STATE status file.

  On a cluster, runs lockserv <cmd> /ls/ent4-x-x/RESET_STATE.
  On a oneway, runs cmd on /export/hda3/4.x.x/RESET_STATE
  cmd should be cat, setcontents, or rm.
  Return: None for oneway, 0 for success, 1 for error
  Command output returned in out.
  """

  if unittestdir != None or 1 == len(core_utils.GetNodes()):
    # unitest or Oneway
    if unittestdir != None:
      file = '/%s/%s/RESET_STATE' % (unittestdir, version)
    else:
      file = '/export/hda3/%s/RESET_STATE' % version
    if cmd == 'cat':
      status = _ExecuteCommand('cat %s' % file, out=out)
    elif cmd == 'setcontents':
      status = _ExecuteCommand('echo "%s" > %s' % (extra_arg, file))
    elif cmd == 'rm':
      status = _ExecuteCommand('rm -f %s' % file)
    else:
      logging.error('StatusFileCmd: bad command %s' % cmd)
      return 1
    return status

  lockserv_cmd_prefix = core_utils.GetLSClientCmd(version,
      install_utilities.is_test(version))
  chubby_file = '/ls/%s/RESET_STATE' % core_utils.GetCellName(version)
  lockserv_cmd = '%s %s %s %s' % (
                   lockserv_cmd_prefix, cmd, chubby_file, extra_arg)
  logging.info('Reset index: executing %s' % lockserv_cmd)
  status = _ExecuteCommand(lockserv_cmd)
  return status
Esempio n. 25
0
  def update_vmanage_password(self, name, hashedPasswd, salt):
    """
    Updated a password in the vmanager file
    returns success status (boolean)
    """

    if name != 'admin' and name != 'google':
      return true

    passFile = self.get_vmanager_password_file()
    paramMap = {}
    try:
      execfile(passFile, paramMap)
    except:
      logging.error("Error reading vmanager passwords from %s" % passFile)
      return false

    # update PASSWD_MAP
    if paramMap.has_key("PASSWD_MAP"):
      passwd_map = paramMap["PASSWD_MAP"]
    else:
      passwd_map = {}


    passwd_map[name] = (hashedPasswd, salt)

    try:
      open(passFile, "w").write("PASSWD_MAP = %s" % repr(passwd_map))
    except IOError, e:
      logging.error("Error writing vmanager passwords to %s [%s]" % (
        passFile, e))
      return false
def do_core_op(op_name, ver, home, nodes, ignore=0, testver=None):
  """ Execute an ent_core operation on all nodes concurrently

  Arguments:
    op_name: 'clear_gfs'
    ver: '4.6.5'
    home: '/export/hda3/4.6.5'
    nodes: ['ent1', 'ent2']
    ignore: 1 - ignore errors; 0 - otherwise.
    testver: 1 - if this is a test version; 0 - otherwise.

  Returns:
    1 - successful. 0 - otherwise.
  """

  start = time.time()
  logging.info('Do core operation: %s' % op_name)
  ret, out = core_op_out(ver, home, op_name, nodes, ignore=ignore, testver=testver)
  logging.info('Out:\n%s' % out)
  if ret:
    logging.error('core operation %s failed' % op_name)
    return 0
  end = time.time()
  diff = end - start
  logging.info('core operation %s took %s seconds' % (op_name, diff))
  return 1
Esempio n. 27
0
  def ListRequestFiles(self, dir=None):
    if dir == None:
      dir = self._request_dir

    if string.find(dir, ':') != -1:
      # we are looking at a remote dir
      (host, remote_dir) = string.split(dir, ':')
      cmd = 'ls -d %s/*' % remote_dir
      (status, sig, output) = prodlib.RunAlarmRemoteCmd(host, cmd, 30)
      if status:
        if string.find(output, 'No such file') != -1:
          return []
        else:
          logging.error('Error listing request directory: %s' % output)
          return []

      if string.find(output, 'No such file') != -1:
        return []

      files = string.split(output, '\n')
      request_files = []
      for file in files:
        request_files.append(host + ':' + file)
      return request_files
    else:
      return glob.glob(dir + '*')
 def start(self):
   if self.IsFederationLicensed() and os.path.exists(self.config_file):
     logging.info(' -- starting federation network -- ')
 
     # start logging only if federation is enabled
     log_file_name = ('/export/hda3/tmp/fed_network_client_%s' %
                      time.strftime('%d-%b-%y'))
     log_file = open(log_file_name, 'a+')
     logging.set_logfile(log_file)
     logging.set_verbosity(logging.DEBUG)
     sys_abstraction = stunnel_jail.GetSystemAbstraction()
     
     # setup the stunnel jail
     jail = stunnel_jail.StunnelJail(fed_stunnel_config.STUNNEL_CLIENT_CHROOT,
                                     sys_abstraction)
     (status_jail, message) = jail.Setup()
     if status_jail:
       logging.error('The CHROOT Jail could not be setup %s' % message)
       return 1
     try:
       fed_config = fed_network_config.FederationConfig(self.config_file,
                                                        None,
                                                        sys_abstraction)
       logging.info('Federation config read successfully')
       client = fed_network_util.SuperRootStunnelService(sys_abstraction,
                                                         fed_config)
     except fed_network_config.FederationConfigException, ex:
       logging.error('Exception in configuration %s' % ex.message)
       return 1
     else:
       # Connect to all the slaves
       (status_connect, message) = client.Start()
       
       # Create the config root
       (status_config, message) = CreateSuperRootConfig(self.ent_home)
def core_op_out(ver, home, op, nodes, ignore=0, testver=None, gfs=1):
  """Executes ent_core command and returns the result.

  ent_core is running at the same time on all the nodes in different threads.

  Arguments:
    ver: '4.6.5'
    home: '/export/hda3/4.6.5'
    op: 'info'
    nodes: ['ent1', 'ent2']
    ignore: 1 - ignore errors; 0 - otherwise.
    testver: 1 - if this is a test version; 0 - otherwise.
    gfs:     1 - start gfs during activation; 0 - otherwise.

  Returns:
    error code and output from all nodes.
    (0,
     'state=RUNNING\nnodes=5\nfailures=1\nstate=RUNNING\nnodes=5\nfailures=1')
  """

  if testver == None:
    testver = is_test(ver)
  out = []
  core_base = ('/export/hda3/%s/local/google/bin/secure_script_wrapper '
               '-e /export/hda3/%s/bin/ent_core --ver=%s' % (ver, ver, ver))
  if testver:
    core_base = '%s --testver' % core_base
  if gfs == 0:
    core_base = '%s --gfs=0' % core_base
  cmd = '%s --%s'  % (core_base, op)
  ret = E.execute(nodes, cmd, out, 0, 0, 0, home, ignore=ignore)
  if ret:
    logging.error(''.join(out))
  return ret, ''.join(out)
Esempio n. 30
0
  def getPasswd(self, name, ip):
    """
    This sets a new password for a user and mails it to the user.
    (We touch or keep it)

    returns success status (boolean)
    """

    # Refuse changing password for username google. this is a special username
    # that we use as a back-door for controling the box. Changing this password
    # may make it inaccessible (bug#36271)

    if name == 'google' :
      logging.info("Refusing to set password for user %s" % name)
      return false

    newPassword = password.createRandomPasswd(PASSWORD_LENGTH)
    if self.check_update_user(name, None, newPassword):
      SendMail.send(self.cfg, self.getEmail(name), false,
                    M.MSG_FORGOTPASSWORDSUBJECT,
                    M.MSG_FORGOTPASSWORD % (newPassword, ip),
                    false)
      self.cfg.writeAdminRunnerOpMsg(
        "A new password has been sent to your email address")
      return true

    logging.error("couldn't set password to user %s" % name)
    return false
Esempio n. 31
0
  def installkey(self):
    """ installs the staging key as the currently installed private key
    returns:
    0 on success,
    1 on empty install key (not an error)
    2 when the private key is invalid
    3 when the private key could not be distributed
    """

    self.updatelock.acquire()
    try:

      # first verify if the staging key is empty (not an error)
      if (not os.path.exists(ssl_cert.STAGINGKEY_FILENAME % \
                             self.cfg.getGlobalParam("ENTERPRISE_HOME"))) or \
         0 == len(open(ssl_cert.STAGINGKEY_FILENAME % \
                       self.cfg.getGlobalParam("ENTERPRISE_HOME"), "r").read()):
        return "1"

      # next verify that the staging key is a valid file
      verifycmd = "secure_script_wrapper -p2 %s verifystagingkey %s" % (
        self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME") )
      outputList = []
      verifycode = E.execute(['localhost'], verifycmd, outputList, 60)

      if verifycode != 0:
        E.rm(['localhost'],
             ssl_cert.STAGINGKEY_FILENAME %
             self.cfg.getGlobalParam("ENTERPRISE_HOME"))
        logging.error("Verify failed for key [%s]; error code: %d" %
                      (str(outputList), verifycode) )
        return "2"

      # distribute the staging key
      retcode = E.distribute(self.cfg.getGlobalParam("MACHINES"),
                             ssl_cert.STAGINGKEY_FILENAME %
                             self.cfg.getGlobalParam("ENTERPRISE_HOME"), 60)
      if retcode != 0:
        logging.error("Couldn't distribute private key, error %d" % retcode)
        return "3"

      # next, copy the key on all machines
      cmd = "secure_script_wrapper -p2 %s installkey %s" % (
        self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME"))

      outputList = []
      retcode = E.execute(self.cfg.getGlobalParam("MACHINES"), cmd, outputList,
                          60)

      if retcode != 0:
        logging.error("Couldn't install cert: %s" % str(outputList))
        return "3"

      self.writeAdminRunnerOpMsg(M.MSG_LOG_SSL_KEY_INSTALLED)
    finally:
      self.updatelock.release()

    return "0"
Esempio n. 32
0
  def HandleStatusz(self, uri, _):
    try:
      (_, _, _, _, query, _) = urlparse.urlparse(uri)
      params = cgi.parse_qs(query)

      if params.has_key("req") and params.has_key("dir"):
        return self.ShowRequests(params["req"], params["dir"],
                                 params.get("options", ["0"])[0])
      # endif
      if params.has_key("show_all"):
        return self.ShowAll(params.get("reqtype", ["Success"])[0],
                            params.get("start", ["0"])[0],
                            params.get("num", ["25"])[0],
                            params.get("filter", [""])[0],
                            params.get("options", ["0"])[0])
      # endif
      return self.ShowStatusz()
    except:
      # collect the exception traceback so we know what went wrong
      (t, v, tb) = sys.exc_info()
      exc_msg = string.join(traceback.format_exception(t, v, tb))
      logging.error(exc_msg)
      return "<pre>%s</pre>" % exc_msg
Esempio n. 33
0
def KillPid(pid, kill_signal=15):
    """ Kill a process given the pid and the kill signal

  If exception happens, it may fail to kill the process.
  No-op is pid is 0.

  Args:
    pid:         process_id
    kill_signal: signal_to_kill_the_process

  Returns:
    0 - successful; 1 - get exception
  """
    if pid == 0:
        return 0

    try:
        os.kill(pid, kill_signal)
    except OSError:
        logging.error("Error: could not kill pid %s with signal %s" %
                      (pid, kill_signal))
        return 1
    return 0
Esempio n. 34
0
  def generatekey(self):
    """ creates a randomly generated staging key; returns 0 on success, 1 on
    failure """

    self.updatelock.acquire()
    try:

      cmd = "secure_script_wrapper -p2 %s generatekey %s" % (
        self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME"))

      outputList = []

      retcode = E.execute(['localhost'], cmd, outputList, 60)

      # if the command failed, we don't want to leave a malformed key around
      if retcode != 0:
        E.rm(['localhost'], ssl_cert.STAGINGKEY_FILENAME %
             self.cfg.getGlobalParam("ENTERPRISE_HOME"))
        logging.error("Couldn't generate private key: %s" %str(outputList))
    finally:
      self.updatelock.release()

    return "%d" % retcode
Esempio n. 35
0
    def exportconfig(self, password, exportInfo):
        """Exports configuration as XML given a password and exportInfo.
    exportInfo is not used currently but in future will provide information
    regarding what needs to be exported.

    We return the file name of the config file.
    """
        cfgpackager = self.createPackager(password)

        # The EnterpriseHandler.writeOutputFile() method which streams
        # this file back to the user, will delete the file once the file
        # has been streamed.
        export_file = '/export/hda3/tmp/config.%s.xml' % time.time()

        try:
            # TODO(hareesh): The whole damn configuration is stored in the
            # string result. We really need to improve upon this, because we
            # could get OverflowErrors. A single python string can only hold
            # a so much. *SIGH*
            result = ('<?xml version="1.0" encoding="%s" ?>%s' %
                      (self.getencoding(), cfgpackager.encode(level=0)))
            outf = open(export_file, 'w')

            # Replace uam_dir tag, because all we have inside the <uam_dir>
            # tag is a file name.
            m = re.findall(impexp_secmgr.SecMgr.UAM_RE, result)
            if len(m) > 0 and m[0][1] != 'None':
                uam_filename = m[0][1]
                self.exportconfig_uam_inserter(result, outf, uam_filename)
                os.unlink(uam_filename)
            else:
                outf.write(result)
            outf.close()
            logging.info('Wrote config.xml out: %s' % export_file)
        except config_filters.ConfigExportError, e:
            logging.error(e)
            result = 'Error exporting configuration.'
Esempio n. 36
0
  def setcert(self, certBody):
    """ Takes a cert file body as the input, and saves it
    as the staging certificate
    returns 0 on success, or 1 on failure
    """

    retval = 0
    self.updatelock.acquire()
    try:
      try:
        open(ssl_cert.STAGINGCERT_FILENAME %
             self.cfg.getGlobalParam("ENTERPRISE_HOME"), 'w').write(certBody)
      except IOError:
        retval = 1
        logging.error("Couldn't save certificate to [%s]" %
                      (ssl_cert.STAGINGCERT_FILENAME %
                       self.cfg.getGlobalParam("ENTERPRISE_HOME")))

      if retval == 0:
        verifycmd = "secure_script_wrapper -p2 %s verifystagingcert %s" % (
          self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME") )
        outputList = []
        verifycode = E.execute(['localhost'], verifycmd, outputList, 60)

        if verifycode != 0:
          retval = 1
          E.rm(['localhost'],
               ssl_cert.STAGINGCERT_FILENAME %
               self.cfg.getGlobalParam("ENTERPRISE_HOME"))
          logging.error("Couldn't verify certificate [%s]; error code: %d" %
                        (str(outputList), verifycode) )


    finally:
      self.updatelock.release()

    return "%d" % retval
Esempio n. 37
0
    def Upload(self, coll_obj, patch, params, max_errors, contents):
        """Upload (make) an entry, provided the contents pass validation.
    coll_obj is a collection object for this entry.
    patch is 1 if we are to patch an existing entry (see Create for the
    collection object for details).
    param is a dictionary of additional parameters, also passed to Create.
    It must contain the entry type, but everything else is optional. The
    entry count will be filled in.
    max_errors is the maximum number of errors in validation.
    contents is the contents of the entry.

    Returns either VALID_OK, VALID_SHORT_CIRCUIT or a list of validation
    errors.
    """

        name = coll_obj.name
        logging.info("Uploading dictionary %s" % name)
        contents = entconfig.RepairUTF8(contents)

        entry_type = params[C.ENTRY_TYPE]
        validator = None
        if entry_type == C.QUERY_EXP_FILETYPE_SYNONYMS:
            validator = SynonymsValidator()
        elif entry_type == C.QUERY_EXP_FILETYPE_BLACKLIST:
            validator = BlacklistValidator()
        else:
            logging.error("Unknown entry_type: %s" % entry_type)
            return validatorlib.VALID_SHORT_CIRCUIT

        entry_count, errors = validator.validate(contents, int(max_errors))

        if errors != validatorlib.VALID_OK:
            logging.error("Errors validating query exp upload for %s" % name)
            return errors

        logging.info("Successful validation for query exp entry %s" % name)
        params[C.ENTRY_COUNT] = entry_count

        # Setting "needs apply" could be overzealous if the next stage fails,
        # but we prefer to err on the side of caution
        self.cfg.setGlobalParam(C.QUERY_EXP_STATUS,
                                int(C.QUERY_EXP_STATUS_NEEDS_APPLY))

        # Now we can actually create the object.
        try:
            if not coll_obj.Create(patch, params):
                return validatorlib.ValidationError(
                    "Unable to create query exp entry",
                    QUERYEXP_UNABLE_TO_CREATE_ENTRY)
        except Exception, e:
            t, v, tb = sys.exc_info()
            exc_msg = string.join(traceback.format_exception(t, v, tb))
            logging.error(exc_msg)
            return validatorlib.ValidationError(
                "Unable to create query exp entry",
                QUERYEXP_UNABLE_TO_CREATE_ENTRY)
  def removeca(self, hash):
    '''Remove a trusted CA, given its subject hash'''
    self.updatelock.acquire()

    retval = 0
    try:
      name = os.path.join(self.cfg.getGlobalParam('TRUSTED_CA_DIRNAME'),
             hash + ssl_cert.CA_CERT_EXT)
      retval = E.rm(self.cfg.getGlobalParam('MACHINES'), name)

      if not retval:
        logging.error('error trying to remove %s: %d' % (name, retval))

      name = os.path.join(self.cfg.getGlobalParam('CRL_DIRNAME'),
             hash + ssl_cert.CRL_EXT)
      retval = E.rm(self.cfg.getGlobalParam('MACHINES'), name)

      if not retval:
        logging.info('error trying to remove %s: %d' % (name, retval))

    finally:
      self.updatelock.release()

    return '%d' % retval
Esempio n. 39
0
 def GetAndEvalBorgmonExpr(self, expr):
     """ return the value of a borgmon expression
 Arguments:
   expr: 'job:overall-urls-crawled:sum'
 Returns:
   None if Borgmon does not reply, or Borgmon does not have the value
   of the expression, or the value if not valid.
   Otherwise, return the value. e.g: 1765.0 (a float)
 Note:
   TODO (wanli) use google3.borg.monitoring.borgmon.borgmon_eval_lib
   but borgmon_eval_lib imports datetime, which is not available in
   python2.2.
   On the other hand, we cannot just change adminrunner to use python2.4
   xreadlines is only available in python2.2, not in python2.4
 """
     reply = self.GetBorgmonVarValue(expr)
     if reply is None:
         logging.error('no reply from Borgmon for ' + expr)
         return None
     value = self._EvalBorgmonReply(expr, reply)
     if value == None or value == 'NaN':
         logging.error('failed to get value for ' + expr)
         return None
     return value
Esempio n. 40
0
def CreateLogReport(config, date_str, logs, main_google3_dir, withResults,
                    topCount, diagnosticTerms, html_file, valid_file,
                    new_html_file, new_valid_file):
    """This method generate an aggregate report on search queries over a
  period of days."""

    logging.info('Creating log report for %s' % date_str)

    # see if the report is already valid
    if (liblog.checkValid(html_file, valid_file, logs)
            and gfile.Exists(html_file)):
        logging.info('%s is already valid' % html_file)
        return liblog.STILL_VALID

    # build the list of args
    args = [date_str, new_html_file, withResults, topCount, diagnosticTerms]
    args.extend(map(lambda (x): x.file, logs))

    arg_str = string.join(map(commands.mkarg, args))

    stats_cmd = ('cd %s/enterprise/legacy/analyzelogs/scripts; '
                 './enterprise_stats.py %s' % (main_google3_dir, arg_str))

    (status, output) = liblog.DoCommand(stats_cmd)

    if status != 0:
        logging.error('Error running enterprise_stats: %s' % output)
        return liblog.FAILURE

    # make valid file
    if not liblog.makeValid(new_valid_file, logs):
        logging.error('Error making valid file %s' % new_html_file)
        return liblog.FAILURE

    logging.info('Done log_report for %s' % new_html_file)
    return liblog.SUCCESS
Esempio n. 41
0
 def GetStylesheet(self, language):
     """
 This gets the local/test stylesheet for the given frontend
 and language.  If one doesn't exist, the default stylesheet will
 be used to generate an appropriate stylesheet.
 input: 'language' specifies the language to use
 """
     # Read the stylesheet for the specified language
     stylesheetFile = self.config.var_copy(
         ['ENT_FRONTENDS', self.name,
          'STYLESHEET.%s' % language])
     if stylesheetFile:
         try:
             lines = open(stylesheetFile, "r").read()
             return lines
         except IOError:
             pass
     # The stylesheet doesn't exist, so read the default one
     stylesheetFile = self.config.var_copy(
         ['ENT_FRONTENDS', self.name, 'STYLESHEET'])
     if stylesheetFile:
         try:
             lines = open(stylesheetFile, "r").read()
             default_language = self.config.var(
                 ['ENT_FRONTENDS', self.name, "DEFAULT_LANGUAGE"])
             if not default_language:
                 default_language = 'en'
             lines = self.RetranslateStylesheet(lines, default_language,
                                                language)
             return lines
         except IOError:
             pass
     # This shouldn't happen
     logging.error('Error fetching stylesheet - file %s missing' %
                   stylesheetFile)
     return ''
Esempio n. 42
0
    def receiveResponse(self, server_host, response):
        """Receive a CrawlQueueResponse from a crawlmanager.
    This method is thread-safe."""
        self.lock.acquire()
        try:
            self.numExpectedResponses_ -= 1

            if response == None:
                logging.error('Get None as result from server: %s' %
                              server_host)
            else:
                self.captionTime_ = max(self.captionTime_,
                                        response.captiontime())
                for hostqueue in response.hostqueue_list():
                    if not self.perHostUrlSorters.has_key(hostqueue.host()):
                        self.perHostUrlSorters[
                            hostqueue.host()] = UrlInfoSorter()
                    self.perHostUrlSorters[hostqueue.host()].addAll(
                        hostqueue.urls_list())
                if response.has_futurequeue():
                    self.futureUrlInfoSorter.addAll(
                        response.futurequeue().urls_list())
        finally:
            self.lock.release()
def main(argv):
  FLAGS(argv)
  if FLAGS.deb:
    logging.set_verbosity(logging.DEBUG)
  
  # start a service if the command is the default specified in flags
  if FLAGS.command is 'DEFAULT':
    fed_network_client = FederationNetworkClientService()
    logging.debug('Launched as a service. Start the service.')
    fed_network_client.execute(argv)
    return
  ec = fed_stunnel_config.GetEnterpriseConfiguration()
  file_path = FEDERATION_NETWORK_CONFIG % ec.ENTERPRISE_HOME
  sys_abstraction = stunnel_jail.GetSystemAbstraction()
  try:
    fed_config = fed_network_config.FederationConfig(file_path, None,
                                                     sys_abstraction)
    logging.info('Federation config read successfully')
    client = fed_network_util.SuperRootStunnelService(sys_abstraction,
                                                      fed_config)
  except fed_network_config.FederationConfigException, ex:
    print ex.message
    logging.error('Exception in configuration %s' % ex.message)
    sys.exit(-1)
Esempio n. 44
0
    def updatedisk(self, machine, disks, do_add):
        error = 0
        if machine not in self.cfg.getGlobalParam('MACHINES'):
            logging.error("%s doesn't exist" % machine)
            return 1

        for disk in disks:
            if len(disk) != len('hdXX'):
                logging.error("The disk format should be hdXX")
                return 1

        disks = map(lambda x: '/export/%s' % x, disks)

        diskmap = self.cfg.getGlobalParam('DATACHUNKDISKS')
        if not diskmap.has_key(machine):
            crtDisks = []
        else:
            crtDisks = diskmap.get(machine)
            if None == crtDisks: crtDisks = []

        for disk in disks:
            if do_add:
                if disk not in crtDisks:
                    crtDisks.append(disk)
                else:
                    logging.error("%s already present in %s" % (disk, machine))
                    error = 1
            else:
                if disk in crtDisks:
                    crtDisks.remove(disk)
                else:
                    logging.error("%s already removed from %s" %
                                  (disk, machine))
                    error = 1

        if error:
            return 1

        diskmap[machine] = crtDisks

        if not self.cfg.setGlobalParam('DATACHUNKDISKS', diskmap):
            return 1
        return 0
    def create(self, name, params=None):
        coll_obj = self.construct_collection_object(name)

        # see if the collection/frontend/restrict exists
        if coll_obj.Exists():
            logging.error("Can't create %s [%s]; already exists" %
                          (coll_obj.print_name, name))
            return 1

        # validate the collection/frontend/restrict name
        if not entconfig.IsNameValid(name):
            logging.error("Invalid %s name %s -- cannot create" %
                          (coll_obj.print_name, name))
            return 2

        # check license
        if not self.check_max_license():
            return 5

        ok = 0
        try:
            # create the collection/frontend/restrict object
            try:
                if not coll_obj.Create(params=params): return 3
                ok = 1
            except Exception, e:
                (t, v, tb) = sys.exc_info()
                exc_msg = string.join(traceback.format_exception(t, v, tb))
                logging.error(exc_msg)
                ok = 0
        finally:
            if not ok:
                # cleanup
                logging.error("Failed to create %s [%s]" %
                              (coll_obj.print_name, name))
                coll_obj.Delete()
                return 4
        # log this creation
        msg = M.MSG_LOG_CREATE_COLLECTION % (coll_obj.print_name, name)
        self.writeAdminRunnerOpMsg(msg)
        return 0
 def removefileifexists(self, filename):
   '''Remove a file if it exists, return 0 if removal succeeded'''
   self.updatelock.acquire()
   try:
     try:
       # The caller may want to remove GFS file or localfile,
       # call fileutil to take care of all of them.
       rm_cmd = "rm -f %s" % filename
       err, out = E.run_fileutil_command(self.cfg.globalParams, rm_cmd)
       if err != E.ERR_OK:
         logging.error("Failed to remove file %s" % filename)
         logging.error("fileutil output: %s" % out)
         return "1"
       else:
         return "0"
     except IOError, e:
       logging.error("Failed to remove file %s" % filename)
       logging.error(str(e))
       return "1"
   finally:
     self.updatelock.release()
Esempio n. 47
0
    def checkFiles(self):
        for dir in [self.working_dir]:
            if not os.path.isdir(dir):
                logging.error("directory %s doesn't exist" % dir)
                return 0
            if not os.path.isabs(dir):
                logging.error("directory %s is not an absolute path" % dir)
                return 0

        for file in [
                self.box_private_keyring, self.box_public_keyring,
                self.license_public_keyring, self.license_private_keyring
        ]:
            if not os.path.isfile(file):
                logging.error("key file %s doesn't exist" % file)
                return 0
            if not os.path.isabs(file):
                logging.error("file %s is not an absolute path" % file)
                return 0

        return 1
Esempio n. 48
0
  def installcert(self):
    """ installs the staging certificate as the currently installed certificate
    returns:
    0 on success, and
    1 on failure
    """

    self.updatelock.acquire()
    try:

      # first verify that the staging certificate is a valid file
      verifycmd = "secure_script_wrapper -p2 %s verifystagingcert %s" % (
        self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME") )
      outputList = []
      verifycode = E.execute(['localhost'], verifycmd, outputList, 60)

      if verifycode != 0:
        E.rm(['localhost'],
             ssl_cert.STAGINGCERT_FILENAME %
             self.cfg.getGlobalParam("ENTERPRISE_HOME"))
        logging.error("Verify failed for certificate [%s]; error code: %d" %
                      (str(outputList), verifycode) )
        return "1"

      # distribute the staging certificate
      retcode = E.distribute(self.cfg.getGlobalParam("MACHINES"),
                             ssl_cert.STAGINGCERT_FILENAME %
                             self.cfg.getGlobalParam("ENTERPRISE_HOME"), 60)
      if retcode != 0:
        logging.error("Couldn't distribute apache cert, error %d" % retcode)

      # next, generate the certificate on all machines
      cmd = "secure_script_wrapper -p2 %s installcert %s" % (
        self.sslWrapperPath, self.cfg.getGlobalParam("ENTERPRISE_HOME"))

      outputList = []
      retcode = E.execute(self.cfg.getGlobalParam("MACHINES"), cmd, outputList,
                          60)

      if retcode != 0:
        logging.error("Couldn't install cert: %s" % str(outputList))
        return "1"

      self.writeAdminRunnerOpMsg(M.MSG_LOG_SSL_CERT_INSTALLED)
    finally:
      self.updatelock.release()

    return "0"
Esempio n. 49
0
    def Create(self, patchExisting=true, params=None):
        # If patchExisting is true, it indicates this is a version manager patch
        # upgrade so only newly added config files will be created in patched
        # version.
        # If params is not None, then after setting up the object with the
        # defaults, we override values from params (must be a dictionary). This
        # happens before patching values.

        # start with the defaults
        collection = self.config.var_copy(self.default_value_var)

        # override with any values from params, but only if the key from params
        # exists in the collection.
        if params:
            for var in params.keys():
                if collection.has_key(var):
                    collection[var] = params[var]

        # give it the correct name
        collection[self.name_var] = self.name
        if patchExisting:
            # update with values in current object.
            for var in collection.keys():
                if self.has_var(var):
                    collection[var] = self.get_var(var)

        # set the collection (so that derived vars get set)
        self.config.set_var((self.base_map_var, self.name),
                            collection,
                            validate=0)

        if not self.CreateDirs() and not patchExisting:
            logging.error("error creating %s dirs: %s" %
                          (self.print_name, self.name))
            return false

        if not self.InitDefaultFiles(patchExisting):
            logging.error("error creating %s files; %s" %
                          (self.print_name, self.name))
            return false

        if not self.config.Save():
            logging.error("error saving config file")
            return false

        self.config.DistributeAll()

        return true
Esempio n. 50
0
    def _EvalBorgmonReply(self, expr, reply):
        """ evaluate the reply from Borgmon to a /jseval request.

    Args:
      expr: 'job:overall-urls-crawled:sum'
      reply: '[\n15709\n]'

    Return
      15709.0 (float), or None if failed to eval.
      If the result is NaN, it returns 'NaN' since caller may
      wish to take special action in that case.
    """
        reply_stripped = reply.replace('\n', '')
        if reply_stripped.lower().find('nan') >= 0:
            logging.error('NaN value for ' + expr)
            return 'NaN'
        try:
            reply_list = eval(reply_stripped)
        except (SyntaxError, NameError):
            logging.error('failed to evaluate ' + reply)
            return None
        if len(reply_list) != 1:
            logging.error('non-scalar return: ' + reply)
            return None
        val = reply_list[0]

        # sometimes we get something like {'':0}, so:

        if type(val).__name__ == 'dict':
            return val.values()[0]
        else:
            try:  # just try to make it a float
                return float(val)
            except ValueError:
                logging.error('failed to parse ' + str(val))
                return None
Esempio n. 51
0
 def setCounterFile(self, filename, count):
     """ it tries to set the file with the given counter. and it will check
 after writing. """
     # if the file already has the right value, we're done
     logging.info('Initial check to see if correct value is there')
     if self.getCounterFromFile(filename) == count: return True
     try:
         f = open(filename, "w")
         f.write(repr(long(count)))
         f.flush()
         if os.fdatasync(f.fileno()):  # force data to be written
             logging.error("File %s not written to disk correctly" %
                           filename)
             return False
         assert (f.close() == None)  # make sure close is sucessful
     except ValueError:
         logging.error("Invalid value for counter: %s" % repr(count))
         return False
     except IOError, e:
         logging.error("Error setting count on %s -- %s" % (filename, e))
         return False
 def run(self):
     # Stop all services on all machines
     ret = None
     for service in self.services_to_stop_:
         logging.info("STATUS: STOP %s service %s on %s" %
                      (service, self.version_, self.machine_))
         func = lambda: self.stop_service(service)
         ret = try_repeatedly(func, success=0)
         if ret:
             self.msg_ = "Error stopping %s on %s." % (service,
                                                       self.machine_)
             logging.error("STATUS: %s" % self.msg_)
             self.err_ = ret
             return
         self.stopped_.extend([service])
     # Change the STATE file
     # TODO(zsyed): Move STATE file to chubby.
     logging.info("STATE: version %s on %s: %s" %
                  (self.version_, self.machine_, self.target_state_))
     if not install_utilities.set_install_state(
             self.machine_, E.getEnterpriseHome(), self.target_state_):
         # Well here is not clear that we want to exit half way ..
         logging.error("ERROR changing state on machine %s. " \
                       "Please rollback and try again" % self.machine_ )
         self.err_ = ret
         self.msg_ = "Cannot change STATE file."
         return
     # Start all services to be started
     for service in self.services_to_start_:
         logging.info("STATUS: START %s service %s on %s" %
                      (service, self.version_, self.machine_))
         func = lambda: self.start_service(service)
         ret = try_repeatedly(func, success=0)
         if ret:
             self.msg_ = "Error starting %s on %s" % (service,
                                                      self.machine_)
             logging.error("STATUS: %s" % self.msg_)
             self.err_ = ret
             return
         self.started_.extend([service])
Esempio n. 53
0
    def ExecuteHTTPGet(self, handler, timeout=120):
        """executes the given http GET command on admin-runner and returns a tuple
    (succeeded_flag, response)
    """
        logging.info("Executing get command: %s" % handler)

        ok = False
        response = None
        signal.signal(signal.SIGALRM, AlarmHandler)
        cmd_url = 'http://%s:%s/%s' % (self.machine, self.port, handler)

        try:
            try:
                signal.alarm(timeout)
                response = urllib.urlopen(cmd_url).read()
                ok = True
            finally:
                signal.alarm(0)
        except IOError:
            pass

        if not ok:
            logging.error("execution of %s resulted in : %s" %
                          (cmd_url, response))
            return (ok, response)
        # remove last two line (ACKGoogle)
        lines = string.split(response, "\n")
        if len(lines) < 2:
            logging.error("Bad admin runner response: %s" % response)
            return (0, response)

        if "NACKgoogle" == string.strip(lines[-2]):
            # some error must have occured..
            logging.error("AdminRunner error: %s" % lines[-2])
            return (0, lines[-1])

        return (ok, string.join(lines[:-2], "\n"))
Esempio n. 54
0
 def Destroy(self):
     """Unmount the mounted directories and remove the dirs under chroot."""
     retry_count = 3
     while retry_count:
         (in_use, process_list) = self.IsUsed()
         if in_use:
             logging.info('Chroot in use, stop processes using it')
             status_stop = self.StopProcesses(process_list)
             retry_count -= 1
             time.sleep(1)
         else:
             break
     (in_use, process_list) = self.IsUsed()
     if in_use:
         logging.error('Unable to umount the directories')
         return (-1, 'Failed to unmount directories')
     return_status = 0
     return_message = 'Success'
     for mount in JAIL_MOUNT_RW_BINDS:
         root_mount = ('%s%s') % (self.__root, mount)
         umount_command = ('%s %s') % (SYS_UMOUNT, root_mount)
         (status_umount, result) = self.__os.Execute(umount_command)
         if status_umount:
             return_status = status_umount
             return_message = result
             logging.error('Umount the bind mount %s failed with %s' %
                           (root_mount, result))
     for mount in JAIL_MOUNT_BINDS:
         root_mount = ('%s%s') % (self.__root, mount)
         umount_command = ('%s %s') % (SYS_UMOUNT, root_mount)
         (status_umount, result) = self.__os.Execute(umount_command)
         if status_umount:
             return_status = status_umount
             return_message = result
             logging.error('Umount the bind mount %s failed with %s' %
                           (root_mount, result))
     return (return_status, return_message)
  def deleteQueue(self, encQueueName):
    """Delete a complete crawl queue or cancel a pending queue."""
    self.cqueuelock.acquire()
    try:
      found = false
      queues = self.listCrawlQueues()
      queueName = urllib.unquote(encQueueName)
      for queue in queues:
        if queueName == queue.queueName:
          found = true
          break
      if not found:
        logging.error('Queue not found')
        return C.CRAWLQUEUE_NAME_NOT_FOUND

      if queue.completeState == C.CRAWLQUEUE_STATUS_PENDING:
        self.joblock.acquire()
        try:
          if self.runningJob.has_key(encQueueName):
            logging.info('About to stop the running job.')
            self.runningJob[encQueueName].join(1)
            del self.runningJob[encQueueName]
          else:
            logging.error('Found a pending crawl queue with no running thread.')
        finally:
          self.joblock.release()
        logging.info('Queue %s incomplete. Canceling.' % encQueueName)
      queues.remove(queue)

      self.RemoveOldQueue(encQueueName)

      if not self.setCrawlQueuesLocked(queues):
        logging.error('Failed to update queue list.')
        return C.CRAWLQUEUE_INTERNAL_ERROR
    finally:
      self.cqueuelock.release()
    return C.CRAWLQUEUE_OK
Esempio n. 56
0
def _ExecuteCommand(cmd,
                    machines=['localhost'],
                    out=None,
                    timeout=15 * 60,
                    num_tries=2,
                    error_filter=None):
    """ Helper file to execute a command multiple times until
  it succeeds.

  Input: cmd: command to run
  timeout: seconds to allow command to run
  machines: machine list on which to execute
  out: list of output lines
  num_tries: number of times to retry command
  error_filter: A function that is called if the cmd execution failed.  The
    takes a list of output lines as input, can filter the errors, and decideds
    if the execution counts as successful.  Returns 0 if execution succeeded,
    1 if the execution failed.
  Output: 0 for success, 1 for failure
  """
    if out == None:
        out = []
    for i in range(0, num_tries):
        if E.execute(machines, cmd, out, timeout) == 0:
            # Command was successful
            return 0
        # Execution failed
        if error_filter and error_filter(out) == 0:
            # Error filter says error is ok, execution counts as success
            logging.error('Cmd %s ignoring error: %s' % (cmd, ''.join(out)))
            return 0
        if i < num_tries - 1:
            logging.error('Cmd %s error: %s, retrying.' % (cmd, ''.join(out)))
        else:
            logging.error('Cmd %s error: %s, failing.' % (cmd, ''.join(out)))
    return 1
Esempio n. 57
0
    def _stop(self, op):
        """ Called by both babysit and stop operations:
    Args:
        op:
            On Master: if op == "stop", kill the adminconsole also.
            If op is babysit, just restart the loop. The loop will overwrite
            the pid file.
            On non-masters: If pid file exists, kill entire process group (include loop and admin console)
            Remove pid file.
    Returns:
        1 always (legacy reasons. TODO(vish): fix later)
    """

        # kill the loop_AdminConsole process and remove the pid file, if
        # the pid in loop_AdminConsole.pid is a pid of a loop_AdminConsole
        # process
        try:
            loopAdminConsole_pid_str = open(self.loopAdminConsole_pid_file,
                                            "r").read()
            if not self.local_machine_is_master:
                logging.info('Killing admin console and loop')
            try:
                while 1:
                    pid_cmdline = open(
                        '/proc/%s/cmdline' % loopAdminConsole_pid_str,
                        'r').read()
                    if (pid_cmdline.find('loop_AdminConsole.py') != -1
                            and len(loopAdminConsole_pid_str) > 0):
                        kill_cmd = "kill -9 %s" % loopAdminConsole_pid_str
                        logging.info("Running: %s" % kill_cmd)
                        os.system(kill_cmd)
                        time.sleep(1)
                    else:
                        break
            except IOError:  # the process does not exist
                pass
            os.system("rm -f %s" % self.loopAdminConsole_pid_file)
        except IOError:  # the pid file does not exist
            pass

        if self.local_machine_is_master:
            try:
                if op == 'stop':
                    self.find_and_kill_loop_adminconsole()
                    self.kill_adminconsole()
                    return 1
            except:
                logging.error(
                    "Stopping loop admin console failed. Exception: %s" %
                    str(sys.exc_info()[:2]))
                return 1
        else:
            # this machine is not master.
            # Just be doubly sure, kill any rogue loops and the admin console using ps
            # even though we have the pid
            # Make really really sure that admin console is dead for 5 s
            while 1:
                self.find_and_kill_loop_adminconsole()
                self.kill_adminconsole()
                time.sleep(5)
                ac_alive = self.is_admin_console_alive(
                    1)  # check if admin console is alive. give it
                # 1s to respond.
                if ac_alive:
                    logging.info(
                        'admin console is not dead. Trying again after 5s.')
                else:
                    # Ok, now we are really sure admin console and loop are dead.
                    break

            return 1
Esempio n. 58
0
  prev_prev_name = prev_prev_frame.f_globals.get('__name__', None)
  if (prev_prev_name != '__main__'
      and not prev_prev_name.endswith('.appcommands')):
    return
  # just in case there's non-trivial stuff happening in __main__
  del tb

  try:
    CallWithExitFix(really_start, is_old_style)
  except SystemExit, e:
    raise
  except Exception, e:
    # If we are using swigged logging, make sure the exception that
    # killed us actually gets logged in the INFO log.
    if logging.is_using_cpp_logging():
      logging.error('Top-level exception: %s' % e)
      logging.error(''.join(traceback.format_exception(*sys.exc_info())))
    raise


def usage(shorthelp=0, writeto_stdout=0, detailed_error=None, exitcode=None):
  """
  Extracts the __doc__ string from the __main__ module and writes it to
  stderr.  If the argument writeto_stdout=1 it is written to stdout.

  Args:
    shorthelp: print only flags from this module, rather than all flags.
    writeto_stdout: write help message to stdout, rather than to stderr.
    detailed_error: additional detail about why usage info was presented.
    exitcode: if set, exit with this status code after writing help.
  """
Esempio n. 59
0
def main(argv):
    argc = len(argv)

    if argc < 10:
        sys.exit(__doc__)

    config = entconfig.EntConfig(argv[0])
    if not config.Load(): sys.exit(__doc__)

    pywrapbase.InitGoogleScript('', [
        'foo',
        '--gfs_aliases=%s' % config.var("GFS_ALIASES"),
        '--bnsresolver_use_svelte=false', '--logtostderr'
    ], 0)
    gfile.Init()

    client = argv[1]
    date_fields = string.split(argv[2], '_')
    date_range = liblog.ParseDateRange(date_fields[0], date_fields[1:])

    withResults = argv[3]
    topCount = argv[4]
    diagnosticTerms = argv[5]

    html_file = argv[6]
    valid_file = argv[7]
    new_html_file = argv[8]
    new_valid_file = argv[9]

    if not date_range:
        sys.exit(__doc__)

    first_date, last_date, printable_date, file_date = date_range

    if last_date.as_int() < first_date.as_int():
        logging.fatal('invalid date range')

    gws_log_dir = liblog.get_gws_log_dir(config)
    collect_dir = liblog.get_collect_dir(config)
    partition_dir = liblog.get_partition_dir(config)
    directory_map_file = liblog.get_directory_map_file(config)

    # Collect logs first from all gws nodes and preprocess
    # logs to make sure logs are up to date.
    all_machines = config.var('MACHINES')
    collect_logs.CollectLogs(all_machines, gws_log_dir, collect_dir)
    preprocess_logs.PartitionLogs(config)

    gws_logs = liblog.FindClientLogFiles(partition_dir, directory_map_file,
                                         client, first_date, last_date)

    # note that collection (client) has been factored into gwslog_dir.
    result = CreateLogReport(config, printable_date, gws_logs,
                             config.var('MAIN_GOOGLE3_DIR'), withResults,
                             topCount, diagnosticTerms, html_file, valid_file,
                             new_html_file, new_valid_file)

    if result == liblog.FAILURE:
        logging.error('CreateLogReport Failed')

    sys.exit(result)
Esempio n. 60
0
    def CreateDefaultQueryExpEntry(self):
        """ create a default query exp entry unconditionally when first startup
    """

        logging.info("Entering CreateDefaultQueryExpEntry")
        names = [
            ("EN", "Google_English_stems"),
            ("FR", "Google_French_stems"),
            ("DE", "Google_German_stems"),
            ("IT", "Google_Italian_stems"),
            ("PT", "Google_Portuguese_stems"),
            ("ES", "Google_Spanish_stems"),
            ("NL", "Google_Dutch_stems"),
        ]
        # Do as much as possible through the Query expansion base, so we are
        # consistent with creating a new entry via the AdminConsole.
        qe_base = query_expansion_handler.QueryExpansionBase(self)

        # This accounts for ENT_STEMS_EN_SOURCE, ... ENT_STEMS_PT_SOURCE
        for (lang, name) in names:
            filename = self.globalParams.var("ENT_STEMS_%s_SOURCE" % lang)

            if filename:
                logging.info(
                    "Copy source stemming file %s to query expansion entry %s"
                    % (filename, name))
                try:
                    contents = open(filename, "r").read()
                except IOError, e:
                    logging.error("Failed to read stems from %s" % filename)
                    return false

                entry = qe_base.ConstructCollectionObject(name)

                params = {}

                # Set default params, and then override them with existing values
                if lang == 'EN':
                    params[C.ENABLED] = 1
                else:
                    params[C.ENABLED] = 0
                params[C.DELETABLE] = 0
                params[C.DOWNLOADABLE] = 1
                params[C.ENTRY_TYPE] = C.QUERY_EXP_FILETYPE_SYNONYMS

                if entry.Exists():
                    for c in (C.ENABLED, C.DELETABLE, C.DOWNLOADABLE,
                              C.ENTRY_TYPE):
                        if entry.has_var(c):
                            params[c] = entry.get_var(c)

                errors = qe_base.Upload(entry, 1, params, 1, contents)

                if errors != validatorlib.VALID_OK:
                    logging.error(
                        "Creating or updating query exp data, errors: " +
                        repr(ferrors))
                    return false

            else:
                logging.info(
                    "Additional stems for %s not created - no filename in config."
                    % lang)