Beispiel #1
0
    def FillMessageRepository(self):
        """This function fills the MessageRepository with random values.
       It could be useful to test performance of the database.
    """
        self.__CreateAuxiliaryLists()
        LogLevels = [
            'ALWAYS', 'INFO', 'VERB', 'DEBUG', 'WARN', 'ERROR', 'EXCEPT',
            'FATAL'
        ]
        initialDate = dateTime()

        for i in range(1, 800):
            limitDate = toString(initialDate - randrange(0, 1680) * hour -
                                 randrange(0, 60) * minute -
                                 randrange(0, 60) * second)
            message = tupleToMessage([
                self.systemNames[randrange(0, 5)], LogLevels[randrange(0, 8)],
                limitDate, self.fixedMessages[randrange(0, 6)],
                'variable text %s' % randrange(0, 6), '',
                self.subSystemNames[randrange(0,
                                              5)], self.sites[randrange(0, 5)]
            ])
            userId = randrange(0, 12)
            result = self.insertMessageIntoDB(message, self.users[userId][0],
                                              self.users[userId][1],
                                              self.clientIPs[randrange(0, 20)])
            if not result['OK']:
                print result['Value']
Beispiel #2
0
    def getProduction(self, productionID, printOutput=False):
        """Returns the metadata associated with a given production ID. Protects against
       LFN: being prepended and different types of production ID.
    """
        if not isinstance(productionID, (int, long, str)):
            return self._errorReport(
                'Expected string, long or int for production ID')

        result = self.transformationClient.getTransformation(int(productionID))
        if not result['OK']:
            return result

        # to fix TODO
        if printOutput:
            adj = self.prodAdj
            prodInfo = result['Value']
            top = ''
            for i in self.prodHeaders.itervalues():
                top += i.ljust(adj)
            message = ['ProductionID'.ljust(adj) + top + '\n']
            # very painful to make this consistent, better improved first on the server side
            productionID = str(productionID)
            info = productionID.ljust(adj) + prodInfo['Status'].ljust(adj) + prodInfo['Type'].ljust(adj) +\
                prodInfo['AgentType'].ljust(adj) + toString(prodInfo['CreationDate']).ljust(adj) +\
                prodInfo['TransformationName'].ljust(adj)
            message.append(info)
            print '\n'.join(message)
        return S_OK(result['Value'])
Beispiel #3
0
    def getProdJobMetadata(self,
                           productionID,
                           status=None,
                           minorStatus=None,
                           site=None):
        """Function to get the WMS job metadata for selected fields. Given a production ID will return
       the current WMS status information for all jobs in that production starting from the creation
       date.
    """
        result = self.transformationClient.getTransformationParameters(
            long(productionID), ['CreationDate'])
        if not result['OK']:
            self.log.warn(
                'Problem getting production metadata for ID %s:\n%s' %
                (productionID, result))
            return result

        creationDate = toString(result['Value']).split()[0]
        result = self.selectProductionJobs(productionID,
                                           status=status,
                                           minorStatus=minorStatus,
                                           site=site,
                                           date=creationDate)
        if not result['OK']:
            self.log.warn('Problem selecting production jobs for ID %s:\n%s' %
                          (productionID, result))
            return result

        jobsList = result['Value']
        return self.status(jobsList)
    def execute(self):
        """ The main agent execution method
    """
        limitDate = toString(dateTime() - self.period)
        limitDate = limitDate[:limitDate.find('.')]

        commonString = 'FROM MessageRepository WHERE messageTime <'
        cmd = "SELECT count(*) %s '%s'" % (commonString, limitDate)
        result = self.SystemLoggingDB._query(cmd)
        if not result['OK']:
            return result
        recordsToErase = result['Value'][0][0]

        if recordsToErase == 0:
            self.log.info('No records to erase')
            return S_OK('No records to erase')

        cmd = "DELETE LOW_PRIORITY %s '%s'" % (commonString, limitDate)
        result = self.SystemLoggingDB._update(cmd)
        if not result['OK']:
            self.log.error('Could not erase the requested records',
                           'those older than %s' % limitDate)
            return result

        self.log.info('%s records have been erased' % recordsToErase)
        return result
Beispiel #5
0
  def FillMessageRepository(self):
    """This function fills the MessageRepository with random values.
       It could be useful to test performance of the database.
    """
    self.__CreateAuxiliaryLists()
    LogLevels = [ 'ALWAYS' , 'INFO', 'VERB', 'DEBUG', 'WARN',
                  'ERROR', 'EXCEPT', 'FATAL' ]
    initialDate=dateTime()

    for _i in range( 1, 800 ):
      limitDate = toString( initialDate - randrange(0,1680) * hour -
                            randrange( 0, 60) * minute -
                            randrange( 0, 60) * second )
      message = tupleToMessage ( [ self.systemNames[ randrange( 0, 5 ) ],
                          LogLevels[ randrange( 0, 8 ) ], limitDate,
                          self.fixedMessages[ randrange( 0, 6 ) ],
                          'variable text %s' % randrange( 0, 6 ), '',
                          self.subSystemNames[ randrange( 0, 5 ) ],
                          self.sites[ randrange( 0, 5 ) ] ] )
      userId = randrange( 0, 12 )
      result = self.insertMessageIntoDB( message, self.users[ userId ][ 0 ],
                                         self.users[ userId ][ 1 ],
                                         self.clientIPs[ randrange( 0, 20 ) ] )
      if not result['OK']:
        print result['Value']
  def execute(self):
    """ The main agent execution method
    """
    limitDate = toString(dateTime() - self.period)
    limitDate = limitDate[:limitDate.find('.')]

    commonString = 'FROM MessageRepository WHERE messageTime <'
    cmd = "SELECT count(*) %s '%s'" % (commonString, limitDate)
    result = self.SystemLoggingDB._query(cmd)
    if not result['OK']:
      return result
    recordsToErase = result['Value'][0][0]

    if recordsToErase == 0:
      self.log.info('No records to erase')
      return S_OK('No records to erase')

    cmd = "DELETE LOW_PRIORITY %s '%s'" % (commonString, limitDate)
    result = self.SystemLoggingDB._update(cmd)
    if not result['OK']:
      self.log.error('Could not erase the requested records',
                     'those older than %s' % limitDate)
      return result

    self.log.info('%s records have been erased' % recordsToErase)
    return result
Beispiel #7
0
  def test_addAndRemove(self):
    """ Some test cases
    """

    systemName = 'TestSystem'
    subSystemName = 'TestSubSystem'
    level = 10
    time = toString()
    msgTest = 'Hello'
    variableText = time
    frameInfo = ""
    message = tupleToMessage((systemName, level, time, msgTest, variableText, frameInfo, subSystemName))
    site = 'somewehere'
    longSite = 'somewehere1234567890123456789012345678901234567890123456789012345678901234567890'
    nodeFQDN = '127.0.0.1'
    userDN = 'Yo'
    userGroup = 'Us'
    remoteAddress = 'elsewhere'

    records = 10

    db = SystemLoggingDB()
    res = db._connect()
    self.assertTrue(res['OK'])

    gLogger.info('\n Inserting some records\n')
    for k in xrange(records):
      result = db.insertMessage(message, site, nodeFQDN, userDN, userGroup, remoteAddress)
      self.assertTrue(result['OK'])
      self.assertEqual(result['lastRowId'], k + 1)
      self.assertEqual(result['Value'], 1)

    result = db._queryDB(showFieldList=['SiteName'])
    self.assertTrue(result['OK'])
    self.assertEqual(result['Value'][0][0], site)

    result = db._queryDB(showFieldList=['SystemName'])
    self.assertTrue(result['OK'])
    self.assertEqual(result['Value'][0][0], systemName)

    result = db._queryDB(showFieldList=['SubSystemName'])
    self.assertTrue(result['OK'])
    self.assertEqual(result['Value'][0][0], subSystemName)

    result = db._queryDB(showFieldList=['OwnerGroup'])
    self.assertTrue(result['OK'])
    self.assertEqual(result['Value'][0][0], userGroup)

    result = db._queryDB(showFieldList=['FixedTextString'])
    self.assertTrue(result['OK'])
    self.assertEqual(result['Value'][0][0], msgTest)

    result = db._queryDB(showFieldList=['VariableText', 'SiteName'], count=True, groupColumn='VariableText')
    self.assertTrue(result['OK'])
    self.assertEqual(result['Value'][0][1], site)
    self.assertEqual(result['Value'][0][2], records)

    result = db.insertMessage(message, longSite, nodeFQDN, userDN, userGroup, remoteAddress)
    self.assertFalse(result['OK'])
Beispiel #8
0
  def test_addAndRemove(self):
    """ Some test cases
    """

    systemName = 'TestSystem'
    subSystemName = 'TestSubSystem'
    level = 10
    time = toString()
    msgTest = 'Hello'
    variableText = time
    frameInfo = ""
    message = tupleToMessage((systemName, level, time, msgTest, variableText, frameInfo, subSystemName))
    site = 'somewehere'
    longSite = 'somewehere1234567890123456789012345678901234567890123456789012345678901234567890'
    nodeFQDN = '127.0.0.1'
    userDN = 'Yo'
    userGroup = 'Us'
    remoteAddress = 'elsewhere'

    records = 10

    db = SystemLoggingDB()
    res = db._connect()
    self.assertTrue(res['OK'])

    gLogger.info('\n Inserting some records\n')
    for k in xrange(records):
      result = db.insertMessage(message, site, nodeFQDN, userDN, userGroup, remoteAddress)
      self.assertTrue(result['OK'])
      self.assertEqual(result['lastRowId'], k + 1)
      self.assertEqual(result['Value'], 1)

    result = db._queryDB(showFieldList=['SiteName'])
    self.assertTrue(result['OK'])
    self.assertEqual(result['Value'][0][0], site)

    result = db._queryDB(showFieldList=['SystemName'])
    self.assertTrue(result['OK'])
    self.assertEqual(result['Value'][0][0], systemName)

    result = db._queryDB(showFieldList=['SubSystemName'])
    self.assertTrue(result['OK'])
    self.assertEqual(result['Value'][0][0], subSystemName)

    result = db._queryDB(showFieldList=['OwnerGroup'])
    self.assertTrue(result['OK'])
    self.assertEqual(result['Value'][0][0], userGroup)

    result = db._queryDB(showFieldList=['FixedTextString'])
    self.assertTrue(result['OK'])
    self.assertEqual(result['Value'][0][0], msgTest)

    result = db._queryDB(showFieldList=['VariableText', 'SiteName'], count=True, groupColumn='VariableText')
    self.assertTrue(result['OK'])
    self.assertEqual(result['Value'][0][1], site)
    self.assertEqual(result['Value'][0][2], records)

    result = db.insertMessage(message, longSite, nodeFQDN, userDN, userGroup, remoteAddress)
    self.assertFalse(result['OK'])
Beispiel #9
0
def main():
    Script.registerSwitch("f:", "File=",
                          "Get status for jobs with IDs from the file")
    Script.registerSwitch("g:", "JobGroup=",
                          "Get status for jobs in the given group")
    # Registering arguments will automatically add their description to the help menu
    Script.registerArgument(["JobID:    DIRAC Job ID"], mandatory=False)
    sws, args = Script.parseCommandLine(ignoreErrors=True)

    from DIRAC import exit as DIRACExit
    from DIRAC.Core.Utilities.Time import toString, date, day
    from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments

    dirac = Dirac()
    exitCode = 0

    jobs = []
    for key, value in sws:
        if key.lower() in ("f", "file"):
            if os.path.exists(value):
                jFile = open(value)
                jobs += jFile.read().split()
                jFile.close()
        elif key.lower() in ("g", "jobgroup"):
            jobDate = toString(date() - 30 * day)
            # Choose jobs no more than 30 days old
            result = dirac.selectJobs(jobGroup=value, date=jobDate)
            if not result["OK"]:
                print("Error:", result["Message"])
                DIRACExit(-1)
            jobs += result["Value"]

    if len(args) < 1 and not jobs:
        Script.showHelp(exitCode=1)

    if len(args) > 0:
        jobs += parseArguments(args)

    result = dirac.getJobStatus(jobs)
    if result["OK"]:
        for job in result["Value"]:
            print("JobID=" + str(job), end=" ")
            for status in result["Value"][job].items():
                print("%s=%s;" % status, end=" ")
            print()
    else:
        exitCode = 2
        print("ERROR: %s" % result["Message"])

    DIRACExit(exitCode)
Beispiel #10
0
def main():
    Script.registerSwitch("f:", "File=",
                          "Get status for jobs with IDs from the file")
    Script.registerSwitch("g:", "JobGroup=",
                          "Get status for jobs in the given group")

    Script.parseCommandLine(ignoreErrors=True)
    args = Script.getPositionalArgs()

    from DIRAC import exit as DIRACExit
    from DIRAC.Core.Utilities.Time import toString, date, day
    from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments

    dirac = Dirac()
    exitCode = 0

    jobs = []
    for key, value in Script.getUnprocessedSwitches():
        if key.lower() in ('f', 'file'):
            if os.path.exists(value):
                jFile = open(value)
                jobs += jFile.read().split()
                jFile.close()
        elif key.lower() in ('g', 'jobgroup'):
            jobDate = toString(date() - 30 * day)
            # Choose jobs no more than 30 days old
            result = dirac.selectJobs(jobGroup=value, date=jobDate)
            if not result['OK']:
                print("Error:", result['Message'])
                DIRACExit(-1)
            jobs += result['Value']

    if len(args) < 1 and not jobs:
        Script.showHelp(exitCode=1)

    if len(args) > 0:
        jobs += parseArguments(args)

    result = dirac.getJobStatus(jobs)
    if result['OK']:
        for job in result['Value']:
            print('JobID=' + str(job), end=' ')
            for status in result['Value'][job].items():
                print('%s=%s;' % status, end=' ')
            print()
    else:
        exitCode = 2
        print("ERROR: %s" % result['Message'])

    DIRACExit(exitCode)
Beispiel #11
0
def main():
    Script.registerSwitch("f:", "File=",
                          "Get output for jobs with IDs from the file")
    Script.registerSwitch("g:", "JobGroup=",
                          "Get output for jobs in the given group")
    # Registering arguments will automatically add their description to the help menu
    Script.registerArgument(["JobID:    DIRAC Job ID"], mandatory=False)
    sws, args = Script.parseCommandLine(ignoreErrors=True)

    import DIRAC
    from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
    from DIRAC.Core.Utilities.Time import toString, date, day

    dirac = Dirac()

    jobs = []
    for sw, value in sws:
        if sw.lower() in ("f", "file"):
            if os.path.exists(value):
                jFile = open(value)
                jobs += jFile.read().split()
                jFile.close()
        elif sw.lower() in ("g", "jobgroup"):
            group = value
            jobDate = toString(date() - 30 * day)
            result = dirac.selectJobs(jobGroup=value, date=jobDate)
            if not result["OK"]:
                if "No jobs selected" not in result["Message"]:
                    print("Error:", result["Message"])
                    DIRAC.exit(-1)
            else:
                jobs += result["Value"]

    for arg in parseArguments(args):
        jobs.append(arg)

    if not jobs:
        print("Warning: no jobs selected")
        Script.showHelp()
        DIRAC.exit(0)

    result = dirac.deleteJob(jobs)
    if result["OK"]:
        print("Deleted jobs %s" % ",".join([str(j) for j in result["Value"]]))
        exitCode = 0
    else:
        print(result["Message"])
        exitCode = 2

    DIRAC.exit(exitCode)
Beispiel #12
0
def main():
    Script.registerSwitch("f:", "File=",
                          "Get output for jobs with IDs from the file")
    Script.registerSwitch("g:", "JobGroup=",
                          "Get output for jobs in the given group")

    Script.parseCommandLine(ignoreErrors=True)
    args = Script.getPositionalArgs()

    import DIRAC
    from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
    from DIRAC.Core.Utilities.Time import toString, date, day
    dirac = Dirac()

    jobs = []
    for sw, value in Script.getUnprocessedSwitches():
        if sw.lower() in ('f', 'file'):
            if os.path.exists(value):
                jFile = open(value)
                jobs += jFile.read().split()
                jFile.close()
        elif sw.lower() in ('g', 'jobgroup'):
            group = value
            jobDate = toString(date() - 30 * day)
            result = dirac.selectJobs(jobGroup=value, date=jobDate)
            if not result['OK']:
                if "No jobs selected" not in result['Message']:
                    print("Error:", result['Message'])
                    DIRAC.exit(-1)
            else:
                jobs += result['Value']

    for arg in parseArguments(args):
        jobs.append(arg)

    if not jobs:
        print("Warning: no jobs selected")
        Script.showHelp()
        DIRAC.exit(0)

    result = dirac.deleteJob(jobs)
    if result['OK']:
        print('Deleted jobs %s' % ','.join([str(j) for j in result['Value']]))
        exitCode = 0
    else:
        print(result['Message'])
        exitCode = 2

    DIRAC.exit(exitCode)
Beispiel #13
0
    def getProductionLoggingInfo(self, productionID, printOutput=False):
        """The logging information for the given production is returned.  This includes
       the operation performed, any messages associated with the operation and the
       DN of the production manager performing it.
    """
        if not isinstance(productionID, (int, long, str)):
            return self._errorReport(
                'Expected string, long or int for production ID')

        result = self.transformationClient.getTransformationLogging(
            int(productionID))
        if not result['OK']:
            self.log.warn(
                'Could not get transformation logging information for productionID %s'
                % (productionID))
            return result
        if not result['Value']:
            self.log.warn('No logging information found for productionID %s' %
                          (productionID))
            return result

        if not printOutput:
            return result

        infoM = 'ProdID'.ljust(int(0.5 * self.prodAdj)) + 'Message'.ljust(3 * self.prodAdj) +\
            'DateTime [UTC]'.ljust(self.prodAdj) + 'AuthorCN'.ljust(2 * self.prodAdj)
        message = [infoM]
        for line in result['Value']:
            infoL = str(line['TransformationID']).ljust(int(0.5 * self.prodAdj)) +\
                line['Message'].ljust(3 * self.prodAdj) + toString(line['MessageDate']).ljust(self.prodAdj) +\
                line['AuthorDN'].split('/')[-1].ljust(2 * self.prodAdj)
            message.append(infoL)

        print '\nLogging summary for productionID ' + str(
            productionID) + '\n\n' + '\n'.join(message)

        return result
errorList = []

outputDir = None
group = None
jobs = []
for sw, value in Script.getUnprocessedSwitches():
  if sw in ( 'D', 'Dir' ):
    outputDir = value
  elif sw.lower() in ( 'f', 'file' ):
    if os.path.exists( value ):
      jFile = open( value )
      jobs += jFile.read().split()
      jFile.close()
  elif sw.lower() in ( 'g', 'jobgroup' ):
    group = value
    jobDate = toString( date() - 30 * day )

    # Choose jobs in final state, no more than 30 days old
    result = dirac.selectJobs( jobGroup = value, date = jobDate, status = 'Done' )
    if not result['OK']:
      if not "No jobs selected" in result['Message']:
        print "Error:", result['Message']
        DIRAC.exit( -1 )
    else:
      jobs += result['Value']
    result = dirac.selectJobs( jobGroup = value, date = jobDate, status = 'Failed' )
    if not result['OK']:
      if not "No jobs selected" in result['Message']:
        print "Error:", result['Message']
        DIRAC.exit( -1 )
    else:
def main():
    Script.registerSwitch(
        "C", "country",
        "Sort site names by country postfix (i.e. LCG.IHEP.cn, LCG.IN2P3.fr, LCG.IHEP.su)",
        sortBy)
    Script.registerSwitch("R", "reverse", "Reverse the sort order", isReverse)
    # Registering arguments will automatically add their description to the help menu
    Script.registerArgument([
        "Section: Name of the subsection in '/Resources/Sites/' for sort (i.e. LCG DIRAC)"
    ],
                            mandatory=False)

    Script.parseCommandLine(ignoreErrors=True)
    args = Script.getPositionalArgs()

    result = getProxyInfo()
    if not result["OK"]:
        gLogger.error("Failed to get proxy information", result["Message"])
        DIRACExit(2)
    proxy = result["Value"]
    if proxy["secondsLeft"] < 1:
        gLogger.error("Your proxy has expired, please create new one")
        DIRACExit(2)
    group = proxy["group"]
    if "CSAdministrator" not in getPropertiesForGroup(group):
        gLogger.error(
            "You must be CSAdministrator user to execute this script")
        gLogger.notice(
            "Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'"
        )
        DIRACExit(2)

    cs = CSAPI()
    result = cs.getCurrentCFG()
    if not result["OK"]:
        gLogger.error("Failed to get copy of CS", result["Message"])
        DIRACExit(2)
    cfg = result["Value"]

    if not cfg.isSection("Resources"):
        gLogger.error("Section '/Resources' is absent in CS")
        DIRACExit(2)

    if not cfg.isSection("Resources/Sites"):
        gLogger.error("Subsection '/Resources/Sites' is absent in CS")
        DIRACExit(2)

    if args and len(args) > 0:
        resultList = args[:]
    else:
        resultList = cfg["Resources"]["Sites"].listSections()

    hasRun = False
    isDirty = False
    for i in resultList:
        if not cfg.isSection("Resources/Sites/%s" % i):
            gLogger.error("Subsection /Resources/Sites/%s does not exists" % i)
            continue
        hasRun = True
        if SORTBYNAME:
            dirty = cfg["Resources"]["Sites"][i].sortAlphabetically(
                ascending=not REVERSE)
        else:
            dirty = cfg["Resources"]["Sites"][i].sortByKey(key=country,
                                                           reverse=REVERSE)
        if dirty:
            isDirty = True

    if not hasRun:
        gLogger.notice(
            "Failed to find suitable subsections with site names to sort")
        DIRACExit(0)

    if not isDirty:
        gLogger.notice("Nothing to do, site names are already sorted")
        DIRACExit(0)

    timestamp = toString(dateTime())
    stamp = "Site names are sorted by %s script at %s" % (Script.scriptName,
                                                          timestamp)
    cs.setOptionComment("/Resources/Sites", stamp)

    result = cs.commit()
    if not result["OK"]:
        gLogger.error("Failed to commit changes to CS", result["Message"])
        DIRACExit(2)
    gLogger.notice("Site names are sorted and committed to CS")
    DIRACExit(0)
Beispiel #16
0
if __name__ == "__main__":

    from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
    from DIRAC.Core.Utilities.Time import toString, date, day
    dirac = Dirac()

    jobs = []
    for sw, value in Script.getUnprocessedSwitches():
        if sw.lower() in ('f', 'file'):
            if os.path.exists(value):
                jFile = open(value)
                jobs += jFile.read().split()
                jFile.close()
        elif sw.lower() in ('g', 'jobgroup'):
            group = value
            jobDate = toString(date() - 30 * day)
            result = dirac.selectJobs(jobGroup=value, date=jobDate)
            if not result['OK']:
                if "No jobs selected" not in result['Message']:
                    print("Error:", result['Message'])
                    DIRAC.exit(-1)
            else:
                jobs += result['Value']

    for arg in parseArguments(args):
        jobs.append(arg)

    if not jobs:
        print("Warning: no jobs selected")
        Script.showHelp()
        DIRAC.exit(0)
    hasRun = True
    if SORTBYNAME:
        dirty = cfg["Resources"]["Sites"][i].sortAlphabetically(
            ascending=not REVERSE)
    else:
        dirty = cfg["Resources"]["Sites"][i].sortByKey(key=country,
                                                       reverse=REVERSE)
    if dirty:
        isDirty = True

if not hasRun:
    gLogger.notice(
        "Failed to find suitable subsections with site names to sort")
    DIRAC.exit(0)

if not isDirty:
    gLogger.notice("Nothing to do, site names are already sorted")
    DIRAC.exit(0)

timestamp = toString(dateTime())
stamp = "Site names are sorted by %s script at %s" % (Script.scriptName,
                                                      timestamp)
cs.setOptionComment("/Resources/Sites", stamp)

result = cs.commit()
if not result["OK"]:
    gLogger.error("Failed to commit changes to CS", result["Message"])
    DIRAC.exit(2)
gLogger.notice("Site names are sorted and committed to CS")
DIRAC.exit(0)
  def execute(self):
    """ The main agent execution method
    """
    limitDate = date() - self._period
    tableList = ["MessageRepository", "FixedTextMessages", "Systems",
                 "SubSystems"]
    columnsList = ["SystemName", "SubSystemName", "count(*) as entries",
                   "FixedTextString"]
    cmd = "SELECT " + ', '.join(columnsList) + " FROM " \
          + " NATURAL JOIN ".join(tableList) \
          + " WHERE MessageTime > '%s'" % limitDate \
          + " AND LogLevel in ('ERROR','FATAL','EXCEPT')" \
          + " GROUP BY FixedTextID,SystemName,SubSystemName HAVING entries > %s" % self._threshold \
          + " ORDER BY entries DESC LIMIT %i;" % self._limit

    result = self.systemLoggingDB._query(cmd)
    if not result['OK']:
      return result

    messageList = result['Value']

    if messageList == 'None' or messageList == ():
      self.log.warn('The DB query returned an empty result')
      return S_OK()

    mailBody = '\n'
    for message in messageList:
      mailBody = mailBody + "Count: " + str(message[2]) + "\tError: '"\
          + message[3] + "'\tSystem: '" + message[0]\
          + "'\tSubsystem: '" + message[1] + "'\n"

    mailBody = mailBody + "\n\n-------------------------------------------------------\n"\
        + "Please do not reply to this mail. It was automatically\n"\
        + "generated by a Dirac Agent.\n"

    result = self.systemLoggingDB._getDataFromAgentTable(self.agentName)
    self.log.debug(result)
    if not result['OK']:
      errorString = "Could not get the date when the last mail was sent"
      self.log.error(errorString)
      return S_ERROR(errorString)
    else:
      if result['Value']:
        self.log.debug("date value: %s" % fromString(result['Value'][0][0][1:-1]))
        lastMailSentDate = fromString(result['Value'][0][0][1:-1])
      else:
        lastMailSentDate = limitDate - 1 * day
        result = self.systemLoggingDB._insertDataIntoAgentTable(self.agentName, lastMailSentDate)
        if not result['OK']:
          errorString = "Could not insert data into the DB"
          self.log.error(errorString, result['Message'])
          return S_ERROR(errorString + ": " + result['Message'])

    self.log.debug("limitDate: %s\t" % limitDate + "lastMailSentDate: %s\n" % lastMailSentDate)
    if lastMailSentDate > limitDate:
      self.log.info("The previous report was sent less " + " than %s days ago" % self.__days)
      return S_OK()

    dateSent = toString(date())
    self.log.info("The list with the top errors has been sent")

    result = self.systemLoggingDB._insertDataIntoAgentTable(self.agentName, dateSent)
    if not result['OK']:
      errorString = "Could not insert data into the DB"
      self.log.error(errorString, result['Message'])
      return S_ERROR(errorString + ": " + result['Message'])

    result = self.notification.sendMail(self._mailAddress, self._subject,
                                        mailBody)
    if not result['OK']:
      self.log.warn("The notification could not be sent")
      return S_OK()

    return S_OK("The list with the top errors has been sent")
Beispiel #19
0
  def execute(self):
    """ The main agent execution method
    """
    limitDate = date() - self._period
    tableList = ["MessageRepository", "FixedTextMessages", "Systems",
                 "SubSystems"]
    columnsList = ["SystemName", "SubSystemName", "count(*) as entries",
                   "FixedTextString"]
    cmd = "SELECT " + ', '.join(columnsList) + " FROM " \
          + " NATURAL JOIN ".join(tableList) \
          + " WHERE MessageTime > '%s'" % limitDate \
          + " AND LogLevel in ('ERROR','FATAL','EXCEPT')" \
          + " GROUP BY FixedTextID,SystemName,SubSystemName HAVING entries > %s" % self._threshold \
          + " ORDER BY entries DESC LIMIT %i;" % self._limit

    result = self.systemLoggingDB._query(cmd)
    if not result['OK']:
      return result

    messageList = result['Value']

    if messageList == 'None' or messageList == ():
      self.log.warn('The DB query returned an empty result')
      return S_OK()

    mailBody = '\n'
    for message in messageList:
      mailBody = mailBody + "Count: " + str(message[2]) + "\tError: '"\
          + message[3] + "'\tSystem: '" + message[0]\
          + "'\tSubsystem: '" + message[1] + "'\n"

    mailBody = mailBody + "\n\n-------------------------------------------------------\n"\
        + "Please do not reply to this mail. It was automatically\n"\
        + "generated by a Dirac Agent.\n"

    result = self.systemLoggingDB._getDataFromAgentTable(self.agentName)
    self.log.debug(result)
    if not result['OK']:
      errorString = "Could not get the date when the last mail was sent"
      self.log.error(errorString)
      return S_ERROR(errorString)
    else:
      if result['Value']:
        self.log.debug("date value: %s" % fromString(result['Value'][0][0][1:-1]))
        lastMailSentDate = fromString(result['Value'][0][0][1:-1])
      else:
        lastMailSentDate = limitDate - 1 * day
        result = self.systemLoggingDB._insertDataIntoAgentTable(self.agentName, lastMailSentDate)
        if not result['OK']:
          errorString = "Could not insert data into the DB"
          self.log.error(errorString, result['Message'])
          return S_ERROR(errorString + ": " + result['Message'])

    self.log.debug("limitDate: %s\t" % limitDate + "lastMailSentDate: %s\n" % lastMailSentDate)
    if lastMailSentDate > limitDate:
      self.log.info("The previous report was sent less " + " than %s days ago" % self.__days)
      return S_OK()

    dateSent = toString(date())
    self.log.info("The list with the top errors has been sent")

    result = self.systemLoggingDB._insertDataIntoAgentTable(self.agentName, dateSent)
    if not result['OK']:
      errorString = "Could not insert data into the DB"
      self.log.error(errorString, result['Message'])
      return S_ERROR(errorString + ": " + result['Message'])

    result = self.notification.sendMail(self._mailAddress, self._subject,
                                        mailBody)
    if not result['OK']:
      self.log.warn("The notification could not be sent")
      return S_OK()

    return S_OK("The list with the top errors has been sent")
for i in resultList:
    if not cfg.isSection("Resources/Sites/%s" % i):
        gLogger.error("Subsection /Resources/Sites/%s does not exists" % i)
        continue
    hasRun = True
    if SORTBYNAME:
        dirty = cfg["Resources"]["Sites"][i].sortAlphabetically(ascending=not REVERSE)
    else:
        dirty = cfg["Resources"]["Sites"][i].sortByKey(key=country, reverse=REVERSE)
    if dirty:
        isDirty = True

if not hasRun:
    gLogger.notice("Failed to find suitable subsections with site names to sort")
    DIRAC.exit(0)

if not isDirty:
    gLogger.notice("Nothing to do, site names are already sorted")
    DIRAC.exit(0)

timestamp = toString(dateTime())
stamp = "Site names are sorted by %s script at %s" % (Script.scriptName, timestamp)
cs.setOptionComment("/Resources/Sites", stamp)

result = cs.commit()
if not result["OK"]:
    gLogger.error("Failed to commit changes to CS", result["Message"])
    DIRAC.exit(2)
gLogger.notice("Site names are sorted and committed to CS")
DIRAC.exit(0)
Beispiel #21
0
def main():
    Script.registerSwitch("D:", "Dir=", "Store the output in this directory")
    Script.registerSwitch("f:", "File=", "Get output for jobs with IDs from the file")
    Script.registerSwitch("g:", "JobGroup=", "Get output for jobs in the given group")
    # Registering arguments will automatically add their description to the help menu
    Script.registerArgument(["JobID: DIRAC Job ID or a name of the file with JobID per line"], mandatory=False)
    sws, args = Script.parseCommandLine(ignoreErrors=True)

    from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
    from DIRAC.Core.Utilities.Time import toString, date, day
    from DIRAC.Core.Utilities.File import mkDir

    dirac = Dirac()
    exitCode = 0
    errorList = []

    outputDir = None
    group = None
    jobs = []
    for sw, value in sws:
        if sw in ("D", "Dir"):
            outputDir = value
        elif sw.lower() in ("f", "file"):
            if os.path.exists(value):
                jFile = open(value)
                jobs += jFile.read().split()
                jFile.close()
        elif sw.lower() in ("g", "jobgroup"):
            group = value
            jobDate = toString(date() - 30 * day)

            # Choose jobs in final state, no more than 30 days old
            result = dirac.selectJobs(jobGroup=value, date=jobDate, status="Done")
            if not result["OK"]:
                if "No jobs selected" not in result["Message"]:
                    print("Error:", result["Message"])
                    DIRAC.exit(-1)
            else:
                jobs += result["Value"]
            result = dirac.selectJobs(jobGroup=value, date=jobDate, status="Failed")
            if not result["OK"]:
                if "No jobs selected" not in result["Message"]:
                    print("Error:", result["Message"])
                    DIRAC.exit(-1)
            else:
                jobs += result["Value"]

    for arg in parseArguments(args):
        if os.path.isdir(arg):
            print("Output for job %s already retrieved, remove the output directory to redownload" % arg)
        else:
            jobs.append(arg)

    if not jobs:
        print("No jobs selected")
        DIRAC.exit(0)

    if group:
        if outputDir:
            outputDir = os.path.join(outputDir, group)
        else:
            outputDir = group

    if outputDir:
        mkDir(outputDir)
    else:
        outputDir = os.getcwd()

    jobs = [str(job) for job in jobs]
    doneJobs = os.listdir(outputDir)
    todoJobs = [job for job in jobs if job not in doneJobs]

    for job in todoJobs:

        result = dirac.getOutputSandbox(job, outputDir=outputDir)

        jobDir = str(job)
        if outputDir:
            jobDir = os.path.join(outputDir, job)
        if result["OK"]:
            if os.path.exists(jobDir):
                print("Job output sandbox retrieved in %s/" % (jobDir))
        else:
            if os.path.exists("%s" % jobDir):
                shutil.rmtree(jobDir)
            errorList.append((job, result["Message"]))
            exitCode = 2

    for error in errorList:
        print("ERROR %s: %s" % error)

    DIRAC.exit(exitCode)
Beispiel #22
0
def main():
  Script.registerSwitch("D:", "Dir=", "Store the output in this directory")
  Script.registerSwitch("f:", "File=", "Get output for jobs with IDs from the file")
  Script.registerSwitch("g:", "JobGroup=", "Get output for jobs in the given group")

  Script.parseCommandLine(ignoreErrors=True)
  args = Script.getPositionalArgs()

  from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
  from DIRAC.Core.Utilities.Time import toString, date, day
  from DIRAC.Core.Utilities.File import mkDir

  dirac = Dirac()
  exitCode = 0
  errorList = []

  outputDir = None
  group = None
  jobs = []
  for sw, value in Script.getUnprocessedSwitches():
    if sw in ('D', 'Dir'):
      outputDir = value
    elif sw.lower() in ('f', 'file'):
      if os.path.exists(value):
        jFile = open(value)
        jobs += jFile.read().split()
        jFile.close()
    elif sw.lower() in ('g', 'jobgroup'):
      group = value
      jobDate = toString(date() - 30 * day)

      # Choose jobs in final state, no more than 30 days old
      result = dirac.selectJobs(jobGroup=value, date=jobDate, status='Done')
      if not result['OK']:
        if "No jobs selected" not in result['Message']:
          print("Error:", result['Message'])
          DIRAC.exit(-1)
      else:
        jobs += result['Value']
      result = dirac.selectJobs(jobGroup=value, date=jobDate, status='Failed')
      if not result['OK']:
        if "No jobs selected" not in result['Message']:
          print("Error:", result['Message'])
          DIRAC.exit(-1)
      else:
        jobs += result['Value']

  for arg in parseArguments(args):
    if os.path.isdir(arg):
      print("Output for job %s already retrieved, remove the output directory to redownload" % arg)
    else:
      jobs.append(arg)

  if not jobs:
    print("No jobs selected")
    DIRAC.exit(0)

  if group:
    if outputDir:
      outputDir = os.path.join(outputDir, group)
    else:
      outputDir = group

  if outputDir:
    mkDir(outputDir)
  else:
    outputDir = os.getcwd()

  jobs = [str(job) for job in jobs]
  doneJobs = os.listdir(outputDir)
  todoJobs = [job for job in jobs if job not in doneJobs]

  for job in todoJobs:

    result = dirac.getOutputSandbox(job, outputDir=outputDir)

    jobDir = str(job)
    if outputDir:
      jobDir = os.path.join(outputDir, job)
    if result['OK']:
      if os.path.exists(jobDir):
        print('Job output sandbox retrieved in %s/' % (jobDir))
    else:
      if os.path.exists('%s' % jobDir):
        shutil.rmtree(jobDir)
      errorList.append((job, result['Message']))
      exitCode = 2

  for error in errorList:
    print("ERROR %s: %s" % error)

  DIRAC.exit(exitCode)