def aggregateLogSummary():
    """
    Create an aggregate of log summary files, write it in an aggregate log 
    summary file and in the end return the values
    """

    global monitorAggregatorConfig

    # initialize global counters
    global_total={'Current':{},'Entered':{},'Exited':{},'CompletedCounts':{'Sum':{},'Waste':{},'WasteTime':{},'Lasted':{},'JobsNr':{},'JobsDuration':{}}}

    for s in ('Wait','Idle','Running','Held'):
        for k in ['Current','Entered','Exited']:
            global_total[k][s]=0

    for s in ('Completed','Removed'):
        for k in ['Entered']:
            global_total[k][s]=0

    for k in glideFactoryMonitoring.getAllJobTypes():
        for w in ("Waste","WasteTime"):
            el={}
            for t in glideFactoryMonitoring.getAllMillRanges():
                el[t]=0
            global_total['CompletedCounts'][w][k]=el

    el={}
    for t in glideFactoryMonitoring.getAllTimeRanges():
        el[t]=0
    global_total['CompletedCounts']['Lasted']=el

    el={}
    for t in glideFactoryMonitoring.getAllJobRanges():
        el[t]=0
    global_total['CompletedCounts']['JobsNr']=el

    el={}
    # KEL - why is the same el used twice (see above)
    for t in glideFactoryMonitoring.getAllTimeRanges():
        el[t]=0
    global_total['CompletedCounts']['JobsDuration']=el

    global_total['CompletedCounts']['Sum']={'Glideins':0,
                                            'Lasted':0,
                                            'FailedNr':0,
                                            'JobsNr':0,
                                            'JobsLasted':0,
                                            'JobsGoodput':0,
                                            'JobsTerminated':0,
                                            'CondorLasted':0}

    fe_total=copy.deepcopy(global_total) # same as above but for frontend totals
    
    #
    status={'entries':{},'total':global_total}
    status_fe={'frontends':{}} #analogous to above but for frontend totals

    nr_entries=0
    nr_feentries={} #dictionary for nr entries per fe
    for entry in monitorAggregatorConfig.entries:
        # load entry log summary file
        status_fname=os.path.join(os.path.join(monitorAggregatorConfig.monitor_dir,'entry_'+entry),
                                  monitorAggregatorConfig.logsummary_relname)
        try:
            entry_data=xmlParse.xmlfile2dict(status_fname,always_singular_list=['Fraction','TimeRange','Range'])
        except IOError:
            continue # file not found, ignore

        # update entry
        out_data={}
        for frontend in entry_data['frontends'].keys():
            fe_el=entry_data['frontends'][frontend]
            out_fe_el={}
            for k in ['Current','Entered','Exited']:
                out_fe_el[k]={}
                for s in fe_el[k].keys():
                    out_fe_el[k][s]=int(fe_el[k][s])
            out_fe_el['CompletedCounts']={'Waste':{},'WasteTime':{},'Lasted':{},'JobsNr':{},'JobsDuration':{},'Sum':{}}
            for tkey in fe_el['CompletedCounts']['Sum'].keys():
                out_fe_el['CompletedCounts']['Sum'][tkey]=int(fe_el['CompletedCounts']['Sum'][tkey])
            for k in glideFactoryMonitoring.getAllJobTypes():
                for w in ("Waste","WasteTime"):
                    out_fe_el['CompletedCounts'][w][k]={}
                    for t in glideFactoryMonitoring.getAllMillRanges():
                        out_fe_el['CompletedCounts'][w][k][t]=int(fe_el['CompletedCounts'][w][k][t]['val'])
            for t in glideFactoryMonitoring.getAllTimeRanges():
                out_fe_el['CompletedCounts']['Lasted'][t]=int(fe_el['CompletedCounts']['Lasted'][t]['val'])
            out_fe_el['CompletedCounts']['JobsDuration']={}
            for t in glideFactoryMonitoring.getAllTimeRanges():
                out_fe_el['CompletedCounts']['JobsDuration'][t]=int(fe_el['CompletedCounts']['JobsDuration'][t]['val'])
            for t in glideFactoryMonitoring.getAllJobRanges():
                out_fe_el['CompletedCounts']['JobsNr'][t]=int(fe_el['CompletedCounts']['JobsNr'][t]['val'])
            out_data[frontend]=out_fe_el

        status['entries'][entry]={'frontends':out_data}

        # update total
        if entry_data.has_key('total'):
            nr_entries+=1
            local_total={}

            for k in ['Current','Entered','Exited']:
                local_total[k]={}
                for s in global_total[k].keys():
                    local_total[k][s]=int(entry_data['total'][k][s])
                    global_total[k][s]+=int(entry_data['total'][k][s])
            local_total['CompletedCounts']={'Sum':{},'Waste':{},'WasteTime':{},'Lasted':{},'JobsNr':{},'JobsDuration':{}}
            for tkey in entry_data['total']['CompletedCounts']['Sum'].keys():
                local_total['CompletedCounts']['Sum'][tkey]=int(entry_data['total']['CompletedCounts']['Sum'][tkey])
                global_total['CompletedCounts']['Sum'][tkey]+=int(entry_data['total']['CompletedCounts']['Sum'][tkey])
            for k in glideFactoryMonitoring.getAllJobTypes():
                for w in ("Waste","WasteTime"):
                    local_total['CompletedCounts'][w][k]={}
                    for t in glideFactoryMonitoring.getAllMillRanges():
                        local_total['CompletedCounts'][w][k][t]=int(entry_data['total']['CompletedCounts'][w][k][t]['val'])
                        global_total['CompletedCounts'][w][k][t]+=int(entry_data['total']['CompletedCounts'][w][k][t]['val'])

            for t in glideFactoryMonitoring.getAllTimeRanges():
                local_total['CompletedCounts']['Lasted'][t]=int(entry_data['total']['CompletedCounts']['Lasted'][t]['val'])
                global_total['CompletedCounts']['Lasted'][t]+=int(entry_data['total']['CompletedCounts']['Lasted'][t]['val'])
            local_total['CompletedCounts']['JobsDuration']={}
            for t in glideFactoryMonitoring.getAllTimeRanges():
                local_total['CompletedCounts']['JobsDuration'][t]=int(entry_data['total']['CompletedCounts']['JobsDuration'][t]['val'])
                global_total['CompletedCounts']['JobsDuration'][t]+=int(entry_data['total']['CompletedCounts']['JobsDuration'][t]['val'])

            for t in glideFactoryMonitoring.getAllJobRanges():
                local_total['CompletedCounts']['JobsNr'][t]=int(entry_data['total']['CompletedCounts']['JobsNr'][t]['val'])
                global_total['CompletedCounts']['JobsNr'][t]+=int(entry_data['total']['CompletedCounts']['JobsNr'][t]['val'])

            status['entries'][entry]['total']=local_total
        
        # update frontends
        for fe in out_data:
            #compare each to the list of fe's accumulated so far
            if not (fe in status_fe['frontends']):
                status_fe['frontends'][fe]={}
            if not (fe in nr_feentries):
                nr_feentries[fe]=1 #already found one
            else:
                nr_feentries[fe]+=1

            # sum them up
            sumDictInt(out_data[fe],status_fe['frontends'][fe])

    # Write xml files
    # To do - Igor: Consider adding status_fe to the XML file
    updated=time.time()
    xml_str=('<?xml version="1.0" encoding="ISO-8859-1"?>\n\n'+
             '<glideFactoryLogSummary>\n'+
             xmlFormat.time2xml(updated,"updated", indent_tab=xmlFormat.DEFAULT_TAB,leading_tab=xmlFormat.DEFAULT_TAB)+"\n"+
             xmlFormat.dict2string(status["entries"],dict_name="entries",el_name="entry",
                                   subtypes_params={"class":{"dicts_params":{"frontends":{"el_name":"frontend",
                                                                                          "subtypes_params":{"class":{'subclass_params':{'CompletedCounts':glideFactoryMonitoring.get_completed_stats_xml_desc()}}}}},
                                                             "subclass_params":{"total":{"subclass_params":{'CompletedCounts':glideFactoryMonitoring.get_completed_stats_xml_desc()}}}
                                                             }
                                                    },
                                   leading_tab=xmlFormat.DEFAULT_TAB)+"\n"+
             xmlFormat.class2string(status["total"],inst_name="total",subclass_params={'CompletedCounts':glideFactoryMonitoring.get_completed_stats_xml_desc()},leading_tab=xmlFormat.DEFAULT_TAB)+"\n"+
             "</glideFactoryLogSummary>\n")
    glideFactoryMonitoring.monitoringConfig.write_file(monitorAggregatorConfig.logsummary_relname,xml_str)

    # Write rrds
    writeLogSummaryRRDs("total",status["total"])
    
    # Frontend total rrds across all factories
    for fe in status_fe['frontends']:
        writeLogSummaryRRDs("total/%s"%("frontend_"+fe),status_fe['frontends'][fe])

    return status
def verifyRRD(fix_rrd=False):
    """
    Go through all known monitoring rrds and verify that they
    match existing schema (could be different if an upgrade happened)
    If fix_rrd is true, then also attempt to add any missing attributes.
    """
    global rrd_problems_found
    global monitorAggregatorConfig
    dir=monitorAggregatorConfig.monitor_dir
    total_dir=os.path.join(dir,"total")
   
    status_dict={}
    completed_stats_dict={}
    completed_waste_dict={}
    counts_dict={}
 
    # initialize the RRD dictionaries to match the current schema for verification
    for tp in status_attributes.keys():
        if tp in type_strings.keys():
            tp_str=type_strings[tp]
            attributes_tp=status_attributes[tp]
            for a in attributes_tp:
                status_dict["%s%s"%(tp_str,a)]=None

    for jobrange in glideFactoryMonitoring.getAllJobRanges():
        completed_stats_dict["JobsNr_%s"%(jobrange)]=None
    for timerange in glideFactoryMonitoring.getAllTimeRanges():
        completed_stats_dict["Lasted_%s"%(timerange)]=None
        completed_stats_dict["JobsLasted_%s"%(timerange)]=None

    for jobtype in glideFactoryMonitoring.getAllJobTypes():
        for timerange in glideFactoryMonitoring.getAllMillRanges():
            completed_waste_dict["%s_%s"%(jobtype,timerange)]=None

    for jobtype in ('Entered','Exited','Status'):
        for jobstatus in ('Wait','Idle','Running','Held'):
            counts_dict["%s%s"%(jobtype,jobstatus)]=None
    for jobstatus in ('Completed','Removed'):
            counts_dict["%s%s"%('Entered',jobstatus)]=None

    verifyHelper(os.path.join(total_dir,
        "Status_Attributes.rrd"),status_dict, fix_rrd)
    verifyHelper(os.path.join(total_dir,
        "Log_Completed.rrd"),
        glideFactoryMonitoring.getLogCompletedDefaults(), fix_rrd)
    verifyHelper(os.path.join(total_dir,
        "Log_Completed_Stats.rrd"),completed_stats_dict, fix_rrd)
    verifyHelper(os.path.join(total_dir,
        "Log_Completed_WasteTime.rrd"),completed_waste_dict, fix_rrd)
    verifyHelper(os.path.join(total_dir,
        "Log_Counts.rrd"),counts_dict, fix_rrd)
    for filename in os.listdir(dir):
        if filename[:6]=="entry_":
            entrydir=os.path.join(dir,filename)
            for subfilename in os.listdir(entrydir):
                if subfilename[:9]=="frontend_":
                    current_dir=os.path.join(entrydir,subfilename)
                    verifyHelper(os.path.join(current_dir,
                        "Status_Attributes.rrd"),status_dict, fix_rrd)
                    verifyHelper(os.path.join(current_dir,
                        "Log_Completed.rrd"),
                        glideFactoryMonitoring.getLogCompletedDefaults(),fix_rrd)
                    verifyHelper(os.path.join(current_dir,
                        "Log_Completed_Stats.rrd"),completed_stats_dict,fix_rrd)
                    verifyHelper(os.path.join(current_dir,
                        "Log_Completed_WasteTime.rrd"),
                        completed_waste_dict,fix_rrd)
                    verifyHelper(os.path.join(current_dir,
                        "Log_Counts.rrd"),counts_dict,fix_rrd)
    return not rrd_problems_found
Example #3
0
def verifyRRD(fix_rrd=False):
    """
    Go through all known monitoring rrds and verify that they
    match existing schema (could be different if an upgrade happened)
    If fix_rrd is true, then also attempt to add any missing attributes.
    """
    global rrd_problems_found
    global monitorAggregatorConfig
    mon_dir = monitorAggregatorConfig.monitor_dir

    status_dict = {}
    completed_stats_dict = {}
    completed_waste_dict = {}
    counts_dict = {}

    # initialize the RRD dictionaries to match the current schema for verification
    for tp in status_attributes.keys():
        if tp in type_strings.keys():
            tp_str = type_strings[tp]
            attributes_tp = status_attributes[tp]
            for a in attributes_tp:
                status_dict["%s%s" % (tp_str, a)] = None

    for jobrange in glideFactoryMonitoring.getAllJobRanges():
        completed_stats_dict["JobsNr_%s" % (jobrange, )] = None
    for timerange in glideFactoryMonitoring.getAllTimeRanges():
        completed_stats_dict["Lasted_%s" % (timerange, )] = None
        completed_stats_dict["JobsLasted_%s" % (timerange, )] = None

    for jobtype in glideFactoryMonitoring.getAllJobTypes():
        for timerange in glideFactoryMonitoring.getAllMillRanges():
            completed_waste_dict["%s_%s" % (jobtype, timerange)] = None

    for jobtype in ('Entered', 'Exited', 'Status'):
        for jobstatus in ('Wait', 'Idle', 'Running', 'Held'):
            counts_dict["%s%s" % (jobtype, jobstatus)] = None
    for jobstatus in ('Completed', 'Removed'):
        counts_dict["%s%s" % ('Entered', jobstatus)] = None
    # FROM: lib2to3.fixes.fix_ws_comma
    #         completed_waste_dict["%s_%s"%(jobtype, timerange)]=None
    #
    # for jobtype in ('Entered', 'Exited', 'Status'):
    #     for jobstatus in ('Wait', 'Idle', 'Running', 'Held'):
    #         counts_dict["%s%s"%(jobtype, jobstatus)]=None
    # for jobstatus in ('Completed', 'Removed'):
    #         counts_dict["%s%s"%('Entered', jobstatus)]=None
    #
    # verifyHelper(os.path.join(total_dir,
    #     "Status_Attributes.rrd"), status_dict, fix_rrd)
    # verifyHelper(os.path.join(total_dir,
    #     "Log_Completed.rrd"),
    #     glideFactoryMonitoring.getLogCompletedDefaults(), fix_rrd)
    # verifyHelper(os.path.join(total_dir,
    #     "Log_Completed_Stats.rrd"), completed_stats_dict, fix_rrd)
    # verifyHelper(os.path.join(total_dir,
    #     "Log_Completed_WasteTime.rrd"), completed_waste_dict, fix_rrd)
    # verifyHelper(os.path.join(total_dir,
    #     "Log_Counts.rrd"), counts_dict, fix_rrd)
    # for filename in os.listdir(dir):
    #     if filename[:6]=="entry_":
    #         entrydir=os.path.join(dir, filename)
    #         for subfilename in os.listdir(entrydir):
    #             if subfilename[:9]=="frontend_":
    #                 current_dir=os.path.join(entrydir, subfilename)
    #                 verifyHelper(os.path.join(current_dir,
    #                     "Status_Attributes.rrd"), status_dict, fix_rrd)
    #                 verifyHelper(os.path.join(current_dir,
    #                     "Log_Completed.rrd"),
    #                     glideFactoryMonitoring.getLogCompletedDefaults(), fix_rrd)
    #                 verifyHelper(os.path.join(current_dir,
    #                     "Log_Completed_Stats.rrd"), completed_stats_dict, fix_rrd)
    #                 verifyHelper(os.path.join(current_dir,
    #                     "Log_Completed_WasteTime.rrd"),
    #                     completed_waste_dict, fix_rrd)
    #                 verifyHelper(os.path.join(current_dir,
    #                     "Log_Counts.rrd"), counts_dict, fix_rrd)
    # return not rrd_problems_found

    completed_dict = glideFactoryMonitoring.getLogCompletedDefaults()

    rrdict = {
        "Status_Attributes.rrd": status_dict,
        "Log_Completed.rrd": completed_dict,
        "Log_Completed_Stats.rrd": completed_stats_dict,
        "Log_Completed_WasteTime.rrd": completed_waste_dict,
        "Log_Counts.rrd": counts_dict,
    }

    for dir_name, sdir_name, f_list in os.walk(mon_dir):
        for file_name in f_list:
            if file_name in rrdict.keys():
                verifyHelper(os.path.join(dir_name, file_name),
                             rrdict[file_name], fix_rrd)

    return not rrd_problems_found
Example #4
0
def aggregateLogSummary():
    """
    Create an aggregate of log summary files, write it in an aggregate log
    summary file and in the end return the values
    """

    global monitorAggregatorConfig

    # initialize global counters
    global_total = {
        'Current': {},
        'Entered': {},
        'Exited': {},
        'CompletedCounts': {
            'Sum': {},
            'Waste': {},
            'WasteTime': {},
            'Lasted': {},
            'JobsNr': {},
            'JobsDuration': {}
        }
    }

    for s in ('Wait', 'Idle', 'Running', 'Held'):
        for k in ['Current', 'Entered', 'Exited']:
            global_total[k][s] = 0

    for s in ('Completed', 'Removed'):
        for k in ['Entered']:
            global_total[k][s] = 0

    for k in glideFactoryMonitoring.getAllJobTypes():
        for w in ('Waste', 'WasteTime'):
            el = {}
            for t in glideFactoryMonitoring.getAllMillRanges():
                el[t] = 0
            global_total['CompletedCounts'][w][k] = el

    el = {}
    for t in glideFactoryMonitoring.getAllTimeRanges():
        el[t] = 0
    global_total['CompletedCounts']['Lasted'] = el

    el = {}
    for t in glideFactoryMonitoring.getAllJobRanges():
        el[t] = 0
    global_total['CompletedCounts']['JobsNr'] = el

    el = {}
    # KEL - why is the same el used twice (see above)
    for t in glideFactoryMonitoring.getAllTimeRanges():
        el[t] = 0
    global_total['CompletedCounts']['JobsDuration'] = el

    global_total['CompletedCounts']['Sum'] = {
        'Glideins': 0,
        'Lasted': 0,
        'FailedNr': 0,
        'JobsNr': 0,
        'JobsLasted': 0,
        'JobsGoodput': 0,
        'JobsTerminated': 0,
        'CondorLasted': 0
    }

    fe_total = copy.deepcopy(
        global_total)  # same as above but for frontend totals

    #
    status = {'entries': {}, 'total': global_total}
    status_fe = {'frontends': {}}  # analogous to above but for frontend totals

    nr_entries = 0
    nr_feentries = {}  # dictionary for nr entries per fe
    for entry in monitorAggregatorConfig.entries:
        # load entry log summary file
        status_fname = os.path.join(
            os.path.join(monitorAggregatorConfig.monitor_dir,
                         'entry_' + entry),
            monitorAggregatorConfig.logsummary_relname)

        try:
            entry_data = xmlParse.xmlfile2dict(
                status_fname,
                always_singular_list=['Fraction', 'TimeRange', 'Range'])
        except IOError:
            continue  # file not found, ignore

        # update entry
        out_data = {}
        for frontend in entry_data['frontends'].keys():
            fe_el = entry_data['frontends'][frontend]
            out_fe_el = {}
            for k in ['Current', 'Entered', 'Exited']:
                out_fe_el[k] = {}
                for s in fe_el[k].keys():
                    out_fe_el[k][s] = int(fe_el[k][s])
            out_fe_el['CompletedCounts'] = {
                'Waste': {},
                'WasteTime': {},
                'Lasted': {},
                'JobsNr': {},
                'JobsDuration': {},
                'Sum': {}
            }
            for tkey in fe_el['CompletedCounts']['Sum'].keys():
                out_fe_el['CompletedCounts']['Sum'][tkey] = int(
                    fe_el['CompletedCounts']['Sum'][tkey])
            for k in glideFactoryMonitoring.getAllJobTypes():
                for w in ("Waste", "WasteTime"):
                    out_fe_el['CompletedCounts'][w][k] = {}
                    for t in glideFactoryMonitoring.getAllMillRanges():
                        out_fe_el['CompletedCounts'][w][k][t] = int(
                            fe_el['CompletedCounts'][w][k][t]['val'])
            for t in glideFactoryMonitoring.getAllTimeRanges():
                out_fe_el['CompletedCounts']['Lasted'][t] = int(
                    fe_el['CompletedCounts']['Lasted'][t]['val'])
            out_fe_el['CompletedCounts']['JobsDuration'] = {}
            for t in glideFactoryMonitoring.getAllTimeRanges():
                out_fe_el['CompletedCounts']['JobsDuration'][t] = int(
                    fe_el['CompletedCounts']['JobsDuration'][t]['val'])
            for t in glideFactoryMonitoring.getAllJobRanges():
                out_fe_el['CompletedCounts']['JobsNr'][t] = int(
                    fe_el['CompletedCounts']['JobsNr'][t]['val'])
            out_data[frontend] = out_fe_el

        status['entries'][entry] = {'frontends': out_data}

        # update total
        if 'total' in entry_data:
            nr_entries += 1
            local_total = {}

            for k in ['Current', 'Entered', 'Exited']:
                local_total[k] = {}
                for s in global_total[k].keys():
                    local_total[k][s] = int(entry_data['total'][k][s])
                    global_total[k][s] += int(entry_data['total'][k][s])
            local_total['CompletedCounts'] = {
                'Sum': {},
                'Waste': {},
                'WasteTime': {},
                'Lasted': {},
                'JobsNr': {},
                'JobsDuration': {}
            }
            for tkey in entry_data['total']['CompletedCounts']['Sum'].keys():
                local_total['CompletedCounts']['Sum'][tkey] = int(
                    entry_data['total']['CompletedCounts']['Sum'][tkey])
                global_total['CompletedCounts']['Sum'][tkey] += int(
                    entry_data['total']['CompletedCounts']['Sum'][tkey])
            for k in glideFactoryMonitoring.getAllJobTypes():
                for w in ('Waste', 'WasteTime'):
                    local_total['CompletedCounts'][w][k] = {}
                    for t in glideFactoryMonitoring.getAllMillRanges():
                        local_total['CompletedCounts'][w][k][t] = int(
                            entry_data['total']['CompletedCounts'][w][k][t]
                            ['val'])
                        global_total['CompletedCounts'][w][k][t] += int(
                            entry_data['total']['CompletedCounts'][w][k][t]
                            ['val'])

            for t in glideFactoryMonitoring.getAllTimeRanges():
                local_total['CompletedCounts']['Lasted'][t] = int(
                    entry_data['total']['CompletedCounts']['Lasted'][t]['val'])
                global_total['CompletedCounts']['Lasted'][t] += int(
                    entry_data['total']['CompletedCounts']['Lasted'][t]['val'])
            local_total['CompletedCounts']['JobsDuration'] = {}
            for t in glideFactoryMonitoring.getAllTimeRanges():
                local_total['CompletedCounts']['JobsDuration'][t] = int(
                    entry_data['total']['CompletedCounts']['JobsDuration'][t]
                    ['val'])
                global_total['CompletedCounts']['JobsDuration'][t] += int(
                    entry_data['total']['CompletedCounts']['JobsDuration'][t]
                    ['val'])

            for t in glideFactoryMonitoring.getAllJobRanges():
                local_total['CompletedCounts']['JobsNr'][t] = int(
                    entry_data['total']['CompletedCounts']['JobsNr'][t]['val'])
                global_total['CompletedCounts']['JobsNr'][t] += int(
                    entry_data['total']['CompletedCounts']['JobsNr'][t]['val'])

            status['entries'][entry]['total'] = local_total

        # update frontends
        for fe in out_data:
            # compare each to the list of fe's accumulated so far
            if not (fe in status_fe['frontends']):
                status_fe['frontends'][fe] = {}
            if not (fe in nr_feentries):
                nr_feentries[fe] = 1  # already found one
            else:
                nr_feentries[fe] += 1

            # sum them up
            sumDictInt(out_data[fe], status_fe['frontends'][fe])

    # Write xml files
    # To do - Igor: Consider adding status_fe to the XML file
    updated = time.time()
    xml_str = ('<?xml version="1.0" encoding="ISO-8859-1"?>\n\n' +
               '<glideFactoryLogSummary>\n' +
               xmlFormat.time2xml(updated,
                                  "updated",
                                  indent_tab=xmlFormat.DEFAULT_TAB,
                                  leading_tab=xmlFormat.DEFAULT_TAB) + '\n' +
               xmlFormat.dict2string(
                   status["entries"],
                   dict_name="entries",
                   el_name="entry",
                   subtypes_params={
                       "class": {
                           "dicts_params": {
                               "frontends": {
                                   "el_name": "frontend",
                                   "subtypes_params": {
                                       "class": {
                                           'subclass_params': {
                                               'CompletedCounts':
                                               glideFactoryMonitoring.
                                               get_completed_stats_xml_desc()
                                           }
                                       }
                                   }
                               }
                           },
                           "subclass_params": {
                               "total": {
                                   "subclass_params": {
                                       'CompletedCounts':
                                       glideFactoryMonitoring.
                                       get_completed_stats_xml_desc()
                                   }
                               }
                           }
                       }
                   },
                   leading_tab=xmlFormat.DEFAULT_TAB) + '\n' +
               xmlFormat.class2string(
                   status["total"],
                   inst_name="total",
                   subclass_params={
                       'CompletedCounts':
                       glideFactoryMonitoring.get_completed_stats_xml_desc()
                   },
                   leading_tab=xmlFormat.DEFAULT_TAB) + '\n' +
               "</glideFactoryLogSummary>\n")
    glideFactoryMonitoring.monitoringConfig.write_file(
        monitorAggregatorConfig.logsummary_relname, xml_str)

    # Write rrds
    writeLogSummaryRRDs("total", status["total"])

    # Frontend total rrds across all factories
    for fe in status_fe['frontends']:
        writeLogSummaryRRDs("total/%s" % ("frontend_" + fe),
                            status_fe['frontends'][fe])

    return status
def aggregateLogSummary():
    """
    Create an aggregate of log summary files, write it in an aggregate log 
    summary file and in the end return the values
    """

    global monitorAggregatorConfig

    # initialize global counters
    global_total={'Current':{},'Entered':{},'Exited':{},'CompletedCounts':{'Sum':{},'Waste':{},'WasteTime':{},'Lasted':{},'JobsNr':{},'JobsDuration':{}}}

    for s in ('Wait','Idle','Running','Held'):
        for k in ['Current','Entered','Exited']:
            global_total[k][s]=0

    for s in ('Completed','Removed'):
        for k in ['Entered']:
            global_total[k][s]=0

    for k in glideFactoryMonitoring.getAllJobTypes():
        for w in ("Waste","WasteTime"):
            el={}
            for t in glideFactoryMonitoring.getAllMillRanges():
                el[t]=0
            global_total['CompletedCounts'][w][k]=el

    el={}
    for t in glideFactoryMonitoring.getAllTimeRanges():
        el[t]=0
    global_total['CompletedCounts']['Lasted']=el

    el={}
    for t in glideFactoryMonitoring.getAllJobRanges():
        el[t]=0
    global_total['CompletedCounts']['JobsNr']=el

    el={}
    # KEL - why is the same el used twice (see above)
    for t in glideFactoryMonitoring.getAllTimeRanges():
        el[t]=0
    global_total['CompletedCounts']['JobsDuration']=el

    global_total['CompletedCounts']['Sum']={'Glideins':0,
                                            'Lasted':0,
                                            'FailedNr':0,
                                            'JobsNr':0,
                                            'JobsLasted':0,
                                            'JobsGoodput':0,
                                            'JobsTerminated':0,
                                            'CondorLasted':0}

    #
    status={'entries':{},'total':global_total}
    nr_entries=0
    for entry in monitorAggregatorConfig.entries:
        # load entry log summary file
        status_fname=os.path.join(os.path.join(monitorAggregatorConfig.monitor_dir,'entry_'+entry),
                                  monitorAggregatorConfig.logsummary_relname)
        try:
            entry_data=xmlParse.xmlfile2dict(status_fname,always_singular_list=['Fraction','TimeRange','Range'])
        except IOError:
            continue # file not found, ignore

        # update entry
        out_data={}
        for frontend in entry_data['frontends'].keys():
            fe_el=entry_data['frontends'][frontend]
            out_fe_el={}
            for k in ['Current','Entered','Exited']:
                out_fe_el[k]={}
                for s in fe_el[k].keys():
                    out_fe_el[k][s]=int(fe_el[k][s])
            out_fe_el['CompletedCounts']={'Waste':{},'WasteTime':{},'Lasted':{},'JobsNr':{},'JobsDuration':{},'Sum':{}}
            for tkey in fe_el['CompletedCounts']['Sum'].keys():
                out_fe_el['CompletedCounts']['Sum'][tkey]=int(fe_el['CompletedCounts']['Sum'][tkey])
            for k in glideFactoryMonitoring.getAllJobTypes():
                for w in ("Waste","WasteTime"):
                    out_fe_el['CompletedCounts'][w][k]={}
                    for t in glideFactoryMonitoring.getAllMillRanges():
                        out_fe_el['CompletedCounts'][w][k][t]=int(fe_el['CompletedCounts'][w][k][t]['val'])
            for t in glideFactoryMonitoring.getAllTimeRanges():
                out_fe_el['CompletedCounts']['Lasted'][t]=int(fe_el['CompletedCounts']['Lasted'][t]['val'])
            out_fe_el['CompletedCounts']['JobsDuration']={}
            for t in glideFactoryMonitoring.getAllTimeRanges():
                out_fe_el['CompletedCounts']['JobsDuration'][t]=int(fe_el['CompletedCounts']['JobsDuration'][t]['val'])
            for t in glideFactoryMonitoring.getAllJobRanges():
                out_fe_el['CompletedCounts']['JobsNr'][t]=int(fe_el['CompletedCounts']['JobsNr'][t]['val'])
            out_data[frontend]=out_fe_el

        status['entries'][entry]={'frontends':out_data}

        # update total
        if entry_data.has_key('total'):
            nr_entries+=1
            local_total={}

            for k in ['Current','Entered','Exited']:
                local_total[k]={}
                for s in global_total[k].keys():
                    local_total[k][s]=int(entry_data['total'][k][s])
                    global_total[k][s]+=int(entry_data['total'][k][s])
            local_total['CompletedCounts']={'Sum':{},'Waste':{},'WasteTime':{},'Lasted':{},'JobsNr':{},'JobsDuration':{}}
            for tkey in entry_data['total']['CompletedCounts']['Sum'].keys():
                local_total['CompletedCounts']['Sum'][tkey]=int(entry_data['total']['CompletedCounts']['Sum'][tkey])
                global_total['CompletedCounts']['Sum'][tkey]+=int(entry_data['total']['CompletedCounts']['Sum'][tkey])
            for k in glideFactoryMonitoring.getAllJobTypes():
                for w in ("Waste","WasteTime"):
                    local_total['CompletedCounts'][w][k]={}
                    for t in glideFactoryMonitoring.getAllMillRanges():
                        local_total['CompletedCounts'][w][k][t]=int(entry_data['total']['CompletedCounts'][w][k][t]['val'])
                        global_total['CompletedCounts'][w][k][t]+=int(entry_data['total']['CompletedCounts'][w][k][t]['val'])

            for t in glideFactoryMonitoring.getAllTimeRanges():
                local_total['CompletedCounts']['Lasted'][t]=int(entry_data['total']['CompletedCounts']['Lasted'][t]['val'])
                global_total['CompletedCounts']['Lasted'][t]+=int(entry_data['total']['CompletedCounts']['Lasted'][t]['val'])
            local_total['CompletedCounts']['JobsDuration']={}
            for t in glideFactoryMonitoring.getAllTimeRanges():
                local_total['CompletedCounts']['JobsDuration'][t]=int(entry_data['total']['CompletedCounts']['JobsDuration'][t]['val'])
                global_total['CompletedCounts']['JobsDuration'][t]+=int(entry_data['total']['CompletedCounts']['JobsDuration'][t]['val'])

            for t in glideFactoryMonitoring.getAllJobRanges():
                local_total['CompletedCounts']['JobsNr'][t]=int(entry_data['total']['CompletedCounts']['JobsNr'][t]['val'])
                global_total['CompletedCounts']['JobsNr'][t]+=int(entry_data['total']['CompletedCounts']['JobsNr'][t]['val'])

            status['entries'][entry]['total']=local_total

    # Write xml files
    updated=time.time()
    xml_str=('<?xml version="1.0" encoding="ISO-8859-1"?>\n\n'+
             '<glideFactoryLogSummary>\n'+
             xmlFormat.time2xml(updated,"updated", indent_tab=xmlFormat.DEFAULT_TAB,leading_tab=xmlFormat.DEFAULT_TAB)+"\n"+
             xmlFormat.dict2string(status["entries"],dict_name="entries",el_name="entry",
                                   subtypes_params={"class":{"dicts_params":{"frontends":{"el_name":"frontend",
                                                                                          "subtypes_params":{"class":{'subclass_params':{'CompletedCounts':glideFactoryMonitoring.get_completed_stats_xml_desc()}}}}},
                                                             "subclass_params":{"total":{"subclass_params":{'CompletedCounts':glideFactoryMonitoring.get_completed_stats_xml_desc()}}}
                                                             }
                                                    },
                                   leading_tab=xmlFormat.DEFAULT_TAB)+"\n"+
             xmlFormat.class2string(status["total"],inst_name="total",subclass_params={'CompletedCounts':glideFactoryMonitoring.get_completed_stats_xml_desc()},leading_tab=xmlFormat.DEFAULT_TAB)+"\n"+
             "</glideFactoryLogSummary>\n")
    glideFactoryMonitoring.monitoringConfig.write_file(monitorAggregatorConfig.logsummary_relname,xml_str)

    # Write rrds
    fe_dir="total"
    sdata=status["total"]['Current']

    glideFactoryMonitoring.monitoringConfig.establish_dir(fe_dir)
    val_dict_counts={}
    val_dict_counts_desc={}
    val_dict_completed={}
    val_dict_stats={}
    val_dict_waste={}
    val_dict_wastetime={}
    for s in ('Wait','Idle','Running','Held','Completed','Removed'):
        if not (s in ('Completed','Removed')): # I don't have their numbers from inactive logs
            count=sdata[s]
            val_dict_counts["Status%s"%s]=count
            val_dict_counts_desc["Status%s"%s]={'ds_type':'GAUGE'}

            exited=-status["total"]['Exited'][s]
            val_dict_counts["Exited%s"%s]=exited
            val_dict_counts_desc["Exited%s"%s]={'ds_type':'ABSOLUTE'}

        entered=status["total"]['Entered'][s]
        val_dict_counts["Entered%s"%s]=entered
        val_dict_counts_desc["Entered%s"%s]={'ds_type':'ABSOLUTE'}

        if s=='Completed':
            completed_counts=status["total"]['CompletedCounts']
            count_entered_times=completed_counts['Lasted']
            count_jobnrs=completed_counts['JobsNr']
            count_jobs_duration=completed_counts['JobsDuration']
            count_waste_mill=completed_counts['Waste']
            time_waste_mill=completed_counts['WasteTime']
            # save run times
            for timerange in count_entered_times.keys():
                val_dict_stats['Lasted_%s'%timerange]=count_entered_times[timerange]
                # they all use the same indexes
                val_dict_stats['JobsLasted_%s'%timerange]=count_jobs_duration[timerange]

            # save jobsnr
            for jobrange in count_jobnrs.keys():
                val_dict_stats['JobsNr_%s'%jobrange]=count_jobnrs[jobrange]

            # save simple vals
            for tkey in completed_counts['Sum'].keys():
                val_dict_completed[tkey]=completed_counts['Sum'][tkey]

            # save waste_mill
            for w in count_waste_mill.keys():
                count_waste_mill_w=count_waste_mill[w]
                for p in count_waste_mill_w.keys():
                    val_dict_waste['%s_%s'%(w,p)]=count_waste_mill_w[p]

            for w in time_waste_mill.keys():
                time_waste_mill_w=time_waste_mill[w]
                for p in time_waste_mill_w.keys():
                    val_dict_wastetime['%s_%s'%(w,p)]=time_waste_mill_w[p]

    # write the data to disk
    glideFactoryMonitoring.monitoringConfig.write_rrd_multi_hetero("%s/Log_Counts"%fe_dir,
                                                            val_dict_counts_desc,updated,val_dict_counts)
    glideFactoryMonitoring.monitoringConfig.write_rrd_multi("%s/Log_Completed"%fe_dir,
                                                            "ABSOLUTE",updated,val_dict_completed)
    glideFactoryMonitoring.monitoringConfig.write_rrd_multi("%s/Log_Completed_Stats"%fe_dir,
                                                            "ABSOLUTE",updated,val_dict_stats)
    # Disable Waste RRDs... WasteTime much more useful
    #glideFactoryMonitoring.monitoringConfig.write_rrd_multi("%s/Log_Completed_Waste"%fe_dir,
    #                                                        "ABSOLUTE",updated,val_dict_waste)
    glideFactoryMonitoring.monitoringConfig.write_rrd_multi("%s/Log_Completed_WasteTime"%fe_dir,
                                                            "ABSOLUTE",updated,val_dict_wastetime)

    return status