Exemplo n.º 1
0
def updateScanStatus(dumper, recoid, datasetid):
    global dbcursor

    print 'Updateing scan status.'

    lumis = {}
    for run in dumper.getAnalyzedRuns():
        if run not in lumis:
            lumis[run] = []

        for lumi in dumper.getAnalyzedLumis(run):
            lumis[run].append(lumi)

    query = 'UPDATE `scanstatus` SET STATUS = \'done\' WHERE `recoid` = %d AND `datasetid` = %d AND (' % (recoid, datasetid)
    runblocks = []
    for run, ls in lumis.items():
        block  = '(`run` = %d AND `lumi` IN (%s))' % (run, ','.join(map(str, ls)))
        runblocks.append(block)

    query += ' OR '.join(runblocks)
    query += ')'

    dbcursor.execute(query)

    dumper.resetRuns()
Exemplo n.º 2
0
def cleanup(timestamp, jobdir):
    taskdir = config.installdir + '/jobs/' + timestamp + '/' + jobdir
    if os.path.exists(taskdir + '/result/missingLumiSummary.json'):
        with open(taskdir + '/result/missingLumiSummary.json') as json:
            lumiLists = eval(json.read())
    else:
        jsonName = jobdir.replace('crab_', 'lumiMask_') + '.json'
        with open(config.installdir + '/jobs/' + timestamp + '/' + jsonName) as json:
            lumiLists = eval(json.read())

    allLumis = []
    for srun, lumiRanges in lumiLists.items():
        for start, end in lumiRanges:
            allLumis += ['(%s, %d)' % (srun, l) for l in range(start, end + 1)]

    query = 'UPDATE `scanstatus` SET `status` = \'failed\' WHERE `status` LIKE \'scanning\' AND (`run`, `lumi`) IN (%s)' % (', '.join(allLumis))

    dbcursor.execute(query)
    shutil.rmtree(config.installdir + '/jobs/' + timestamp + '/' + jobdir)
Exemplo n.º 3
0
import math
import subprocess

import config
from localdb import dbcursor

#htmlDirs = ['/var/www/html', '/afs/cern.ch/user/y/yiiyama/www/metscan']
htmlDirs = ['/var/www/html']

messages = '    <p><span style="color:red;">The system is currently re-scanning the entire dataset.</span></p>\n'
messages += '    <p><a href="nov18/index.html">Status as of November 18</a></p>\n'
messages += '    <p>Golden JSON used is: ' + config.goldenJson + '</p>\n'
messages += '    <p>Silver JSON used is: ' + config.silverJson + '</p>\n'
messages += '    <p>Page last updated: ' + time.asctime() + '</p>'

dbcursor.execute('SELECT `status`+0 FROM `scanstatus` WHERE `status` LIKE \'done\'')
DONE = dbcursor.fetchall()[0][0]

dbcursor.execute('SELECT `recoid`, `name` FROM `reconstructions` ORDER BY `recoid`')
recos = [(row[0], row[1]) for row in dbcursor]

dbcursor.execute('SELECT `datasetid`, `name` FROM `primarydatasets` ORDER BY `name`')
pds = [(pdid, pdname) for pdid, pdname in dbcursor]

status = dict([(reco[0], dict([(pdid, {}) for pdid, name in pds])) for reco in recos])
dbcursor.execute('SELECT `recoid`, `datasetid`, `run`, `lumi`, `status`+0 FROM `scanstatus`')
for recoid, pdid, run, lumi, st in dbcursor:
    if recoid not in status:
        status[recoid] = {}
    if pdid not in status[recoid]:
        status[recoid][pdid] = {}
Exemplo n.º 4
0
import sys
import os
import re
import subprocess

import config
from das import dasQuery, datasetList
from localdb import dbcursor

### STEP 1 ###################################################
### Find lumisections to be processed from DAS             ###
##############################################################

recoids = {}
for reco in config.reconstructions:
    dbcursor.execute('SELECT `recoid` FROM `reconstructions` WHERE `name` LIKE %s', (reco,))
    if dbcursor.rowcount <= 0:
        # insert new reconstruction version
        dbcursor.execute('INSERT INTO `reconstructions` (name) VALUES (%s)', (reco,))
        recoids[reco] = dbcursor.lastrowid
    else:
        recoids[reco] = dbcursor.fetchall()[0][0]

dbcursor.execute('SELECT `datasetid`, `name` FROM `primarydatasets`')
knownPDs = dict([(name, datasetid) for datasetid, name in dbcursor])

# list of dataset full names (PD + reconstruction version)
# There isn't really a need to query das every time. Providing a hard-coded dataset list is another option..
datasets = datasetList()

# DCS-only JSON mask
Exemplo n.º 5
0
import CRABClient.UserUtilities
from httplib import HTTPException

import config
from das import dasQuery, datasetList
from localdb import dbcursor

### STEP 2 ###################################################
### Submit ntuplizer jobs over all new lumisections        ###
##############################################################

# list of dataset full names (PD + reconstruction version)
# There isn't really a need to query das every time. Providing a hard-coded dataset list is another option..
datasets = datasetList()

dbcursor.execute('SELECT `datasetid`, `name` FROM `primarydatasets`')
knownPDs = dict([(name, datasetid) for datasetid, name in dbcursor])

timestamp = time.strftime('%y%m%d%H%M%S')

crabConfig = CRABClient.UserUtilities.config()
crabConfig.General.workArea = config.installdir + '/jobs/' + timestamp
crabConfig.JobType.pluginName = 'Analysis'
#crabConfig.JobType.outputFiles = ['tags.txt', 'eventdata.txt', 'lumis.txt']
crabConfig.Data.splitting = 'LumiBased'
#crabConfig.Data.totalUnits = 1 # TESTING
crabConfig.Data.outLFNDirBase = config.eosdir.replace('/eos/cms', '') + '/' + timestamp
crabConfig.Site.storageSite = 'T2_CH_CERN'

try:
    os.makedirs(crabConfig.General.workArea)
Exemplo n.º 6
0
import sys
import os
import re
import subprocess

import config
from das import dasQuery, datasetList
from localdb import dbcursor

### STEP 1 ###################################################
### Find lumisections to be processed from DAS             ###
##############################################################

recoids = {}
for reco in config.reconstructions:
    dbcursor.execute('SELECT `recoid` FROM `reconstructions` WHERE `name` LIKE %s', (reco,))
    if dbcursor.rowcount <= 0:
        # insert new reconstruction version
        dbcursor.execute('INSERT INTO `reconstructions` (name) VALUES (%s)', (reco,))
        recoids[reco] = dbcursor.lastrowid
    else:
        recoids[reco] = dbcursor.fetchall()[0][0]

dbcursor.execute('SELECT `datasetid`, `name` FROM `primarydatasets`')
knownPDs = dict([(name, datasetid) for datasetid, name in dbcursor])

# list of dataset full names (PD + reconstruction version)
# There isn't really a need to query das every time. Providing a hard-coded dataset list is another option..
datasets = datasetList()

for reco in config.reconstructions:
Exemplo n.º 7
0
import config
from localdb import dbcursor

query = 'UPDATE `scanstatus` SET STATUS = \'new\' WHERE `recoid` = 1 AND `datasetid` = %s AND `run` = %s AND `lumi` = %s'

replist = open('/data/scratch/reprocess.txt')

for line in replist:
    datasetid, run, lumi = map(int, line.split())
    dbcursor.execute(query, (datasetid, run, lumi))

dbcursor.commit()
replist.close()
Exemplo n.º 8
0
    loadFromFile('datasetrel.txt', 'datasetrel')
    dumper.resetNData()


def loadFromFile(fileName, tableName):
    global config

    query = 'LOAD DATA LOCAL INFILE \'' + config.scratchdir + '/' + fileName + '\' INTO TABLE `' + tableName + '` FIELDS TERMINATED BY \',\' LINES TERMINATED BY \'\\n\''
    proc = subprocess.Popen(['mysql', '-u', config.dbuser, '-p' + config.dbpass, '-D', config.dbname, '-e', query])
    out, err = proc.communicate()


ROOT.gROOT.LoadMacro(config.installdir + '/scripts/dumpASCII.cc+')
dumper = ROOT.ASCIIDumper(config.scratchdir)

dbcursor.execute('SELECT `filterid`, `name` from `filters`')
for filterid, name in dbcursor:
    dumper.addFilter(filterid, name)

sourcePaths = {}
nFiles = 0

class MaxFiles(Exception):
    pass

try:
    for reco in os.listdir('/'.join((config.scratchdir, 'merged'))):
        sourcePaths[reco] = {}
    
        for pd in os.listdir('/'.join((config.scratchdir, 'merged', reco))):
            sourcePaths[reco][pd] = []
Exemplo n.º 9
0
        proc = subprocess.Popen(['hadd', outFile + '.tmp'] + ['root://eoscms.cern.ch/' + f for f in filesToMerge])
        proc.wait()

        if proc.returncode == 0:
            os.rename(outFile + '.tmp', outFile)

            for path in filesToMerge:
                xrd.rm(path)

        else:
            os.remove(outFile + '.tmp')


if __name__ == '__main__':
    dbcursor.execute('SELECT `name` FROM `primarydatasets`')
    for reco in config.reconstructions:
        for name in [row[0] for row in dbcursor]:
            if not os.path.isdir('/'.join((config.scratchdir, 'merged', reco, name))):
                os.mkdir('/'.join((config.scratchdir, 'merged', reco, name)))

    for tsdir in xrd.ls(sourcedir):
        #temporary
        if int(os.path.basename(tsdir)) < 151108000000:
            continue

        for pddir in xrd.ls(tsdir):
            for recovdir in xrd.ls(pddir):
                mergeAndMove(recovdir)

    xrd.cleanup(sourcedir)