def check_resident(self, site, stage_data):
"""
Checks that the requested data is resident on the site
"""
goodToGo = True
# Format site name
locSite = site.replace('_Buffer', '_MSS')
if not locSite.endswith('_MSS'):
locSite += '_MSS'
qParams = {'node': locSite}
# Get block info from PhEDEx
phedex = Requests(url='https://cmsweb.cern.ch', idict={'accept_type':'application/json'})
for data in stage_data:
if data.find('#') < 0:
qParams['dataset'] = data
else:
qParams['block'] = data
self.logger.debug('Checking data residency for %s at %s' % (data, locSite))
try:
pdata = phedex.get('/phedex/datasvc/json/prod/blockreplicas', qParams)[0]
except httplib.HTTPException, he:
self.handleHTTPExcept(he, 'HTTPException for block: %s node: %s' % (data, locSite))
# Parse block info and check > 0 block exist
try:
if len(JsonWrapper.loads(pdata)['phedex']['block']) == 0:
goodToGo = False
self.logger.error('Block %s not resident at site %s' % (data, locSite))
except:
self.logger.debug('error loading json')
goodToGo = False
def process_files(self, stage_data = [], request_id=''):
"""
Contact PhEDEx data service to get a list of files for a given request.
TODO: use Service.PhEDEx
"""
db = self.localcouch.connectDatabase('%s_stagequeue' % self.site)
# TODO: make the phedex URL a configurable!
phedex = Requests(url='https://cmsweb.cern.ch', idict={'accept_type':'application/json'})
totalFiles = 0
totalBytes = 0
# Need to clean up the input a bit
for d in stage_data:
qParams = {'node': self.node}
# Need to pass in blocks
if d.find('#') < 0:
qParams['dataset'] = d
else:
qParams['block'] = d
try:
pdata = phedex.get('/phedex/datasvc/json/prod/filereplicas', qParams)[0]
except httplib.HTTPException, he:
self.handleHTTPExcept(he, 'HTTPException for block: %s node: %s' % (d, self.node))
# Parse block info
try:
p = JsonWrapper.loads(pdata)
if len(p['phedex']['block']) == 0:
self.logger.error('Block %s not resident at site %s' % (d, self.node))
else:
self.logger.debug('Creating stage-in requests for %s' % self.node)
for blk in p['phedex']['block']:
self.logger.debug('Creating stage-in requests for block %s' % blk['name'])
for fle in blk['file']:
self.logger.debug('Creating stage-in requests for file %s' % fle['name'])
checksum = fle['checksum']
f = {'_id': fle['name'],
'bytes': int(fle['bytes']),
'checksum': {checksum.split(':')[0]: checksum.split(':')[1]},
'state': 'new',
'retry_count': [],
'request_id': request_id
}
try:
db.queue(f, timestamp = True, viewlist=['stagequeue/file_state'])
totalFiles += 1
totalBytes += f['bytes']
except httplib.HTTPException, he:
self.handleHTTPExcept(he, 'Could not commit data')
except:
self.logger.debug('error loading json')
