def check_resident(self, site, stage_data): """ Checks that the requested data is resident on the site """ goodToGo = True # Format site name locSite = site.replace('_Buffer', '_MSS') if not locSite.endswith('_MSS'): locSite += '_MSS' qParams = {'node': locSite} # Get block info from PhEDEx phedex = Requests(url='https://cmsweb.cern.ch', idict={'accept_type':'application/json'}) for data in stage_data: if data.find('#') < 0: qParams['dataset'] = data else: qParams['block'] = data self.logger.debug('Checking data residency for %s at %s' % (data, locSite)) try: pdata = phedex.get('/phedex/datasvc/json/prod/blockreplicas', qParams)[0] except httplib.HTTPException, he: self.handleHTTPExcept(he, 'HTTPException for block: %s node: %s' % (data, locSite)) # Parse block info and check > 0 block exist try: if len(JsonWrapper.loads(pdata)['phedex']['block']) == 0: goodToGo = False self.logger.error('Block %s not resident at site %s' % (data, locSite)) except: self.logger.debug('error loading json') goodToGo = False
def process_files(self, stage_data = [], request_id=''): """ Contact PhEDEx data service to get a list of files for a given request. TODO: use Service.PhEDEx """ db = self.localcouch.connectDatabase('%s_stagequeue' % self.site) # TODO: make the phedex URL a configurable! phedex = Requests(url='https://cmsweb.cern.ch', idict={'accept_type':'application/json'}) totalFiles = 0 totalBytes = 0 # Need to clean up the input a bit for d in stage_data: qParams = {'node': self.node} # Need to pass in blocks if d.find('#') < 0: qParams['dataset'] = d else: qParams['block'] = d try: pdata = phedex.get('/phedex/datasvc/json/prod/filereplicas', qParams)[0] except httplib.HTTPException, he: self.handleHTTPExcept(he, 'HTTPException for block: %s node: %s' % (d, self.node)) # Parse block info try: p = JsonWrapper.loads(pdata) if len(p['phedex']['block']) == 0: self.logger.error('Block %s not resident at site %s' % (d, self.node)) else: self.logger.debug('Creating stage-in requests for %s' % self.node) for blk in p['phedex']['block']: self.logger.debug('Creating stage-in requests for block %s' % blk['name']) for fle in blk['file']: self.logger.debug('Creating stage-in requests for file %s' % fle['name']) checksum = fle['checksum'] f = {'_id': fle['name'], 'bytes': int(fle['bytes']), 'checksum': {checksum.split(':')[0]: checksum.split(':')[1]}, 'state': 'new', 'retry_count': [], 'request_id': request_id } try: db.queue(f, timestamp = True, viewlist=['stagequeue/file_state']) totalFiles += 1 totalBytes += f['bytes'] except httplib.HTTPException, he: self.handleHTTPExcept(he, 'Could not commit data') except: self.logger.debug('error loading json')