def scan_files_in_dir(dir_name, patterns=None): fileList = [] for root, dirs, filenames in walk(dir_name): for filename in filenames: # check filename if patterns is not None: matched = False for pattern in patterns: if re.search(pattern, filename) is not None: matched = True break if not matched: continue # make dict tmpFileDict = dict() pfn = os.path.join(root, filename) lfn = os.path.basename(pfn) tmpFileDict['path'] = pfn tmpFileDict['fsize'] = os.stat(pfn).st_size tmpFileDict['type'] = 'es_output' tmpFileDict['guid'] = str(uuid.uuid4()) tmpFileDict['chksum'] = core_utils.calc_adler32(pfn) tmpFileDict['eventRangeID'] = lfn.split('.')[-1] tmpFileDict['eventStatus'] = "finished" fileList.append(tmpFileDict) return fileList
def simple_zip_output(self, jobspec, tmp_log): tmp_log.debug('start') try: for fileSpec in jobspec.outFiles: if self.zipDir == "${SRCDIR}": # the same directory as src zipDir = os.path.dirname( next(iter(fileSpec.associatedFiles)).path) elif self.zipDir == "${WORKDIR}": # work dir workSpec = jobspec.get_workspec_list()[0] zipDir = workSpec.get_access_point() else: zipDir = self.zipDir zipPath = os.path.join(zipDir, fileSpec.lfn) msgStr = 'self.zipDir - {0} zipDir - {1} fileSpec.lfn - {2} zipPath - {3}' \ .format(self.zipDir, zipDir, fileSpec.lfn, zipPath) tmp_log.debug(msgStr) # make zip if doesn't exist if not os.path.exists(zipPath): tmpZipPath = zipPath + '.' + str(uuid.uuid4()) with tarfile.open(tmpZipPath, "w") as zf: for assFileSpec in fileSpec.associatedFiles: zf.add(assFileSpec.path, os.path.basename(assFileSpec.path)) # avoid overwriting lockName = 'zip.lock.{0}'.format(fileSpec.lfn) lockInterval = 60 tmpStat = False # get lock for i in range(lockInterval): tmpStat = self.dbInterface.get_object_lock( lockName, lock_interval=lockInterval) if tmpStat: break time.sleep(1) # failed to lock if not tmpStat: msgStr = 'failed to get zip lock for {0}'.format( fileSpec.lfn) tmp_log.error(msgStr) return None, msgStr if not os.path.exists(zipPath): os.rename(tmpZipPath, zipPath) # release lock self.dbInterface.release_object_lock(lockName) # set path fileSpec.path = zipPath # get size statInfo = os.stat(zipPath) fileSpec.fsize = statInfo.st_size fileSpec.chksum = core_utils.calc_adler32(zipPath) msgStr = 'fileSpec.path - {0}, fileSpec.fsize - {1}, fileSpec.chksum(adler32) - {2}'\ .format(fileSpec.path, fileSpec.fsize, fileSpec.chksum) tmp_log.debug(msgStr) except Exception: errMsg = core_utils.dump_error_message(tmp_log) return False, 'failed to zip with {0}'.format(errMsg) tmp_log.debug('done') return True, ''
def scan_files_in_dir(dir_name, patterns=None, zip_patterns=None): fileList = [] for root, dirs, filenames in walk(dir_name): for filename in filenames: # check if zipped is_zipped = False if zip_patterns: matched = False for pattern in zip_patterns: if re.search(pattern, filename) is not None: matched = True break if matched: is_zipped = True # check filename if not is_zipped and patterns: matched = False for pattern in patterns: if re.search(pattern, filename) is not None: matched = True break if not matched: continue # make dict tmpFileDict = dict() pfn = os.path.join(root, filename) tmpFileDict['path'] = pfn tmpFileDict['fsize'] = os.stat(pfn).st_size tmpFileDict['guid'] = str(uuid.uuid4()) tmpFileDict['chksum'] = core_utils.calc_adler32(pfn) tmpFileDict['eventStatus'] = "finished" if is_zipped: lfns = [] # extract actual event filenames from zip with tarfile.open(pfn) as f: for tar_info in f.getmembers(): lfns.append(os.path.basename(tar_info.name)) tmpFileDict['type'] = 'zip_output' else: lfns = [os.path.basename(pfn)] tmpFileDict['type'] = 'es_output' for lfn in lfns: tmpDict = copy.copy(tmpFileDict) tmpDict['eventRangeID'] = lfn.split('.')[-1] fileList.append(tmpDict) return fileList
def trigger_preparation(self, jobspec): # make logger tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID), method_name='trigger_preparation') tmpLog.debug('start') # loop over all inputs inFileInfo = jobspec.get_input_file_attributes() gucInput = None for tmpFileSpec in jobspec.inFiles: # construct source and destination paths srcPath = mover_utils.construct_file_path( self.srcBasePath, inFileInfo[tmpFileSpec.lfn]['scope'], tmpFileSpec.lfn) dstPath = mover_utils.construct_file_path( self.dstBasePath, inFileInfo[tmpFileSpec.lfn]['scope'], tmpFileSpec.lfn) # local access path accPath = mover_utils.construct_file_path( self.localBasePath, inFileInfo[tmpFileSpec.lfn]['scope'], tmpFileSpec.lfn) if self.checkLocalPath: # check if already exits if os.path.exists(accPath): # calculate checksum checksum = core_utils.calc_adler32(accPath) checksum = 'ad:{0}'.format(checksum) if checksum == inFileInfo[tmpFileSpec.lfn]['checksum']: continue # make directories if needed if not os.path.isdir(os.path.dirname(accPath)): os.makedirs(os.path.dirname(accPath)) # make input for globus-url-copy if gucInput is None: gucInput = tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='_guc_in.tmp') gucInput.write("{0} {1}\n".format(srcPath, dstPath)) tmpFileSpec.attemptNr += 1 # nothing to transfer if gucInput is None: tmpLog.debug('done with no transfers') return True, '' # transfer tmpLog.debug('execute globus-url-copy') gucInput.close() args = ['globus-url-copy', '-f', gucInput.name, '-cd'] if self.gulOpts is not None: args += self.gulOpts.split() try: tmpLog.debug('execute: ' + ' '.join(args)) p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: stdout, stderr = p.communicate(timeout=self.timeout) except subprocess.TimeoutExpired: p.kill() stdout, stderr = p.communicate() tmpLog.warning('command timeout') return_code = p.returncode if stdout is not None: if not isinstance(stdout, str): stdout = stdout.decode() stdout = stdout.replace('\n', ' ') if stderr is not None: if not isinstance(stderr, str): stderr = stderr.decode() stderr = stderr.replace('\n', ' ') tmpLog.debug("stdout: %s" % stdout) tmpLog.debug("stderr: %s" % stderr) except Exception: core_utils.dump_error_message(tmpLog) return_code = 1 os.remove(gucInput.name) if return_code == 0: tmpLog.debug('succeeded') return True, '' else: errMsg = 'failed with {0}'.format(return_code) tmpLog.error(errMsg) # check attemptNr for tmpFileSpec in jobspec.inFiles: if tmpFileSpec.attemptNr >= self.maxAttempts: errMsg = 'gave up due to max attempts' tmpLog.error(errMsg) return (False, errMsg) return None, errMsg
def get_files_to_stage_out(self, workspec): # get logger tmpLog = core_utils.make_logger(_logger, 'workerID={0}'.format( workspec.workerID), method_name='get_files_to_stage_out') fileDict = dict() # look for the json just under the access point for pandaID in workspec.pandaid_list: # look for the json just under the access point accessPoint = self.get_access_point(workspec, pandaID) jsonFilePath = os.path.join(accessPoint, jsonOutputsFileName) readJsonPath = jsonFilePath + suffixReadJson # first look for json.read which is not yet acknowledged tmpLog.debug('looking for output file {0}'.format(readJsonPath)) if os.path.exists(readJsonPath): pass else: tmpLog.debug( 'looking for output file {0}'.format(jsonFilePath)) if not os.path.exists(jsonFilePath): # not found tmpLog.debug('not found') continue try: tmpLog.debug('found') # rename to prevent from being overwritten os.rename(jsonFilePath, readJsonPath) except Exception: tmpLog.error('failed to rename json') continue # load json toSkip = False loadDict = None try: with open(readJsonPath) as jsonFile: loadDict = json.load(jsonFile) except Exception: tmpLog.error('failed to load json') toSkip = True # test validity of data format (ie it should be a Dictionary) if not toSkip: if not isinstance(loadDict, dict): tmpLog.error('loaded data is not a dictionary') toSkip = True # collect files and events nData = 0 if not toSkip: sizeMap = dict() chksumMap = dict() eventsList = dict() for tmpPandaID, tmpEventMapList in iteritems(loadDict): tmpPandaID = long(tmpPandaID) # test if tmpEventMapList is a list if not isinstance(tmpEventMapList, list): tmpLog.error('loaded data item is not a list') toSkip = True break for tmpEventInfo in tmpEventMapList: try: nData += 1 if 'eventRangeID' in tmpEventInfo: tmpEventRangeID = tmpEventInfo['eventRangeID'] else: tmpEventRangeID = None tmpFileDict = dict() pfn = tmpEventInfo['path'] lfn = os.path.basename(pfn) tmpFileDict['path'] = pfn if pfn not in sizeMap: if 'fsize' in tmpEventInfo: sizeMap[pfn] = tmpEventInfo['fsize'] else: sizeMap[pfn] = os.stat(pfn).st_size tmpFileDict['fsize'] = sizeMap[pfn] tmpFileDict['type'] = tmpEventInfo['type'] if tmpEventInfo['type'] in ['log', 'output']: # disable zipping tmpFileDict['isZip'] = 0 elif tmpEventInfo['type'] == 'zip_output': # already zipped tmpFileDict['isZip'] = 1 elif 'isZip' in tmpEventInfo: tmpFileDict['isZip'] = tmpEventInfo['isZip'] # guid if 'guid' in tmpEventInfo: tmpFileDict['guid'] = tmpEventInfo['guid'] else: tmpFileDict['guid'] = str(uuid.uuid4()) # get checksum if pfn not in chksumMap: if 'chksum' in tmpEventInfo: chksumMap[pfn] = tmpEventInfo['chksum'] else: chksumMap[pfn] = core_utils.calc_adler32( pfn) tmpFileDict['chksum'] = chksumMap[pfn] if tmpPandaID not in fileDict: fileDict[tmpPandaID] = dict() if lfn not in fileDict[tmpPandaID]: fileDict[tmpPandaID][lfn] = [] fileDict[tmpPandaID][lfn].append(tmpFileDict) # skip if unrelated to events if tmpFileDict['type'] not in [ 'es_output', 'zip_output' ]: continue tmpFileDict['eventRangeID'] = tmpEventRangeID if tmpPandaID not in eventsList: eventsList[tmpPandaID] = list() eventsList[tmpPandaID].append({ 'eventRangeID': tmpEventRangeID, 'eventStatus': tmpEventInfo['eventStatus'] }) except Exception: core_utils.dump_error_message(tmpLog) # dump events if not toSkip: if len(eventsList) > 0: curName = os.path.join(accessPoint, jsonEventsUpdateFileName) newName = curName + '.new' f = open(newName, 'w') json.dump(eventsList, f) f.close() os.rename(newName, curName) # remove empty file if toSkip or nData == 0: try: os.remove(readJsonPath) except Exception: pass tmpLog.debug('got {0} files for PandaID={1}'.format( nData, pandaID)) return fileDict
def make_one_zip(self, arg_dict): self.zip_tmp_log.debug('start') try: zipPath = arg_dict['zipPath'] lfn = os.path.basename(zipPath) self.zip_tmp_log.debug( '{0} start zipPath={1} with {2} files'.format( lfn, zipPath, len(arg_dict['associatedFiles']))) # make zip if doesn't exist if not os.path.exists(zipPath): # tmp file names tmpZipPath = zipPath + '.' + str(uuid.uuid4()) tmpZipPathIn = tmpZipPath + '.in' with open(tmpZipPathIn, "w") as f: for associatedFile in arg_dict['associatedFiles']: f.write("{0}\n".format(associatedFile)) # make command com = 'tar -c -f {0} -T {1} '.format(tmpZipPath, tmpZipPathIn) com += "--transform 's/.*\///' " # execute p = subprocess.Popen(com, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdOut, stdErr = p.communicate() retCode = p.returncode if retCode != 0: msgStr = 'failed to make zip for {0} with {1}:{2}'.format( lfn, stdOut, stdErr) self.zip_tmp_log.error(msgStr) return None, msgStr, {} # avoid overwriting lockName = 'zip.lock.{0}'.format(lfn) lockInterval = 60 tmpStat = False # get lock for i in range(lockInterval): tmpStat = self.dbInterface.get_object_lock( lockName, lock_interval=lockInterval) if tmpStat: break time.sleep(1) # failed to lock if not tmpStat: msgStr = 'failed to lock for {0}'.format(lfn) self.zip_tmp_log.error(msgStr) return None, msgStr if not os.path.exists(zipPath): os.rename(tmpZipPath, zipPath) # release lock self.dbInterface.release_object_lock(lockName) # make return fileInfo = dict() fileInfo['path'] = zipPath # get size statInfo = os.stat(zipPath) fileInfo['fsize'] = statInfo.st_size fileInfo['chksum'] = core_utils.calc_adler32(zipPath) except Exception: errMsg = core_utils.dump_error_message(self.zip_tmp_log) return False, 'failed to zip with {0}'.format(errMsg) self.zip_tmp_log.debug('{0} done'.format(lfn)) return True, '', fileInfo
def trigger_preparation(self, jobspec): # make logger tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID), method_name='trigger_preparation') tmpLog.debug('Start. Trigger data transfer for job: {0}'.format( jobspec.PandaID)) # check that jobspec.computingSite is defined if jobspec.computingSite is None: # not found tmpLog.error('jobspec.computingSite is not defined') return False, 'jobspec.computingSite is not defined' else: tmpLog.debug('jobspec.computingSite : {0}'.format( jobspec.computingSite)) # get input files files = [] inFiles = jobspec.get_input_file_attributes(skip_ready=True) # set path to each file tmpLog.info( "Prepare files to download (construct path and verifiy existing files)" ) for inLFN, inFile in iteritems(inFiles): inFile['path'] = mover_utils.construct_file_path( self.basePath, inFile['scope'], inLFN) # check if file exist. Skip alrady downoladed files if os.path.exists(inFile['path']): checksum = core_utils.calc_adler32(inFile['path']) checksum = 'ad:%s' % checksum #tmpLog.debug('checksum for file %s is %s' % (inFile['path'], checksum)) if 'checksum' in inFile and inFile['checksum'] and inFile[ 'checksum'] == checksum: #tmpLog.debug('File %s already exists at %s' % (inLFN, inFile['path'])) continue dstpath = os.path.dirname(inFile['path']) # check if path exists if not create it. if not os.access(dstpath, os.F_OK): os.makedirs(dstpath) files.append({ 'scope': inFile['scope'], 'name': inLFN, 'destination': dstpath }) tmpLog.info('Number of files to dowload: {0} for job: {1}'.format( len(files), jobspec.PandaID)) #tmpLog.debug('files {0}'.format(files)) tmpLog.info('Setup of Pilot2 API client') data_client = data.StageInClient(site=jobspec.computingSite) allChecked = True ErrMsg = 'These files failed to download : ' if len(files) > 0: tmpLog.info( "Going to transfer {0} of files with one call to Pilot2 Data API" .format(len(files))) try: result = data_client.transfer(files) except Exception as e: tmpLog.error("Pilot2 Data API rise error: {0}".format( e.message)) tmpLog.debug( 'data_client.transfer(files) result:\n{0}'.format(result)) tmpLog.info("Transfer call to Pilot2 Data API completed") # loop over each file check result all must be true for entire result to be true if result: for answer in result: if answer['errno'] != 0: allChecked = False ErrMsg = ErrMsg + (" %s " % answer['name']) else: tmpLog.info( 'Looks like all files in place. Number of files: {0}'. format(len(files))) # return tmpLog.debug( 'Finished data transfer with {0} files for job {1}'.format( len(files), jobspec.PandaID)) if allChecked: return True, '' else: return False, ErrMsg
def trigger_preparation(self, jobspec): # make logger tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID), method_name='trigger_preparation') tmpLog.debug('start') # check that jobspec.computingSite is defined if jobspec.computingSite is None: # not found tmpLog.error('jobspec.computingSite is not defined') return False, 'jobspec.computingSite is not defined' else: tmpLog.debug('jobspec.computingSite : {0}'.format(jobspec.computingSite)) # get input files files = [] inFiles = jobspec.get_input_file_attributes(skip_ready=True) # set path to each file for inLFN, inFile in iteritems(inFiles): inFile['path'] = mover_utils.construct_file_path(self.basePath, inFile['scope'], inLFN) tmpLog.debug('To check file: %s' % inFile) if os.path.exists(inFile['path']): checksum = core_utils.calc_adler32(inFile['path']) checksum = 'ad:%s' % checksum tmpLog.debug('checksum for file %s is %s' % (inFile['path'], checksum)) if 'checksum' in inFile and inFile['checksum'] and inFile['checksum'] == checksum: tmpLog.debug('File %s already exists at %s' % (inLFN, inFile['path'])) continue dstpath = os.path.dirname(inFile['path']) # check if path exists if not create it. if not os.access(dstpath, os.F_OK): os.makedirs(dstpath) files.append({'scope': inFile['scope'], 'name': inLFN, 'destination': dstpath}) tmpLog.debug('files[] {0}'.format(files)) allChecked = True ErrMsg = 'These files failed to download : ' if files: threads = [] n_files_per_thread = (len(files) + self.n_threads - 1) / self.n_threads tmpLog.debug('num files per thread: %s' % n_files_per_thread) for i in range(0, len(files), n_files_per_thread): sub_files = files[i:i + n_files_per_thread] thread = threading.Thread(target=self.stage_in, kwargs={'tmpLog': tmpLog, 'jobspec': jobspec, 'files': sub_files}) threads.append(thread) [t.start() for t in threads] while len(threads) > 0: time.sleep(1) threads = [t for t in threads if t and t.isAlive()] tmpLog.info('Checking all files: {0}'.format(files)) for file in files: if file['errno'] != 0: allChecked = False ErrMsg = ErrMsg + (" %s " % file['name']) # return tmpLog.debug('stop') if allChecked: tmpLog.info('Looks like all files are successfully downloaded.') return True, '' else: return False, ErrMsg
def trigger_stage_out(self, jobspec): """Trigger the stage-out procedure for the job. Output files are available through jobspec.get_outfile_specs(skip_done=False) which gives a list of FileSpecs not yet done. FileSpec.attemptNr shows how many times transfer was tried for the file so far. :param jobspec: job specifications :type jobspec: JobSpec :return: A tuple of return code (True: success, False: fatal failure, None: temporary failure) and error dialog :rtype: (bool, string) """ # let gc clean up memory gc.collect() # make logger tmpLog = self.make_logger(_logger, 'PandaID={0}'.format(jobspec.PandaID), method_name='trigger_stage_out') tmpLog.debug('start') # get the environment harvester_env = os.environ.copy() #tmpLog.debug('Harvester environment : {}'.format(harvester_env)) xrdcpOutput = None allfiles_transfered = True overall_errMsg = "" fileAttrs = jobspec.get_output_file_attributes() # loop over all output files for fileSpec in jobspec.get_output_file_specs(skip_done=True): # fileSpec.objstoreID = 123 # fileSpec.fileAttributes['guid'] # construct source and destination paths dstPath = mover_utils.construct_file_path( self.dstBasePath, fileAttrs[fileSpec.lfn]['scope'], fileSpec.lfn) # local path localPath = mover_utils.construct_file_path( self.localBasePath, fileAttrs[fileSpec.lfn]['scope'], fileSpec.lfn) tmpLog.debug('fileSpec.path - {0} fileSpec.lfn = {1}'.format( fileSpec.path, fileSpec.lfn)) localPath = fileSpec.path if self.checkLocalPath: # check if already exits if os.path.exists(localPath): # calculate checksum checksum = core_utils.calc_adler32(localPath) checksum = 'ad:{0}'.format(checksum) if checksum == fileAttrs[fileSpec.lfn]['checksum']: continue # collect list of output files if xrdcpOutput is None: xrdcpOutput = [dstPath] else: if dstPath not in xrdcpOutput: xrdcpOutput.append(dstPath) # transfer using xrdcp one file at a time tmpLog.debug('execute xrdcp') args = ['xrdcp', '--nopbar', '--force'] args_files = [localPath, dstPath] if self.xrdcpOpts is not None: args += self.xrdcpOpts.split() args += args_files fileSpec.attemptNr += 1 try: xrdcp_cmd = ' '.join(args) tmpLog.debug('execute: {0}'.format(xrdcp_cmd)) process = subprocess.Popen(xrdcp_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=harvester_env, shell=True) try: stdout, stderr = process.communicate(timeout=self.timeout) except subprocess.TimeoutExpired: process.kill() stdout, stderr = process.communicate() tmpLog.warning('command timeout') return_code = process.returncode if stdout is not None: if not isinstance(stdout, str): stdout = stdout.decode() stdout = stdout.replace('\n', ' ') if stderr is not None: if not isinstance(stderr, str): stderr = stderr.decode() stderr = stderr.replace('\n', ' ') tmpLog.debug("stdout: %s" % stdout) tmpLog.debug("stderr: %s" % stderr) except Exception: core_utils.dump_error_message(tmpLog) return_code = 1 if return_code == 0: fileSpec.status = 'finished' else: overall_errMsg += "file - {0} did not transfer error code {1} ".format( localPath, return_code) allfiles_transfered = False errMsg = 'failed with {0}'.format(return_code) tmpLog.error(errMsg) # check attemptNr if fileSpec.attemptNr >= self.maxAttempts: tmpLog.error( 'reached maxattempts: {0}, marked it as failed'.format( self.maxAttempts)) fileSpec.status = 'failed' # force update fileSpec.force_update('status') tmpLog.debug('file: {0} status: {1}'.format( fileSpec.lfn, fileSpec.status)) del process, stdout, stderr # end loop over output files # nothing to transfer if xrdcpOutput is None: tmpLog.debug('done with no transfers') return True, '' # check if all files were transfered tmpLog.debug('done') if allfiles_transfered: return True, '' else: return None, overall_errMsg
def get_files_to_stage_out(self, workspec): # get logger tmpLog = core_utils.make_logger(_logger, 'workerID={0}'.format(workspec.workerID), method_name='get_files_to_stage_out') fileDict = dict() # look for the json just under the access point for pandaID in workspec.pandaid_list: # look for the json just under the access point accessPoint = self.get_access_point(workspec, pandaID) jsonFilePath = os.path.join(accessPoint, jsonOutputsFileName) readJsonPath = jsonFilePath + suffixReadJson # first look for json.read which is not yet acknowledged tmpLog.debug('looking for output file {0}'.format(readJsonPath)) if os.path.exists(readJsonPath): pass else: tmpLog.debug('looking for output file {0}'.format(jsonFilePath)) if not os.path.exists(jsonFilePath): # not found tmpLog.debug('not found') continue try: tmpLog.debug('found') # rename to prevent from being overwritten os.rename(jsonFilePath, readJsonPath) except Exception: tmpLog.error('failed to rename json') continue # load json toSkip = False loadDict = None try: with open(readJsonPath) as jsonFile: loadDict = json.load(jsonFile) except Exception: tmpLog.error('failed to load json') toSkip = True # test validity of data format (ie it should be a Dictionary) if not toSkip: if not isinstance(loadDict, dict): tmpLog.error('loaded data is not a dictionary') toSkip = True # collect files and events nData = 0 if not toSkip: sizeMap = dict() chksumMap = dict() eventsList = dict() for tmpPandaID, tmpEventMapList in iteritems(loadDict): tmpPandaID = long(tmpPandaID) # test if tmpEventMapList is a list if not isinstance(tmpEventMapList, list): tmpLog.error('loaded data item is not a list') toSkip = True break for tmpEventInfo in tmpEventMapList: try: nData += 1 if 'eventRangeID' in tmpEventInfo: tmpEventRangeID = tmpEventInfo['eventRangeID'] else: tmpEventRangeID = None tmpFileDict = dict() pfn = tmpEventInfo['path'] lfn = os.path.basename(pfn) tmpFileDict['path'] = pfn if pfn not in sizeMap: if 'fsize' in tmpEventInfo: sizeMap[pfn] = tmpEventInfo['fsize'] else: sizeMap[pfn] = os.stat(pfn).st_size tmpFileDict['fsize'] = sizeMap[pfn] tmpFileDict['type'] = tmpEventInfo['type'] if tmpEventInfo['type'] in ['log', 'output']: # disable zipping tmpFileDict['isZip'] = 0 elif tmpEventInfo['type'] == 'zip_output': # already zipped tmpFileDict['isZip'] = 1 elif 'isZip' in tmpEventInfo: tmpFileDict['isZip'] = tmpEventInfo['isZip'] # guid if 'guid' in tmpEventInfo: tmpFileDict['guid'] = tmpEventInfo['guid'] else: tmpFileDict['guid'] = str(uuid.uuid4()) # get checksum if pfn not in chksumMap: if 'chksum' in tmpEventInfo: chksumMap[pfn] = tmpEventInfo['chksum'] else: chksumMap[pfn] = core_utils.calc_adler32(pfn) tmpFileDict['chksum'] = chksumMap[pfn] if tmpPandaID not in fileDict: fileDict[tmpPandaID] = dict() if lfn not in fileDict[tmpPandaID]: fileDict[tmpPandaID][lfn] = [] fileDict[tmpPandaID][lfn].append(tmpFileDict) # skip if unrelated to events if tmpFileDict['type'] not in ['es_output', 'zip_output']: continue tmpFileDict['eventRangeID'] = tmpEventRangeID if tmpPandaID not in eventsList: eventsList[tmpPandaID] = list() eventsList[tmpPandaID].append({'eventRangeID': tmpEventRangeID, 'eventStatus': tmpEventInfo['eventStatus']}) except Exception: core_utils.dump_error_message(tmpLog) # dump events if not toSkip: if len(eventsList) > 0: curName = os.path.join(accessPoint, jsonEventsUpdateFileName) newName = curName + '.new' f = open(newName, 'w') json.dump(eventsList, f) f.close() os.rename(newName, curName) # remove empty file if toSkip or nData == 0: try: os.remove(readJsonPath) except Exception: pass tmpLog.debug('got {0} files for PandaID={1}'.format(nData, pandaID)) return fileDict
def trigger_preparation(self, jobspec): # make logger tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID), method_name='trigger_preparation') tmpLog.debug('Start. Trigger data transfer for job: {0}'.format(jobspec.PandaID)) # check that jobspec.computingSite is defined if jobspec.computingSite is None: # not found tmpLog.error('jobspec.computingSite is not defined') return False, 'jobspec.computingSite is not defined' else: tmpLog.debug('jobspec.computingSite : {0}'.format(jobspec.computingSite)) # get input files files = [] inFiles = jobspec.get_input_file_attributes(skip_ready=True) # set path to each file tmpLog.info("Prepare files to download (construct path and verifiy existing files)") for inLFN, inFile in iteritems(inFiles): inFile['path'] = mover_utils.construct_file_path(self.basePath, inFile['scope'], inLFN) # check if file exist. Skip alrady downoladed files if os.path.exists(inFile['path']): checksum = core_utils.calc_adler32(inFile['path']) checksum = 'ad:%s' % checksum #tmpLog.debug('checksum for file %s is %s' % (inFile['path'], checksum)) if 'checksum' in inFile and inFile['checksum'] and inFile['checksum'] == checksum: #tmpLog.debug('File %s already exists at %s' % (inLFN, inFile['path'])) continue dstpath = os.path.dirname(inFile['path']) # check if path exists if not create it. if not os.access(dstpath, os.F_OK): os.makedirs(dstpath) files.append({'scope': inFile['scope'], 'name': inLFN, 'destination': dstpath}) tmpLog.info('Number of files to dowload: {0} for job: {1}'.format(len(files), jobspec.PandaID)) #tmpLog.debug('files {0}'.format(files)) tmpLog.info('Setup of Pilot2 API client') data_client = data.StageInClient(site=jobspec.computingSite) allChecked = True ErrMsg = 'These files failed to download : ' if len(files) > 0: tmpLog.info("Going to transfer {0} of files with one call to Pilot2 Data API".format(len(files))) try: result = data_client.transfer(files) except Exception as e: tmpLog.error("Pilot2 Data API rise error: {0}".format(e.message)) tmpLog.debug('data_client.transfer(files) result:\n{0}'.format(result)) tmpLog.info("Transfer call to Pilot2 Data API completed") # loop over each file check result all must be true for entire result to be true if result: for answer in result: if answer['errno'] != 0: allChecked = False ErrMsg = ErrMsg + (" %s " % answer['name']) else: tmpLog.info('Looks like all files in place. Number of files: {0}'.format(len(files))) # return tmpLog.debug('Finished data transfer with {0} files for job {1}'.format(len(files), jobspec.PandaID)) if allChecked: return True, '' else: return False, ErrMsg
def trigger_preparation(self, jobspec): # make logger tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID), method_name='trigger_preparation') tmpLog.debug('start') # get the environment harvester_env = os.environ.copy() #tmpLog.debug('Harvester environment : {}'.format(harvester_env)) # loop over all inputs inFileInfo = jobspec.get_input_file_attributes() xrdcpInput = None allfiles_transfered = True overall_errMsg = "" for tmpFileSpec in jobspec.inFiles: # construct source and destination paths srcPath = mover_utils.construct_file_path( self.srcBasePath, inFileInfo[tmpFileSpec.lfn]['scope'], tmpFileSpec.lfn) # local path localPath = mover_utils.construct_file_path( self.localBasePath, inFileInfo[tmpFileSpec.lfn]['scope'], tmpFileSpec.lfn) if self.checkLocalPath: # check if already exits if os.path.exists(localPath): # calculate checksum checksum = core_utils.calc_adler32(localPath) checksum = 'ad:{0}'.format(checksum) if checksum == inFileInfo[tmpFileSpec.lfn]['checksum']: continue # make directories if needed if not os.path.isdir(os.path.dirname(localPath)): os.makedirs(os.path.dirname(localPath)) tmpLog.debug('Make directory - {0}'.format( os.path.dirname(localPath))) # collect list of input files if xrdcpInput is None: xrdcpInput = [srcPath] else: xrdcpInput.append[srcPath] # transfer using xrdcp one file at a time tmpLog.debug('execute xrdcp') args = ['xrdcp', '--nopbar', '--force'] args_files = [srcPath, localPath] if self.xrdcpOpts is not None: args += self.xrdcpOpts.split() args += args_files tmpFileSpec.attemptNr += 1 try: xrdcp_cmd = ' '.join(args) tmpLog.debug('execute: {0}'.format(xrdcp_cmd)) p = subprocess.Popen(xrdcp_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=harvester_env, shell=True) try: stdout, stderr = p.communicate(timeout=self.timeout) except subprocess.TimeoutExpired: p.kill() stdout, stderr = p.communicate() tmpLog.warning('command timeout') return_code = p.returncode if stdout is not None: if not isinstance(stdout, str): stdout = stdout.decode() stdout = stdout.replace('\n', ' ') if stderr is not None: if not isinstance(stderr, str): stderr = stderr.decode() stderr = stderr.replace('\n', ' ') tmpLog.debug("stdout: %s" % stdout) tmpLog.debug("stderr: %s" % stderr) except Exception: core_utils.dump_error_message(tmpLog) return_code = 1 if return_code != 0: overall_errMsg += "file - {0} did not transfer error code {1} ".format( localPath, return_code) allfiles_transfered = False errMsg = 'failed with {0}'.format(return_code) tmpLog.error(errMsg) # check attemptNr if tmpFileSpec.attemptNr >= self.maxAttempts: errMsg = 'gave up due to max attempts' tmpLog.error(errMsg) return (False, errMsg) # end loop over input files # nothing to transfer if xrdcpInput is None: tmpLog.debug('done with no transfers') return True, '' # check if all files were transfered if allfiles_transfered: return True, '' else: return None, overall_errMsg