def _get_gc_block_list(self, use_phedex):
dataset_name_list = self.get_dataset_name_list()
progress_ds = ProgressActivity('Getting dataset', len(dataset_name_list))
for dataset_idx, dataset_path in enumerate(dataset_name_list):
progress_ds.update_progress(dataset_idx, msg='Getting dataset %s' % dataset_path)
counter = 0
blockinfo_list = list(self._filter_cms_blockinfo_list(dataset_path, not use_phedex))
progress_block = ProgressActivity('Getting block information', len(blockinfo_list))
for (block_path, replica_infos) in blockinfo_list:
result = {}
result[DataProvider.Dataset] = block_path.split('#')[0]
result[DataProvider.BlockName] = block_path.split('#')[1]
progress_block.update_progress(counter,
msg='Getting block information for ' + result[DataProvider.BlockName])
if use_phedex: # Start parallel phedex query
replicas_dict = {}
phedex_thread = start_thread('Query phedex site info for %s' % block_path,
self._get_phedex_replica_list, block_path, replicas_dict)
self._fill_cms_fi_list(result, block_path)
phedex_thread.join()
replica_infos = replicas_dict.get(block_path)
else:
self._fill_cms_fi_list(result, block_path)
result[DataProvider.Locations] = self._process_replica_list(block_path, replica_infos)
if len(result[DataProvider.FileList]):
counter += 1
yield result
progress_block.finish()
if counter == 0:
raise DatasetError('Dataset %s does not contain any valid blocks!' % dataset_path)
progress_ds.finish()
def download_monitored(jobNum, output, fileIdx, checkPath, sourcePath,
targetPath):
copyAbortLock = threading.Lock()
monitorLock = threading.Lock()
monitorLock.acquire()
monitor = start_thread('Download monitor %s' % jobNum, transfer_monitor,
output, fileIdx, checkPath, monitorLock,
copyAbortLock)
result = -1
procCP = se_copy(sourcePath, targetPath, tmp=checkPath)
while True:
if not copyAbortLock.acquire(False):
monitor.join()
break
copyAbortLock.release()
result = procCP.status(timeout=0)
if result is not None:
monitorLock.release()
monitor.join()
break
time.sleep(0.02)
if result != 0:
output.error('Unable to copy file from SE!')
utils.eprint(
'%s\n%s\n' %
(procCP.stdout.read(timeout=0), procCP.stderr.read(timeout=0)))
return False
return True
def getGCBlocks(self, usePhedex):
for datasetPath in self.getDatasets():
counter = 0
for (blockPath,
listSE) in self.getCMSBlocks(datasetPath,
getSites=not usePhedex):
result = {}
result[DataProvider.Dataset] = blockPath.split('#')[0]
result[DataProvider.BlockName] = blockPath.split('#')[1]
if usePhedex: # Start parallel phedex query
dictSE = {}
tPhedex = start_thread(
'Query phedex site info for %s' % blockPath,
self._getPhedexSEList, blockPath, dictSE)
self.fillCMSFiles(result, blockPath)
tPhedex.join()
listSE = dictSE.get(blockPath)
else:
self.fillCMSFiles(result, blockPath)
result[DataProvider.Locations] = listSE
if len(result[DataProvider.FileList]):
counter += 1
yield result
if counter == 0:
raise DatasetError(
'Dataset %s does not contain any valid blocks!' %
datasetPath)
def tchain(iterables, timeout = None): # Combines multiple, threaded generators into single generator
import time
from grid_control.utils.thread_tools import start_thread, GCQueue
threads = []
result = GCQueue()
for idx, it in enumerate(iterables):
def generator_thread(iterator):
try:
for item in iterator:
result.put(item)
finally:
result.put(GCQueue) # Use GCQueue as end-of-generator marker
threads.append(start_thread('generator thread %d' % idx, generator_thread, it))
if timeout is not None:
t_end = time.time() + timeout
try:
while len(threads):
if timeout is not None:
timeout = max(0, t_end - time.time())
try:
tmp = result.get(timeout)
except IndexError: # Empty queue after waiting for timeout
break
if tmp == GCQueue:
threads.pop() # which thread is irrelevant - only used as counter
else:
yield tmp
except Exception:
result.finish()
def getGCBlocks(self, usePhedex):
for datasetPath in self.getDatasets():
counter = 0
for (blockPath, listSE) in self.getCMSBlocks(datasetPath, getSites = not usePhedex):
result = {}
result[DataProvider.Dataset] = blockPath.split('#')[0]
result[DataProvider.BlockName] = blockPath.split('#')[1]
if usePhedex: # Start parallel phedex query
dictSE = {}
tPhedex = start_thread('Query phedex site info for %s' % blockPath, self.getPhedexSEList, blockPath, dictSE)
if self._lumi_query:
result[DataProvider.Metadata] = ['Runs', 'Lumi']
result[DataProvider.FileList] = list(self.getCMSFiles(blockPath))
if usePhedex:
tPhedex.join()
listSE = dictSE.get(blockPath)
result[DataProvider.Locations] = listSE
if len(result[DataProvider.FileList]):
counter += 1
yield result
if counter == 0:
raise DatasetError('Dataset %s does not contain any valid blocks!' % datasetPath)
def tchain(iterables, timeout = None): # Combines multiple, threaded generators into single generator
import time
from grid_control.utils.thread_tools import start_thread, GCQueue
threads = []
result = GCQueue()
ec = ExceptionCollector()
for idx, it in enumerate(iterables):
def generator_thread(iterator):
try:
try:
for item in iterator:
result.put(item)
finally:
result.put(GCQueue) # Use GCQueue as end-of-generator marker
except Exception:
ec.collect()
threads.append(start_thread('generator thread %d' % idx, generator_thread, it))
if timeout is not None:
t_end = time.time() + timeout
try:
while len(threads):
if timeout is not None:
timeout = max(0, t_end - time.time())
try:
tmp = result.get(timeout)
except IndexError: # Empty queue after waiting for timeout
break
if tmp == GCQueue:
threads.pop() # which thread is irrelevant - only used as counter
else:
yield tmp
except Exception:
result.finish()
ec.raise_any(NestedException('Caught exception during threaded chain'))
def _get_gc_block_list(self, use_phedex):
dataset_name_list = self.get_dataset_name_list()
progress_ds = ProgressActivity('Getting dataset',
len(dataset_name_list))
for dataset_idx, dataset_path in enumerate(dataset_name_list):
progress_ds.update_progress(dataset_idx,
msg='Getting dataset %s' %
dataset_path)
counter = 0
blockinfo_list = list(
self._filter_cms_blockinfo_list(dataset_path, not use_phedex))
progress_block = ProgressActivity('Getting block information',
len(blockinfo_list))
for (block_path, replica_infos) in blockinfo_list:
result = {}
result[DataProvider.Dataset] = block_path.split('#')[0]
result[DataProvider.BlockName] = block_path.split('#')[1]
progress_block.update_progress(
counter,
msg='Getting block information for ' +
result[DataProvider.BlockName])
if use_phedex and self._allow_phedex: # Start parallel phedex query
replicas_dict = {}
phedex_thread = start_thread(
'Query phedex site info for %s' % block_path,
self._get_phedex_replica_list, block_path,
replicas_dict)
self._fill_cms_fi_list(result, block_path)
phedex_thread.join()
replica_infos = replicas_dict.get(block_path)
else:
self._fill_cms_fi_list(result, block_path)
result[DataProvider.Locations] = self._process_replica_list(
block_path, replica_infos)
if len(result[DataProvider.FileList]):
counter += 1
yield result
progress_block.finish()
if counter == 0:
raise DatasetError(
'Dataset %s does not contain any valid blocks!' %
dataset_path)
progress_ds.finish()
def download_multithreaded_main(opts, workDir, jobList, incInfo, jobDB, token, DisplayClass, screen, errorOutput):
(active, todo) = ([], list(jobList))
todo.reverse()
screen.move(0, 0)
screen.savePos()
while True:
screen.erase()
screen.loadPos()
active = lfilter(lambda thread_display: thread_display[0].isAlive(), active)
while len(active) < int(opts.threads) and len(todo):
display = DisplayClass()
active.append((start_thread('Download %s' % todo[-1], download_job_output,
opts, incInfo, workDir, jobDB, token, todo.pop(), display), display))
for (_, display) in active:
sys.stdout.write(str.join('\n', display.output))
sys.stdout.write(str.join('\n', ['=' * 50] + errorOutput))
sys.stdout.flush()
if len(active) == 0:
break
time.sleep(0.01)
def download_monitored(jobNum, output, fileIdx, checkPath, sourcePath, targetPath):
copyAbortLock = GCLock()
monitorLock = GCLock()
monitorLock.acquire()
monitor = start_thread('Download monitor %s' % jobNum, transfer_monitor, output, fileIdx, checkPath, monitorLock, copyAbortLock)
result = -1
procCP = se_copy(sourcePath, targetPath, tmp = checkPath)
while True:
if not copyAbortLock.acquire(False):
monitor.join()
break
copyAbortLock.release()
result = procCP.status(timeout = 0)
if result is not None:
monitorLock.release()
monitor.join()
break
time.sleep(0.02)
if result != 0:
output.error('Unable to copy file from SE!')
log.critical('%s\n%s\n', procCP.stdout.read(timeout = 0), procCP.stderr.read(timeout = 0))
return False
return True
def _getGCBlocks(self, usePhedex):
for datasetPath in self.getDatasets():
counter = 0
for (blockPath, replica_infos) in self._getCMSBlocks(datasetPath, getSites = not usePhedex):
result = {}
result[DataProvider.Dataset] = blockPath.split('#')[0]
result[DataProvider.BlockName] = blockPath.split('#')[1]
if usePhedex: # Start parallel phedex query
dictReplicas = {}
tPhedex = start_thread('Query phedex site info for %s' % blockPath, self._getPhedexReplicas, blockPath, dictReplicas)
self._fillCMSFiles(result, blockPath)
tPhedex.join()
replica_infos = dictReplicas.get(blockPath)
else:
self._fillCMSFiles(result, blockPath)
result[DataProvider.Locations] = self._processReplicas(blockPath, replica_infos)
if len(result[DataProvider.FileList]):
counter += 1
yield result
if counter == 0:
raise DatasetError('Dataset %s does not contain any valid blocks!' % datasetPath)
def download_multithreaded_main(opts, workDir, jobList, incInfo, jobDB, token,
DisplayClass, screen, errorOutput):
(active, todo) = ([], list(jobList))
todo.reverse()
screen.move(0, 0)
screen.savePos()
while True:
screen.erase()
screen.loadPos()
active = lfilter(lambda thread_display: thread_display[0].isAlive(),
active)
while len(active) < int(opts.threads) and len(todo):
display = DisplayClass()
active.append((start_thread('Download %s' % todo[-1],
download_job_output, opts,
incInfo, workDir, jobDB, token,
todo.pop(), display), display))
for (_, display) in active:
sys.stdout.write(str.join('\n', display.output))
sys.stdout.write(str.join('\n', ['=' * 50] + errorOutput))
sys.stdout.flush()
if len(active) == 0:
break
time.sleep(0.01)
def _start_watcher(self, desc, daemon, pid, *args): desc += ' (%d:%r)' % (pid, [self._cmd] + self._args) if daemon: return start_daemon(desc, *args) return start_thread(desc, *args)
def _schedule_update_layout(self, sig=None, frame=None):
start_thread(
'update layout', self._draw,
self._update_layout) # using new thread to ensure RLock is free
def _schedule_update_layout(self, sig = None, frame = None):
start_thread('update layout', self._draw, self._update_layout) # using new thread to ensure RLock is free
def _start_watcher(self, desc, daemon, pid, *args):
desc += ' (%d:%r)' % (pid, [self._cmd] + self._args)
if daemon:
return start_daemon(desc, *args)
return start_thread(desc, *args)