def __call__(self, jobNum, jobObj):
onTerm = lambda term: term(
jobNum, jobObj
) # [[f1], [f2,f3]] => f1(...) || (f2(...) && f3(...))
return reduce(
operator.or_,
imap(lambda andTerm: reduce(operator.and_, imap(onTerm, andTerm)),
self.js))
def __call__(self, jobnum, job_obj): def _check_id(job_range): if (job_range[0] == '') or (jobnum >= job_range[0]): if (job_range[1] == '') or (jobnum <= job_range[1]): return True return False return reduce(operator.or_, imap(_check_id, self._ranges))
def __call__(self, jobNum, jobObj): def checkID(jobRange): if (jobRange[0] == '') or (jobNum >= jobRange[0]): if (jobRange[1] == '') or (jobNum <= jobRange[1]): return True return False return reduce(operator.or_, imap(checkID, self.ranges))
def split_partitions(self, block_iter, entry_first=0):
for block in block_iter:
for sub_block in reduce(lambda x, y: y.divide_blocks(x),
self._part_list, [block]):
for partition in self._part_final.split_partitions([sub_block
]):
yield partition
def __call__(self, jobNum, jobObj):
def checkID(jobRange):
if (jobRange[0] == '') or (jobNum >= jobRange[0]):
if (jobRange[1] == '') or (jobNum <= jobRange[1]):
return True
return False
return reduce(operator.or_, imap(checkID, self._ranges))
def splitDatasetInternal(self, blocks, firstEvent = 0): for block in blocks: splitterList = self._setup(self._splitstack, block) subSplitter = imap(lambda x: FileLevelSplitter.createInstance(x, self._config), splitterList[:-1]) endSplitter = DataSplitter.createInstance(splitterList[-1], self._config) for subBlock in reduce(lambda x, y: y.splitBlocks(x), subSplitter, [block]): for splitting in endSplitter.splitDatasetInternal([subBlock]): yield splitting
def lfnTrans(self, lfn):
if lfn and self._looseMatch: # return looseMatch path elements in reverse order
def trunkPath(x, y):
return (lambda s:
(s[0], os.path.join(x[1], s[1])))(os.path.split(x[0]))
return reduce(trunkPath, irange(self._looseMatch), (lfn, ''))[1]
return lfn
def parse_time(usertime):
if usertime is None or usertime == '':
return -1
tmp = lmap(int, usertime.split(':'))
while len(tmp) < 3:
tmp.append(0)
if tmp[2] > 59 or tmp[1] > 59 or len(tmp) > 3:
raise Exception('Invalid time format: %s' % usertime)
return reduce(lambda x, y: x * 60 + y, tmp)
def parse_time(usertime):
if usertime is None or usertime == '':
return -1
tmp = lmap(int, usertime.split(':'))
while len(tmp) < 3:
tmp.append(0)
if tmp[2] > 59 or tmp[1] > 59 or len(tmp) > 3:
raise Exception('Invalid time format: %s' % usertime)
return reduce(lambda x, y: x * 60 + y, tmp)
def _init_psrc_max(self): self._psrc_info_list = [] psrc_group_size = 1 for (psrc, psrc_max) in izip(self._psrc_list, self._psrc_max_list): self._psrc_info_list.append((psrc, psrc_max, psrc_group_size)) if psrc_max: psrc_group_size *= psrc_max psrc_max_list = lfilter(lambda n: n is not None, self._psrc_max_list) if psrc_max_list: return reduce(lambda a, b: a * b, psrc_max_list)
def _init_psrc_max(self):
self._psrc_info_list = []
psrc_group_size = 1
for (psrc, psrc_max) in izip(self._psrc_list, self._psrc_max_list):
self._psrc_info_list.append((psrc, psrc_max, psrc_group_size))
if psrc_max:
psrc_group_size *= psrc_max
psrc_max_list = lfilter(lambda n: n is not None, self._psrc_max_list)
if psrc_max_list:
return reduce(lambda a, b: a * b, psrc_max_list)
def initMaxParameters(self): self.quickFill = [] prev = 1 for (psource, maxN) in izip(self._psourceList, self._psourceMaxList): self.quickFill.append((psource, maxN, prev)) if maxN: prev *= maxN maxList = lfilter(lambda n: n is not None, self._psourceMaxList) if maxList: return reduce(lambda a, b: a * b, maxList)
def _combineSources(self, clsName, args):
repeat = reduce(lambda a, b: a * b, ifilter(lambda expr: isinstance(expr, int), args), 1)
args = lfilter(lambda expr: not isinstance(expr, int), args)
if args:
result = ParameterSource.createInstance(clsName, *args)
if repeat > 1:
return ParameterSource.createInstance('RepeatParameterSource', result, repeat)
return result
elif repeat > 1:
return repeat
return NullParameterSource()
def __init__(self, arg, **kwargs):
predef = {
'TODO': 'SUBMITTED,WAITING,READY,QUEUED,UNKNOWN',
'ALL': str.join(',', Job.enumNames)
}
RegExSelector.__init__(self, predef.get(arg.upper(), arg), None,
lambda x: '^%s.*' % x.upper())
stateList = reduce(
operator.add,
imap(lambda x: lfilter(x.match, Job.enumNames), self._rxList))
self._states = lmap(Job.str2enum, stateList)
def splitDatasetInternal(self, blocks, firstEvent=0):
for block in blocks:
splitterList = self._setup(self._splitstack, block)
subSplitter = imap(
lambda x: FileLevelSplitter.createInstance(x, self._config),
splitterList[:-1])
endSplitter = DataSplitter.createInstance(splitterList[-1],
self._config)
for subBlock in reduce(lambda x, y: y.splitBlocks(x), subSplitter,
[block]):
for splitting in endSplitter.splitDatasetInternal([subBlock]):
yield splitting
def combineSources(self, PSourceClass, args): repeat = reduce(lambda a, b: a * b, ifilter(lambda expr: isinstance(expr, int), args), 1) args = lfilter(lambda expr: not isinstance(expr, int), args) if len(args) > 1: result = PSourceClass(*args) elif len(args) > 0: result = args[0] else: return utils.QM(repeat > 1, [repeat], []) if repeat > 1: return [RepeatParameterSource(result, repeat)] return [result]
class JobClass(object):
mkJobClass = lambda *fList: (reduce(operator.add,
imap(lambda f: 1 << f, fList)), fList)
ATWMS = mkJobClass(Job.SUBMITTED, Job.WAITING, Job.READY, Job.QUEUED)
RUNNING = mkJobClass(Job.RUNNING)
PROCESSING = mkJobClass(Job.SUBMITTED, Job.WAITING, Job.READY, Job.QUEUED,
Job.RUNNING)
READY = mkJobClass(Job.INIT, Job.FAILED, Job.ABORTED, Job.CANCELLED)
DONE = mkJobClass(Job.DONE)
SUCCESS = mkJobClass(Job.SUCCESS)
DISABLED = mkJobClass(Job.DISABLED)
ENDSTATE = mkJobClass(Job.SUCCESS, Job.DISABLED)
PROCESSED = mkJobClass(Job.SUCCESS, Job.FAILED, Job.CANCELLED, Job.ABORTED)
def optSplit(opt, delim, empty = ''):
""" Split option strings into fixed tuples
>>> optSplit('abc : ghi # def', ['#', ':'])
('abc', 'def', 'ghi')
>>> optSplit('abc:def', '::')
('abc', 'def', '')
"""
def getDelimeterPart(oldResult, prefix):
try:
tmp = oldResult[0].split(prefix)
new = tmp.pop(1)
try: # Find position of other delimeters in string
otherDelim = min(ifilter(lambda idx: idx >= 0, imap(new.find, delim)))
tmp[0] += new[otherDelim:]
except Exception:
otherDelim = None
return [str.join(prefix, tmp)] + oldResult[1:] + [new[:otherDelim]]
except Exception:
return oldResult + ['']
result = lmap(str.strip, reduce(getDelimeterPart, delim, [opt]))
return tuple(imap(lambda x: QM(x == '', empty, x), result))
def split_opt(opt, delim):
""" Split option strings into fixed tuples
>>> split_opt('abc : ghi # def', ['#', ':'])
('abc', 'def', 'ghi')
>>> split_opt('abc:def', '::')
('abc', 'def', '')
"""
def _get_delimeter_part(old_result, prefix):
try:
tmp = old_result[0].split(prefix)
new = tmp.pop(1)
try: # Find position of other delimeters in string
other_delim = min(ifilter(lambda idx: idx >= 0, imap(new.find, delim)))
tmp[0] += new[other_delim:]
except Exception:
other_delim = None
return [str.join(prefix, tmp)] + old_result[1:] + [new[:other_delim]]
except Exception:
return old_result + ['']
result = imap(str.strip, reduce(_get_delimeter_part, delim, [opt]))
return tuple(result)
def optSplit(opt, delim, empty=''):
""" Split option strings into fixed tuples
>>> optSplit('abc : ghi # def', ['#', ':'])
('abc', 'def', 'ghi')
>>> optSplit('abc:def', '::')
('abc', 'def', '')
"""
def getDelimeterPart(oldResult, prefix):
try:
tmp = oldResult[0].split(prefix)
new = tmp.pop(1)
try: # Find position of other delimeters in string
otherDelim = min(
ifilter(lambda idx: idx >= 0, imap(new.find, delim)))
tmp[0] += new[otherDelim:]
except Exception:
otherDelim = None
return [str.join(prefix, tmp)] + oldResult[1:] + [new[:otherDelim]]
except Exception:
return oldResult + ['']
result = lmap(str.strip, reduce(getDelimeterPart, delim, [opt]))
return tuple(imap(lambda x: QM(x == '', empty, x), result))
def split_opt(opt, delim):
""" Split option strings into fixed tuples
>>> split_opt('abc : ghi # def', ['#', ':'])
('abc', 'def', 'ghi')
>>> split_opt('abc:def', '::')
('abc', 'def', '')
"""
def _get_delimeter_part(old_result, prefix):
try:
tmp = old_result[0].split(prefix)
new = tmp.pop(1)
try: # Find position of other delimeters in string
other_delim = min(
ifilter(lambda idx: idx >= 0, imap(new.find, delim)))
tmp[0] += new[other_delim:]
except Exception:
other_delim = None
return [str.join(prefix, tmp)
] + old_result[1:] + [new[:other_delim]]
except Exception:
return old_result + ['']
result = imap(str.strip, reduce(_get_delimeter_part, delim, [opt]))
return tuple(result)
def add_pairs_in_sub_chunk(self, local_pair_angles, pairs):
local_angular_separation_bins = \
calc_angular_separation(pairs, local_pair_angles, self.ar_extinction, self.extinction_mean)
mpi_helper.l_print('local pair count:',
local_angular_separation_bins[1].sum())
local_pair_separation_bins_array = local_angular_separation_bins
local_pair_separation_bins_metadata = None
local_array_shape = local_pair_separation_bins_array.shape
array_block_size = np.prod(local_array_shape[1:])
comm.Barrier()
mpi_helper.r_print("BEGIN GATHER")
mpi_helper.l_print_no_barrier('local array shape:', local_array_shape)
array_counts = comm.allgather(local_array_shape[0])
pair_separation_bins_array = None
array_endings = np.cumsum(array_counts)
array_displacements = array_endings - np.array(array_counts)
if comm.rank == 0:
mpi_helper.r_print('array count:', array_counts)
root_array_shape = (np.sum(array_counts), ) + local_array_shape[1:]
mpi_helper.r_print('root array shape:', root_array_shape)
pair_separation_bins_array = np.ones(shape=root_array_shape,
dtype=np.float64)
send_buf = [
local_pair_separation_bins_array,
local_array_shape[0] * array_block_size
]
receive_buf = [
pair_separation_bins_array,
np.multiply(array_counts, array_block_size),
np.multiply(array_displacements, array_block_size), MPI.DOUBLE
]
# mpi_helper.l_print(send_buf)
comm.Gatherv(sendbuf=send_buf, recvbuf=receive_buf)
list_pair_separation_bins_metadata = comm.gather(
local_pair_separation_bins_metadata)
comm.Barrier()
mpi_helper.r_print("END_GATHER")
if comm.rank == 0:
# mpi_helper.r_print(receive_buf[0][0][0:10])
list_pair_separation_bins = [
pair_separation_bins_array[
array_displacements[rank]:array_endings[rank]] for rank,
metadata in enumerate(list_pair_separation_bins_metadata)
]
# initialize bins only if this is the first time we get here
# for now use a function level static variable
if self.angular_separation_bins is None:
self.angular_separation_bins = np.zeros_like(
local_angular_separation_bins)
# add new results to existing bins
if list_pair_separation_bins:
self.angular_separation_bins = reduce(
lambda x, y: x + y, list_pair_separation_bins,
self.angular_separation_bins)
mpi_helper.r_print('total number of pixel pairs in bins:',
self.angular_separation_bins[1].sum())
np.save("../../data/extinction_correlation.npy",
self.angular_separation_bins)
# pixel_pairs.significant_qso_pairs.save(settings.get_significant_qso_pairs_npy())
else:
print('no results received.')
def _get_proposal(splitter): return reduce(lambda prop, prov: prov.check_splitter(prop), self._provider_list, splitter)
def lfnTrans(self, lfn): if lfn and self._looseMatch: # return looseMatch path elements in reverse order def trunkPath(x, y): return (lambda s: (s[0], os.path.join(x[1], s[1])))(os.path.split(x[0])) return reduce(trunkPath, irange(self._looseMatch), (lfn, ''))[1] return lfn
def __call__(self, jobNum, jobObj):
return reduce(
operator.and_,
imap(lambda selector: selector(jobNum, jobObj), self._selectors))
def __call__(self, jobNum, jobObj): onTerm = lambda term: term(jobNum, jobObj) # [[f1], [f2,f3]] => f1(...) || (f2(...) && f3(...)) return reduce(operator.or_, imap(lambda andTerm: reduce(operator.and_, imap(onTerm, andTerm)), self.js))
def getProposal(x): return reduce(lambda prop, prov: prov.checkSplitter(prop), self._providerList, x)
def split_partitions(self, block_iter, entry_first=0): for block in block_iter: for sub_block in reduce(lambda x, y: y.divide_blocks(x), self._part_list, [block]): for partition in self._part_final.split_partitions([sub_block]): yield partition
def _separate_repeat(psrc_list): repeat = reduce(operator.mul, ifilter(lambda p: not isinstance(p, ParameterSource), psrc_list), 1) return (repeat, lfilter(lambda p: isinstance(p, ParameterSource), psrc_list))
def __call__(self, jobnum, job_obj): def _on_term(term): return term(jobnum, job_obj) # [[f1], [f2,f3]] => f1(...) || (f2(...) && f3(...)) return reduce(operator.or_, imap(lambda andTerm: reduce(operator.and_, imap(_on_term, andTerm)), self._js))
def _separate_repeat(psrc_list):
repeat = reduce(
operator.mul,
ifilter(lambda p: not isinstance(p, ParameterSource), psrc_list), 1)
return (repeat, lfilter(lambda p: isinstance(p, ParameterSource),
psrc_list))
def __init__(self, arg, **kwargs):
predef = {'TODO': 'SUBMITTED,WAITING,READY,QUEUED', 'ALL': str.join(',', Job.enumNames)}
RegExSelector.__init__(self, predef.get(arg.upper(), arg), None, lambda x: '^%s.*' % x.upper())
stateList = reduce(operator.add, imap(lambda x: lfilter(x.match, Job.enumNames), self.rxList))
self.states = lmap(Job.str2enum, stateList)
def __call__(self, jobNum, jobObj): def match(var, rx): return rx.search(self.jobCfg(jobNum, var)) is not None return reduce(operator.and_, ismap(match, self.rxDict))
def add_pairs_in_sub_chunk(self, delta_t_file, local_pair_angles, pairs,
pixel_pairs):
local_pair_separation_bins = \
pixel_pairs.add_qso_pairs_to_bins(pairs, local_pair_angles, delta_t_file)
mpi_helper.l_print('local pair count:',
local_pair_separation_bins.get_pair_count())
local_pair_separation_bins_array = local_pair_separation_bins.get_data_as_array(
)
local_pair_separation_bins_metadata = local_pair_separation_bins.get_metadata(
)
local_array_shape = local_pair_separation_bins_array.shape
array_block_size = np.prod(local_array_shape[1:])
comm.Barrier()
mpi_helper.r_print("BEGIN GATHER")
mpi_helper.l_print_no_barrier('local array shape:', local_array_shape)
array_counts = comm.allgather(local_array_shape[0])
pair_separation_bins_array = None
array_endings = np.cumsum(array_counts)
array_displacements = array_endings - np.array(array_counts)
if comm.rank == 0:
mpi_helper.r_print('array count:', array_counts)
root_array_shape = (np.sum(array_counts), ) + local_array_shape[1:]
mpi_helper.r_print('root array shape:', root_array_shape)
pair_separation_bins_array = np.ones(shape=root_array_shape,
dtype=np.float64)
send_buf = [
local_pair_separation_bins_array,
local_array_shape[0] * array_block_size
]
receive_buf = [
pair_separation_bins_array,
np.multiply(array_counts, array_block_size),
np.multiply(array_displacements, array_block_size), MPI.DOUBLE
]
# mpi_helper.l_print(send_buf)
comm.Gatherv(sendbuf=send_buf, recvbuf=receive_buf)
list_pair_separation_bins_metadata = comm.gather(
local_pair_separation_bins_metadata)
comm.Barrier()
mpi_helper.r_print("END_GATHER")
if comm.rank == 0:
# mpi_helper.r_print(receive_buf[0][0][0:10])
list_pair_separation_bins = [
type(local_pair_separation_bins).load_from(
pair_separation_bins_array[
array_displacements[rank]:array_endings[rank]],
metadata) for rank, metadata in enumerate(
list_pair_separation_bins_metadata)
]
# initialize bins only if this is the first time we get here
if not self.pair_separation_bins:
self.pair_separation_bins = local_pair_separation_bins.init_as(
local_pair_separation_bins)
# add new results to existing bins
if list_pair_separation_bins:
for i in list_pair_separation_bins:
for g in i.dict_bins_3d_data.keys():
mpi_helper.l_print_no_barrier(
np.sum(i.dict_bins_3d_data[g].ar_count))
self.pair_separation_bins = reduce(lambda x, y: x + y,
list_pair_separation_bins,
self.pair_separation_bins)
mpi_helper.r_print('total number of pixel pairs in bins:',
self.pair_separation_bins.get_pair_count())
self.pair_separation_bins.flush()
pixel_pairs.significant_qso_pairs.save(
settings.get_significant_qso_pairs_npy())
else:
print('no results received.')
def __call__(self, jobNum, jobObj): return reduce(operator.and_, imap(lambda selector: selector(jobNum, jobObj), self.selectors))
def __call__(self, jobNum, jobObj):
def match(var, rx):
return rx.search(self._jobCfg(jobNum, var)) is not None
return reduce(operator.and_, ismap(match, self._rxDict))
def getProposal(x):
return reduce(lambda prop, prov: prov.checkSplitter(prop),
self._providerList, x)
def __call__(self, jobnum, job_obj): def _match(var, regex_obj): return regex_obj.search(self._job_config(jobnum, var)) is not None return reduce(operator.and_, ismap(_match, self._regex_obj_list))
