def __call__(self, jobNum, jobObj): onTerm = lambda term: term( jobNum, jobObj ) # [[f1], [f2,f3]] => f1(...) || (f2(...) && f3(...)) return reduce( operator.or_, imap(lambda andTerm: reduce(operator.and_, imap(onTerm, andTerm)), self.js))
def __call__(self, jobnum, job_obj): def _check_id(job_range): if (job_range[0] == '') or (jobnum >= job_range[0]): if (job_range[1] == '') or (jobnum <= job_range[1]): return True return False return reduce(operator.or_, imap(_check_id, self._ranges))
def __call__(self, jobNum, jobObj): def checkID(jobRange): if (jobRange[0] == '') or (jobNum >= jobRange[0]): if (jobRange[1] == '') or (jobNum <= jobRange[1]): return True return False return reduce(operator.or_, imap(checkID, self.ranges))
def split_partitions(self, block_iter, entry_first=0): for block in block_iter: for sub_block in reduce(lambda x, y: y.divide_blocks(x), self._part_list, [block]): for partition in self._part_final.split_partitions([sub_block ]): yield partition
def __call__(self, jobNum, jobObj): def checkID(jobRange): if (jobRange[0] == '') or (jobNum >= jobRange[0]): if (jobRange[1] == '') or (jobNum <= jobRange[1]): return True return False return reduce(operator.or_, imap(checkID, self._ranges))
def splitDatasetInternal(self, blocks, firstEvent = 0): for block in blocks: splitterList = self._setup(self._splitstack, block) subSplitter = imap(lambda x: FileLevelSplitter.createInstance(x, self._config), splitterList[:-1]) endSplitter = DataSplitter.createInstance(splitterList[-1], self._config) for subBlock in reduce(lambda x, y: y.splitBlocks(x), subSplitter, [block]): for splitting in endSplitter.splitDatasetInternal([subBlock]): yield splitting
def lfnTrans(self, lfn): if lfn and self._looseMatch: # return looseMatch path elements in reverse order def trunkPath(x, y): return (lambda s: (s[0], os.path.join(x[1], s[1])))(os.path.split(x[0])) return reduce(trunkPath, irange(self._looseMatch), (lfn, ''))[1] return lfn
def parse_time(usertime): if usertime is None or usertime == '': return -1 tmp = lmap(int, usertime.split(':')) while len(tmp) < 3: tmp.append(0) if tmp[2] > 59 or tmp[1] > 59 or len(tmp) > 3: raise Exception('Invalid time format: %s' % usertime) return reduce(lambda x, y: x * 60 + y, tmp)
def parse_time(usertime): if usertime is None or usertime == '': return -1 tmp = lmap(int, usertime.split(':')) while len(tmp) < 3: tmp.append(0) if tmp[2] > 59 or tmp[1] > 59 or len(tmp) > 3: raise Exception('Invalid time format: %s' % usertime) return reduce(lambda x, y: x * 60 + y, tmp)
def _init_psrc_max(self): self._psrc_info_list = [] psrc_group_size = 1 for (psrc, psrc_max) in izip(self._psrc_list, self._psrc_max_list): self._psrc_info_list.append((psrc, psrc_max, psrc_group_size)) if psrc_max: psrc_group_size *= psrc_max psrc_max_list = lfilter(lambda n: n is not None, self._psrc_max_list) if psrc_max_list: return reduce(lambda a, b: a * b, psrc_max_list)
def _init_psrc_max(self): self._psrc_info_list = [] psrc_group_size = 1 for (psrc, psrc_max) in izip(self._psrc_list, self._psrc_max_list): self._psrc_info_list.append((psrc, psrc_max, psrc_group_size)) if psrc_max: psrc_group_size *= psrc_max psrc_max_list = lfilter(lambda n: n is not None, self._psrc_max_list) if psrc_max_list: return reduce(lambda a, b: a * b, psrc_max_list)
def initMaxParameters(self): self.quickFill = [] prev = 1 for (psource, maxN) in izip(self._psourceList, self._psourceMaxList): self.quickFill.append((psource, maxN, prev)) if maxN: prev *= maxN maxList = lfilter(lambda n: n is not None, self._psourceMaxList) if maxList: return reduce(lambda a, b: a * b, maxList)
def _combineSources(self, clsName, args): repeat = reduce(lambda a, b: a * b, ifilter(lambda expr: isinstance(expr, int), args), 1) args = lfilter(lambda expr: not isinstance(expr, int), args) if args: result = ParameterSource.createInstance(clsName, *args) if repeat > 1: return ParameterSource.createInstance('RepeatParameterSource', result, repeat) return result elif repeat > 1: return repeat return NullParameterSource()
def __init__(self, arg, **kwargs): predef = { 'TODO': 'SUBMITTED,WAITING,READY,QUEUED,UNKNOWN', 'ALL': str.join(',', Job.enumNames) } RegExSelector.__init__(self, predef.get(arg.upper(), arg), None, lambda x: '^%s.*' % x.upper()) stateList = reduce( operator.add, imap(lambda x: lfilter(x.match, Job.enumNames), self._rxList)) self._states = lmap(Job.str2enum, stateList)
def splitDatasetInternal(self, blocks, firstEvent=0): for block in blocks: splitterList = self._setup(self._splitstack, block) subSplitter = imap( lambda x: FileLevelSplitter.createInstance(x, self._config), splitterList[:-1]) endSplitter = DataSplitter.createInstance(splitterList[-1], self._config) for subBlock in reduce(lambda x, y: y.splitBlocks(x), subSplitter, [block]): for splitting in endSplitter.splitDatasetInternal([subBlock]): yield splitting
def combineSources(self, PSourceClass, args): repeat = reduce(lambda a, b: a * b, ifilter(lambda expr: isinstance(expr, int), args), 1) args = lfilter(lambda expr: not isinstance(expr, int), args) if len(args) > 1: result = PSourceClass(*args) elif len(args) > 0: result = args[0] else: return utils.QM(repeat > 1, [repeat], []) if repeat > 1: return [RepeatParameterSource(result, repeat)] return [result]
class JobClass(object): mkJobClass = lambda *fList: (reduce(operator.add, imap(lambda f: 1 << f, fList)), fList) ATWMS = mkJobClass(Job.SUBMITTED, Job.WAITING, Job.READY, Job.QUEUED) RUNNING = mkJobClass(Job.RUNNING) PROCESSING = mkJobClass(Job.SUBMITTED, Job.WAITING, Job.READY, Job.QUEUED, Job.RUNNING) READY = mkJobClass(Job.INIT, Job.FAILED, Job.ABORTED, Job.CANCELLED) DONE = mkJobClass(Job.DONE) SUCCESS = mkJobClass(Job.SUCCESS) DISABLED = mkJobClass(Job.DISABLED) ENDSTATE = mkJobClass(Job.SUCCESS, Job.DISABLED) PROCESSED = mkJobClass(Job.SUCCESS, Job.FAILED, Job.CANCELLED, Job.ABORTED)
def optSplit(opt, delim, empty = ''): """ Split option strings into fixed tuples >>> optSplit('abc : ghi # def', ['#', ':']) ('abc', 'def', 'ghi') >>> optSplit('abc:def', '::') ('abc', 'def', '') """ def getDelimeterPart(oldResult, prefix): try: tmp = oldResult[0].split(prefix) new = tmp.pop(1) try: # Find position of other delimeters in string otherDelim = min(ifilter(lambda idx: idx >= 0, imap(new.find, delim))) tmp[0] += new[otherDelim:] except Exception: otherDelim = None return [str.join(prefix, tmp)] + oldResult[1:] + [new[:otherDelim]] except Exception: return oldResult + [''] result = lmap(str.strip, reduce(getDelimeterPart, delim, [opt])) return tuple(imap(lambda x: QM(x == '', empty, x), result))
def split_opt(opt, delim): """ Split option strings into fixed tuples >>> split_opt('abc : ghi # def', ['#', ':']) ('abc', 'def', 'ghi') >>> split_opt('abc:def', '::') ('abc', 'def', '') """ def _get_delimeter_part(old_result, prefix): try: tmp = old_result[0].split(prefix) new = tmp.pop(1) try: # Find position of other delimeters in string other_delim = min(ifilter(lambda idx: idx >= 0, imap(new.find, delim))) tmp[0] += new[other_delim:] except Exception: other_delim = None return [str.join(prefix, tmp)] + old_result[1:] + [new[:other_delim]] except Exception: return old_result + [''] result = imap(str.strip, reduce(_get_delimeter_part, delim, [opt])) return tuple(result)
def optSplit(opt, delim, empty=''): """ Split option strings into fixed tuples >>> optSplit('abc : ghi # def', ['#', ':']) ('abc', 'def', 'ghi') >>> optSplit('abc:def', '::') ('abc', 'def', '') """ def getDelimeterPart(oldResult, prefix): try: tmp = oldResult[0].split(prefix) new = tmp.pop(1) try: # Find position of other delimeters in string otherDelim = min( ifilter(lambda idx: idx >= 0, imap(new.find, delim))) tmp[0] += new[otherDelim:] except Exception: otherDelim = None return [str.join(prefix, tmp)] + oldResult[1:] + [new[:otherDelim]] except Exception: return oldResult + [''] result = lmap(str.strip, reduce(getDelimeterPart, delim, [opt])) return tuple(imap(lambda x: QM(x == '', empty, x), result))
def split_opt(opt, delim): """ Split option strings into fixed tuples >>> split_opt('abc : ghi # def', ['#', ':']) ('abc', 'def', 'ghi') >>> split_opt('abc:def', '::') ('abc', 'def', '') """ def _get_delimeter_part(old_result, prefix): try: tmp = old_result[0].split(prefix) new = tmp.pop(1) try: # Find position of other delimeters in string other_delim = min( ifilter(lambda idx: idx >= 0, imap(new.find, delim))) tmp[0] += new[other_delim:] except Exception: other_delim = None return [str.join(prefix, tmp) ] + old_result[1:] + [new[:other_delim]] except Exception: return old_result + [''] result = imap(str.strip, reduce(_get_delimeter_part, delim, [opt])) return tuple(result)
def add_pairs_in_sub_chunk(self, local_pair_angles, pairs): local_angular_separation_bins = \ calc_angular_separation(pairs, local_pair_angles, self.ar_extinction, self.extinction_mean) mpi_helper.l_print('local pair count:', local_angular_separation_bins[1].sum()) local_pair_separation_bins_array = local_angular_separation_bins local_pair_separation_bins_metadata = None local_array_shape = local_pair_separation_bins_array.shape array_block_size = np.prod(local_array_shape[1:]) comm.Barrier() mpi_helper.r_print("BEGIN GATHER") mpi_helper.l_print_no_barrier('local array shape:', local_array_shape) array_counts = comm.allgather(local_array_shape[0]) pair_separation_bins_array = None array_endings = np.cumsum(array_counts) array_displacements = array_endings - np.array(array_counts) if comm.rank == 0: mpi_helper.r_print('array count:', array_counts) root_array_shape = (np.sum(array_counts), ) + local_array_shape[1:] mpi_helper.r_print('root array shape:', root_array_shape) pair_separation_bins_array = np.ones(shape=root_array_shape, dtype=np.float64) send_buf = [ local_pair_separation_bins_array, local_array_shape[0] * array_block_size ] receive_buf = [ pair_separation_bins_array, np.multiply(array_counts, array_block_size), np.multiply(array_displacements, array_block_size), MPI.DOUBLE ] # mpi_helper.l_print(send_buf) comm.Gatherv(sendbuf=send_buf, recvbuf=receive_buf) list_pair_separation_bins_metadata = comm.gather( local_pair_separation_bins_metadata) comm.Barrier() mpi_helper.r_print("END_GATHER") if comm.rank == 0: # mpi_helper.r_print(receive_buf[0][0][0:10]) list_pair_separation_bins = [ pair_separation_bins_array[ array_displacements[rank]:array_endings[rank]] for rank, metadata in enumerate(list_pair_separation_bins_metadata) ] # initialize bins only if this is the first time we get here # for now use a function level static variable if self.angular_separation_bins is None: self.angular_separation_bins = np.zeros_like( local_angular_separation_bins) # add new results to existing bins if list_pair_separation_bins: self.angular_separation_bins = reduce( lambda x, y: x + y, list_pair_separation_bins, self.angular_separation_bins) mpi_helper.r_print('total number of pixel pairs in bins:', self.angular_separation_bins[1].sum()) np.save("../../data/extinction_correlation.npy", self.angular_separation_bins) # pixel_pairs.significant_qso_pairs.save(settings.get_significant_qso_pairs_npy()) else: print('no results received.')
def _get_proposal(splitter): return reduce(lambda prop, prov: prov.check_splitter(prop), self._provider_list, splitter)
def lfnTrans(self, lfn): if lfn and self._looseMatch: # return looseMatch path elements in reverse order def trunkPath(x, y): return (lambda s: (s[0], os.path.join(x[1], s[1])))(os.path.split(x[0])) return reduce(trunkPath, irange(self._looseMatch), (lfn, ''))[1] return lfn
def __call__(self, jobNum, jobObj): return reduce( operator.and_, imap(lambda selector: selector(jobNum, jobObj), self._selectors))
def __call__(self, jobNum, jobObj): onTerm = lambda term: term(jobNum, jobObj) # [[f1], [f2,f3]] => f1(...) || (f2(...) && f3(...)) return reduce(operator.or_, imap(lambda andTerm: reduce(operator.and_, imap(onTerm, andTerm)), self.js))
def getProposal(x): return reduce(lambda prop, prov: prov.checkSplitter(prop), self._providerList, x)
def split_partitions(self, block_iter, entry_first=0): for block in block_iter: for sub_block in reduce(lambda x, y: y.divide_blocks(x), self._part_list, [block]): for partition in self._part_final.split_partitions([sub_block]): yield partition
def _separate_repeat(psrc_list): repeat = reduce(operator.mul, ifilter(lambda p: not isinstance(p, ParameterSource), psrc_list), 1) return (repeat, lfilter(lambda p: isinstance(p, ParameterSource), psrc_list))
def __call__(self, jobnum, job_obj): def _on_term(term): return term(jobnum, job_obj) # [[f1], [f2,f3]] => f1(...) || (f2(...) && f3(...)) return reduce(operator.or_, imap(lambda andTerm: reduce(operator.and_, imap(_on_term, andTerm)), self._js))
def _separate_repeat(psrc_list): repeat = reduce( operator.mul, ifilter(lambda p: not isinstance(p, ParameterSource), psrc_list), 1) return (repeat, lfilter(lambda p: isinstance(p, ParameterSource), psrc_list))
def __init__(self, arg, **kwargs): predef = {'TODO': 'SUBMITTED,WAITING,READY,QUEUED', 'ALL': str.join(',', Job.enumNames)} RegExSelector.__init__(self, predef.get(arg.upper(), arg), None, lambda x: '^%s.*' % x.upper()) stateList = reduce(operator.add, imap(lambda x: lfilter(x.match, Job.enumNames), self.rxList)) self.states = lmap(Job.str2enum, stateList)
def __call__(self, jobNum, jobObj): def match(var, rx): return rx.search(self.jobCfg(jobNum, var)) is not None return reduce(operator.and_, ismap(match, self.rxDict))
def add_pairs_in_sub_chunk(self, delta_t_file, local_pair_angles, pairs, pixel_pairs): local_pair_separation_bins = \ pixel_pairs.add_qso_pairs_to_bins(pairs, local_pair_angles, delta_t_file) mpi_helper.l_print('local pair count:', local_pair_separation_bins.get_pair_count()) local_pair_separation_bins_array = local_pair_separation_bins.get_data_as_array( ) local_pair_separation_bins_metadata = local_pair_separation_bins.get_metadata( ) local_array_shape = local_pair_separation_bins_array.shape array_block_size = np.prod(local_array_shape[1:]) comm.Barrier() mpi_helper.r_print("BEGIN GATHER") mpi_helper.l_print_no_barrier('local array shape:', local_array_shape) array_counts = comm.allgather(local_array_shape[0]) pair_separation_bins_array = None array_endings = np.cumsum(array_counts) array_displacements = array_endings - np.array(array_counts) if comm.rank == 0: mpi_helper.r_print('array count:', array_counts) root_array_shape = (np.sum(array_counts), ) + local_array_shape[1:] mpi_helper.r_print('root array shape:', root_array_shape) pair_separation_bins_array = np.ones(shape=root_array_shape, dtype=np.float64) send_buf = [ local_pair_separation_bins_array, local_array_shape[0] * array_block_size ] receive_buf = [ pair_separation_bins_array, np.multiply(array_counts, array_block_size), np.multiply(array_displacements, array_block_size), MPI.DOUBLE ] # mpi_helper.l_print(send_buf) comm.Gatherv(sendbuf=send_buf, recvbuf=receive_buf) list_pair_separation_bins_metadata = comm.gather( local_pair_separation_bins_metadata) comm.Barrier() mpi_helper.r_print("END_GATHER") if comm.rank == 0: # mpi_helper.r_print(receive_buf[0][0][0:10]) list_pair_separation_bins = [ type(local_pair_separation_bins).load_from( pair_separation_bins_array[ array_displacements[rank]:array_endings[rank]], metadata) for rank, metadata in enumerate( list_pair_separation_bins_metadata) ] # initialize bins only if this is the first time we get here if not self.pair_separation_bins: self.pair_separation_bins = local_pair_separation_bins.init_as( local_pair_separation_bins) # add new results to existing bins if list_pair_separation_bins: for i in list_pair_separation_bins: for g in i.dict_bins_3d_data.keys(): mpi_helper.l_print_no_barrier( np.sum(i.dict_bins_3d_data[g].ar_count)) self.pair_separation_bins = reduce(lambda x, y: x + y, list_pair_separation_bins, self.pair_separation_bins) mpi_helper.r_print('total number of pixel pairs in bins:', self.pair_separation_bins.get_pair_count()) self.pair_separation_bins.flush() pixel_pairs.significant_qso_pairs.save( settings.get_significant_qso_pairs_npy()) else: print('no results received.')
def __call__(self, jobNum, jobObj): return reduce(operator.and_, imap(lambda selector: selector(jobNum, jobObj), self.selectors))
def __call__(self, jobNum, jobObj): def match(var, rx): return rx.search(self._jobCfg(jobNum, var)) is not None return reduce(operator.and_, ismap(match, self._rxDict))
def getProposal(x): return reduce(lambda prop, prov: prov.checkSplitter(prop), self._providerList, x)
def __call__(self, jobnum, job_obj): def _match(var, regex_obj): return regex_obj.search(self._job_config(jobnum, var)) is not None return reduce(operator.and_, ismap(_match, self._regex_obj_list))