def load_dataset(enc, path, args, combine=50000): """Returns list of strings. Multiple files can be merged into one string. Whenever a string's len is more than combine, it's separated into multiple strings, or something like that.""" paths = [] if os.path.isfile(path): # Simple file paths.append(path) elif os.path.isdir(path): # Directory for (dirpath, _, fnames) in os.walk(path): for fname in fnames: paths.append(os.path.join(dirpath, fname)) else: # Assume glob paths = glob.glob(path) token_chunks = [] raw_text = '' for path in tqdm.tqdm(paths): if path.endswith('.npz'): # Pre-encoded with np.load(path) as npz: for item in npz.files: token_chunks.append(npz[item]) else: # Plain text with open(path, 'r') as fp: text = fp.read() if args.min_file_len and len(text) < args.min_file_len: continue if args.max_file_len and len(text) > args.max_file_len: continue raw_text += text if len(raw_text) >= combine: subtexts = raw_text.split("<|endoftext|>") subtexts_as_tokens = [ np.stack(enc.encode(s)) for s in subtexts if s ] foo = ["<|endoftext|>"] + list( intersperse(np.array([enc.encoder["<|endoftext|>"]]), subtexts_as_tokens)) token_chunks.append(np.concatenate(foo)) raw_text = '' else: raw_text += '<|endoftext|>' if raw_text: subtexts = raw_text.split("<|endoftext|>") subtexts_as_tokens = [np.stack(enc.encode(s)) for s in subtexts if s] foo = list( intersperse(np.array([enc.encoder["<|endoftext|>"]]), subtexts_as_tokens)) token_chunks.append(np.concatenate(foo)) return token_chunks
def combine_wavs(self): """Concat wavs of same speaker, so that video of speaker can be made easily""" wavs_dicts = [] wavs_dict = {} last_speaker = '' for i, sentence_dict in enumerate(self.sentence_dicts): wav = sentence_dict['wav'] # Add silence between lines if sentence_dict['begin']: wav = np.pad(wav, (get_silence(0.5), 0), 'constant') # Every line has 0.5s silence if i != 0 and last_speaker != sentence_dict['speaker']: wavs_dict['speaker'] = last_speaker # Add silence between each sentence within a line, default 0.15s wavs_dict['wav'] = np.concatenate([ *intersperse(np.zeros(get_silence(0.15), dtype=np.int16), wavs_dict['wav']) ], axis=None) # pad silence at the end wavs_dict['wav'] = np.pad(wavs_dict['wav'], (0, get_silence(0.5)), 'constant') wavs_dicts.append(wavs_dict) wavs_dict = {} if 'wav' not in wavs_dict: wavs_dict['wav'] = [wav] else: wavs_dict['wav'].append(wav) last_speaker = sentence_dict['speaker'] if wavs_dict: wavs_dict['speaker'] = last_speaker # Add silence between each sentence within a line, default 0.15s wavs_dict['wav'] = np.concatenate([ *intersperse(np.zeros(get_silence(0.15), dtype=np.int16), wavs_dict['wav']) ], axis=None) # pad silence at the end wavs_dict['wav'] = np.pad(wavs_dict['wav'], (0, get_silence(0.5)), 'constant') wavs_dicts.append(wavs_dict) # TODO: add silence according to punctuation self.wav = np.concatenate( [wavs_dict['wav'] for wavs_dict in wavs_dicts], axis=None) self.wavs_dicts = wavs_dicts
def combine(voice): if not os.path.exists(f'{voice}_combined'): os.mkdir(f'{voice}_combined') audio_list = glob.glob(f'{voice}_output/*.wav') audio_df = pd.DataFrame(audio_list, columns=['path']) audio_df['filename'] = audio_df['path'].str.split( '.', expand=True)[0].apply(lambda x: os.path.split(x)[1]) audio_df['name'], audio_df['number'] = zip( *audio_df['filename'].str.split('_').tolist()) audio_df.to_csv(f'{voice}_audio_df.csv', encoding='utf-8', index=False, sep='|', quoting=csv.QUOTE_NONE) for name, name_df in audio_df.groupby('name'): path_list = name_df.sort_values('number')['path'].to_list() if len(path_list) > 1: audio_slice = intersperse( np.zeros(int(sr * 1500 / 10000), dtype=np.int16), [wav.read(path)[1] for path in path_list]) # name_audio = trim_custom(np.concatenate([*audio_slice], axis=None)) name_audio = np.concatenate([*audio_slice], axis=None) wav.write(f'{voice}_combined/{name}.wav', sr, name_audio.astype(np.int16)) else: shutil.copy2(path_list[0], f'{voice}_combined/{name}.wav')
def _join(arrays: Sequence[np.ndarray], axis: int, separation: int) -> np.ndarray: """ Join a sequence of drawings along the given axis with the given separation. Each drawing is padded along the other axis with :func:`pad` to ensure that they all have the same width (axis=0) or height (axis=1) so that they can be concatenated together. :param arrays: Sequence of arrays to join. :param axis: Axis along which to join - 0: rows, 1: columns. :param separation: Number of blank rows (axis=0) or columns (axis=1) separating each drawing in `arrays` in the final result. :return: A new array consisting of the concatenation of `arrays` with the given separation between each. """ if not arrays: return empty_canvas(0, 0) other_axis = (axis + 1) % 2 other_length = max(a.shape[other_axis] for a in arrays) joint_shape = np.roll([separation, other_length], shift=axis) joint = empty_canvas(*joint_shape) pad_shape = dict( zip(["cols", "rows"], np.roll([other_length, None], shift=axis))) padder = functools.partial(pad, **pad_shape) return np.concatenate( list(more_itertools.intersperse(joint, map(padder, arrays))), axis=axis, )
def concatenate_files(file_list: list[pathlib.Path]) -> list[str]: file_contents = [] for file_location in sorted(file_list): with open(file_location, 'r') as file: file_contents.append(file.read()) # TODO: make this file separator configurable return _flatten(list(intersperse(['\n-------------\n'], file_contents)))
def mk_sql_list(ls): """ Turn Python list of strings into an SQL list :param ls: a list of strings :return: SQL list """ res = "(" + ' '.join([str(elem) for elem in intersperse(",", ls)]) + ")" return res
def render(self, circle_free: bool = False) -> str: blocs = [] commands = list(self._dump_commands()) if commands: blocs.append(commands) notes = list(self._dump_notes(circle_free)) if notes: blocs.append(notes) return "\n".join(collapse(intersperse("", blocs)))
def fscalc_mean(niis, output, freesurfer_version=DEFAULT_FREESURFER_VERSION): if not niis or len(niis) < 2: raise Exception( "Need at least 2 nifti volumes to compute mean. Got: " + '\n'.join(niis)) cmd = ['fscalc'] + \ list(intersperse('add', niis)) + \ ['div', str(len(niis)), '-o', output] return run(cmd)
def create_paragraph_breaks(lines: list[str]) -> list[str]: """Ensures that there is a spare line between each line of text""" halfway_lines_list = [] for line in lines: line = line.split('\n') line = [sub_line.replace('\n', '').strip() for sub_line in line] line = [sub_line + '\n' for sub_line in line if sub_line != ''] halfway_lines_list.append(line) flattened_lines = _flatten(halfway_lines_list) return list(intersperse('\n', flattened_lines))
def render(self, circle_free: bool = False) -> str: blocs = [] commands = list(self._dump_commands()) if commands: blocs.append(commands) symbols = list(self._dump_symbol_definitions()) if symbols: blocs.append(symbols) notes = list(self._dump_notes(circle_free)) if notes: blocs.append(notes) return "\n".join(collapse([intersperse("", blocs), "--"]))
def apply_selection_from_bs_time_series(bsa_master_time_series, bsl_master_time_series, bse_master_time_series, data_store): """ selecting a point or points in the time series updates the transaction table to show all transactions up to that point """ ctx = dash.callback_context click = ctx.triggered[0]['prop_id'].split('.')[0] if not data_store or len(click) == 0: raise PreventUpdate inputs = { 'bsa_master_time_series': bsa_master_time_series, 'bsl_master_time_series': bsl_master_time_series, 'bse_master_time_series': bse_master_time_series } selection = inputs[click] trans, eras, account_tree, earliest_trans, latest_trans = data_from_json_store( data_store, ACCOUNTS) trans_filter: dict = {} sel_trans: pd.DataFrame = pd.DataFrame() sel_text: list = [] for point in selection['points']: account = point['customdata'] end_date = point['x'] try: trans_filter[account].append(end_date) except (KeyError, AttributeError): trans_filter[account] = [end_date] for account in trans_filter.keys(): if not trans_filter[account]: continue end_date = max(trans_filter[account]) new_trans = trans.loc[(trans['account'] == account) & (trans['date'] <= end_date)] sel_trans = sel_trans.append(new_trans) new_text = f'{account}: {len(sel_trans)} records through {pretty_date(end_date)}' sel_text = sel_text + [new_text] if len(sel_trans) == 0: raise PreventUpdate sel_trans = sel_trans.set_index('date').sort_index() sel_trans['total'] = sel_trans['amount'].cumsum() sel_trans['date'] = pd.DatetimeIndex(sel_trans.index).strftime("%Y-%m-%d") sel_output = [html.Span(children=x) for x in sel_text] final_label = list(intersperse(html.Br(), sel_output)) return [sel_trans.to_dict('records'), final_label]
def __init__(self, *args, steps=None): if args and steps: print( "Wrong args: instantiating CachedProduct is not recommended, use operator ** instead." ) exit() if args: steps = args # noinspection PyUnresolvedReferences from more_itertools import intersperse if "cache" not in CACHE: raise Exception("Missing call to setcache(...).") steps = list(intersperse(CACHE["cache"], steps)) op.Operator.__init__(self, *steps) ContainerN.__init__(self, steps)
def run2String(jobs: List[Simulation.Job]) -> str: #assumption: complete Run #assumption: starts at 0 #whoops #assumption: 0 <= id < 10 #assumption: ids unique firstQ: int = min(jobs, key=(lambda j: j.queueingT)).queueingT lastCompletion: int = max( jobs, key=(lambda j: j.realCompletionT)).realCompletionT #legende: #id,qtime,paral legend: string = "queueintT, processingT, realProcessingT, degreeOfParallelism\n" + "".join( list( map( lambda j: "id: %d, qT: %d, pT: %d, rPT: %d, doP: %d\n" % (j.id, j.queueingT, j.processingT, j.realProcessingT, j. degreeOP), list(sorted(jobs, key=lambda j: j.id))))) #nodes: #QTimes später #make a dict of all the nodes running works #add a reduced form of all the jobs to that id #------------------------------start,end,id nodesInRun: Dict[int, List[Tuple[int, int, int]]] = {} for j in jobs: for n in j.runningOn: if n.id in nodesInRun: nodesInRun[n.id].append( (j.startRunning, j.realCompletionT, j.id)) else: nodesInRun[n.id] = [(j.startRunning, j.realCompletionT, j.id)] for l in nodesInRun: nodesInRun[l].sort(key=lambda x: x[0]) #sorted by start #store the strings for each node here: nodeStrings: Dict[int, str] = {} for l in nodesInRun: nodeStrings[l] = ['-'] * (lastCompletion ) #how does last completion become float?? for j in nodesInRun[l]: nodeStrings[l][j[0]:j[1]] = [str(j[2])] * (j[1] - j[0]) nodeStrings[l] = ["[", str(l), "]", ":"] + list( intersperse("|", nodeStrings[l], n=3)) + ['\n'] return legend + "".join( list(map(lambda x: "".join(x), list(sorted(nodeStrings.values())))))
def _abbreviate(word: str, abbrs: List[str], opt=False): abbrs, dashed = partition(lambda abbr: '-' in abbr, abbrs) dashed = map( lambda a: rule(*map(caseless, intersperse('-', a.split('-')))), dashed) original_word = rule(normalized(word)) dashed_sequence = rule(or_(*dashed)) abbr_with_dot = rule( or_(*map(caseless, abbrs)), eq('.').optional(), ) result = or_(original_word, dashed_sequence, abbr_with_dot) \ .interpretation(interpretation.const(word)) return result.optional() if opt else result
def c_layout(i, definition, template): c_name = layer_names[i] pretty_name = c_name.strip('_').capitalize() layout = d['layout'] surround = lambda s: ''.join(interleave_longest(['│']*(len(s)+1), s)) layer = list(map(uni, definition)) layer[41] = layer[41].center(11) layer = chunked(layer, 12) rows = intersperse(mid, map(surround, layer)) pretty = '\n'.join(itertools.chain([top], rows, [bottom])) surround = lambda s: ', '.join(s) layer = list(map(lambda k: layer_name.get(k, k), definition)) layer = chunked(layer, 12) rows = map(surround, layer) c_layer = ',\n '.join(itertools.chain([], rows, [])) return template.format(pretty_name, pretty, c_name, layout, c_layer)
def c_layout(i, definition, template): c_name = layer_names[i] pretty_name = c_name.strip('_').capitalize() layout = d['layout'] surround = lambda s: ''.join(interleave_longest(['│'] * (len(s) + 1), s)) layer = list(map(uni, definition)) layer[41] = layer[41].center(11) layer = chunked(layer, 12) rows = intersperse(mid, map(surround, layer)) pretty = '\n'.join(itertools.chain([top], rows, [bottom])) surround = lambda s: ', '.join(s) layer = list(map(lambda k: layer_name.get(k, k), definition)) layer = chunked(layer, 12) rows = map(surround, layer) c_layer = ',\n '.join(itertools.chain([], rows, [])) return template.format(pretty_name, pretty, c_name, layout, c_layer)
def mk_value_tests(self, value_test_conditions, study): """ Helper function used to creat a Where clause to find measurements that fail the conditions :param value_test_conditions: a list where each element is takes the following form: (measurementType,condition) where condition is a string holding the condition against which the value in each measurement is compared. :param study: study id :return: """ all_conditions = [] for (measurement_type, condition) in value_test_conditions: mt_info = self.getMeasurementTypeInfo(study, measurement_type) val_type = mt_info[0][2] cond = "((measurement.measurementtype = " + str(measurement_type) + ") AND NOT " + \ self.makeValueTest(val_type, condition) + ")" all_conditions = all_conditions + [cond] result = ' '.join([elem for elem in intersperse(" OR ", all_conditions)]) return result
def _dump_notes( self, circle_free: bool = False, ) -> Iterator[str]: frames: List[Dict[NotePosition, str]] = [] frame: Dict[NotePosition, str] = {} for note in self.notes: time_in_section = note.time - self.current_beat symbol = self.symbols[time_in_section] if isinstance(note, LongNote): needed_positions = set(note.positions_covered()) if needed_positions & frame.keys(): frames.append(frame) frame = {} direction = note.tail_direction() arrow = DIRECTION_TO_ARROW[direction] line = DIRECTION_TO_LINE[direction] for is_first, is_last, pos in mark_ends( note.positions_covered()): if is_first: frame[pos] = symbol elif is_last: frame[pos] = arrow else: frame[pos] = line elif isinstance(note, TapNote): if note.position in frame: frames.append(frame) frame = {} frame[note.position] = symbol elif isinstance(note, LongNoteEnd): if note.position in frame: frames.append(frame) frame = {} if circle_free and symbol in NOTE_TO_CIRCLE_FREE_SYMBOL: symbol = NOTE_TO_CIRCLE_FREE_SYMBOL[symbol] frame[note.position] = symbol frames.append(frame) dumped_frames = map(self._dump_frame, frames) yield from collapse(intersperse("", dumped_frames))
def find_args(root: Path, follow: bool, ignore: List[str]=[]) -> List[str]: prune_dir_args = [] ignore_file_args = [] if ignore: # -name {name} for all the file/directories in ignore ignore_names = [['-name', n] for n in ignore] # OR (-o) all the names together and flatten ignore_names_l = list(itertools.chain(*intersperse(['-o'], ignore_names))) # Prune all of those directories, and make the entire clause evaluate to false # (so that it doesn't match anything and make find print) prune_dir_args = ['-type', 'd', '-a', '(', *ignore_names_l, ')', '-prune', '-false', '-o'] # Also ignore any files with the names as well ignore_file_args = ['-a', '-not', '(', *ignore_names_l, ')'] return [ *(['-L'] if follow else []), str(root), *prune_dir_args, '-type', 'f', *ignore_file_args ]
def _dump_notes(self, circle_free: bool = False) -> Iterator[str]: # Split notes into bars notes_by_bar: Dict[int, List[AnyNote]] = defaultdict(list) for note in self.notes: time_in_section = note.time - self.current_beat bar_index = int(time_in_section) notes_by_bar[bar_index].append(note) # Pre-render timing bars bars: Dict[int, List[str]] = defaultdict(list) chosen_symbols: Dict[BeatsTime, str] = {} symbols_iterator = iter(NOTE_SYMBOLS) for bar_index in range(ceil(self.length)): notes = notes_by_bar.get(bar_index, []) bar_length = lcm(*((note.time - self.current_beat).denominator for note in notes)) if bar_length < 3: bar_length = 4 bar_dict: Dict[int, str] = {} for note in notes: time_in_section = note.time - self.current_beat time_in_bar = time_in_section % Fraction(1) time_index = time_in_bar.numerator * (bar_length // time_in_bar.denominator) if time_index not in bar_dict: symbol = next(symbols_iterator) chosen_symbols[time_in_section] = symbol bar_dict[time_index] = symbol bar = [ bar_dict.get(i, EMPTY_BEAT_SYMBOL) for i in range(bar_length) ] bars[bar_index] = bar # Create frame by bar frames_by_bar: Dict[int, List[Frame]] = defaultdict(list) for bar_index in range(ceil(self.length)): bar = bars.get(bar_index, []) frame = Frame() frame.bars[bar_index] = bar for note in notes_by_bar[bar_index]: time_in_section = note.time - self.current_beat symbol = chosen_symbols[time_in_section] if isinstance(note, TapNote): if note.position in frame.positions: frames_by_bar[bar_index].append(frame) frame = Frame() frame.positions[note.position] = symbol elif isinstance(note, LongNote): needed_positions = set(note.positions_covered()) if needed_positions & frame.positions.keys(): frames_by_bar[bar_index].append(frame) frame = Frame() direction = note.tail_direction() arrow = DIRECTION_TO_ARROW[direction] line = DIRECTION_TO_LINE[direction] for is_first, is_last, pos in mark_ends( note.positions_covered()): if is_first: frame.positions[pos] = symbol elif is_last: frame.positions[pos] = arrow else: frame.positions[pos] = line elif isinstance(note, LongNoteEnd): if note.position in frame.positions: frames_by_bar[bar_index].append(frame) frame = Frame() if circle_free and symbol in NOTE_TO_CIRCLE_FREE_SYMBOL: symbol = NOTE_TO_CIRCLE_FREE_SYMBOL[symbol] frame.positions[note.position] = symbol frames_by_bar[bar_index].append(frame) # Merge bar-specific frames is possible final_frames: List[Frame] = [] for bar_index in range(ceil(self.length)): frames = frames_by_bar[bar_index] # Merge if : # - No split in current bar (only one frame) # - There is a previous frame # - The previous frame is not a split frame (it holds a bar) # - The previous and current bars are all in the same 4-bar group # - The note positions in the previous frame do not clash with the current frame if (len(frames) == 1 and final_frames and final_frames[-1].bars and max(final_frames[-1].bars.keys()) // 4 == min(frames[0].bars.keys()) // 4 and (not (final_frames[-1].positions.keys() & frames[0].positions.keys()))): final_frames[-1].bars.update(frames[0].bars) final_frames[-1].positions.update(frames[0].positions) else: final_frames.extend(frames) dumped_frames = map(lambda f: f.dump(self.length), final_frames) yield from collapse(intersperse("", dumped_frames))
def multiband_parsl_driver(configuration, rerun_in, rerun_out, tract_id, patch_id_no_parens, filter_list, logbase="", inputs=[], wrap=None): """This is a parsl-level replacement for multiBandDriver.py. There is no single final task for multiband driver - instead there is one per filter. So the future returned by this is a `combine` of those final filter tasks. Something following on from this might instead be able to depend only on the final filter tasks.""" repo_dir = configuration.repo_dir # mergeCoaddDetections (one, for all filters on this patch) # deblendCoaddSources (per filter) - depends on single mergeCoaddDetections # measureCoaddSources (per filter) - depends on corresponding filter deblend # mergeCoaddMeasurements - depends on all filters # forcedPhotCoadd (per filter) - depends on mergeCoaddMeasurements # 'combine' - future combinator to generate the final status future filter_ids_for_dm = "" for el in intersperse("^", filter_list): filter_ids_for_dm += el merge_coadd_det_fut = merge_coadd_detections( repo_dir, rerun_in, rerun_out, tract_id, patch_id_no_parens, obs_lsst_configs=configuration.obs_lsst_configs, filters=filter_ids_for_dm, wrap=wrap, inputs=inputs, stdout="{logbase}.merge_coadd_detections.stdout".format( logbase=logbase), stderr="{logbase}.merge_coadd_detections.stderr".format( logbase=logbase), parsl_resource_specification={ "priority": (6001, tract_id, patch_id_no_parens) }) measure_futs = [] for filter_id in filter_list: deblend_coadd_sources_fut = deblend_coadd_sources( repo_dir, rerun_out, tract_id, patch_id_no_parens, filter_id, wrap=wrap, inputs=[merge_coadd_det_fut], stdout="{logbase}-filter-{filter_id}-deblend_coadd_sources.stdout". format(logbase=logbase, filter_id=filter_id), stderr="{logbase}-filter-{filter_id}-deblend_coadd_sources.stderr". format(logbase=logbase, filter_id=filter_id), parsl_resource_specification={ "priority": (6002, tract_id, patch_id_no_parens) }) measure_coadd_sources_fut = measure_coadd_sources( repo_dir, rerun_out, tract_id, patch_id_no_parens, filter_id, obs_lsst_configs=configuration.obs_lsst_configs, wrap=wrap, inputs=[deblend_coadd_sources_fut], stdout="{logbase}-filter-{filter_id}-measure_coadd_sources.stdout". format(logbase=logbase, filter_id=filter_id), stderr="{logbase}-filter-{filter_id}-measure_coadd_sources.stderr". format(logbase=logbase, filter_id=filter_id), parsl_resource_specification={ "priority": (6003, tract_id, patch_id_no_parens) }) measure_futs.append(measure_coadd_sources_fut) merge_fut = merge_coadd_measurements( repo_dir, rerun_out, tract_id, patch_id_no_parens, obs_lsst_configs=configuration.obs_lsst_configs, wrap=wrap, inputs=measure_futs, stdout="{logbase}-merge-coadd-measurements.stdout".format( logbase=logbase), stderr="{logbase}-merge-coadd-measurements.stderr".format( logbase=logbase), parsl_resource_specification={ "priority": (6004, tract_id, patch_id_no_parens) }) forced_phot_coadd_futs = [] for filter_id in filter_list: forced_phot_coadd_future = forced_phot_coadd( repo_dir, rerun_out, tract_id, patch_id_no_parens, filter_id, obs_lsst_configs=configuration.obs_lsst_configs, wrap=wrap, inputs=[merge_fut], stdout="{logbase}-filter-{filter_id}-forced_phot_coadd.stdout". format(filter_id=filter_id, logbase=logbase), stderr="{logbase}-filter-{filter_id}-forced_phot_coadd.stderr". format(filter_id=filter_id, logbase=logbase), parsl_resource_specification={ "priority": (6005, tract_id, patch_id_no_parens) }) forced_phot_coadd_futs.append(forced_phot_coadd_future) return combine(inputs=forced_phot_coadd_futs)
def coadd_parsl_driver(configuration, rerun_in, rerun_out, tract_id, patch_id, filter_id, visit_file, visit_futs, inputs=None, wrap=None, logbase=""): """visit_futs should be: a dict of visit id -> visit processing completed future, or None if no waiting for visits should happen (for example because should be assumed to be completed by some other mechanism. This implementation assumes that visit_file is already generated at the time of invocation, which is not true in general. """ repo_dir = configuration.repo_dir patch_id_no_parens = re.sub("[() ]", "", patch_id) visitLines = read_and_strip(visit_file) if visitLines == []: # skip if no visits logger.warning( f'No visits for tract {tract_id}, patch {patch_id}, filter {filter_id}' ) return const_future(None) ## 9/22/2020 - New visit file format, each line contains: <visitID> <det1^det2^det3...> visits = [] dets = [] for visitLine in visitLines: visits.append(visitLine.split(' ')[0]) dets.append(visitLine.split(' ')[1]) pass logger.info("WFLOWx: There are " + str(len(visits)) + " visits for filter/tract/patch " + str(filter_id) + '/' + str(tract_id) + '/' + str(patch_id)) visit_ids_for_dm = "" for el in intersperse("^", visits): visit_ids_for_dm += el per_visit_futures = [] for visit, det in zip(visits, dets): input_deps = [] if visit_futs: input_deps.append(visit_futs[visit]) pass #### 9/11/2020 [POSSIBLY TEMPORARY] Check if warp files exist #### # warpath = configuration.repo_dir/rerun/<rerun>/deepCoadd/<filter>/<tract>/<patch> # w1 = <warpath>/psfMatchedWarp-<filter>-<tract>-<patch>-<visit>.fits # w2 = <warpath>/warp-<filter>-<tract>-<patch>-<visit>.fits # e.g. /global/cscratch1/sd/descdm/DC2/DR2/repo/rerun/Tom3-1.2.3.4/deepCoadd/u/5063/6,4/warp-u-5063-6,4-2249.fits warpath = os.path.join(configuration.repo_dir, 'rerun', rerun_out, 'deepCoadd', filter_id, tract_id, patch_id) fpart = '-'.join([filter_id, tract_id, patch_id, str(visit)]) w1 = os.path.join(warpath, 'psfMatchedWarp-' + fpart + '.fits') w2 = os.path.join(warpath, 'warp-' + fpart + '.fits') logger.info('WFLOWx: Searching for warp files for: ' + fpart) logger.info('WFLOWx: w1 = ' + w1) logger.info('WFLOWx: w2 = ' + w2) if os.path.exists(w1) and os.path.exists(w2): logger.info('WFLOWx: Warp files already exist for: ' + fpart + ', skipping makeCoaddTempExp...') else: logger.info('WFLOWx: makeCoaddTempExp for: ' + fpart) per_visit_futures.append( make_coadd_temp_exp( repo_dir, rerun_in, rerun_out, tract_id, patch_id_no_parens, filter_id, visit, det, inputs=input_deps, obs_lsst_configs=configuration.obs_lsst_configs, wrap=wrap, stdout="{logbase}-visit-{visit}.stdout".format( logbase=logbase, visit=visit), stderr="{logbase}-visit-{visit}.stderr".format( logbase=logbase, visit=visit), parsl_resource_specification={ "priority": (5000, tract_id, patch_id_no_parens, filter_id) })) pass pass fut2 = assemble_coadd( repo_dir, rerun_out, tract_id, patch_id_no_parens, filter_id, visit_ids_for_dm, inputs=per_visit_futures, obs_lsst_configs=configuration.obs_lsst_configs, wrap=wrap, stdout="{logbase}.assemble_coadd.stdout".format(logbase=logbase), stderr="{logbase}.assemble_coadd.stderr".format(logbase=logbase), parsl_resource_specification={ "priority": (5010, tract_id, patch_id_no_parens, filter_id) }) fut3 = detect_coadd_sources( repo_dir, rerun_out, tract_id, patch_id_no_parens, filter_id, visit_file, inputs=[fut2], wrap=wrap, stdout="{logbase}.detect_coadd_sources.stdout".format(logbase=logbase), stderr="{logbase}.detect_coadd_sources.stderr".format(logbase=logbase), parsl_resource_specification={ "priority": (5020, tract_id, patch_id_no_parens, filter_id) }) return fut3