def load_dataset(enc, path, args, combine=50000):
"""Returns list of strings. Multiple files can be merged into one string.
Whenever a string's len is more than combine, it's separated into multiple
strings, or something like that."""
paths = []
if os.path.isfile(path):
# Simple file
paths.append(path)
elif os.path.isdir(path):
# Directory
for (dirpath, _, fnames) in os.walk(path):
for fname in fnames:
paths.append(os.path.join(dirpath, fname))
else:
# Assume glob
paths = glob.glob(path)
token_chunks = []
raw_text = ''
for path in tqdm.tqdm(paths):
if path.endswith('.npz'):
# Pre-encoded
with np.load(path) as npz:
for item in npz.files:
token_chunks.append(npz[item])
else:
# Plain text
with open(path, 'r') as fp:
text = fp.read()
if args.min_file_len and len(text) < args.min_file_len:
continue
if args.max_file_len and len(text) > args.max_file_len:
continue
raw_text += text
if len(raw_text) >= combine:
subtexts = raw_text.split("<|endoftext|>")
subtexts_as_tokens = [
np.stack(enc.encode(s)) for s in subtexts if s
]
foo = ["<|endoftext|>"] + list(
intersperse(np.array([enc.encoder["<|endoftext|>"]]),
subtexts_as_tokens))
token_chunks.append(np.concatenate(foo))
raw_text = ''
else:
raw_text += '<|endoftext|>'
if raw_text:
subtexts = raw_text.split("<|endoftext|>")
subtexts_as_tokens = [np.stack(enc.encode(s)) for s in subtexts if s]
foo = list(
intersperse(np.array([enc.encoder["<|endoftext|>"]]),
subtexts_as_tokens))
token_chunks.append(np.concatenate(foo))
return token_chunks
def combine_wavs(self):
"""Concat wavs of same speaker, so that video of speaker can be made easily"""
wavs_dicts = []
wavs_dict = {}
last_speaker = ''
for i, sentence_dict in enumerate(self.sentence_dicts):
wav = sentence_dict['wav']
# Add silence between lines
if sentence_dict['begin']:
wav = np.pad(wav, (get_silence(0.5), 0),
'constant') # Every line has 0.5s silence
if i != 0 and last_speaker != sentence_dict['speaker']:
wavs_dict['speaker'] = last_speaker
# Add silence between each sentence within a line, default 0.15s
wavs_dict['wav'] = np.concatenate([
*intersperse(np.zeros(get_silence(0.15), dtype=np.int16),
wavs_dict['wav'])
],
axis=None)
# pad silence at the end
wavs_dict['wav'] = np.pad(wavs_dict['wav'],
(0, get_silence(0.5)), 'constant')
wavs_dicts.append(wavs_dict)
wavs_dict = {}
if 'wav' not in wavs_dict:
wavs_dict['wav'] = [wav]
else:
wavs_dict['wav'].append(wav)
last_speaker = sentence_dict['speaker']
if wavs_dict:
wavs_dict['speaker'] = last_speaker
# Add silence between each sentence within a line, default 0.15s
wavs_dict['wav'] = np.concatenate([
*intersperse(np.zeros(get_silence(0.15), dtype=np.int16),
wavs_dict['wav'])
],
axis=None)
# pad silence at the end
wavs_dict['wav'] = np.pad(wavs_dict['wav'], (0, get_silence(0.5)),
'constant')
wavs_dicts.append(wavs_dict)
# TODO: add silence according to punctuation
self.wav = np.concatenate(
[wavs_dict['wav'] for wavs_dict in wavs_dicts], axis=None)
self.wavs_dicts = wavs_dicts
def combine(voice):
if not os.path.exists(f'{voice}_combined'):
os.mkdir(f'{voice}_combined')
audio_list = glob.glob(f'{voice}_output/*.wav')
audio_df = pd.DataFrame(audio_list, columns=['path'])
audio_df['filename'] = audio_df['path'].str.split(
'.', expand=True)[0].apply(lambda x: os.path.split(x)[1])
audio_df['name'], audio_df['number'] = zip(
*audio_df['filename'].str.split('_').tolist())
audio_df.to_csv(f'{voice}_audio_df.csv',
encoding='utf-8',
index=False,
sep='|',
quoting=csv.QUOTE_NONE)
for name, name_df in audio_df.groupby('name'):
path_list = name_df.sort_values('number')['path'].to_list()
if len(path_list) > 1:
audio_slice = intersperse(
np.zeros(int(sr * 1500 / 10000), dtype=np.int16),
[wav.read(path)[1] for path in path_list])
# name_audio = trim_custom(np.concatenate([*audio_slice], axis=None))
name_audio = np.concatenate([*audio_slice], axis=None)
wav.write(f'{voice}_combined/{name}.wav', sr,
name_audio.astype(np.int16))
else:
shutil.copy2(path_list[0], f'{voice}_combined/{name}.wav')
def _join(arrays: Sequence[np.ndarray], axis: int,
separation: int) -> np.ndarray:
"""
Join a sequence of drawings along the given axis with the given separation.
Each drawing is padded along the other axis with :func:`pad` to ensure that they
all have the same width (axis=0) or height (axis=1) so that they can be
concatenated together.
:param arrays: Sequence of arrays to join.
:param axis: Axis along which to join - 0: rows, 1: columns.
:param separation: Number of blank rows (axis=0) or columns (axis=1) separating
each drawing in `arrays` in the final result.
:return: A new array consisting of the concatenation of `arrays` with the
given separation between each.
"""
if not arrays:
return empty_canvas(0, 0)
other_axis = (axis + 1) % 2
other_length = max(a.shape[other_axis] for a in arrays)
joint_shape = np.roll([separation, other_length], shift=axis)
joint = empty_canvas(*joint_shape)
pad_shape = dict(
zip(["cols", "rows"], np.roll([other_length, None], shift=axis)))
padder = functools.partial(pad, **pad_shape)
return np.concatenate(
list(more_itertools.intersperse(joint, map(padder, arrays))),
axis=axis,
)
def concatenate_files(file_list: list[pathlib.Path]) -> list[str]:
file_contents = []
for file_location in sorted(file_list):
with open(file_location, 'r') as file:
file_contents.append(file.read())
# TODO: make this file separator configurable
return _flatten(list(intersperse(['\n-------------\n'], file_contents)))
def mk_sql_list(ls):
"""
Turn Python list of strings into an SQL list
:param ls: a list of strings
:return: SQL list
"""
res = "(" + ' '.join([str(elem) for elem in intersperse(",", ls)]) + ")"
return res
def render(self, circle_free: bool = False) -> str:
blocs = []
commands = list(self._dump_commands())
if commands:
blocs.append(commands)
notes = list(self._dump_notes(circle_free))
if notes:
blocs.append(notes)
return "\n".join(collapse(intersperse("", blocs)))
def fscalc_mean(niis, output, freesurfer_version=DEFAULT_FREESURFER_VERSION):
if not niis or len(niis) < 2:
raise Exception(
"Need at least 2 nifti volumes to compute mean. Got: " +
'\n'.join(niis))
cmd = ['fscalc'] + \
list(intersperse('add', niis)) + \
['div', str(len(niis)), '-o', output]
return run(cmd)
def create_paragraph_breaks(lines: list[str]) -> list[str]:
"""Ensures that there is a spare line between each line of text"""
halfway_lines_list = []
for line in lines:
line = line.split('\n')
line = [sub_line.replace('\n', '').strip() for sub_line in line]
line = [sub_line + '\n' for sub_line in line if sub_line != '']
halfway_lines_list.append(line)
flattened_lines = _flatten(halfway_lines_list)
return list(intersperse('\n', flattened_lines))
def render(self, circle_free: bool = False) -> str:
blocs = []
commands = list(self._dump_commands())
if commands:
blocs.append(commands)
symbols = list(self._dump_symbol_definitions())
if symbols:
blocs.append(symbols)
notes = list(self._dump_notes(circle_free))
if notes:
blocs.append(notes)
return "\n".join(collapse([intersperse("", blocs), "--"]))
def apply_selection_from_bs_time_series(bsa_master_time_series,
bsl_master_time_series,
bse_master_time_series, data_store):
"""
selecting a point or points in the time series updates the transaction table to show
all transactions up to that point
"""
ctx = dash.callback_context
click = ctx.triggered[0]['prop_id'].split('.')[0]
if not data_store or len(click) == 0:
raise PreventUpdate
inputs = {
'bsa_master_time_series': bsa_master_time_series,
'bsl_master_time_series': bsl_master_time_series,
'bse_master_time_series': bse_master_time_series
}
selection = inputs[click]
trans, eras, account_tree, earliest_trans, latest_trans = data_from_json_store(
data_store, ACCOUNTS)
trans_filter: dict = {}
sel_trans: pd.DataFrame = pd.DataFrame()
sel_text: list = []
for point in selection['points']:
account = point['customdata']
end_date = point['x']
try:
trans_filter[account].append(end_date)
except (KeyError, AttributeError):
trans_filter[account] = [end_date]
for account in trans_filter.keys():
if not trans_filter[account]:
continue
end_date = max(trans_filter[account])
new_trans = trans.loc[(trans['account'] == account)
& (trans['date'] <= end_date)]
sel_trans = sel_trans.append(new_trans)
new_text = f'{account}: {len(sel_trans)} records through {pretty_date(end_date)}'
sel_text = sel_text + [new_text]
if len(sel_trans) == 0:
raise PreventUpdate
sel_trans = sel_trans.set_index('date').sort_index()
sel_trans['total'] = sel_trans['amount'].cumsum()
sel_trans['date'] = pd.DatetimeIndex(sel_trans.index).strftime("%Y-%m-%d")
sel_output = [html.Span(children=x) for x in sel_text]
final_label = list(intersperse(html.Br(), sel_output))
return [sel_trans.to_dict('records'), final_label]
def __init__(self, *args, steps=None):
if args and steps:
print(
"Wrong args: instantiating CachedProduct is not recommended, use operator ** instead."
)
exit()
if args:
steps = args
# noinspection PyUnresolvedReferences
from more_itertools import intersperse
if "cache" not in CACHE:
raise Exception("Missing call to setcache(...).")
steps = list(intersperse(CACHE["cache"], steps))
op.Operator.__init__(self, *steps)
ContainerN.__init__(self, steps)
def run2String(jobs: List[Simulation.Job]) -> str:
#assumption: complete Run
#assumption: starts at 0 #whoops
#assumption: 0 <= id < 10
#assumption: ids unique
firstQ: int = min(jobs, key=(lambda j: j.queueingT)).queueingT
lastCompletion: int = max(
jobs, key=(lambda j: j.realCompletionT)).realCompletionT
#legende:
#id,qtime,paral
legend: string = "queueintT, processingT, realProcessingT, degreeOfParallelism\n" + "".join(
list(
map(
lambda j: "id: %d, qT: %d, pT: %d, rPT: %d, doP: %d\n" %
(j.id, j.queueingT, j.processingT, j.realProcessingT, j.
degreeOP), list(sorted(jobs, key=lambda j: j.id)))))
#nodes: #QTimes später
#make a dict of all the nodes running works
#add a reduced form of all the jobs to that id
#------------------------------start,end,id
nodesInRun: Dict[int, List[Tuple[int, int, int]]] = {}
for j in jobs:
for n in j.runningOn:
if n.id in nodesInRun:
nodesInRun[n.id].append(
(j.startRunning, j.realCompletionT, j.id))
else:
nodesInRun[n.id] = [(j.startRunning, j.realCompletionT, j.id)]
for l in nodesInRun:
nodesInRun[l].sort(key=lambda x: x[0]) #sorted by start
#store the strings for each node here:
nodeStrings: Dict[int, str] = {}
for l in nodesInRun:
nodeStrings[l] = ['-'] * (lastCompletion
) #how does last completion become float??
for j in nodesInRun[l]:
nodeStrings[l][j[0]:j[1]] = [str(j[2])] * (j[1] - j[0])
nodeStrings[l] = ["[", str(l), "]", ":"] + list(
intersperse("|", nodeStrings[l], n=3)) + ['\n']
return legend + "".join(
list(map(lambda x: "".join(x), list(sorted(nodeStrings.values())))))
def _abbreviate(word: str, abbrs: List[str], opt=False):
abbrs, dashed = partition(lambda abbr: '-' in abbr, abbrs)
dashed = map(
lambda a: rule(*map(caseless, intersperse('-', a.split('-')))), dashed)
original_word = rule(normalized(word))
dashed_sequence = rule(or_(*dashed))
abbr_with_dot = rule(
or_(*map(caseless, abbrs)),
eq('.').optional(),
)
result = or_(original_word, dashed_sequence, abbr_with_dot) \
.interpretation(interpretation.const(word))
return result.optional() if opt else result
def c_layout(i, definition, template):
c_name = layer_names[i]
pretty_name = c_name.strip('_').capitalize()
layout = d['layout']
surround = lambda s: ''.join(interleave_longest(['│']*(len(s)+1), s))
layer = list(map(uni, definition))
layer[41] = layer[41].center(11)
layer = chunked(layer, 12)
rows = intersperse(mid, map(surround, layer))
pretty = '\n'.join(itertools.chain([top], rows, [bottom]))
surround = lambda s: ', '.join(s)
layer = list(map(lambda k: layer_name.get(k, k), definition))
layer = chunked(layer, 12)
rows = map(surround, layer)
c_layer = ',\n '.join(itertools.chain([], rows, []))
return template.format(pretty_name, pretty, c_name, layout, c_layer)
def c_layout(i, definition, template):
c_name = layer_names[i]
pretty_name = c_name.strip('_').capitalize()
layout = d['layout']
surround = lambda s: ''.join(interleave_longest(['│'] * (len(s) + 1), s))
layer = list(map(uni, definition))
layer[41] = layer[41].center(11)
layer = chunked(layer, 12)
rows = intersperse(mid, map(surround, layer))
pretty = '\n'.join(itertools.chain([top], rows, [bottom]))
surround = lambda s: ', '.join(s)
layer = list(map(lambda k: layer_name.get(k, k), definition))
layer = chunked(layer, 12)
rows = map(surround, layer)
c_layer = ',\n '.join(itertools.chain([], rows, []))
return template.format(pretty_name, pretty, c_name, layout, c_layer)
def mk_value_tests(self, value_test_conditions, study):
"""
Helper function used to creat a Where clause to find measurements that fail the conditions
:param value_test_conditions: a list where each element is takes the following form:
(measurementType,condition)
where condition is a string holding the condition
against which the value in each measurement is compared.
:param study: study id
:return:
"""
all_conditions = []
for (measurement_type, condition) in value_test_conditions:
mt_info = self.getMeasurementTypeInfo(study, measurement_type)
val_type = mt_info[0][2]
cond = "((measurement.measurementtype = " + str(measurement_type) + ") AND NOT " + \
self.makeValueTest(val_type, condition) + ")"
all_conditions = all_conditions + [cond]
result = ' '.join([elem for elem in intersperse(" OR ", all_conditions)])
return result
def _dump_notes(
self,
circle_free: bool = False,
) -> Iterator[str]:
frames: List[Dict[NotePosition, str]] = []
frame: Dict[NotePosition, str] = {}
for note in self.notes:
time_in_section = note.time - self.current_beat
symbol = self.symbols[time_in_section]
if isinstance(note, LongNote):
needed_positions = set(note.positions_covered())
if needed_positions & frame.keys():
frames.append(frame)
frame = {}
direction = note.tail_direction()
arrow = DIRECTION_TO_ARROW[direction]
line = DIRECTION_TO_LINE[direction]
for is_first, is_last, pos in mark_ends(
note.positions_covered()):
if is_first:
frame[pos] = symbol
elif is_last:
frame[pos] = arrow
else:
frame[pos] = line
elif isinstance(note, TapNote):
if note.position in frame:
frames.append(frame)
frame = {}
frame[note.position] = symbol
elif isinstance(note, LongNoteEnd):
if note.position in frame:
frames.append(frame)
frame = {}
if circle_free and symbol in NOTE_TO_CIRCLE_FREE_SYMBOL:
symbol = NOTE_TO_CIRCLE_FREE_SYMBOL[symbol]
frame[note.position] = symbol
frames.append(frame)
dumped_frames = map(self._dump_frame, frames)
yield from collapse(intersperse("", dumped_frames))
def find_args(root: Path, follow: bool, ignore: List[str]=[]) -> List[str]:
prune_dir_args = []
ignore_file_args = []
if ignore:
# -name {name} for all the file/directories in ignore
ignore_names = [['-name', n] for n in ignore]
# OR (-o) all the names together and flatten
ignore_names_l = list(itertools.chain(*intersperse(['-o'], ignore_names)))
# Prune all of those directories, and make the entire clause evaluate to false
# (so that it doesn't match anything and make find print)
prune_dir_args = ['-type', 'd', '-a', '(', *ignore_names_l, ')', '-prune', '-false', '-o']
# Also ignore any files with the names as well
ignore_file_args = ['-a', '-not', '(', *ignore_names_l, ')']
return [
*(['-L'] if follow else []),
str(root),
*prune_dir_args,
'-type', 'f',
*ignore_file_args
]
def _dump_notes(self, circle_free: bool = False) -> Iterator[str]:
# Split notes into bars
notes_by_bar: Dict[int, List[AnyNote]] = defaultdict(list)
for note in self.notes:
time_in_section = note.time - self.current_beat
bar_index = int(time_in_section)
notes_by_bar[bar_index].append(note)
# Pre-render timing bars
bars: Dict[int, List[str]] = defaultdict(list)
chosen_symbols: Dict[BeatsTime, str] = {}
symbols_iterator = iter(NOTE_SYMBOLS)
for bar_index in range(ceil(self.length)):
notes = notes_by_bar.get(bar_index, [])
bar_length = lcm(*((note.time - self.current_beat).denominator
for note in notes))
if bar_length < 3:
bar_length = 4
bar_dict: Dict[int, str] = {}
for note in notes:
time_in_section = note.time - self.current_beat
time_in_bar = time_in_section % Fraction(1)
time_index = time_in_bar.numerator * (bar_length //
time_in_bar.denominator)
if time_index not in bar_dict:
symbol = next(symbols_iterator)
chosen_symbols[time_in_section] = symbol
bar_dict[time_index] = symbol
bar = [
bar_dict.get(i, EMPTY_BEAT_SYMBOL) for i in range(bar_length)
]
bars[bar_index] = bar
# Create frame by bar
frames_by_bar: Dict[int, List[Frame]] = defaultdict(list)
for bar_index in range(ceil(self.length)):
bar = bars.get(bar_index, [])
frame = Frame()
frame.bars[bar_index] = bar
for note in notes_by_bar[bar_index]:
time_in_section = note.time - self.current_beat
symbol = chosen_symbols[time_in_section]
if isinstance(note, TapNote):
if note.position in frame.positions:
frames_by_bar[bar_index].append(frame)
frame = Frame()
frame.positions[note.position] = symbol
elif isinstance(note, LongNote):
needed_positions = set(note.positions_covered())
if needed_positions & frame.positions.keys():
frames_by_bar[bar_index].append(frame)
frame = Frame()
direction = note.tail_direction()
arrow = DIRECTION_TO_ARROW[direction]
line = DIRECTION_TO_LINE[direction]
for is_first, is_last, pos in mark_ends(
note.positions_covered()):
if is_first:
frame.positions[pos] = symbol
elif is_last:
frame.positions[pos] = arrow
else:
frame.positions[pos] = line
elif isinstance(note, LongNoteEnd):
if note.position in frame.positions:
frames_by_bar[bar_index].append(frame)
frame = Frame()
if circle_free and symbol in NOTE_TO_CIRCLE_FREE_SYMBOL:
symbol = NOTE_TO_CIRCLE_FREE_SYMBOL[symbol]
frame.positions[note.position] = symbol
frames_by_bar[bar_index].append(frame)
# Merge bar-specific frames is possible
final_frames: List[Frame] = []
for bar_index in range(ceil(self.length)):
frames = frames_by_bar[bar_index]
# Merge if :
# - No split in current bar (only one frame)
# - There is a previous frame
# - The previous frame is not a split frame (it holds a bar)
# - The previous and current bars are all in the same 4-bar group
# - The note positions in the previous frame do not clash with the current frame
if (len(frames) == 1 and final_frames and final_frames[-1].bars
and max(final_frames[-1].bars.keys()) // 4
== min(frames[0].bars.keys()) // 4
and (not (final_frames[-1].positions.keys()
& frames[0].positions.keys()))):
final_frames[-1].bars.update(frames[0].bars)
final_frames[-1].positions.update(frames[0].positions)
else:
final_frames.extend(frames)
dumped_frames = map(lambda f: f.dump(self.length), final_frames)
yield from collapse(intersperse("", dumped_frames))
def multiband_parsl_driver(configuration,
rerun_in,
rerun_out,
tract_id,
patch_id_no_parens,
filter_list,
logbase="",
inputs=[],
wrap=None):
"""This is a parsl-level replacement for multiBandDriver.py. There is no single final
task for multiband driver - instead there is one per filter. So the future returned
by this is a `combine` of those final filter tasks. Something following on from this
might instead be able to depend only on the final filter tasks."""
repo_dir = configuration.repo_dir
# mergeCoaddDetections (one, for all filters on this patch)
# deblendCoaddSources (per filter) - depends on single mergeCoaddDetections
# measureCoaddSources (per filter) - depends on corresponding filter deblend
# mergeCoaddMeasurements - depends on all filters
# forcedPhotCoadd (per filter) - depends on mergeCoaddMeasurements
# 'combine' - future combinator to generate the final status future
filter_ids_for_dm = ""
for el in intersperse("^", filter_list):
filter_ids_for_dm += el
merge_coadd_det_fut = merge_coadd_detections(
repo_dir,
rerun_in,
rerun_out,
tract_id,
patch_id_no_parens,
obs_lsst_configs=configuration.obs_lsst_configs,
filters=filter_ids_for_dm,
wrap=wrap,
inputs=inputs,
stdout="{logbase}.merge_coadd_detections.stdout".format(
logbase=logbase),
stderr="{logbase}.merge_coadd_detections.stderr".format(
logbase=logbase),
parsl_resource_specification={
"priority": (6001, tract_id, patch_id_no_parens)
})
measure_futs = []
for filter_id in filter_list:
deblend_coadd_sources_fut = deblend_coadd_sources(
repo_dir,
rerun_out,
tract_id,
patch_id_no_parens,
filter_id,
wrap=wrap,
inputs=[merge_coadd_det_fut],
stdout="{logbase}-filter-{filter_id}-deblend_coadd_sources.stdout".
format(logbase=logbase, filter_id=filter_id),
stderr="{logbase}-filter-{filter_id}-deblend_coadd_sources.stderr".
format(logbase=logbase, filter_id=filter_id),
parsl_resource_specification={
"priority": (6002, tract_id, patch_id_no_parens)
})
measure_coadd_sources_fut = measure_coadd_sources(
repo_dir,
rerun_out,
tract_id,
patch_id_no_parens,
filter_id,
obs_lsst_configs=configuration.obs_lsst_configs,
wrap=wrap,
inputs=[deblend_coadd_sources_fut],
stdout="{logbase}-filter-{filter_id}-measure_coadd_sources.stdout".
format(logbase=logbase, filter_id=filter_id),
stderr="{logbase}-filter-{filter_id}-measure_coadd_sources.stderr".
format(logbase=logbase, filter_id=filter_id),
parsl_resource_specification={
"priority": (6003, tract_id, patch_id_no_parens)
})
measure_futs.append(measure_coadd_sources_fut)
merge_fut = merge_coadd_measurements(
repo_dir,
rerun_out,
tract_id,
patch_id_no_parens,
obs_lsst_configs=configuration.obs_lsst_configs,
wrap=wrap,
inputs=measure_futs,
stdout="{logbase}-merge-coadd-measurements.stdout".format(
logbase=logbase),
stderr="{logbase}-merge-coadd-measurements.stderr".format(
logbase=logbase),
parsl_resource_specification={
"priority": (6004, tract_id, patch_id_no_parens)
})
forced_phot_coadd_futs = []
for filter_id in filter_list:
forced_phot_coadd_future = forced_phot_coadd(
repo_dir,
rerun_out,
tract_id,
patch_id_no_parens,
filter_id,
obs_lsst_configs=configuration.obs_lsst_configs,
wrap=wrap,
inputs=[merge_fut],
stdout="{logbase}-filter-{filter_id}-forced_phot_coadd.stdout".
format(filter_id=filter_id, logbase=logbase),
stderr="{logbase}-filter-{filter_id}-forced_phot_coadd.stderr".
format(filter_id=filter_id, logbase=logbase),
parsl_resource_specification={
"priority": (6005, tract_id, patch_id_no_parens)
})
forced_phot_coadd_futs.append(forced_phot_coadd_future)
return combine(inputs=forced_phot_coadd_futs)
def coadd_parsl_driver(configuration,
rerun_in,
rerun_out,
tract_id,
patch_id,
filter_id,
visit_file,
visit_futs,
inputs=None,
wrap=None,
logbase=""):
"""visit_futs should be:
a dict of visit id -> visit processing completed future,
or None if no waiting for visits should happen (for example because
should be assumed to be completed by some other mechanism.
This implementation assumes that visit_file is already generated at the
time of invocation, which is not true in general.
"""
repo_dir = configuration.repo_dir
patch_id_no_parens = re.sub("[() ]", "", patch_id)
visitLines = read_and_strip(visit_file)
if visitLines == []: # skip if no visits
logger.warning(
f'No visits for tract {tract_id}, patch {patch_id}, filter {filter_id}'
)
return const_future(None)
## 9/22/2020 - New visit file format, each line contains: <visitID> <det1^det2^det3...>
visits = []
dets = []
for visitLine in visitLines:
visits.append(visitLine.split(' ')[0])
dets.append(visitLine.split(' ')[1])
pass
logger.info("WFLOWx: There are " + str(len(visits)) +
" visits for filter/tract/patch " + str(filter_id) + '/' +
str(tract_id) + '/' + str(patch_id))
visit_ids_for_dm = ""
for el in intersperse("^", visits):
visit_ids_for_dm += el
per_visit_futures = []
for visit, det in zip(visits, dets):
input_deps = []
if visit_futs:
input_deps.append(visit_futs[visit])
pass
#### 9/11/2020 [POSSIBLY TEMPORARY] Check if warp files exist ####
# warpath = configuration.repo_dir/rerun/<rerun>/deepCoadd/<filter>/<tract>/<patch>
# w1 = <warpath>/psfMatchedWarp-<filter>-<tract>-<patch>-<visit>.fits
# w2 = <warpath>/warp-<filter>-<tract>-<patch>-<visit>.fits
# e.g. /global/cscratch1/sd/descdm/DC2/DR2/repo/rerun/Tom3-1.2.3.4/deepCoadd/u/5063/6,4/warp-u-5063-6,4-2249.fits
warpath = os.path.join(configuration.repo_dir, 'rerun', rerun_out,
'deepCoadd', filter_id, tract_id, patch_id)
fpart = '-'.join([filter_id, tract_id, patch_id, str(visit)])
w1 = os.path.join(warpath, 'psfMatchedWarp-' + fpart + '.fits')
w2 = os.path.join(warpath, 'warp-' + fpart + '.fits')
logger.info('WFLOWx: Searching for warp files for: ' + fpart)
logger.info('WFLOWx: w1 = ' + w1)
logger.info('WFLOWx: w2 = ' + w2)
if os.path.exists(w1) and os.path.exists(w2):
logger.info('WFLOWx: Warp files already exist for: ' + fpart +
', skipping makeCoaddTempExp...')
else:
logger.info('WFLOWx: makeCoaddTempExp for: ' + fpart)
per_visit_futures.append(
make_coadd_temp_exp(
repo_dir,
rerun_in,
rerun_out,
tract_id,
patch_id_no_parens,
filter_id,
visit,
det,
inputs=input_deps,
obs_lsst_configs=configuration.obs_lsst_configs,
wrap=wrap,
stdout="{logbase}-visit-{visit}.stdout".format(
logbase=logbase, visit=visit),
stderr="{logbase}-visit-{visit}.stderr".format(
logbase=logbase, visit=visit),
parsl_resource_specification={
"priority":
(5000, tract_id, patch_id_no_parens, filter_id)
}))
pass
pass
fut2 = assemble_coadd(
repo_dir,
rerun_out,
tract_id,
patch_id_no_parens,
filter_id,
visit_ids_for_dm,
inputs=per_visit_futures,
obs_lsst_configs=configuration.obs_lsst_configs,
wrap=wrap,
stdout="{logbase}.assemble_coadd.stdout".format(logbase=logbase),
stderr="{logbase}.assemble_coadd.stderr".format(logbase=logbase),
parsl_resource_specification={
"priority": (5010, tract_id, patch_id_no_parens, filter_id)
})
fut3 = detect_coadd_sources(
repo_dir,
rerun_out,
tract_id,
patch_id_no_parens,
filter_id,
visit_file,
inputs=[fut2],
wrap=wrap,
stdout="{logbase}.detect_coadd_sources.stdout".format(logbase=logbase),
stderr="{logbase}.detect_coadd_sources.stderr".format(logbase=logbase),
parsl_resource_specification={
"priority": (5020, tract_id, patch_id_no_parens, filter_id)
})
return fut3