def get_metapixels_filedata(self, metapixels):
""" """
subdata = self.data[[
"Source", "filefracday", "fieldid", "ccdid", "qid", "filterid"
]][self.data['u_digit,v_digit'].isin(metapixels)]
subdata["filename"] = buildurl.build_filename_from_dataframe(subdata)
return subdata[["filename", "Source"]]
def fetch_metapixel_data(self, metapixel, datakey):
""" """
subdata = self.get_metapixel_data(metapixel,
columns=[
"Source", "filefracday",
"fieldid", "ccdid", "qid",
"filterid"
])
subdata["filename"] = buildurl.build_filename_from_dataframe(subdata)
fdata = subdata[["filename", "Source"]]
def get_metapixel_sources(self,
metapixel,
columns=["filename", "Source"],
compute=True):
""" """
metapixeldata = self.grouped_digit.get_group(tuple(metapixel))
metapixeldata["filename"] = buildurl.build_filename_from_dataframe(
metapixeldata)
if columns is not None and compute:
return metapixeldata[columns].compute()
return metapixeldata
def cget_median_stamp(self,
metapixels,
client=None,
on="stars",
gather=True):
"""
Parameters
----------
on: [string] -optional-
on could be stars or residual
"""
print("TO BE TESTED TO CHECK IF DEPRECATED, SEE cget_metapixels_data")
client = self.get_client(client)
if client is None:
raise ValueError("client requested for this method.")
# dmetapixeldata is lazy
all_meta = []
for p_ in metapixels:
metapixeldata = self.grouped_digit[[
"filefracday", "fieldid", "ccdid", "qid", "filterid", "Source"
]].get_group(tuple(metapixel))
metapixeldata["filename"] = buildurl.build_filename_from_dataframe(
metapixeldata)
dmetapixeldata = [
self.get_metapixel_sources(l_, compute=False) for l_ in metapixels
]
# all metapixeldata are computed but still distribution inside the cluster
# they are 'futures'
# They are computed together for the share the same data files
f_metapixeldata = client.compute(dmetapixeldata)
# Logic. Then work on the distributed data
# Grab all the stars for each of them. Computation made on the respective cluster's computer
f_stamps = client.map(fetch_parquetsource_data,
f_metapixeldata,
datakey=on)
if gather:
return client.gather(f_stamps)
return f_stamps
def cget_median_stampsky(self,
metapixels,
client,
on="stars",
buffer=2,
gather=True):
"""
Parameters
----------
on: [string] -optional-
on could be stars or residual
"""
# dmetapixeldata is lazy
all_meta = []
for p_ in metapixels:
metapixeldata = self.grouped_digit[[
"filefracday", "fieldid", "ccdid", "qid", "filterid", "Source"
]].get_group(tuple(metapixel))
metapixeldata["filename"] = buildurl.build_filename_from_dataframe(
metapixeldata)
dmetapixeldata = [
self.get_metapixel_sources(l_, compute=False) for l_ in metapixels
]
# all metapixeldata are computed but still distribution inside the cluster
# they are 'futures'
# They are computed together for the share the same data files
f_metapixeldata = client.compute(dmetapixeldata)
# Logic. Then work on the distributed data
# Grab all the stars for each of them. Computation made on the respective cluster's computer
f_stamps = client.map(_fetch_residuals_, f_metapixeldata, datakey=on)
# Compute the sky study on them
f_skies = client.map(_residual_to_skydata_, f_stamps, buffer=buffer)
if gather:
return client.gather(f_skies)
return f_skies