def attach_time_diff_for_spice_versions(data):
docs = list(data)
for doc in docs:
start_unix = doc['data_start_unix_time']
end_unix = doc['data_stop_unix_time']
start_scet = doc['data_start_scet']
end_scet = doc['data_end_scet']
if start_scet == 0 or end_scet == 0:
continue
new_start_unix = stix_datetime.scet2unix(start_scet)
new_end_unix = stix_datetime.scet2unix(end_scet)
doc['delta_start'] = abs(start_unix - new_start_unix)
doc['delta_end'] = abs(end_unix - new_end_unix)
return docs
def merge(self, cursor):
packet_utc = ''
start_time = 0
merged_packet = sdt.Packet.merge(cursor, 54125)
scet_coarse = np.array(merged_packet['NIX00445'])
scet_fine = np.array(merged_packet['NIX00446'])
num_samples = merged_packet['NIX00089']
scet = scet_coarse + scet_fine / 65536.
readouts = [[]] * 4
times = []
dt = np.array(merged_packet['NIX00490']) * np.array(
merged_packet['NIX00088']) * 1e-3
readouts = np.array([merged_packet[f'NIX0009{i}'] for i in range(4)])
start_unix_time = [stix_datetime.scet2unix(x) for x in scet]
obs_times = []
for delta_t, start_t, ns in zip(dt.tolist(), start_unix_time,
num_samples):
obs_time = start_t + delta_t * np.arange(ns)
obs_times.extend(obs_time.tolist())
res = {
'readouts': readouts.tolist(),
'obs_time': obs_times,
'status': 'OK',
}
return res
def parse(cls, packets, dlt=0):
if not packets:
return {'error': 'Data not available!'}
lightcurves = {}
unix_time = []
energy_bins = {}
last_time = 0
for pkt in packets:
packet = sdt.Packet(pkt)
if not packet.isa(QLLC_SPID):
continue
#fig = None
scet_coarse = packet[1].raw
scet_fine = packet[2].raw
start_scet = scet_coarse + scet_fine / 65536.
if start_scet <= last_time:
continue
last_time = start_scet
int_duration = (packet[3].raw + 1) * 0.1
detector_mask = packet[4].raw
pixel_mask = packet[6].raw
num_lc = packet[17].raw
compression_s = packet[8].raw
compression_k = packet[9].raw
compression_m = packet[10].raw
if not energy_bins:
energy_bin_mask = packet[16].raw
energy_bins = get_energy_bins(energy_bin_mask)
num_lc_points = packet.get('NIX00270/NIX00271')[0]
lc = packet.get('NIX00270/NIX00271/*.eng')[0]
rcr = packet.get('NIX00275/*.raw')
UTC = packet['header']['UTC']
for i in range(len(lc)):
if i not in lightcurves:
lightcurves[i] = []
lightcurves[i].extend(lc[i])
unix_time.extend([
stix_datetime.scet2unix(start_scet + dlt + x * int_duration)
for x in range(num_lc_points[0])
])
if not lightcurves:
return {'error': 'Data not available!'}
return {
'unix_time': unix_time,
'light_curves': {x: lightcurves[x]
for x in lightcurves},
'energy_bins': energy_bins,
'num': len(unix_time),
'start_unix': unix_time[0],
'start_utc': stix_datetime.unix2utc(unix_time[0]),
'end_unix': unix_time[-1],
'end_utc': stix_datetime.unix2utc(unix_time[-1])
}
def parse(cls, packets):
if not packets:
return {'status': 'Background data not available!'}
results = []
for pkt in packets:
packet = sdt.Packet(pkt)
if not packet.isa(QLBKG_SPID):
continue
scet_coarse = packet[1].raw
scet_fine = packet[2].raw
start_unix = stix_datetime.scet2unix(scet_coarse, scet_fine)
tbin = (packet[3].raw + 1) * 0.1
num_samples = packet[14].raw
samples = packet[14].children
for i in range(num_samples):
detector = samples[0 + 35 * i][1]
spectra = [samples[j + 35 * i][2] for j in range(32)]
trig = samples[33 + 35 * i][2]
t = start_unix + samples[34 + 35 * i][
1] * tbin #number of integrations after the first one
struct = {
'detector': detector,
'spectra': spectra,
'triggers': trig,
'obs_time': t,
'tbin': tbin,
'packet_id': pkt['_id']
}
results.push(struct)
return results
def merge(self, cursor):
readouts = [[], [], [], []]
read_time = []
packet_utc = ''
start_time = 0
for pkt in cursor:
packet = sdt.Packet(pkt)
packet_utc = packet['UTC']
T0 = stix_datetime.scet2unix(
packet[1].raw) + packet[2].raw / 65536.
if start_time == 0:
start_time = T0
dt = packet[4].raw / 64 * (1 / 64.) # has to be fixed
children = packet[5].children
for i, param in enumerate(children):
readouts[i % 4].append(param[1])
if i % 4 == 0:
read_time.append(dt * int(i / 4) + T0)
return {
'packet_utc': packet_utc,
'readouts': readouts,
'read_time': read_time,
'start_time': start_time,
'status': 'OK',
}
def time_conversion_scet2unix(value):
result = {}
try:
result = {'data': sdt.scet2unix(float(value))}
except Exception as e:
result = {'error': str(e)}
return json_util.dumps(result)
def merge(self, cursor):
for pkt in cursor:
packet = sdt.Packet(pkt)
self.request_id = packet[3].raw
self.packet_unix = packet['unix_time']
T0 = stix_datetime.scet2unix(packet[12].raw)
num_structures = packet[13].raw
self.eacc_SKM = (packet[5].raw, packet[6].raw, packet[7].raw)
self.trig_SKM = (packet[9].raw, packet[10].raw, packet[11].raw)
trig_idx = 1
# uncompressed counts
if sum(self.trig_SKM) > 0:
trig_idx = 2
counts_idx = 1 # raw value
if sum(self.eacc_SKM) > 0:
counts_idx = 2
children = packet[13].children
group = {}
for i in range(0, num_structures):
offset = i * 31
time = children[offset][1] * 0.1 + T0
rcr = children[offset + 1][1]
integrations = children[offset + 2][1]
pixel_mask = [
children[offset + 4][1], children[offset + 6][1],
children[offset + 8][1]
]
detector_mask = children[offset + 13][1]
triggers = [children[k + 14][trig_idx] for k in range(0, 16)]
# id 6-22, 16 trigger accumulators
num_samples = children[30 + offset][1]
samples = children[30 + offset][3]
subgroups = []
for j in range(0, num_samples):
k = j * 5
E1_low = samples[k + 1][1]
E2_high = samples[k + 2][1]
flux = samples[k + 3][1]
num_vis = samples[k + 4][1]
visiblity_root = samples[k + 4][3]
visibilities = [
(visiblity_root[m][1], visiblity_root[m + 1][1],
visiblity_root[m + 2][1]) for m in range(0, num_vis)
]
subgroups.append([E1_low, E2_high, flux, visibilities])
group = {
'time': time,
'rcr': rcr,
'pixel_mask': pixel_mask,
'detector_mask': detector_mask,
'triggers': triggers,
'integrations': integrations,
'subgroups': subgroups,
}
self.groups.append(group)
return self.format_report()
def merge(self, cursor):
for pkt in cursor:
packet = sdt.Packet(pkt)
self.request_id = packet[3].raw
# print(self.request_id)
self.packet_unix = packet['unix_time']
T0 = stix_datetime.scet2unix(packet[12].raw)
num_structures = packet[13].raw
eacc_SKM = (packet[5].raw, packet[6].raw, packet[7].raw)
trig_SKM = (packet[9].raw, packet[10].raw, packet[11].raw)
trig_idx = 1
# uncompressed counts
if sum(trig_SKM) > 0:
trig_idx = 2
counts_idx = 1 # raw value
if sum(eacc_SKM) > 0:
counts_idx = 2
children = packet[13].children
# print(len(children))
for i in range(0, num_structures):
offset = i * 23
unix_time = children[offset][1] * 0.1 + T0
self.auxiliary['time'].append(unix_time)
self.auxiliary['rcr'].append(children[offset + 1][1])
dt = children[offset + 2][1] * 0.1
self.auxiliary['dt'].append(dt)
self.auxiliary['pixel_mask'].append(children[offset + 4][1])
self.auxiliary['detector_mask'].append(children[offset + 5][1])
self.auxiliary['triggers'].append(
[children[k + 6][trig_idx] for k in range(0, 16)])
# id 6-22, 16 trigger accumulators
num_samples = children[22][1]
samples = children[22][3]
for j in range(0, num_samples):
k = j * 5
pixel = samples[k + 1][1]
detector = samples[k + 2][1]
energy_bin = samples[k + 3][1]
num_bits = samples[k + 4][1]
continous_counts = samples[k + 4][3]
counts = 1
if num_bits == 1:
counts = continous_counts[0][counts_idx]
if num_bits == 2:
counts = continous_counts[0][counts_idx] << 8
counts += continous_counts[1][counts_idx]
self.boxes['detector'].append(detector)
self.boxes['pixel'].append(pixel)
self.boxes['energy'].append(energy_bin)
self.boxes['time'].append(unix_time)
self.boxes['counts'].append(counts)
return self.format_report()
def validate_time(file_id):
msg = {'msg': 'File not found'}
doc = STIX_MDB.get_raw_file_info(file_id)
if doc:
start_unix = doc['data_start_unix_time']
end_unix = doc['data_stop_unix_time']
start_scet = doc['data_start_scet']
end_scet = doc['data_end_scet']
if start_scet == 0 or end_scet == 0:
msg = {'msg': 'Invalid SCET'}
else:
new_start_unix = stix_datetime.scet2unix(start_scet)
new_end_unix = stix_datetime.scet2unix(end_scet)
delta_start = abs(start_unix - new_start_unix)
delta_end = abs(end_unix - new_end_unix)
msg = {
'msg': 'OK',
'data': {
'_id': doc['_id'],
'dt_start': delta_start,
'dt_end': delta_end
}
}
return json_util.dumps(msg)
def minify(packets):
data = {
'num_packets': 0,
#'scet_coarse':[],
#'scet_fine':[],
'obs_beg_unix': [],
'integration_time': None,
'detector_mask': None,
'pixel_mask': None,
'energy_bin_mask': None,
'energy_bin_mask_upper': None,
'skm_counts': None,
'skm_trig': None,
'num_energies': None,
'num_samples': [],
'light_curves': [],
'triggers': [],
'packet_ids': [],
'rcr': []
}
hashes = []
for pkt in packets:
if pkt['hash'] in hashes:
continue
hashes.append(pkt['hash'])
data['packet_ids'].append(pkt['_id'])
data['num_packets'] += 1
parameters = pkt['parameters']
stix_packet = sdt.Packet.merge([pkt],
54118,
'eng',
remove_duplicates=True)
#data['scet_fine'].append( stix_packet['NIX00446'][0])
#data['scet_coarse'].append( pkt[1][0])
data['obs_beg_unix'].append(
stix_datetime.scet2unix(parameters[1][1], parameters[2][1]))
data['integration_time'] = (stix_packet['NIX00405'][0] +
1) * 0.1 # Seconds
data['detector_mask'] = stix_packet['NIX00407'][0]
data['pixel_mask'] = stix_packet['NIXD0407'][0]
data['energy_bin_mask'] = stix_packet['NIX00266'][0]
data['energy_bin_mask_upper'] = stix_packet['NIXD0107'][0]
data['skm_counts'] = (stix_packet['NIXD0101'][0],
stix_packet['NIXD0102'][0],
stix_packet['NIXD0103'][0])
data['skm_trig'] = (stix_packet['NIXD0104'][0],
stix_packet['NIXD0105'][0],
stix_packet['NIXD0106'][0])
# DATA
num_energies = stix_packet['NIX00270'][0]
data['num_energies'] = num_energies
# TODO Check why this is necessary
samples = stix_packet['NIX00271'][::5]
num_samples = sum(samples)
data['num_samples'].append(num_samples)
counts = stix_packet['NIX00272']
flat_indcies = np.cumsum([0, *samples]) * num_energies
lc = []
for i, sample in enumerate(samples):
lc.append(
np.array(
counts[flat_indcies[i]:flat_indcies[i + 1]]).reshape(
num_energies, sample).tolist())
data['light_curves'].append(lc)
data['triggers'].append(stix_packet['NIX00274'])
data['rcr'].append(stix_packet['NIX00276'])
return data
def minify(packets):
times = []
time_hk1 = []
time_hk2 = []
raw = {}
eng = {}
result = {
'hk1': 0,
'hk2': 0,
'time_hk1': [],
'time_hk2': [],
'raw': {},
'eng': {},
'num': 0,
'eng_valid': {}
}
i = 0
hashs = []
start_unix = math.inf
end_unix = -math.inf
for pkt in packets:
header = pkt['header']
if pkt['hash'] in hashs:
continue
hashs.append(pkt['hash'])
spid = header['SPID']
coarse = header['coarse_time']
fine = header['fine_time']
unix = stix_datetime.scet2unix(coarse, fine)
if unix > end_unix:
end_unix = unix
if unix < start_unix:
start_unix = unix
if spid == 54101:
result['hk1'] += 1
time_hk1.append(unix)
else:
result['hk2'] += 1
time_hk2.append(unix)
result['num'] += 1
times.append(unix)
i += 1
parameters = pkt['parameters']
for p in parameters:
if 'NIXG' in p[0]:
continue
if p[0] not in raw:
raw[p[0]] = []
if p[0] not in eng:
result['eng_valid'][p[0]] = True
eng[p[0]] = []
raw[p[0]].append(p[1])
try:
tmp = float(p[2])
eng[p[0]].append(p[2])
except ValueError:
result['eng_valid'][p[0]] = False
eng[p[0]].append('')
result['eng'] = eng
result['raw'] = raw
result['time'] = times
result['start_unix'] = start_unix
result['end_unix'] = end_unix
if len(time_hk1) != len(times):
result['time_hk1'] = time_hk1
if len(time_hk2) != len(times):
result['time_hk2'] = time_hk2
return result
def merge(self, cursor):
last_timestamp = None
for pkt in cursor:
packet = sdt.Packet(pkt)
self.request_id = packet[3].raw
self.packet_unix = packet['unix_time']
T0 = stix_datetime.scet2unix(packet[12].raw)
# if self.start_time == 0:
# self.start_time = T0
num_structures = packet[13].raw
self.eacc_SKM = (packet[5].raw, packet[6].raw, packet[7].raw)
self.trig_SKM = (packet[9].raw, packet[10].raw, packet[11].raw)
trig_idx = 1
# uncompressed counts
if sum(self.trig_SKM) > 0:
trig_idx = 2
counts_idx = 1 # raw value
if sum(self.eacc_SKM) > 0:
counts_idx = 2
children = packet[13].children
group = {}
if last_timestamp is None:
last_timestamp = T0
self.num_time_bins = 0
for i in range(0, num_structures):
offset = i * 10
pixel_mask = children[offset + 1][1]
detector_mask = children[offset + 2][1]
rcr = children[offset + 3][1]
E1 = children[offset + 5][1]
E2 = children[offset + 6][1]
Eunit = children[offset + 7][1]
num_samples = children[8 + offset][1]
samples = children[8 + offset][3]
subgroups = []
for j in range(0, num_samples):
k = j * 3
timestamp = samples[k + 0][1] * 0.1 + T0
dT = timestamp - last_timestamp
if dT > 0:
self.num_time_bins += 1
last_timestamp = timestamp
triggers = samples[k + 1][trig_idx]
num_energies = samples[k + 2][1]
energy_children = samples[k + 2][3]
lcs = [m[counts_idx] for m in energy_children]
subgroups.append((timestamp, triggers, lcs, dT))
group = {
'rcr': rcr,
'pixel_mask': pixel_mask,
'detector_mask': detector_mask,
'E1': E1,
'E2': E2,
'Eunit': Eunit,
'subgroups': subgroups,
}
self.groups.append(group)
return self.format_report()
def merge(self, cursor):
yield '{"groups":['
for pkt in cursor:
packet = sdt.Packet(pkt)
self.request_id = packet[3].raw
self.packet_unix = packet['unix_time']
T0 = stix_datetime.scet2unix(packet[12].raw)
num_structures = packet[13].raw
self.eacc_SKM = (packet[5].raw, packet[6].raw, packet[7].raw)
self.trig_SKM = (packet[9].raw, packet[10].raw, packet[11].raw)
trig_idx = 1
# uncompressed counts
if sum(self.trig_SKM) > 0:
trig_idx = 2
counts_idx = 1 # raw value
if sum(self.eacc_SKM) > 0:
counts_idx = 2
children = packet[13].children
group = {}
last_time_bin = 0
self.num_time_bins = 0
for i in range(0, num_structures):
offset = i * 22
time = children[offset][1] * 0.1 + T0
if time != last_time_bin:
self.num_time_bins += 1
last_time_bin = time
rcr = children[offset + 1][1]
pixel_mask = [
e[1] for e in children[offset + 2][3] if 'NIXG' not in e[0]
]
# exclude NIXG parameters
detector_mask = children[offset + 3][1]
integrations = children[offset + 4][1]
triggers = [children[k + 5][trig_idx] for k in range(0, 16)]
# id 6-22, 16 trigger accumulators
num_samples = children[21 + offset][1]
samples = children[21 + offset][3]
energies = []
for j in range(0, num_samples):
k = j * 4
E1_low = samples[k + 1][1]
E2_high = samples[k + 2][1]
pixel_counts = [e[counts_idx] for e in samples[k + 3][3]]
energies.append(
[E1_low, E2_high,
sum(pixel_counts), pixel_counts])
pixel_indexes = self.get_spectrum_pixel_indexes(
detector_mask, pixel_mask)
group = {
'time': time,
'rcr': rcr,
'pixel_mask': pixel_mask,
'detector_mask': detector_mask,
'triggers': triggers,
'integrations': integrations,
'pixel_indexes': pixel_indexes,
'energies': energies,
}
yield json.dumps(group)
yield ','
yield '{}],'
yield f'"request_id": {self.request_id},'
yield f'"packet_unix": {self.packet_unix},'
yield '"status": "OK",'
yield f'"trig_skm": {list(self.trig_SKM)},'
yield f'"eacc_skm": {list(self.eacc_SKM)},'
yield f'"num_time_bins": {self.num_time_bins}' + '}'
def merge(self, cursor):
#print('Analyzing data...')
for pkt in cursor:
packet = sdt.Packet(pkt)
self.request_id = packet[3].raw
self.packet_unix = packet['unix_time']
T0 = stix_datetime.scet2unix(packet[12].raw)
num_structures = packet[13].raw
self.eacc_SKM = (packet[5].raw, packet[6].raw, packet[7].raw)
self.trig_SKM = (packet[9].raw, packet[10].raw, packet[11].raw)
trig_idx = 1
# uncompressed counts
if sum(self.trig_SKM) > 0:
trig_idx = 2
counts_idx = 1 # raw value
if sum(self.eacc_SKM) > 0:
counts_idx = 2
children = packet[13].children
group = {}
last_time_bin = 0
self.num_time_bins = 0
for i in range(0, num_structures):
offset = i * 22
time = children[offset][1] * 0.1 + T0
if time != last_time_bin:
self.num_time_bins += 1
last_time_bin = time
rcr = children[offset + 1][1]
pixel_mask = [
e[1] for e in children[offset + 2][3] if 'NIXG' not in e[0]
]
# exclude NIXG parameters
detector_mask = children[offset + 3][1]
integrations = children[offset + 4][1]
triggers = [children[k + 5][trig_idx] for k in range(0, 16)]
# id 6-22, 16 trigger accumulators
num_samples = children[21 + offset][1]
samples = children[21 + offset][3]
energies = []
for j in range(0, num_samples):
k = j * 4
E1_low = samples[k + 1][1]
E2_high = samples[k + 2][1]
pixel_counts = [e[counts_idx] for e in samples[k + 3][3]]
energies.append(
[E1_low, E2_high,
sum(pixel_counts), pixel_counts])
group = {
'time': time,
'rcr': rcr,
'pixel_mask': pixel_mask,
'detector_mask': detector_mask,
'triggers': triggers,
'integrations': integrations,
'energies': energies,
}
self.groups.append(group)
return self.format_report()
