def attach_time_diff_for_spice_versions(data): docs = list(data) for doc in docs: start_unix = doc['data_start_unix_time'] end_unix = doc['data_stop_unix_time'] start_scet = doc['data_start_scet'] end_scet = doc['data_end_scet'] if start_scet == 0 or end_scet == 0: continue new_start_unix = stix_datetime.scet2unix(start_scet) new_end_unix = stix_datetime.scet2unix(end_scet) doc['delta_start'] = abs(start_unix - new_start_unix) doc['delta_end'] = abs(end_unix - new_end_unix) return docs
def merge(self, cursor): packet_utc = '' start_time = 0 merged_packet = sdt.Packet.merge(cursor, 54125) scet_coarse = np.array(merged_packet['NIX00445']) scet_fine = np.array(merged_packet['NIX00446']) num_samples = merged_packet['NIX00089'] scet = scet_coarse + scet_fine / 65536. readouts = [[]] * 4 times = [] dt = np.array(merged_packet['NIX00490']) * np.array( merged_packet['NIX00088']) * 1e-3 readouts = np.array([merged_packet[f'NIX0009{i}'] for i in range(4)]) start_unix_time = [stix_datetime.scet2unix(x) for x in scet] obs_times = [] for delta_t, start_t, ns in zip(dt.tolist(), start_unix_time, num_samples): obs_time = start_t + delta_t * np.arange(ns) obs_times.extend(obs_time.tolist()) res = { 'readouts': readouts.tolist(), 'obs_time': obs_times, 'status': 'OK', } return res
def parse(cls, packets, dlt=0): if not packets: return {'error': 'Data not available!'} lightcurves = {} unix_time = [] energy_bins = {} last_time = 0 for pkt in packets: packet = sdt.Packet(pkt) if not packet.isa(QLLC_SPID): continue #fig = None scet_coarse = packet[1].raw scet_fine = packet[2].raw start_scet = scet_coarse + scet_fine / 65536. if start_scet <= last_time: continue last_time = start_scet int_duration = (packet[3].raw + 1) * 0.1 detector_mask = packet[4].raw pixel_mask = packet[6].raw num_lc = packet[17].raw compression_s = packet[8].raw compression_k = packet[9].raw compression_m = packet[10].raw if not energy_bins: energy_bin_mask = packet[16].raw energy_bins = get_energy_bins(energy_bin_mask) num_lc_points = packet.get('NIX00270/NIX00271')[0] lc = packet.get('NIX00270/NIX00271/*.eng')[0] rcr = packet.get('NIX00275/*.raw') UTC = packet['header']['UTC'] for i in range(len(lc)): if i not in lightcurves: lightcurves[i] = [] lightcurves[i].extend(lc[i]) unix_time.extend([ stix_datetime.scet2unix(start_scet + dlt + x * int_duration) for x in range(num_lc_points[0]) ]) if not lightcurves: return {'error': 'Data not available!'} return { 'unix_time': unix_time, 'light_curves': {x: lightcurves[x] for x in lightcurves}, 'energy_bins': energy_bins, 'num': len(unix_time), 'start_unix': unix_time[0], 'start_utc': stix_datetime.unix2utc(unix_time[0]), 'end_unix': unix_time[-1], 'end_utc': stix_datetime.unix2utc(unix_time[-1]) }
def parse(cls, packets): if not packets: return {'status': 'Background data not available!'} results = [] for pkt in packets: packet = sdt.Packet(pkt) if not packet.isa(QLBKG_SPID): continue scet_coarse = packet[1].raw scet_fine = packet[2].raw start_unix = stix_datetime.scet2unix(scet_coarse, scet_fine) tbin = (packet[3].raw + 1) * 0.1 num_samples = packet[14].raw samples = packet[14].children for i in range(num_samples): detector = samples[0 + 35 * i][1] spectra = [samples[j + 35 * i][2] for j in range(32)] trig = samples[33 + 35 * i][2] t = start_unix + samples[34 + 35 * i][ 1] * tbin #number of integrations after the first one struct = { 'detector': detector, 'spectra': spectra, 'triggers': trig, 'obs_time': t, 'tbin': tbin, 'packet_id': pkt['_id'] } results.push(struct) return results
def merge(self, cursor): readouts = [[], [], [], []] read_time = [] packet_utc = '' start_time = 0 for pkt in cursor: packet = sdt.Packet(pkt) packet_utc = packet['UTC'] T0 = stix_datetime.scet2unix( packet[1].raw) + packet[2].raw / 65536. if start_time == 0: start_time = T0 dt = packet[4].raw / 64 * (1 / 64.) # has to be fixed children = packet[5].children for i, param in enumerate(children): readouts[i % 4].append(param[1]) if i % 4 == 0: read_time.append(dt * int(i / 4) + T0) return { 'packet_utc': packet_utc, 'readouts': readouts, 'read_time': read_time, 'start_time': start_time, 'status': 'OK', }
def time_conversion_scet2unix(value): result = {} try: result = {'data': sdt.scet2unix(float(value))} except Exception as e: result = {'error': str(e)} return json_util.dumps(result)
def merge(self, cursor): for pkt in cursor: packet = sdt.Packet(pkt) self.request_id = packet[3].raw self.packet_unix = packet['unix_time'] T0 = stix_datetime.scet2unix(packet[12].raw) num_structures = packet[13].raw self.eacc_SKM = (packet[5].raw, packet[6].raw, packet[7].raw) self.trig_SKM = (packet[9].raw, packet[10].raw, packet[11].raw) trig_idx = 1 # uncompressed counts if sum(self.trig_SKM) > 0: trig_idx = 2 counts_idx = 1 # raw value if sum(self.eacc_SKM) > 0: counts_idx = 2 children = packet[13].children group = {} for i in range(0, num_structures): offset = i * 31 time = children[offset][1] * 0.1 + T0 rcr = children[offset + 1][1] integrations = children[offset + 2][1] pixel_mask = [ children[offset + 4][1], children[offset + 6][1], children[offset + 8][1] ] detector_mask = children[offset + 13][1] triggers = [children[k + 14][trig_idx] for k in range(0, 16)] # id 6-22, 16 trigger accumulators num_samples = children[30 + offset][1] samples = children[30 + offset][3] subgroups = [] for j in range(0, num_samples): k = j * 5 E1_low = samples[k + 1][1] E2_high = samples[k + 2][1] flux = samples[k + 3][1] num_vis = samples[k + 4][1] visiblity_root = samples[k + 4][3] visibilities = [ (visiblity_root[m][1], visiblity_root[m + 1][1], visiblity_root[m + 2][1]) for m in range(0, num_vis) ] subgroups.append([E1_low, E2_high, flux, visibilities]) group = { 'time': time, 'rcr': rcr, 'pixel_mask': pixel_mask, 'detector_mask': detector_mask, 'triggers': triggers, 'integrations': integrations, 'subgroups': subgroups, } self.groups.append(group) return self.format_report()
def merge(self, cursor): for pkt in cursor: packet = sdt.Packet(pkt) self.request_id = packet[3].raw # print(self.request_id) self.packet_unix = packet['unix_time'] T0 = stix_datetime.scet2unix(packet[12].raw) num_structures = packet[13].raw eacc_SKM = (packet[5].raw, packet[6].raw, packet[7].raw) trig_SKM = (packet[9].raw, packet[10].raw, packet[11].raw) trig_idx = 1 # uncompressed counts if sum(trig_SKM) > 0: trig_idx = 2 counts_idx = 1 # raw value if sum(eacc_SKM) > 0: counts_idx = 2 children = packet[13].children # print(len(children)) for i in range(0, num_structures): offset = i * 23 unix_time = children[offset][1] * 0.1 + T0 self.auxiliary['time'].append(unix_time) self.auxiliary['rcr'].append(children[offset + 1][1]) dt = children[offset + 2][1] * 0.1 self.auxiliary['dt'].append(dt) self.auxiliary['pixel_mask'].append(children[offset + 4][1]) self.auxiliary['detector_mask'].append(children[offset + 5][1]) self.auxiliary['triggers'].append( [children[k + 6][trig_idx] for k in range(0, 16)]) # id 6-22, 16 trigger accumulators num_samples = children[22][1] samples = children[22][3] for j in range(0, num_samples): k = j * 5 pixel = samples[k + 1][1] detector = samples[k + 2][1] energy_bin = samples[k + 3][1] num_bits = samples[k + 4][1] continous_counts = samples[k + 4][3] counts = 1 if num_bits == 1: counts = continous_counts[0][counts_idx] if num_bits == 2: counts = continous_counts[0][counts_idx] << 8 counts += continous_counts[1][counts_idx] self.boxes['detector'].append(detector) self.boxes['pixel'].append(pixel) self.boxes['energy'].append(energy_bin) self.boxes['time'].append(unix_time) self.boxes['counts'].append(counts) return self.format_report()
def validate_time(file_id): msg = {'msg': 'File not found'} doc = STIX_MDB.get_raw_file_info(file_id) if doc: start_unix = doc['data_start_unix_time'] end_unix = doc['data_stop_unix_time'] start_scet = doc['data_start_scet'] end_scet = doc['data_end_scet'] if start_scet == 0 or end_scet == 0: msg = {'msg': 'Invalid SCET'} else: new_start_unix = stix_datetime.scet2unix(start_scet) new_end_unix = stix_datetime.scet2unix(end_scet) delta_start = abs(start_unix - new_start_unix) delta_end = abs(end_unix - new_end_unix) msg = { 'msg': 'OK', 'data': { '_id': doc['_id'], 'dt_start': delta_start, 'dt_end': delta_end } } return json_util.dumps(msg)
def minify(packets): data = { 'num_packets': 0, #'scet_coarse':[], #'scet_fine':[], 'obs_beg_unix': [], 'integration_time': None, 'detector_mask': None, 'pixel_mask': None, 'energy_bin_mask': None, 'energy_bin_mask_upper': None, 'skm_counts': None, 'skm_trig': None, 'num_energies': None, 'num_samples': [], 'light_curves': [], 'triggers': [], 'packet_ids': [], 'rcr': [] } hashes = [] for pkt in packets: if pkt['hash'] in hashes: continue hashes.append(pkt['hash']) data['packet_ids'].append(pkt['_id']) data['num_packets'] += 1 parameters = pkt['parameters'] stix_packet = sdt.Packet.merge([pkt], 54118, 'eng', remove_duplicates=True) #data['scet_fine'].append( stix_packet['NIX00446'][0]) #data['scet_coarse'].append( pkt[1][0]) data['obs_beg_unix'].append( stix_datetime.scet2unix(parameters[1][1], parameters[2][1])) data['integration_time'] = (stix_packet['NIX00405'][0] + 1) * 0.1 # Seconds data['detector_mask'] = stix_packet['NIX00407'][0] data['pixel_mask'] = stix_packet['NIXD0407'][0] data['energy_bin_mask'] = stix_packet['NIX00266'][0] data['energy_bin_mask_upper'] = stix_packet['NIXD0107'][0] data['skm_counts'] = (stix_packet['NIXD0101'][0], stix_packet['NIXD0102'][0], stix_packet['NIXD0103'][0]) data['skm_trig'] = (stix_packet['NIXD0104'][0], stix_packet['NIXD0105'][0], stix_packet['NIXD0106'][0]) # DATA num_energies = stix_packet['NIX00270'][0] data['num_energies'] = num_energies # TODO Check why this is necessary samples = stix_packet['NIX00271'][::5] num_samples = sum(samples) data['num_samples'].append(num_samples) counts = stix_packet['NIX00272'] flat_indcies = np.cumsum([0, *samples]) * num_energies lc = [] for i, sample in enumerate(samples): lc.append( np.array( counts[flat_indcies[i]:flat_indcies[i + 1]]).reshape( num_energies, sample).tolist()) data['light_curves'].append(lc) data['triggers'].append(stix_packet['NIX00274']) data['rcr'].append(stix_packet['NIX00276']) return data
def minify(packets): times = [] time_hk1 = [] time_hk2 = [] raw = {} eng = {} result = { 'hk1': 0, 'hk2': 0, 'time_hk1': [], 'time_hk2': [], 'raw': {}, 'eng': {}, 'num': 0, 'eng_valid': {} } i = 0 hashs = [] start_unix = math.inf end_unix = -math.inf for pkt in packets: header = pkt['header'] if pkt['hash'] in hashs: continue hashs.append(pkt['hash']) spid = header['SPID'] coarse = header['coarse_time'] fine = header['fine_time'] unix = stix_datetime.scet2unix(coarse, fine) if unix > end_unix: end_unix = unix if unix < start_unix: start_unix = unix if spid == 54101: result['hk1'] += 1 time_hk1.append(unix) else: result['hk2'] += 1 time_hk2.append(unix) result['num'] += 1 times.append(unix) i += 1 parameters = pkt['parameters'] for p in parameters: if 'NIXG' in p[0]: continue if p[0] not in raw: raw[p[0]] = [] if p[0] not in eng: result['eng_valid'][p[0]] = True eng[p[0]] = [] raw[p[0]].append(p[1]) try: tmp = float(p[2]) eng[p[0]].append(p[2]) except ValueError: result['eng_valid'][p[0]] = False eng[p[0]].append('') result['eng'] = eng result['raw'] = raw result['time'] = times result['start_unix'] = start_unix result['end_unix'] = end_unix if len(time_hk1) != len(times): result['time_hk1'] = time_hk1 if len(time_hk2) != len(times): result['time_hk2'] = time_hk2 return result
def merge(self, cursor): last_timestamp = None for pkt in cursor: packet = sdt.Packet(pkt) self.request_id = packet[3].raw self.packet_unix = packet['unix_time'] T0 = stix_datetime.scet2unix(packet[12].raw) # if self.start_time == 0: # self.start_time = T0 num_structures = packet[13].raw self.eacc_SKM = (packet[5].raw, packet[6].raw, packet[7].raw) self.trig_SKM = (packet[9].raw, packet[10].raw, packet[11].raw) trig_idx = 1 # uncompressed counts if sum(self.trig_SKM) > 0: trig_idx = 2 counts_idx = 1 # raw value if sum(self.eacc_SKM) > 0: counts_idx = 2 children = packet[13].children group = {} if last_timestamp is None: last_timestamp = T0 self.num_time_bins = 0 for i in range(0, num_structures): offset = i * 10 pixel_mask = children[offset + 1][1] detector_mask = children[offset + 2][1] rcr = children[offset + 3][1] E1 = children[offset + 5][1] E2 = children[offset + 6][1] Eunit = children[offset + 7][1] num_samples = children[8 + offset][1] samples = children[8 + offset][3] subgroups = [] for j in range(0, num_samples): k = j * 3 timestamp = samples[k + 0][1] * 0.1 + T0 dT = timestamp - last_timestamp if dT > 0: self.num_time_bins += 1 last_timestamp = timestamp triggers = samples[k + 1][trig_idx] num_energies = samples[k + 2][1] energy_children = samples[k + 2][3] lcs = [m[counts_idx] for m in energy_children] subgroups.append((timestamp, triggers, lcs, dT)) group = { 'rcr': rcr, 'pixel_mask': pixel_mask, 'detector_mask': detector_mask, 'E1': E1, 'E2': E2, 'Eunit': Eunit, 'subgroups': subgroups, } self.groups.append(group) return self.format_report()
def merge(self, cursor): yield '{"groups":[' for pkt in cursor: packet = sdt.Packet(pkt) self.request_id = packet[3].raw self.packet_unix = packet['unix_time'] T0 = stix_datetime.scet2unix(packet[12].raw) num_structures = packet[13].raw self.eacc_SKM = (packet[5].raw, packet[6].raw, packet[7].raw) self.trig_SKM = (packet[9].raw, packet[10].raw, packet[11].raw) trig_idx = 1 # uncompressed counts if sum(self.trig_SKM) > 0: trig_idx = 2 counts_idx = 1 # raw value if sum(self.eacc_SKM) > 0: counts_idx = 2 children = packet[13].children group = {} last_time_bin = 0 self.num_time_bins = 0 for i in range(0, num_structures): offset = i * 22 time = children[offset][1] * 0.1 + T0 if time != last_time_bin: self.num_time_bins += 1 last_time_bin = time rcr = children[offset + 1][1] pixel_mask = [ e[1] for e in children[offset + 2][3] if 'NIXG' not in e[0] ] # exclude NIXG parameters detector_mask = children[offset + 3][1] integrations = children[offset + 4][1] triggers = [children[k + 5][trig_idx] for k in range(0, 16)] # id 6-22, 16 trigger accumulators num_samples = children[21 + offset][1] samples = children[21 + offset][3] energies = [] for j in range(0, num_samples): k = j * 4 E1_low = samples[k + 1][1] E2_high = samples[k + 2][1] pixel_counts = [e[counts_idx] for e in samples[k + 3][3]] energies.append( [E1_low, E2_high, sum(pixel_counts), pixel_counts]) pixel_indexes = self.get_spectrum_pixel_indexes( detector_mask, pixel_mask) group = { 'time': time, 'rcr': rcr, 'pixel_mask': pixel_mask, 'detector_mask': detector_mask, 'triggers': triggers, 'integrations': integrations, 'pixel_indexes': pixel_indexes, 'energies': energies, } yield json.dumps(group) yield ',' yield '{}],' yield f'"request_id": {self.request_id},' yield f'"packet_unix": {self.packet_unix},' yield '"status": "OK",' yield f'"trig_skm": {list(self.trig_SKM)},' yield f'"eacc_skm": {list(self.eacc_SKM)},' yield f'"num_time_bins": {self.num_time_bins}' + '}'
def merge(self, cursor): #print('Analyzing data...') for pkt in cursor: packet = sdt.Packet(pkt) self.request_id = packet[3].raw self.packet_unix = packet['unix_time'] T0 = stix_datetime.scet2unix(packet[12].raw) num_structures = packet[13].raw self.eacc_SKM = (packet[5].raw, packet[6].raw, packet[7].raw) self.trig_SKM = (packet[9].raw, packet[10].raw, packet[11].raw) trig_idx = 1 # uncompressed counts if sum(self.trig_SKM) > 0: trig_idx = 2 counts_idx = 1 # raw value if sum(self.eacc_SKM) > 0: counts_idx = 2 children = packet[13].children group = {} last_time_bin = 0 self.num_time_bins = 0 for i in range(0, num_structures): offset = i * 22 time = children[offset][1] * 0.1 + T0 if time != last_time_bin: self.num_time_bins += 1 last_time_bin = time rcr = children[offset + 1][1] pixel_mask = [ e[1] for e in children[offset + 2][3] if 'NIXG' not in e[0] ] # exclude NIXG parameters detector_mask = children[offset + 3][1] integrations = children[offset + 4][1] triggers = [children[k + 5][trig_idx] for k in range(0, 16)] # id 6-22, 16 trigger accumulators num_samples = children[21 + offset][1] samples = children[21 + offset][3] energies = [] for j in range(0, num_samples): k = j * 4 E1_low = samples[k + 1][1] E2_high = samples[k + 2][1] pixel_counts = [e[counts_idx] for e in samples[k + 3][3]] energies.append( [E1_low, E2_high, sum(pixel_counts), pixel_counts]) group = { 'time': time, 'rcr': rcr, 'pixel_mask': pixel_mask, 'detector_mask': detector_mask, 'triggers': triggers, 'integrations': integrations, 'energies': energies, } self.groups.append(group) return self.format_report()