def _extract_channel_props(self):
"""Extracts channel data from a TDM file.
"""
temp = self._xmltree.find(QNAME + 'include').find('file')
blocks = temp.findall('block_bm')
if len(blocks) == 0:
blocks = temp.findall('block')
channel_names = self._xmltree.find(QNAME + 'data').findall(
'tdm_channel')
self.num_channels = len(channel_names)
assert(len(blocks) >= len(channel_names))
formats = []
names = []
self.channels = []
for i in range(self.num_channels):
chan = ChannelData()
chan.byte_offset = int(blocks[i].get('byteOffset'))
chan.length = int(blocks[i].get('length'))
try:
chan.dtype = self._convert_dtypes(blocks[i].get('valueType'))
except KeyError:
raise TypeError(
'Unknown data type in TDM file. Channel ' + str(i))
chan.name = channel_names[i].find('name').text
if chan.name is None:
chan.name = ''
self.channels.append(chan)
formats.append(chan.dtype)
names.append(chan.name)
#self.dtype = np.format_parser(formats, names, []).dtype
#Names can not be cept here, otherwise importing of files with duplicate column names would not be possible
self.dtype = np.format_parser(formats, [], []).dtype
def define_weather_dtype(filename):
f = open(filename,'rU')
names = re.split(r',\s*',f.readline().strip())
# print names, len(names)
formats = [np.sctype2char(np.float),]*len(names)
for index in weather_dtype_dict:
formats[index] = weather_dtype_dict[index]
# print formats
return np.format_parser(formats,names,[])
def get_dtype(table_desc, get_title=get_title):
"""Example:
with SqliteWrapper() as db:
desc = db.get_table_info(your_table) # persists after cursor and database close()
dtype = get_dtype(desc) # ''
with db.ccursor() as cur:
rows = cur.execute('select * from your_table') # iterable
faster_table = np.from_iter(rows, dtype) # rows dies with the cursor
"""
table_desc.sort(key=lambda row: row.order)
names = [ r.name for r in table_desc ]
formats = [ lookup_numpy_format(r.type) for r in table_desc ]
if get_title:
titles = [ get_title(n) for n in names ]
else:
titles = names
return np.format_parser(names=names, formats=formats, titles=titles)
def _parse_layout_csv(layout_csv):
""" Interpret the layout csv file """
with open(layout_csv, "r") as fhandle:
header = fhandle.readline()
header = [h.strip() for h in header.split()]
dt = np.format_parser(
["U10", "i4", "i4", "f8", "f8", "f8"],
["name", "number", "beamid", "e", "n", "u"],
header,
)
return np.genfromtxt(layout_csv,
autostrip=True,
skip_header=1,
dtype=dt.dtype)
def test_point_catalog_reader():
catfile = os.path.join(SIM_DATA_PATH, 'test_config',
'pointsource_catalog.txt')
catalog = pyuvsim.simsetup.point_sources_from_params(catfile)
header = open(catfile, 'r').readline()
header = [h.strip() for h in header.split()]
dt = np.format_parser(
['a10', 'f8', 'f8', 'f8', 'f8'],
['source_id', 'ra_j2000', 'dec_j2000', 'flux_density_I', 'frequency'],
header)
catalog_table = np.genfromtxt(catfile,
autostrip=True,
skip_header=1,
dtype=dt.dtype)
for src in catalog:
nt.assert_true(src.name in catalog_table['source_id'])
nt.assert_true(src.ra.hour in catalog_table['ra_j2000'])
nt.assert_true(src.dec.deg in catalog_table['dec_j2000'])
nt.assert_true(src.stokes[0] in catalog_table['flux_density_I'])
nt.assert_true(src.freq.to("Hz").value in catalog_table['frequency'])
def __new__(
subtype,
shape,
buf=None,
offset=0,
strides=None,
formats=None,
names=None,
titles=None,
byteorder=None,
aligned=False,
order="C",
) -> "PointArray":
dtype = subtype._record_type.dtype
if dtype is not None:
descr = np.dtype(dtype)
else:
descr = np.format_parser(formats, names, titles, aligned, byteorder)._descr
if buf is None:
self = np.ndarray.__new__(
subtype, shape, (subtype._record_type, descr), order=order
)
else:
self = np.ndarray.__new__(
subtype,
shape,
(subtype._record_type, descr),
buffer=buf,
offset=offset,
strides=strides,
order=order,
)
return self
import sys
import matplotlib.pyplot as plt
import numpy as np
dat = np.genfromtxt(sys.argv[1], names=True, max_rows=2, delimiter=',')
fields = [
'JobID', 'Start', 'MaxRSS_GB', 'NNodes', 'NProcs', 'Nbls', 'Ntimes',
'Nchan', 'Nsrc', 'Beam', 'Ntasks', 'Runtime_Second'
]
titles = [f.lower() for f in fields]
fmts = [
'U10', 'U10', 'f8', 'i4', 'i4', 'i4', 'i4', 'i4', 'i4', 'U10', 'i4', 'f8'
]
dt = np.format_parser(fmts, dat.dtype.names, titles)
dat = np.genfromtxt(sys.argv[1],
autostrip=True,
dtype=dt.dtype,
delimiter=',',
skip_header=1)
Ncpus = np.unique(dat['NProcs'])
beams = np.unique(dat['Beam'])
Ntasks, inv, counts = np.unique(dat['Ntasks'],
return_inverse=True,
return_counts=True)
NNodes = np.unique(dat['NNodes'])
markers = ['.', 'o', '+', '<', '>', '*', 'v', '^', 'h', 'p']
cmap = plt.get_cmap('tab10')
import numpy as np
np.format_parser(['f8', 'i4', 'a5'], ['col1', 'col2', 'col3'],
['T1', 'T2', 'T3']).dtype
np.format_parser(['f8', 'i4', 'a5'], ['col1', 'col2', 'col3'],
[]).dtype
np.format_parser(['f8', 'i4', 'a5'], [], []).dtype
import numpy as np
from matplotlib.mlab import rec2csv
from numpy.lib.recfunctions import append_fields
slurmids = []
Nall = [] # Each entry is a list of Nbl, Ntimes, Nchan, Nsrcs
fname = sys.argv[1]
with open(fname, 'a+') as fhandle:
header = fhandle.readline()
header = [h.strip().upper() for h in header.split(',')]
dt = np.format_parser(
['i4', 'i4', 'i4', 'i4', 'U8', 'i4'],
['Nsrcs', 'Ntimes', 'Nfreqs', 'Nbls', 'beam', 'slurm_id'], header)
filedat = np.genfromtxt(fname,
autostrip=True,
skip_header=1,
delimiter=',',
dtype=dt.dtype)
slurmids = filedat['slurm_id'].astype(str)
slurmids = [sid + '_0.0' for sid in slurmids]
p = subprocess.Popen(
'sacct --jobs=\"' + ",".join(slurmids) +
'\" --format=\"JobID, Start, Elapsed, MaxRSS, NNodes, NTasks, NCPUS\"',
shell=True,
import numpy as np
np.format_parser(['f8', 'i4', 'a5'], ['col1', 'col2', 'col3'],
['T1', 'T2', 'T3']).dtype
np.format_parser(['f8', 'i4', 'a5'], ['col1', 'col2', 'col3'], []).dtype
np.format_parser(['f8', 'i4', 'a5'], [], []).dtype
