def test_values(self):
strings = '''
This is one time where television really fails to capture
the true excitement of a large squirrel predicting the weather.
'''.split()
with open('test/data/multi.xpt', 'rb') as f:
for (i, s), (x, y) in zip(enumerate(strings, 1), xport.Reader(f)):
assert (x, y) == (s, i)
def xpt_to_csv(xpt_file, output_path=None):
import xport
xpt_file = os.path.abspath(xpt_file)
if output_path is None:
output_path = os.path.splitext(xpt_file)[0] + ".csv"
writer = csv.writer(open(output_path, "w"))
with open(xpt_file, "rb") as fpr:
reader = xport.Reader(fpr)
writer.writerow(list(reader.fields))
for line in reader:
writer.writerow(line)
def test_header(self):
with open('test/data/known_values.xpt', 'rb') as f:
reader = xport.Reader(f)
x, = reader._variables
assert reader.fields == ('X', )
assert x.name == 'X'
assert x.numeric == True
assert x.position == 0
assert x.size == 8
def iterload(self):
import xport
with open(self.source, 'rb') as fp:
self.rdr = xport.Reader(fp)
self.columns = []
for i, var in enumerate(self.rdr._variables):
self.addColumn(
ColumnItem(var.name, i,
type=float if var.numeric else str))
yield from self.rdr
def test_csvs(self):
for csvfile in glob.glob('test/data/*.csv'):
directory, filename = os.path.split(csvfile)
xptfile = os.path.join(directory, filename[:-4] + '.xpt')
with open(csvfile) as fcsv, open(xptfile, 'rb') as fxpt:
csvreader = csv.reader(fcsv)
xptreader = xport.Reader(fxpt)
self.assertEqual(tuple(next(csvreader)), xptreader.fields)
values = (self.convert_types(row) for row in csvreader)
list(map(self.assertEqual, values, xptreader))
def reload(self):
import xport
with open(self.source.resolve(), 'rb') as fp:
self.rdr = xport.Reader(fp)
self.columns = []
for i, var in enumerate(self.rdr._variables):
self.addColumn(
ColumnItem(var.name, i,
type=float if var.numeric else str))
self.rows = []
for row in self.rdr:
self.rows.append(row)
def test_header(self):
with open('test/data/multi.xpt', 'rb') as f:
reader = xport.Reader(f)
x, y = reader._variables
assert reader.fields == ('X', 'Y')
assert x.name == 'X'
assert x.numeric == False
assert x.position == 0
assert x.size == 10
assert y.name == 'Y'
assert y.numeric == True
assert y.position == 10
assert y.size == 8
def _convert_xpt_to_csv(self):
if not os.path.exists(self._csv_path):
os.mkdir(self._csv_path)
for i in range(len(self._cdcfiles)):
f = self._cdcfiles[i]
finfo = self._cdcfileinfo[i]
xptfile = os.path.join(self._dirpath, f)
csvfile = os.path.join(self._csv_path, f)
csvfile = os.path.splitext(csvfile)[0]
csvfile = csvfile + ".csv"
if not os.path.exists(csvfile):
print("converting ", finfo, ": ", xptfile, " to ", csvfile)
with open(xptfile, 'rb') as in_xpt:
with open(csvfile, 'w',newline='') as out_csv:
reader = xport.Reader(in_xpt)
writer = csv.writer(out_csv, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow(reader.fields)
for row in reader:
writer.writerow(row)
def test_length(self):
with open('test/data/known_values.xpt', 'rb') as f:
assert len(list(xport.Reader(f))) == 2123
def test_length(self):
with open('test/data/strings.xpt', 'rb') as f:
assert len(list(xport.Reader(f))) == 2
import xport
from xport import Reader as RR
XPT_FILE = '/Users/amitn/Documents/ml_codes/brfss/LLCP2016.XPT'
x = 0
with open(XPT_FILE, 'rb') as f:
for row in xport.Reader(f):
print(row)
x += 1
if x == 3:
break
import xport, math, random
from numpy.random import uniform
from py2neo import Graph, Node, Relationship
file = '../data/source/dm.xpt'
with open(file, 'rb') as f:
fields = xport.Reader(f).fields
with open(file, 'rb') as f:
dm = xport.to_dataframe(f)
# access local neo4j instance on port 7474 with username "phusecss" and password "Phuse1!"
graph = Graph("http://*****:*****@localhost:7474/")
tx = graph.begin()
def checkNAN(inVar):
if isinstance(inVar, float) and math.isnan(inVar) == True:
return ''
else:
return inVar
def write2neo(what2write, dupSubs):
for i in what2write:
if dupSubs == True:
studies = ['1', '2', '3']
studies.remove(dm['STUDYID'][i][-1:])
dm.set_value(i, 'STUDYID', dm['STUDYID'][i][:-1] + random.choice(studies))
