def loaditem(self, itemname):
'''Loads the data file given in itemname, which is the
filename of the data file. This performs some replacements on
the data. For example, it replaces the informant information
with just the infid.'''
filename = _os.path.join(self.datadir, self.project, itemname)
data = []
if _os.path.exists(filename):
infs = self.getInformants()
infd = {}
for i in infs:
infd[i[2]] = i[0]
for line in open(filename).xreadlines():
line = _fcsv.split(line)
if not line:
continue
data.append(
tuple(line[:1] + [infd.get(line[1], '')] + line[3:8] +
[_simplify(line[7])] + line[8:]))
#'blew,NY1,50,V,N,-0-,-0-,bluw,MS,1'
self._cache[itemname] = CsvSet(data=data,
fields=self.fieldnames(
_tables.TableFields.ITEM),
source=self,
filename=filename)
def loadFile(self, filename, fields=()):
'''Returns a CsvSet object of the data in filename, containing
fields fields.'''
return CsvSet(data=[
tuple(_fcsv.split(line)) for line in open(filename).xreadlines()
],
fields=fields,
source=self,
filename=filename)
def convert(filename):
# read in the data
f = file(filename)
data = [fcsv.split(line) for line in f]
f.close()
# write it back out
f = file(filename, 'wb')
csv.writer(f).writerows(data)
f.close()
def loadtables(self):
'''Loads the TABLES file.'''
tfile = _os.path.join(self.datadir, self.project,
_tables.TableNames.TABLES)
data = []
if _os.path.exists(tfile):
for line in open(tfile).xreadlines():
line = _fcsv.split(line)
if not line:
continue
data.append(tuple(line))
self._cache[_tables.TableNames.TABLES] = CsvSet(
data=data,
fields=self.fieldnames(_tables.TableFields.TABLES),
source=self,
filename=tfile)
def loadpage(self, page):
filename = _os.path.join(self.datadir, self.project,
_tables.TableNames.PAGE % page)
data = []
if _os.path.exists(filename):
infs = self.getInformants()
infd = {}
for i in infs:
infd[i[2]] = i[0]
for line in open(filename).xreadlines():
line = _fcsv.split(line)
if not line:
continue
data.append(
tuple(line[:1] + [infd.get(line[1], '')] + line[3:10] +
[line[11], '', line[10], '']))
self._cache[page] = CsvSet(data=data,
fields=self.fieldnames(
tables.TableFields.PAGE),
source=self,
filename=filename)
def loadinfs(self):
'''Loads the informants, putting the data into the INFORMANTS,
COMMUNITIES, FIELDWORKERS, and WORKSHEETS directories.'''
infs = []
fws = {}
wss = {}
coms = {}
tnames = _tables.TableNames
fields = _tables.TableFields
inffile = _os.path.join(self.datadir, self.project, tnames.INFORMANTS)
if _os.path.exists(inffile):
for line in open(inffile).xreadlines():
line = _fcsv.split(line)
if not line:
continue
com = coms.setdefault(line[1], tuple([line[1]] + line[-3:]))
fw = fws.setdefault(line[5], (len(fws), line[5], ''))
ws = wss.setdefault(line[6], (len(wss), line[6], ''))
infs.append(
tuple(line[:1] + [com[0]] + line[2:5] + [fw[0], ws[0]] +
line[7:16]))
cache = self._cache
fn = self.fieldnames
cache[tnames.INFORMANTS] = CsvSet(data=infs,
fields=fn(fields.INFORMANTS),
source=self,
filename=inffile)
cache[tnames.COMMUNITIES] = CsvSet(data=coms.values(),
fields=fn(fields.COMMUNITIES),
source=self)
cache[tnames.FIELDWORKERS] = CsvSet(data=fws.values(),
fields=fn(fields.FIELDWORKERS),
source=self)
cache[tnames.WORKSHEETS] = CsvSet(data=wss.values(),
fields=fn(fields.WORKSHEETS),
source=self)