def loaditem(self, itemname): '''Loads the data file given in itemname, which is the filename of the data file. This performs some replacements on the data. For example, it replaces the informant information with just the infid.''' filename = _os.path.join(self.datadir, self.project, itemname) data = [] if _os.path.exists(filename): infs = self.getInformants() infd = {} for i in infs: infd[i[2]] = i[0] for line in open(filename).xreadlines(): line = _fcsv.split(line) if not line: continue data.append( tuple(line[:1] + [infd.get(line[1], '')] + line[3:8] + [_simplify(line[7])] + line[8:])) #'blew,NY1,50,V,N,-0-,-0-,bluw,MS,1' self._cache[itemname] = CsvSet(data=data, fields=self.fieldnames( _tables.TableFields.ITEM), source=self, filename=filename)
def loadFile(self, filename, fields=()): '''Returns a CsvSet object of the data in filename, containing fields fields.''' return CsvSet(data=[ tuple(_fcsv.split(line)) for line in open(filename).xreadlines() ], fields=fields, source=self, filename=filename)
def convert(filename): # read in the data f = file(filename) data = [fcsv.split(line) for line in f] f.close() # write it back out f = file(filename, 'wb') csv.writer(f).writerows(data) f.close()
def loadtables(self): '''Loads the TABLES file.''' tfile = _os.path.join(self.datadir, self.project, _tables.TableNames.TABLES) data = [] if _os.path.exists(tfile): for line in open(tfile).xreadlines(): line = _fcsv.split(line) if not line: continue data.append(tuple(line)) self._cache[_tables.TableNames.TABLES] = CsvSet( data=data, fields=self.fieldnames(_tables.TableFields.TABLES), source=self, filename=tfile)
def loadpage(self, page): filename = _os.path.join(self.datadir, self.project, _tables.TableNames.PAGE % page) data = [] if _os.path.exists(filename): infs = self.getInformants() infd = {} for i in infs: infd[i[2]] = i[0] for line in open(filename).xreadlines(): line = _fcsv.split(line) if not line: continue data.append( tuple(line[:1] + [infd.get(line[1], '')] + line[3:10] + [line[11], '', line[10], ''])) self._cache[page] = CsvSet(data=data, fields=self.fieldnames( tables.TableFields.PAGE), source=self, filename=filename)
def loadinfs(self): '''Loads the informants, putting the data into the INFORMANTS, COMMUNITIES, FIELDWORKERS, and WORKSHEETS directories.''' infs = [] fws = {} wss = {} coms = {} tnames = _tables.TableNames fields = _tables.TableFields inffile = _os.path.join(self.datadir, self.project, tnames.INFORMANTS) if _os.path.exists(inffile): for line in open(inffile).xreadlines(): line = _fcsv.split(line) if not line: continue com = coms.setdefault(line[1], tuple([line[1]] + line[-3:])) fw = fws.setdefault(line[5], (len(fws), line[5], '')) ws = wss.setdefault(line[6], (len(wss), line[6], '')) infs.append( tuple(line[:1] + [com[0]] + line[2:5] + [fw[0], ws[0]] + line[7:16])) cache = self._cache fn = self.fieldnames cache[tnames.INFORMANTS] = CsvSet(data=infs, fields=fn(fields.INFORMANTS), source=self, filename=inffile) cache[tnames.COMMUNITIES] = CsvSet(data=coms.values(), fields=fn(fields.COMMUNITIES), source=self) cache[tnames.FIELDWORKERS] = CsvSet(data=fws.values(), fields=fn(fields.FIELDWORKERS), source=self) cache[tnames.WORKSHEETS] = CsvSet(data=wss.values(), fields=fn(fields.WORKSHEETS), source=self)