def before(self, req, fp): self.writer = UnicodeWriter(fp) self.writer.__enter__() self.writer.writerow([ f if isinstance(f, string_types) else f[1] for f in self.get_fields(req) ])
def test_UnicodeWriter(tmpdir, row, expected): with UnicodeWriter() as writer: writer.writerows([row]) assert writer.read() == expected filepath = tmpdir / 'test.csv' with UnicodeWriter(str(filepath)) as writer: writer.writerow(row) assert filepath.read_binary() == expected
def rename(args): # pragma: no cover api = Concepticon(args.repos) from_, to_ = args.args assert CONCEPTLIST_ID_PATTERN.match(to_) cl = api.conceptlists[from_] # write the adapted concept list to the new path: with UnicodeWriter(cl.path.parent / cl.path.name.replace(from_, to_), delimiter='\t') as writer: header = [] for i, row in enumerate(reader(cl.path, delimiter='\t')): if i == 0: header = row writer.writerow(row) header = {v: k for k, v in enumerate(header) } # Map col name to row index else: oid = row[header['ID']] assert oid.startswith(from_) nid = oid.replace(from_, to_) api.add_retirement( 'Concept', dict(id=oid, comment='renaming', replacement=nid)) row[header['ID']] = nid writer.writerow(row) # write adapted metadata to the new path: fname = cl.path.name.replace(from_, to_) + MD_SUFFIX md = jsonlib.load(cl.path.parent / (cl.path.name + MD_SUFFIX), object_pairs_hook=OrderedDict) md['tables'][0]['url'] = fname jsonlib.dump(md, cl.path.parent / fname, indent=4) # remove obsolete concept list and metadata: cl.path.unlink() cl.path.parent.joinpath(cl.path.name + MD_SUFFIX).unlink() # adapt conceptlists.tsv rows = [] for row in reader(api.data_path('conceptlists.tsv'), delimiter='\t'): rows.append([col.replace(from_, to_) if col else col for col in row]) with UnicodeWriter(api.data_path('conceptlists.tsv'), delimiter='\t') as writer: writer.writerows(rows) api.add_retirement('Conceptlist', dict(id=from_, comment='renaming', replacement=to_)) print("""Please run grep -r "{0}" concepticondata/ | grep -v retired.json to confirm the renaming was complete!""".format(from_))
def run(args): report, counts = [], {} api = args.repos if args.filename: sheets = [Sheet(args.filename)] else: sheets = [(s, list(s.itervalues(api))) for s in api.iter_sheets()] sheets = (s[0] for s in iterunique(sheets, verbose=args.verbose)) for sheet in sorted(sheets, key=lambda s: s.path): n = sheet.check(api, report=report) if (sheet.glottocode not in counts) or (n > counts[sheet.glottocode][0]): counts[sheet.glottocode] = (n, sheet.path.stem) selected = set(v[1] for v in counts.values()) for row in report: row.insert(1, row[0] in selected) if report and args.report: with UnicodeWriter(args.report, delimiter='\t') as w: w.writerow( ['sheet', 'selected', 'level', 'line', 'feature', 'message']) w.writerows(report) args.log.info('Report written to {0}'.format(args.report)) args.log.error('Repos check found WARNINGs or ERRORs')
def _write_items(self, what, attr=None, items=None): items = items if items is not None else getattr(self, attr or what) if items: with UnicodeWriter(self._path(what)) as writer: writer.writerow(items[0].__class__.fields()) for item in items: writer.writerow(item.astuple())
def cmd_download(self, args): from docx import Document fname = self.raw_dir / "Table_S2_Supplementary_Mennecier_et_al..doc" self.raw_dir.download_and_unpack( "https://ndownloader.figshare.com/articles/3443090/versions/1", fname, log=args.log, ) check_call( "libreoffice --headless --convert-to docx %s --outdir %s" % (fname, self.raw_dir), shell=True, ) doc = Document(self.raw_dir / "Table_S2_Supplementary_Mennecier_et_al..docx") for i, table in enumerate(doc.tables): with UnicodeWriter( self.raw_dir.joinpath("%s.csv" % (i + 1, )).as_posix()) as writer: for row in table.rows: # This code fixes a wrong gloss in the raw source, # where the `itʲerʊ` set is glossed as "to pull" # instead of the correct "to push". See discussion # at https://github.com/lexibank/cals/pull/7 row_data = map(text_and_color, row.cells) if i == 11: row_data = [ cell if cell != "to pull" else "to push" for cell in row_data ] writer.writerow(row_data)
def get_profile(self, clts=None, filename=None): """ Compute an orthography profile with LingPy's function. :param filename: Write the computed profile to a file in addition to returning it. :return: `segments.Profile` instance. """ clts = clts.bipa if clts else None D = {0: ['doculect', 'concept', 'ipa']} for i, key in enumerate(self._concordances.form, start=1): D[i] = ['dummy', key[1], key[0]] wordlist = lingpy.basic.wordlist.Wordlist(D) if not filename: with tempfile.NamedTemporaryFile(delete=FileExistsError) as fp: pass p = pathlib.Path(fp.name) else: p = pathlib.Path(filename) with UnicodeWriter(p, delimiter='\t') as w: w.writerow(['Grapheme', 'IPA', 'Example', 'Count', 'Unicode']) for line in lingpy.sequence.profile.context_profile(wordlist, ref='ipa', clts=clts): w.writerow([line[0], line[1], line[2], line[4], line[5]]) res = segments.Profile.from_file(p) if not filename: p.unlink() return res
def test_roundtrip_with_keyword_dialect(tmpdir, rows=[['1', 'y'], [' "1 ', '3\t4']], dialect='excel'): filename = str(tmpdir / 'test.csv') with UnicodeWriter(filename, dialect=dialect) as w: w.writerows(rows) assert list(iterrows(filename, dialect=dialect)) == rows
def iso2codes(args): """ Map ISO codes to the list of all Glottolog languages and dialects subsumed "under" it. """ nodes = list(args.repos.languoids()) res = {} for node in nodes: if node.iso: res[node.id] = (node.iso, set()) for node in nodes: if node.level == args.repos.languoid_levels.family or node.id in res: continue for nid in res: matched = False for l in node.lineage: if l[1] == nid: res[nid][1].add(node.id) matched = True break if matched: break outdir = Path('.') if not args.args else Path(args.args[0]) with UnicodeWriter(outdir / 'iso2glottocodes.csv') as writer: writer.writerow(['iso', 'glottocodes']) for gc, (iso, gcs) in res.items(): writer.writerow([iso, ';'.join([gc] + list(gcs))])
def create(self, table=None, dbase=None, ignore=None): """ Upload triple-data to sqlite3-db. Thereby, delete the previous table if it is still in the database. """ dbase = pathlib.Path(dbase or self.dbase) table = table or dbase.stem ignore = ignore or [] # write a log for the blacklist with UnicodeWriter( dbase.parent.joinpath( lingpy.rc('timestamp') + '-blacklist.log')) as w: w.writerow(['ID'] + sorted(self.header, key=lambda x: self.header[x])) w.writerows([[str(k)] + [stringval(e) for e in self[k]] for k in self.blacklist]) with self.cursor(dbase) as cu: cu.execute( "CREATE TABLE IF NOT EXISTS backup " "(file TEXT, id INT, col TEXT, val TEXT, date TEXT, user TEXT)" ) cu.execute( "CREATE TABLE IF NOT EXISTS {0} (id INT, col TEXT, val TEXT)". format(table)) cu.execute("DELETE FROM {0}".format(table)) self.update(table=table, dbase=dbase, ignore=ignore)
def run(args): bipa = args.clts.api.bipa func = profile.simple_profile cols = ['Grapheme', 'IPA', 'Frequence', 'Codepoints'] kw = {'ref': 'form', 'clts': bipa} if args.context: func = profile.context_profile cols = [ 'Grapheme', 'IPA', 'Examples', 'Languages', 'Frequence', 'Codepoints' ] kw['col'] = 'language_id' ds = get_dataset(args) profile_path = ds.etc_dir / 'orthography.tsv' if profile_path.exists() and not args.force: raise ParserError( 'Orthography profile exists already. To overwrite, pass "-f" flag') header, D = [], {} for i, row in enumerate(ds.cldf_reader()['FormTable'], start=1): if i == 1: header = [f for f in row.keys() if f != 'ID'] D = {0: ['lid'] + [h.lower() for h in header]} row['Segments'] = ' '.join(row['Segments']) D[i] = [row['ID']] + [row[h] for h in header] with UnicodeWriter(profile_path, delimiter='\t') as writer: writer.writerow(cols) for row in func(Wordlist(D, row='parameter_id', col='language_id'), **kw): writer.writerow(row) args.log.info('Orthography profile written to {0}'.format(profile_path))
def run(args): if args.out == FNAME: # pragma: no cover name = args.repos.path(args.out) else: name = pathlib.Path(args.out) rows = collections.OrderedDict([ ('Feature_ID', lambda f: f.id), ('Feature', lambda f: f.wiki['title']), ('Possible Values', lambda f: f['Possible Values']), ('Language_ID', lambda f: ''), ('Value', lambda f: ''), ('Source', lambda f: ''), ('Comment', lambda f: ''), ('Contributed_Datapoints', lambda f: ''), ('Clarifying comments', lambda f: f.wiki['Summary'].replace('\n', ' ')), ('Relevant unit(s)', lambda f: f['Relevant unit(s)']), ('Function', lambda f: f['Function']), ('Form', lambda f: f['Form']), ('Patron', lambda f: f.wiki['Patron']), ]) with UnicodeWriter(name, delimiter='\t') as w: w.writerow(rows.keys()) for feature in args.repos.features.values(): w.writerow([v(feature) for v in rows.values()])
def cmd_download(self, args): self.raw_dir.write('sources.bib', "\n".join(self.tob_sources.values())) # download data all_records = [] for i in pb(list(range(1, 20 * self.pages + 1, 20))): with self.raw_dir.temp_download(self._url(i), 'file-{0}'.format(i), log=args.log) as fname: soup = BeautifulSoup( fname.open(encoding='utf8').read(), 'html.parser') for record in soup.findAll(name='div', attrs={"class": "results_record"}): if isinstance(record, bs4.element.Tag): children = list(record.children) number = children[0].findAll('span')[1].text.strip() concept = children[1].findAll('span')[1].text for child in children[2:]: if isinstance(child, bs4.element.Tag): dpoints = child.findAll('span') if len(dpoints) >= 3: lname = dpoints[1].text glottolog = re.findall( 'Glottolog: (........)', str(dpoints[1]))[0] entry = dpoints[2].text cogid = list( child.children)[4].text.strip() all_records.append( (number, concept, lname, glottolog, entry, cogid)) with UnicodeWriter(self.raw_dir / 'output.csv') as f: f.writerows(all_records)
def markconservative(m, trigs, ref, hht, outfn, verbose=True, rank=None): blamefield = "hhtype" mafter = markall(m, trigs, verbose=verbose, rank=rank) ls = lstat(ref, hht) lsafter = lstat_witness(mafter, hht) log = [] no_status = defaultdict(set) for (lg, (stat, wits)) in lsafter.items(): if not ls.get(lg): srctrickles = [mafter[k][1].get('srctrickle') for k in wits] for t in srctrickles: if t and not t.startswith('iso6393'): no_status[lg].add(t) continue if hht[stat] > hht[ls[lg]]: log = log + [(lg, [(mafter[k][1].get( blamefield, "No %s" % blamefield), k, mafter[k][1].get( 'title', 'no title'), mafter[k][1].get( 'srctrickle', 'no srctrickle')) for k in wits], ls[lg])] for k in wits: (t, f) = mafter[k] if blamefield in f: del f[blamefield] mafter[k] = (t, f) for lg in no_status: print('{0} lacks status'.format(lg)) with UnicodeWriter(outfn, dialect='excel-tab') as writer: writer.writerows( ((lg, was) + mis for (lg, miss, was) in log for mis in miss)) return mafter
def run(args): datasets = args.dataset.split(",") if args.dataset else None variables = args.variable.split(",") if args.variable else None societies = args.society.split(",") if args.society else None if args.tree: for tree in args.repos.phylogenies: if args.tree == tree.id: break else: raise SystemExit("Failed to find Tree %s" % args.tree) societies = [ s for sublist in [t.soc_ids for t in tree.taxa] for s in sublist ] with UnicodeWriter(f=args.filename) as out: header = [ 'ID', 'XD_ID', 'Glottocode', 'Name', 'OriginalName', 'FocalYear', 'Latitude', 'Longitude', 'Variable', 'Value' ] out.writerow(header) socs = args.repos.societies for record in args.repos.iter_data(datasets=datasets, variables=variables, societies=societies): s = socs.get(record.soc_id, None) row = [ s.id, s.xd_id, s.glottocode, s.pref_name_for_society, s.ORIG_name_and_ID_in_this_dataset, s.main_focal_year, s.Lat, s.Long, record.var_id, record.code ] out.writerow(row)
def write_concepts(self, ctype, filename=None): def format_form(f): if self.monolingual: return f[0] return '{}: {}'.format(f[1], f[0]) concepts = self.get_concepts(ctype) with UnicodeWriter(filename, delimiter='\t') as w: w.writerow( ['ID', 'ENGLISH', 'OCCURRENCE', 'CONCEPT_IN_SOURCE', 'FORMS', 'PHRASE', 'GLOSS']) for i, (concept, forms) in enumerate( sorted(concepts.items(), key=lambda x: x[1][0][-1], reverse=True), start=1): # Get the IGT containing the first occurrence listed in the concordance as example: igt = self[ self._concordances[ctype][ forms[0][0], concept, forms[0][2], forms[0][1], ][0][0] ] w.writerow([ i, concept, sum([f[3] for f in forms]), ' // '.join(sorted(set([f[2] for f in forms]))), ' // '.join(sorted(set([format_form(f) for f in forms]))), igt.phrase_text, igt.gloss_text, ]) if not filename: print(w.read().decode('utf8'))
def run(args): nodes = list(args.repos.languoids()) res = {} for node in nodes: if node.iso: res[node.id] = (node.iso, set()) for node in nodes: if node.level == args.repos.languoid_levels.family or node.id in res: continue for nid in res: matched = False for lineage in node.lineage: if lineage[1] == nid: res[nid][1].add(node.id) matched = True break if matched: break with UnicodeWriter(args.output / 'iso2glottocodes.csv') as writer: writer.writerow(['iso', 'glottocodes']) for gc, (iso, gcs) in res.items(): writer.writerow([iso, ';'.join([gc] + list(gcs))])
def languoids(api, langs, outdir): _Level = Level or api.languoid_levels with UnicodeWriter(outdir / 'csv' / 'glottolog.csv') as writer: writer.writerow([ 'id', 'name', 'family_id', 'family_name', 'iso_code', 'language_id', 'macroarea', 'lineage', 'level' ]) for lang in sorted(langs, key=lambda l: l.id): if lang.level == _Level.language: lid = lang.id elif lang.level == _Level.dialect: for _, lid, level in reversed(lang.lineage): if level == _Level.language: break else: # pragma: no cover raise ValueError else: lid = None writer.writerow([ lang.id, lang.name, lang.lineage[0][1] if lang.lineage else '', lang.lineage[0][0] if lang.lineage else '', lang.iso or '', lid, getattr(lang.macroareas[0], 'value', lang.macroareas[0].name) if lang.macroareas else '', '/'.join(gc for _, gc, _ in lang.lineage), lang.level.name, ])
def fetch_sheet(sheetname, keyfile=None, output=None, delimiter='\t'): spreadsheet = google_api_client(keyfile).open_by_key(DOCUMENT_ID) for i, worksheet in enumerate(spreadsheet.worksheets()): if worksheet.title == sheetname: output = output or '{0}.tsv'.format(worksheet.title) with UnicodeWriter(output, delimiter=delimiter) as writer: writer.writerows(worksheet.get_all_values())
def locations(glottolog, fid, outpath): with UnicodeWriter(outpath) as writer: writer.writerow(['name', 'glottocode', 'latitude', 'longitude']) for lang in glottolog.languoids(): if lang.level == glottolog.languoid_levels.language and lang.latitude is not None: if fid in [l[1] for l in lang.lineage]: writer.writerow( [lang.name, lang.id, lang.latitude, lang.longitude])
def _write_linking_data(api, lang, args): out, freqs = collections.defaultdict(int), collections.defaultdict(int) # find those concept sets that are wrongly linked, they should not go into # the mapping, so we just make a re-linker here rep = {} for c in api.conceptsets.values(): if c.replacement_id: rep[c.id] = c.replacement_id rep[c.gloss] = api.conceptsets[c.replacement_id].gloss else: rep[c.id] = c.id rep[c.gloss] = c.gloss for clist in api.conceptlists.values(): args.log.info("checking {clist.id}".format(clist=clist)) for row in clist.concepts.values(): if row.concepticon_id: gls = None if lang.iso2 == "en": if row.english: gls = row.english.strip("*$-—+") else: if lang.name in row.attributes and row.attributes[ lang.name]: gls = row.attributes[lang.name].strip("*$-—+") if gls: out[rep[row.concepticon_gloss] + "///" + gls, rep[row.concepticon_id]] += 1 freqs[rep[row.concepticon_id]] += 1 if lang.iso2 == "en": for cset in api.conceptsets.values(): gloss = rep[cset.gloss] cid = rep[cset.id] if cset.ontological_category == "Person/Thing": out[gloss + "///the " + cset.gloss.lower(), cid] = freqs[cid] out[gloss + "///the " + cset.gloss.lower() + "s", cid] = freqs[cid] elif cset.ontological_category == "Action/Process": out[gloss + "///to " + cset.gloss.lower(), cid] = freqs[cid] elif cset.ontological_category == "Property": out[gloss + "///" + cset.gloss.lower() + " (adjective)", cid] = freqs[cid] elif cset.ontological_category == "Classifier": out[gloss + "///" + cset.gloss.lower() + " (classifier)", cid] = freqs[cid] else: out[gloss + "///" + cset.gloss.lower(), cid] = freqs[cid] p = api.path("mappings", "map-{0}.tsv".format(lang.iso2)) if not p.parent.exists(): p.parent.mkdir() with UnicodeWriter(p, delimiter="\t") as f: f.writerow(["ID", "GLOSS", "PRIORITY"]) for i, (gloss, cid) in enumerate(sorted(out)): f.writerow([cid, gloss, out[gloss, cid]])
def render(self, ctx, req): with UnicodeWriter() as writer: rows = iter(ctx.get_query(limit=QUERY_LIMIT)) first = next(rows, None) if first is not None: cols = first.csv_head() writer.writerow(cols) for item in chain([first], rows): writer.writerow(item.to_csv(ctx=ctx, req=req, cols=cols)) return writer.read()
def test_roundtrip_escapechar(tmpdir, quoting, escapechar='\\', row=['\\spam', 'eggs']): filename = str(tmpdir / 'spam.csv') kwargs = {'escapechar': escapechar, 'quoting': quoting} with UnicodeWriter(filename, **kwargs) as writer: writer.writerow(row) with UnicodeReader(filename, **kwargs) as reader: result = next(reader) assert result == row
def test_roundtrip_multibyte(tmpdir, encoding, row=['spam', 'eggs'], expected='spam,eggs\r\n', n=2): filepath = tmpdir / 'spam.csv' kwargs = {'encoding': encoding} with UnicodeWriter(str(filepath), **kwargs) as writer: writer.writerows([row] * n) with UnicodeReader(str(filepath), **kwargs) as reader: result = next(reader) assert result == row assert filepath.read_binary() == (expected * n).encode(encoding)
def _write_csv_to_file(data, file_name, api, header=None, dir_name='cldf'): outdir = api.repos.joinpath(dir_name) if not outdir.exists(): outdir.mkdir() if header is None: try: header = data.keys() except AttributeError: pass with UnicodeWriter(outdir.joinpath(file_name)) as w: if header is not None: w.writerow(header) for row in data: w.writerow(row)
def write(p, rows, features): rows = collections.OrderedDict([(r['Feature_ID'], r) for r in rows]) cols = 'Feature_ID Value Source Contributed_Datapoint Comment'.split() with UnicodeWriter(p, delimiter='\t') as writer: writer.writerow(cols) for feature in features.values(): row = rows.pop(feature.id, None) if not row: row = {k: '' for k in cols} row['Feature_ID'] = feature.id writer.writerow([row.get(col, '') for col in cols]) for row in rows.values(): writer.writerow([row.get(col, '') for col in cols])
def render(self, ctx, req): fid = req.route_url('parameter', id='xxx').replace('xxx', '{0}') lid = req.route_url('language', id='xxx').replace('xxx', '{0}') with UnicodeWriter() as writer: writer.writerow(['Language_ID', 'Feature_ID', 'Value']) for _lid, _fid, v in DBSession.query( Language.id, Parameter.id, DomainElement.name) \ .filter(Language.pk == ValueSet.language_pk) \ .filter(Parameter.pk == ValueSet.parameter_pk) \ .filter(Value.valueset_pk == ValueSet.pk) \ .filter(Value.domainelement_pk == DomainElement.pk) \ .order_by(Parameter.pk, Language.id): writer.writerow([lid.format(_lid), fid.format(_fid), v]) return writer.read()
def run(args): api = args.repos if not args.outdir.exists(): args.outdir.mkdir() # Iterate over groups of sheets for the same glottocode: for gc, sheets in itertools.groupby( sorted(api.iter_sheets(), key=lambda s: s.glottocode), lambda s: s.glottocode): if gc not in ['sina1266', 'meke1243']: continue sheets = list(sheets) if len(sheets) == 1: continue with UnicodeWriter(args.outdir / '{}.tsv'.format(gc), delimiter='\t') as w: w.writerow([ 'Feature_ID', 'Value', 'Conflict', 'Classification of conflict', 'Select', 'Sheet', 'Source', 'Contributed_Datapoint', 'Comment', 'Warnings', ]) # Iterate over rows grouped by feature ID: for fid, rows in grouped_rows(sheets): conflict = len( set(r[0]['Value'] for r in rows if r[0]['Value'])) > 1 for row, sheet in rows: # See what "grambank check" would do with the row: warnings = Warnings() sheet.valid_row(row, api, log=warnings) w.writerow([ fid, row['Value'], str(conflict), '', '', sheet.path.stem, row.get('Source', ''), ' '.join(Row.from_dict(row).contributed_datapoint), row.get('Comment', ''), warnings.messages, ])
def _write_linking_data(api, l, args): out, freqs = defaultdict(int), defaultdict(int) for clist in api.conceptlists.values(): args.log.info("checking {clist.id}".format(clist=clist)) for row in clist.concepts.values(): if row.concepticon_id: gls = None if l.iso2 == "en": if row.english: gls = row.english.strip("*$-—+") else: if l.name in row.attributes and row.attributes[l.name]: gls = row.attributes[l.name].strip("*$-—+") if gls: out[row.concepticon_gloss + "///" + gls, row.concepticon_id] += 1 freqs[row.concepticon_id] += 1 if l.iso2 == "en": for cset in api.conceptsets.values(): gloss = cset.gloss if cset.ontological_category == "Person/Thing": out[gloss + "///the " + cset.gloss.lower(), cset.id] = freqs[cset.id] out[gloss + "///the " + cset.gloss.lower() + "s", cset.id] = freqs[cset.id] elif cset.ontological_category == "Action/Process": out[gloss + "///to " + cset.gloss.lower(), cset.id] = freqs[cset.id] elif cset.ontological_category == "Property": out[gloss + "///" + cset.gloss.lower() + " (adjective)", cset.id] = freqs[cset.id] elif cset.ontological_category == "Classifier": out[gloss + "///" + cset.gloss.lower() + " (classifier)", cset.id] = freqs[cset.id] else: out[gloss + "///" + cset.gloss.lower(), cset.id] = freqs[cset.id] p = api.path("mappings", "map-{0}.tsv".format(l.iso2)) if not p.parent.exists(): p.parent.mkdir() with UnicodeWriter(p, delimiter="\t") as f: f.writerow(["ID", "GLOSS", "PRIORITY"]) for i, (gloss, cid) in enumerate(sorted(out)): f.writerow([cid, gloss, out[gloss, cid]])
def fix(p): if '_history' in p.stem: p.unlink() return rows = list(reader(p, dicts=True)) if not rows: p.unlink() return remove = {'created', 'updated', 'active', 'polymorphic_type'} with UnicodeWriter(p) as w: for i, row in enumerate(rows): if i == 0: w.writerow([c for c in row.keys() if c not in remove]) for c in remove: if c in row: del row[c] w.writerow(row.values())
class CsvDump(Download): """Download of a resource type as csv.""" ext = 'csv' def __init__(self, model, pkg, fields=None, **kw): """Initialize. fields can be a list of column names or a dictionary mapping model attribute names to csv column names. """ super(CsvDump, self).__init__(model, pkg, **kw) self.fields = fields self.writer = None def get_stream(self): return StringIO(newline='') if PY3 else BytesIO() def read_stream(self, fp): res = Download.read_stream(self, fp) if PY3: # pragma: no cover res = res.encode('utf8') return res def get_fields(self, req): if not self.fields: self.fields = ['id', 'name'] return self.fields def before(self, req, fp): self.writer = UnicodeWriter(fp) self.writer.__enter__() self.writer.writerow( [f if isinstance(f, string_types) else f[1] for f in self.get_fields(req)]) def row(self, req, fp, item, index): return [getattr(item, f if isinstance(f, string_types) else f[0]) for f in self.get_fields(req)] def dump(self, req, fp, item, index): self.writer.writerow(self.row(req, fp, item, index))
def before(self, req, fp): self.writer = UnicodeWriter(fp) self.writer.__enter__() self.writer.writerow( [f if isinstance(f, string_types) else f[1] for f in self.get_fields(req)])