def fetch(self): csvreader = csv.reader(open( downloader.path( 'https://static.data.gouv.fr/resources/finess-extraction-du-fichier-des-etablissements/20181011-114801/etalab-cs1100507-stock-20181011-0450.csv', 60), 'rt'), delimiter=';') structureet = [ "nofinesset,nofinessej,rs,rslongue,complrs,compldistrib,numvoie,typvoie,voie,compvoie,lieuditbp,commune,departement,libdepartement,ligneacheminement,telephone,telecopie,categetab,libcategetab,categagretab,libcategagretab,siret,codeape,codemft,libmft,codesph,libsph,dateouv,dateautor,datemaj,numuai,coordxet,coordyet,sourcecoordet,datemajcoord" .split(',') ] geolocalisation = {} for row in csvreader: if row[0] == 'structureet': structureet.append(row[1:]) elif row[0] == 'geolocalisation': geolocalisation[row[1]] = row[2:] for row in structureet: row += geolocalisation.get(row[0], []) fd, tmp_file = tempfile.mkstemp() with open(tmp_file, 'w') as csvfile: writer = csv.writer(csvfile) for row in structureet: writer.writerow(row) return tmp_file
def open(self): if self.file: f = bz2.BZ2File(self.file) elif self.fileUrl: f = downloader.urlopen(self.fileUrl, self.fileUrlCache, mode='rb') if self.zip: z = zipfile.ZipFile(f, 'r').open(self.zip) f = io.BytesIO(z.read()) f.seek(0) elif self.gzip: d = gzip.open(downloader.path(self.fileUrl, self.fileUrlCache), mode='r') f = io.BytesIO(d.read()) f.seek(0) f = io.StringIO(f.read().decode(self.encoding, 'ignore')) f.seek(0) if self.filter: f = io.StringIO(self.filter(f.read())) f.seek(0) return f
def path(self): if self.file: return "merge_data/" + self.file elif self.fileUrl: # Do nothing about ZIP return downloader.path(self.fileUrl, self.fileUrlCache)
def path(self): if self.file: return "merge_data/"+self.file elif self.fileUrl: # Do nothing about ZIP return downloader.path(self.fileUrl, self.fileUrlCache)
def path(self): if self._revolve_file(): return self._revolve_file() elif self.fileUrl: # Do nothing about ZIP return downloader.path(self.fileUrl, self.fileUrlCache)