def fetch(self):
csvreader = csv.reader(open(
downloader.path(
'https://static.data.gouv.fr/resources/finess-extraction-du-fichier-des-etablissements/20181011-114801/etalab-cs1100507-stock-20181011-0450.csv',
60), 'rt'),
delimiter=';')
structureet = [
"nofinesset,nofinessej,rs,rslongue,complrs,compldistrib,numvoie,typvoie,voie,compvoie,lieuditbp,commune,departement,libdepartement,ligneacheminement,telephone,telecopie,categetab,libcategetab,categagretab,libcategagretab,siret,codeape,codemft,libmft,codesph,libsph,dateouv,dateautor,datemaj,numuai,coordxet,coordyet,sourcecoordet,datemajcoord"
.split(',')
]
geolocalisation = {}
for row in csvreader:
if row[0] == 'structureet':
structureet.append(row[1:])
elif row[0] == 'geolocalisation':
geolocalisation[row[1]] = row[2:]
for row in structureet:
row += geolocalisation.get(row[0], [])
fd, tmp_file = tempfile.mkstemp()
with open(tmp_file, 'w') as csvfile:
writer = csv.writer(csvfile)
for row in structureet:
writer.writerow(row)
return tmp_file
def open(self):
if self.file:
f = bz2.BZ2File(self.file)
elif self.fileUrl:
f = downloader.urlopen(self.fileUrl, self.fileUrlCache, mode='rb')
if self.zip:
z = zipfile.ZipFile(f, 'r').open(self.zip)
f = io.BytesIO(z.read())
f.seek(0)
elif self.gzip:
d = gzip.open(downloader.path(self.fileUrl, self.fileUrlCache), mode='r')
f = io.BytesIO(d.read())
f.seek(0)
f = io.StringIO(f.read().decode(self.encoding, 'ignore'))
f.seek(0)
if self.filter:
f = io.StringIO(self.filter(f.read()))
f.seek(0)
return f
def path(self):
if self.file:
return "merge_data/" + self.file
elif self.fileUrl:
# Do nothing about ZIP
return downloader.path(self.fileUrl, self.fileUrlCache)
def path(self):
if self.file:
return "merge_data/"+self.file
elif self.fileUrl:
# Do nothing about ZIP
return downloader.path(self.fileUrl, self.fileUrlCache)
def path(self):
if self._revolve_file():
return self._revolve_file()
elif self.fileUrl:
# Do nothing about ZIP
return downloader.path(self.fileUrl, self.fileUrlCache)
