def time(self):
if self.file:
return int(os.path.getmtime("merge_data/"+self.file)+.5)
elif self.fileUrl:
if self.zip:
f = downloader.urlopen(self.fileUrl, self.fileUrlCache)
date_time = zipfile.ZipFile(f, 'r').getinfo(self.zip).date_time
return time.mktime(date_time + (0, 0, -1))
else:
return int(downloader.urlmtime(self.fileUrl, self.fileUrlCache)+.5)
def time(self):
if self.file:
return int(os.path.getmtime(self.file)+.5)
elif self.fileUrl:
if self.zip:
f = downloader.urlopen(self.fileUrl, self.fileUrlCache, mode='rb')
date_time = zipfile.ZipFile(f, 'r').getinfo(self.zip).date_time
return int(time.mktime(date_time + (0, 0, -1))+.5)
else:
return int(downloader.urlmtime(self.fileUrl, self.fileUrlCache)+.5)
def open(self):
if self.file:
f = bz2.BZ2File("merge_data/" + self.file)
elif self.fileUrl:
f = downloader.urlopen(self.fileUrl, self.fileUrlCache)
if self.zip:
f = zipfile.ZipFile(f, 'r').open(self.zip)
if self.encoding not in ("UTF8", "UTF-8"):
f = io.StringIO(f.read().decode(self.encoding, 'ignore'))
f.seek(0)
if self.filter:
f = io.StringIO(self.filter(f.read()))
f.seek(0)
return f
def open(self):
if self.file:
f = bz2.BZ2File("merge_data/"+self.file)
elif self.fileUrl:
f = downloader.urlopen(self.fileUrl, self.fileUrlCache)
if self.zip:
f = zipfile.ZipFile(f, 'r').open(self.zip)
if self.encoding not in ("UTF8", "UTF-8"):
f = io.StringIO(f.read().decode(self.encoding, 'ignore'))
f.seek(0)
if self.filter:
f = io.StringIO(self.filter(f.read()))
f.seek(0)
return f
def open(self):
if self.file:
f = bz2.BZ2File(self.file)
elif self.fileUrl:
f = downloader.urlopen(self.fileUrl, self.fileUrlCache)
if self.zip:
z = zipfile.ZipFile(f, 'r').open(self.zip)
f = io.BytesIO(z.read())
f.seek(0)
f = io.StringIO(f.read().decode(self.encoding, 'ignore'))
f.seek(0)
if self.filter:
f = io.StringIO(self.filter(f.read()))
f.seek(0)
return f
def fetch(self, url, tmp_file, date_string=None):
f = downloader.urlopen(self.fileUrl, 60)
# 0 1 2 3 4
#idImpianto;descCarburante;prezzo;isSelf;dtComu
csvreader = csv.reader(f, delimiter=';')
next(csvreader) # Skip date
next(csvreader) # Skip header
impianti = {}
for row in csvreader:
impianto = impianti.get(row[0], 0)
carburante = self.FUEL_TYPE_MAP.get(row[1].upper())
if (impianto & carburante) != 0:
continue
dt_price = self.date_format(row[4], '%d/%m/%Y %H:%M:%S')
if not dt_price or self.diff_days(dt_price) > 30:
continue
impianto |= carburante
impianti[row[0]] = impianto
csvfile = open(tmp_file, 'w', encoding='utf-8')
writer = csv.writer(csvfile)
f = self.source.open()
csvreader = csv.reader(f, delimiter=';', quotechar = '~')
next(csvreader) # Skip date
header = next(csvreader)
writer.writerow(header + ['Carburanti'])
for row in csvreader:
impianto = impianti.get(row[0])
if impianto:
writer.writerow(row + [ impianto ])
csvfile.seek(0)
return csvfile