Пример #1
0
 def time(self):
     if self.file:
         return int(os.path.getmtime("merge_data/"+self.file)+.5)
     elif self.fileUrl:
         if self.zip:
             f = downloader.urlopen(self.fileUrl, self.fileUrlCache)
             date_time = zipfile.ZipFile(f, 'r').getinfo(self.zip).date_time
             return time.mktime(date_time + (0, 0, -1))
         else:
             return int(downloader.urlmtime(self.fileUrl, self.fileUrlCache)+.5)
Пример #2
0
 def time(self):
     if self.file:
         return int(os.path.getmtime(self.file)+.5)
     elif self.fileUrl:
         if self.zip:
             f = downloader.urlopen(self.fileUrl, self.fileUrlCache, mode='rb')
             date_time = zipfile.ZipFile(f, 'r').getinfo(self.zip).date_time
             return int(time.mktime(date_time + (0, 0, -1))+.5)
         else:
             return int(downloader.urlmtime(self.fileUrl, self.fileUrlCache)+.5)
Пример #3
0
 def open(self):
     if self.file:
         f = bz2.BZ2File("merge_data/" + self.file)
     elif self.fileUrl:
         f = downloader.urlopen(self.fileUrl, self.fileUrlCache)
         if self.zip:
             f = zipfile.ZipFile(f, 'r').open(self.zip)
     if self.encoding not in ("UTF8", "UTF-8"):
         f = io.StringIO(f.read().decode(self.encoding, 'ignore'))
         f.seek(0)
     if self.filter:
         f = io.StringIO(self.filter(f.read()))
         f.seek(0)
     return f
Пример #4
0
 def open(self):
     if self.file:
         f = bz2.BZ2File("merge_data/"+self.file)
     elif self.fileUrl:
         f = downloader.urlopen(self.fileUrl, self.fileUrlCache)
         if self.zip:
             f = zipfile.ZipFile(f, 'r').open(self.zip)
     if self.encoding not in ("UTF8", "UTF-8"):
         f = io.StringIO(f.read().decode(self.encoding, 'ignore'))
         f.seek(0)
     if self.filter:
         f = io.StringIO(self.filter(f.read()))
         f.seek(0)
     return f
Пример #5
0
 def open(self):
     if self.file:
         f = bz2.BZ2File(self.file)
     elif self.fileUrl:
         f = downloader.urlopen(self.fileUrl, self.fileUrlCache)
         if self.zip:
             z = zipfile.ZipFile(f, 'r').open(self.zip)
             f = io.BytesIO(z.read())
             f.seek(0)
     f = io.StringIO(f.read().decode(self.encoding, 'ignore'))
     f.seek(0)
     if self.filter:
         f = io.StringIO(self.filter(f.read()))
         f.seek(0)
     return f
Пример #6
0
    def fetch(self, url, tmp_file, date_string=None):
        f = downloader.urlopen(self.fileUrl, 60)

        #         0              1      2      3      4
        #idImpianto;descCarburante;prezzo;isSelf;dtComu
        csvreader = csv.reader(f, delimiter=';')
        next(csvreader) # Skip date
        next(csvreader) # Skip header

        impianti = {}
        for row in csvreader:
            impianto = impianti.get(row[0], 0)
            carburante = self.FUEL_TYPE_MAP.get(row[1].upper())
            if (impianto & carburante) != 0:
                continue
            dt_price = self.date_format(row[4], '%d/%m/%Y %H:%M:%S')
            if not dt_price or self.diff_days(dt_price) > 30:
                continue
            impianto |= carburante
            impianti[row[0]] = impianto

        csvfile = open(tmp_file, 'w', encoding='utf-8')
        writer = csv.writer(csvfile)

        f = self.source.open()
        csvreader = csv.reader(f, delimiter=';', quotechar = '~')
        next(csvreader) # Skip date
        header = next(csvreader)
        writer.writerow(header + ['Carburanti'])
        for row in csvreader:
            impianto = impianti.get(row[0])
            if impianto:
                writer.writerow(row + [ impianto ])

        csvfile.seek(0)
        return csvfile