def persist_file(self, file_name):
extension = os.path.splitext(file_name)[1]
if extension != '.gz':
return
Logger.file_parse(self.chain, file_name)
temporary_gz_file_path = os.path.join(self.storage_path, file_name)
self.fetch_temporary_gz_file(temporary_gz_file_path)
Gzip.extract_xml_file_from_gz_file(temporary_gz_file_path)
os.remove(temporary_gz_file_path)
def fetch_temporary_gz_file(self, temporary_gz_file_path):
with open(temporary_gz_file_path, 'wb') as file_ftp:
file_name = ntpath.basename(temporary_gz_file_path)
try:
ftp = FTP_TLS(self.ftp_host, self.ftp_username,
self.ftp_password)
ftp.cwd(self.ftp_path)
ftp.retrbinary('RETR ' + file_name, file_ftp.write)
ftp.quit()
except:
Logger.file_parse(self.chain, file_name)
self.fetch_temporary_gz_file(temporary_gz_file_path)
def store_xml_file(self, file_link):
file_save_path = os.path.join(
self.storage_path, ntpath.basename(urlsplit(file_link).path))
Logger.file_parse(self.chain,
ntpath.basename(urlsplit(file_link).path))
try:
urlretrieve(file_link, file_save_path)
except:
Logger.file_retry(self.chain,
ntpath.basename(urlsplit(file_link).path))
self.store_xml_file(file_link)
Gzip.extract_xml_file_from_gz_file(file_save_path)
os.remove(file_save_path)
def scrape(self):
Logger.start_scrapper("Stop Market")
super(StopMarket, self).scrape()
def scrape(self):
Logger.start_scrapper("Fresh Market")
super(FreshMarket, self).scrape()
def scrape(self):
Logger.start_scrapper("Dor Alon")
super(DorAlon, self).scrape()
def scrape(self):
Logger.start_scrapper("Hazi Hinam")
super(HaziHinam, self).scrape()
def scrape(self):
Logger.start_scrapper("Keshet Taamim")
super(Keshet, self).scrape()
def scrape(self):
Logger.start_scrapper("Tiv Taam")
super(TivTaam, self).scrape()
def scrape(self):
Logger.start_scrapper("Osher Ad")
super(Osherad, self).scrape()
def scrape(self):
Logger.start_scrapper("SuperDosh")
super(SuperDosh, self).scrape()
def scrape(self):
Logger.start_scrapper("Shufersal")
super(Shufersal, self).scrape()
def scrape(self):
Logger.start_scrapper("Rami Levy")
super(RamiLevy, self).scrape()
def scrape(self):
Logger.start_scrapper("SalachDabach")
super(SalachDabach, self).scrape()