def _retrieve_and_process_file(self, file_name, from_line=None):
"""
Scarica il file via SFTP e lo importa in Epas.
Restituisce il numero dell'ultima riga processata del file.
"""
logging.info("Process il file %s", file_name)
self._retrieve_file(file_name)
file_path = "%s/%s" % (STAMPINGS_DIR, file_name)
lines, last_line_processed = self._raw_stampings(file_path, from_line)
stamping_importer = StampingImporter()
bad_stampings, parsing_errors = stamping_importer.sendStampingsOnEpas(
lines)
if len(bad_stampings) > 0:
# Rimuove eventuali duplicati
bad_stampings = set(bad_stampings)
FileUtils.storestamping(self.bad_stampings_path, bad_stampings)
if len(parsing_errors) > 0:
FileUtils.storestamping(self.parsing_errors_path, parsing_errors)
file_size = os.path.getsize(file_path)
self.fileInfoManager.save(file_name, file_size, last_line_processed)
def send_bad_stampings():
logging.info("@@ Invio timbrature con problemi @@")
if os.path.exists(bad_stampings_path):
bad_stampings = []
parsing_errors = []
now = datetime.now()
still_good_stampings = []
oldest_day_allowed = datetime.combine(
now - timedelta(MAX_BAD_STAMPING_DAYS),
now.time().min)
logging.info(
"Bad Stampings: verranno mantenute solo le timbrature più nuove di %s",
oldest_day_allowed)
with open(bad_stampings_path, 'r') as f:
lines = f.read().splitlines()
lines = set(lines)
# butto via le timbrature più vecchie di x giorni
for line in lines:
stamp = StampingImporter._parseLine(line)
stamping_date = datetime(stamp.anno, stamp.mese, stamp.giorno,
stamp.ora, stamp.minuti)
if stamping_date >= oldest_day_allowed:
still_good_stampings.append(line)
removed_lines = len(lines) - len(still_good_stampings)
if removed_lines > 0:
logging.info(
'Rimosse %d timbrature dal file %s perché più vecchie del %s',
removed_lines, BAD_STAMPINGS_FILE, oldest_day_allowed)
os.remove(bad_stampings_path)
logging.info("Rimosso il file %s", BAD_STAMPINGS_FILE)
if still_good_stampings:
bad, errors = StampingImporter.sendStampingsOnEpas(
still_good_stampings)
bad_stampings += bad
parsing_errors += errors
if len(bad_stampings) > 0:
# Rimuove eventuali duplicati
bad_stampings = set(bad_stampings)
FileUtils.storestamping(bad_stampings_path, bad_stampings)
if len(parsing_errors) > 0:
FileUtils.storestamping(parsing_errors_path,
parsing_errors)
else:
logging.info("File %s non presente.", BAD_STAMPINGS_FILE)
def process_stamping_files():
stamping_files = os.listdir(STAMPINGS_DIR)
bad_stampings_path = os.path.join(DATA_DIR, BAD_STAMPINGS_FILE)
parsing_errors_path = os.path.join(DATA_DIR, PARSING_ERROR_FILE)
bad_stampings = []
parsing_errors = []
for stamping_file in stamping_files:
logging.info("Processo il file %s per estrarne le timbrature",
stamping_file)
stamping_file_path = os.path.join(STAMPINGS_DIR, stamping_file)
with open(stamping_file_path, 'r') as f:
# Questo metodo di lettura delle righe toglie anche gli \n di fine riga
lines = f.read().splitlines()
# Rimuove eventuali duplicati
lines = set(lines)
# Se c'è almeno una timbratura..
if lines:
bad, errors = StampingImporter.sendStampingsOnEpas(lines)
bad_stampings += bad
parsing_errors += errors
os.remove(stamping_file_path)
logging.info("Rimosso il file %s", stamping_file_path)
if len(bad_stampings) > 0:
# Rimuove eventuali duplicati
bad_stampings = set(bad_stampings)
FileUtils.storestamping(bad_stampings_path, bad_stampings)
if len(parsing_errors) > 0:
FileUtils.storestamping(parsing_errors_path, parsing_errors)