def setup_logger(cls):
now = datetime.now()
log_date = str(datetime.today().date()).replace('-', '_')
log_hour = now.hour
log_minute = now.minute
# f_name = '%s_%s_%s_meteo.log' % (log_date, log_hour, log_minute)
f_name = '%s_%s_meteo.log' % (log_date, log_hour)
logging_level = ConfigReader.get_logging_level()
logging.basicConfig(filename=os.path.join(LOGS_DIR, f_name),
format='%(asctime)s %(levelname)s: %(message)s',
level=logging_level)
return logging
def download_data():
try:
ftp = FtpConnector.get_ftp_connection(host, user, passwd)
except ftpErrors as e:
logging.error(__name__ + ': ' + str(e))
raise FTPLoginException(e)
except gaierror as e:
logging.error(__name__ + ': ' + str(e))
raise FTPLoginException(e)
new_files_list = getFilesList(ftp, remote_dir)
old_files_list = readOldFilesList(ConfigReader.get_used_meteo_list())
new_files = set(new_files_list) - set(old_files_list)
return new_files
def compareLists():
meteo_host = ConfigReader.get_host()
meteo_user = ConfigReader.get_user()
meteo_pass = ConfigReader.get_pass()
remote_dir = ConfigReader.get_remote_dir()
try:
ftp = FtpConnector.get_ftp_connection(meteo_host, \
meteo_user, meteo_pass)
meteoFilesList = getFilesList(ftp, remote_dir)
except ftpErrors as e:
logError(e)
exit(-1)
except gaierror as e:
logError(e)
exit(-1)
oldMeteoListFile = ConfigReader.get_used_meteo_list()
oldFilesList = readOldFilesList(oldMeteoListFile)
filesDiff = set(meteoFilesList) - set(oldFilesList)
print(type(set(filesDiff)))
print(type(filesDiff))
'''
Created on Dec 28, 2017
@author: arturnowicki
'''
import logging
from utils.config_utils import ConfigReader
from utils.connection_utils import FtpConnector
from ftplib import all_errors as ftpErrors
from socket import gaierror
from custom_exceptions import FTPLoginException
import sys
host = ConfigReader.get_host()
user = ConfigReader.get_user()
passwd = ConfigReader.get_pass()
remote_dir = ConfigReader.get_remote_dir()
def extract_data():
logging.info(__name__ + ': Retrieving new ICM files list.')
try:
new_dataset = get_new_dataset()
except FTPLoginException as e:
sys.exit(1)
if len(new_dataset) < 3:
logging.info(__name__ + ': No new ICM files.')
sys.exit(0)
logging.info(__name__ + ': Downloading ICM files.')
download_data(new_dataset)
logging.info(__name__ + ': Extracting ICM files.')
def test_get_config_when_ok():
config = ConfigReader.get_cfg()
assert config is not None
def test_get_logging_level():
logging_level = ConfigReader().get_logging_level()
assert logging_level in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
def test_get_interp_method():
interp_method = ConfigReader().get_interp_method()
assert interp_method in ['spline', 'linear', 'cubic']
def test_get_user():
user = ConfigReader().get_user()
assert user == 'iopan'
def test_get_host():
host = ConfigReader.get_host()
assert host == 'ftpmeteo.icm.edu.pl'