def get_status_local_machine():
# Get info on the status of the local machine
#
logger.debug("Entering routine %s" % 'get_status_local_machine')
# Get the local systems settings
# systemsettings = functions.getSystemSettings()
# Get status of all services
status_services = functions.getStatusAllServices()
get_eumetcast_status = status_services['eumetcast']
get_internet_status = status_services['internet']
ingestion_status = status_services['ingest']
processing_status = status_services['process']
system_status = status_services['system']
# Get status of postgresql
psql_status = functions.getStatusPostgreSQL()
# Get internet connection
internet_status = functions.internet_on()
# ToDo: check disk status!
status_local_machine = {
'get_eumetcast_status':
get_eumetcast_status,
'get_internet_status':
get_internet_status,
'ingestion_status':
ingestion_status,
'processing_status':
processing_status,
'system_status':
system_status,
'system_execution_time':
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
'postgresql_status':
str(psql_status).lower(),
'internet_connection_status':
str(internet_status).lower(),
'active_version':
systemsettings['active_version'],
'mode':
systemsettings['mode'],
'disk_status':
'true'
}
return status_local_machine
def loop_get_internet(dry_run=False, test_one_source=False):
global processed_list_filename, processed_list
global processed_info_filename, processed_info
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGILL, signal_handler)
logger.info("Starting retrieving data from INTERNET.")
while True:
output_dir = es_constants.get_internet_output_dir
logger.debug("Check if the Ingest Server input directory : %s exists.",
output_dir)
if not os.path.exists(output_dir):
# ToDo: create output_dir - ingest directory
logger.fatal(
"The Ingest Server input directory : %s doesn't exists.",
output_dir)
exit(1)
if not os.path.exists(es_constants.processed_list_int_dir):
os.mkdir(es_constants.processed_list_int_dir)
while 1:
# Check internet connection (or continue)
if not functions.internet_on():
logger.error(
"The computer is not currently connected to the internet. Wait 1 minute."
)
time.sleep(60)
else:
try:
time_sleep = user_def_sleep
logger.debug("Sleep time set to : %s.", time_sleep)
except:
logger.warning(
"Sleep time not defined. Setting to default=1min. Continue."
)
time_sleep = 60
logger.info(
"Reading active INTERNET data sources from database")
internet_sources_list = querydb.get_active_internet_sources()
# Loop over active triggers
for internet_source in internet_sources_list:
try:
if test_one_source and (internet_source.internet_id !=
test_one_source):
logger.info(
"Running in test mode, and source is not %s. Continue.",
test_one_source)
continue
execute_trigger = True
# Get this from the pads database table (move from internet_source 'pull_frequency' to the pads table,
# so that it can be exploited by eumetcast triggers as well). It is in minute
pull_frequency = internet_source.pull_frequency
# Manage the case of files to be continuously downloaded (delay < 0)
if pull_frequency < 0:
do_not_consider_processed_list = True
delay_time_source_minutes = -pull_frequency
else:
do_not_consider_processed_list = False
delay_time_source_minutes = pull_frequency
if sys.platform == 'win32':
internet_id = str(
internet_source.internet_id).replace(':', '_')
else:
internet_id = str(internet_source.internet_id)
logger_spec = log.my_logger('apps.get_internet.' +
internet_id)
logger.info("Processing internet source %s.",
internet_source.descriptive_name)
# Create objects for list and info
processed_info_filename = es_constants.get_internet_processed_list_prefix + str(
internet_id) + '.info'
# Restore/Create Info
processed_info = None
processed_info = functions.restore_obj_from_pickle(
processed_info, processed_info_filename)
if processed_info is not None:
# Check the delay
current_delta = datetime.datetime.now(
) - processed_info['time_latest_exec']
current_delta_minutes = int(current_delta.seconds /
60)
if current_delta_minutes < delay_time_source_minutes:
logger.debug(
"Still waiting up to %i minute - since latest execution.",
delay_time_source_minutes)
execute_trigger = False
else:
# Create processed_info object
processed_info = {
'lenght_proc_list': 0,
'time_latest_exec': datetime.datetime.now(),
'time_latest_copy': datetime.datetime.now()
}
execute_trigger = True
if execute_trigger:
# Restore/Create List
processed_list = []
if not do_not_consider_processed_list:
processed_list_filename = es_constants.get_internet_processed_list_prefix + internet_id + '.list'
processed_list = functions.restore_obj_from_pickle(
processed_list, processed_list_filename)
processed_info[
'time_latest_exec'] = datetime.datetime.now()
logger.debug(
"Create current list of file to process for source %s.",
internet_source.internet_id)
if internet_source.user_name is None:
user_name = "anonymous"
else:
user_name = internet_source.user_name
if internet_source.password is None:
password = "******"
else:
password = internet_source.password
usr_pwd = str(user_name) + ':' + str(password)
logger_spec.debug(" Url is %s.",
internet_source.url)
logger_spec.debug(" usr/pwd is %s.",
usr_pwd)
logger_spec.debug(
" regex is %s.",
internet_source.include_files_expression)
internet_type = internet_source.type
if internet_type == 'ftp' or internet_type == 'http':
# Manage the end_date (added for MODIS_FIRMS)
if (internet_source.end_date != ''):
end_date = internet_source.end_date
else:
end_date = None
# Note that the following list might contain sub-dirs (it reflects full_regex)
try:
current_list = get_list_matching_files(
str(internet_source.url),
str(usr_pwd),
str(internet_source.
include_files_expression),
internet_type,
end_date=end_date)
except:
logger.error(
"Error in creating file lists. Continue"
)
continue
elif internet_type == 'http_tmpl':
# Create the full filename from a 'template' which contains
try:
current_list = build_list_matching_files_tmpl(
str(internet_source.url),
str(internet_source.
include_files_expression),
internet_source.start_date,
internet_source.end_date,
str(internet_source.frequency_id))
except:
logger.error(
"Error in creating date lists. Continue"
)
continue
elif internet_type == 'motu_client':
# Create the full filename from a 'template' which contains
try:
current_list = build_list_matching_files_motu(
str(internet_source.url),
str(internet_source.
include_files_expression),
internet_source.start_date,
internet_source.end_date,
str(internet_source.frequency_id),
str(internet_source.user_name),
str(internet_source.password),
str(internet_source.
files_filter_expression),
)
except:
logger.error(
"Error in creating motu_client lists. Continue"
)
continue
# elif internet_type == 'sentinel_sat':
# # Create the full filename from a 'template' which contains
# try:
# current_list = build_list_matching_files_sentinel_sat(str(internet_source.url),
# str(internet_source.include_files_expression),
# internet_source.start_date,
# internet_source.end_date,
# str(internet_source.frequency_id),
# str(internet_source.user_name),
# str(internet_source.password),
# #str(internet_source.files_filter_expression),
# )
#
# except:
# logger.error("Error in creating sentinel_sat lists. Continue")
# continue
elif internet_type == 'local':
logger.info(
"This internet source is meant to copy data on local filesystem"
)
try:
current_list = get_list_matching_files_dir_local(
str(internet_source.url),
str(internet_source.
include_files_expression))
except:
logger.error(
"Error in creating date lists. Continue"
)
continue
elif internet_type == 'offline':
logger.info(
"This internet source is meant to work offline (GoogleDrive)"
)
current_list = []
else:
logger.error(
"No correct type for this internet source type: %s"
% internet_type)
current_list = []
logger_spec.debug(
"Number of files currently available for source %s is %i",
internet_id, len(current_list))
if len(current_list) > 0:
logger_spec.debug(
"Number of files already copied for trigger %s is %i",
internet_id, len(processed_list))
listtoprocess = []
for current_file in current_list:
if len(processed_list) == 0:
listtoprocess.append(current_file)
else:
#if os.path.basename(current_file) not in processed_list: -> save in .list subdirs as well !!
if current_file not in processed_list:
listtoprocess.append(current_file)
logger_spec.debug(
"Number of files to be copied for trigger %s is %i",
internet_id, len(listtoprocess))
if listtoprocess != set([]):
# # Debug
# toprint=''
# for elem in listtoprocess:
# toprint+=elem+','
# logger_spec.info('List in get_list_matching_files: %s' % toprint)
logger_spec.debug(
"Loop on the found files.")
if not dry_run:
for filename in list(listtoprocess):
logger_spec.debug(
"Processing file: " +
str(internet_source.url) +
os.path.sep + filename)
try:
if internet_type == 'local':
shutil.copyfile(
str(internet_source[
'url']) +
os.path.sep + filename,
es_constants.ingest_dir
+ os.path.basename(
filename))
result = 0
elif internet_type == 'motu_client':
result = get_file_from_motu_command(
str(filename),
#target_file=internet_source.files_filter_expression,
target_dir=es_constants
.ingest_dir,
userpwd=str(usr_pwd))
# elif internet_type == 'sentinel_sat':
# result = get_file_from_sentinelsat_url(str(filename),
# target_dir=es_constants.ingest_dir)
else:
result = get_file_from_url(
str(internet_source.url
) + os.path.sep +
filename,
target_file=os.path.
basename(filename),
target_dir=es_constants
.ingest_dir,
userpwd=str(usr_pwd))
if not result:
logger_spec.info(
"File %s copied.",
filename)
processed_list.append(
filename)
else:
logger_spec.warning(
"File %s not copied: ",
filename)
except:
logger_spec.warning(
"Problem while copying file: %s.",
filename)
else:
logger_spec.info(
'Dry_run is set: do not get files')
if not dry_run:
functions.dump_obj_to_pickle(
processed_list, processed_list_filename)
functions.dump_obj_to_pickle(
processed_info, processed_info_filename)
sleep(float(user_def_sleep))
# Loop over sources
except Exception as inst:
logger.error(
"Error while processing source %s. Continue" %
internet_source.descriptive_name)
sleep(float(user_def_sleep))
exit(0)
def test_internet_connection(self):
status = functions.internet_on()
print status
def test_internet_on(self):
# TODO: System setting type_installation must NOT be Server!
status = functions.internet_on()
self.assertTrue(status)