def Test_get_eumetcast_sources(self):
eumetcast_sources = querydb.get_eumetcast_sources()
logger.info("Eumetcast sources are: %s", eumetcast_sources)
for row in eumetcast_sources:
print row
self.assertEqual(1, 1)
def Test_get_eumetcast_sources(self):
eumetcast_sources = querydb.get_eumetcast_sources()
logger.info("Eumetcast sources are: %s", eumetcast_sources)
for row in eumetcast_sources:
print row
self.assertEqual(1, 1)
def loop_eumetcast_ftp(dry_run=False):
global processed_list_filename, processed_list
global processed_info_filename, processed_info
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGILL, signal_handler)
logger.info("Starting retrieving EUMETCast data.")
ftp_eumetcast_url = es_constants.es2globals['ftp_eumetcast_url']
ftp_eumetcast_userpwd = es_constants.es2globals['ftp_eumetcast_userpwd']
while True:
logger.debug("Check if the Ingest Server input directory : %s exists.",
output_dir)
if not os.path.exists(output_dir):
logger.fatal(
"The Ingest Server input directory : %s doesn't exists.",
output_dir)
exit(1)
if not os.path.exists(es_constants.base_tmp_dir):
os.mkdir(es_constants.base_tmp_dir)
if not os.path.exists(es_constants.processed_list_base_dir):
os.mkdir(es_constants.processed_list_base_dir)
if not os.path.exists(es_constants.processed_list_eum_dir):
os.mkdir(es_constants.processed_list_eum_dir)
while 1:
try:
time_sleep = user_def_sleep
logger.debug("Sleep time set to : %s.", time_sleep)
except:
logger.warning(
"Sleep time not defined. Setting to default=1min. Continue."
)
time_sleep = 60
# try:
logger.debug("Reading active EUMETCAST data sources from database")
eumetcast_sources_list = querydb.get_eumetcast_sources()
logger.debug("N. %i active EUMETCAST data sources found",
len(eumetcast_sources_list))
# Get the EUMETCast MESA_JRC files
try:
get_archives_eumetcast_ftp()
except:
logger.error(
"Error in executing get_archives_eumetcast_ftp. Continue")
# Loop over active triggers
for eumetcast_source in eumetcast_sources_list:
# Define a file_handler logger 'source-specific' (for GUI)
logger_spec = log.my_logger('apps.get_eumetcast.' +
eumetcast_source.eumetcast_id)
logger.info("Processing eumetcast source %s.",
eumetcast_source.eumetcast_id)
if sys.platform == 'win32': #Pierluigi
processed_list_filename = es_constants.get_eumetcast_processed_list_prefix + str(
eumetcast_source.eumetcast_id).replace(':',
'_') + '.list'
processed_info_filename = es_constants.get_eumetcast_processed_list_prefix + str(
eumetcast_source.eumetcast_id).replace(':',
'_') + '.info'
else:
processed_list_filename = es_constants.get_eumetcast_processed_list_prefix + str(
eumetcast_source.eumetcast_id) + '.list'
processed_info_filename = es_constants.get_eumetcast_processed_list_prefix + str(
eumetcast_source.eumetcast_id) + '.info'
# Create objects for list and info
processed_list = []
processed_info = {
'length_proc_list': 0,
'time_latest_exec': datetime.datetime.now(),
'time_latest_copy': datetime.datetime.now()
}
logger.debug("Loading the processed file list for source %s ",
eumetcast_source.eumetcast_id)
# Restore/Create List
processed_list = functions.restore_obj_from_pickle(
processed_list, processed_list_filename)
# Restore/Create Info
processed_info = functions.restore_obj_from_pickle(
processed_info, processed_info_filename)
# Update processing time (in case it is restored)
processed_info['time_latest_exec'] = datetime.datetime.now()
logger.debug(
"Create current list of file to process for trigger %s.",
eumetcast_source.eumetcast_id)
try:
current_list = get_list_matching_files(
ftp_eumetcast_url,
ftp_eumetcast_userpwd,
eumetcast_source.filter_expression_jrc,
'ftp',
my_logger=logger_spec)
except:
logger.error(
"Cannot connect to the PC1 via ftp. Wait 1 minute")
current_list = []
time.sleep(60)
if len(current_list) > 0:
# See ES2-204
logger_spec.debug(
"Number of files currently on PC1 for trigger %s is %i",
eumetcast_source.eumetcast_id, len(current_list))
#logger.debug("Number of files already copied for trigger %s is %i", eumetcast_source.eumetcast_id, len(processed_list))
logger_spec.debug(
"Number of files already copied for trigger %s is %i",
eumetcast_source.eumetcast_id, len(processed_list))
listtoprocess = []
listtoprocess = set(current_list) - set(processed_list)
#logger.debug("Number of files to be copied for trigger %s is %i", eumetcast_source.eumetcast_id, len(listtoprocess))
logger_spec.debug(
"Number of files to be copied for trigger %s is %i",
eumetcast_source.eumetcast_id, len(listtoprocess))
if listtoprocess != set([]):
logger_spec.debug("Loop on the found files.")
for filename in list(listtoprocess):
try:
result = get_file_from_url(
str(ftp_eumetcast_url) + os.path.sep +
filename,
target_file=os.path.basename(filename),
target_dir=es_constants.ingest_dir,
userpwd=str(ftp_eumetcast_userpwd))
if not result:
logger_spec.info("File %s copied.",
filename)
processed_list.append(filename)
else:
logger_spec.warning(
"File %s not copied: ", filename)
except:
logger_spec.warning(
"Problem while copying file: %s.",
filename)
else:
logger.debug(
"Nothing to process - go to next trigger.")
pass
for infile in processed_list:
if not infile in current_list:
processed_list.remove(infile)
if not dry_run:
functions.dump_obj_to_pickle(processed_list,
processed_list_filename)
functions.dump_obj_to_pickle(processed_info,
processed_info_filename)
logger.info("End of Get EUMETCast loop. Sleep")
time.sleep(float(time_sleep))
exit(0)
def loop_eumetcast(dry_run=False):
global processed_list_filename, processed_list
global processed_info_filename, processed_info
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGILL, signal_handler)
logger.info("Starting retrieving EUMETCast data.")
while True:
logger.debug("Check if the EUMETCast input directory : %s exists.",
input_dir)
if not os.path.exists(input_dir):
logger.error(
"The EUMETCast input directory : %s is not yet mounted.",
input_dir)
logger.debug("Check if the Ingest Server input directory : %s exists.",
output_dir)
if not os.path.exists(output_dir):
logger.fatal(
"The Ingest Server input directory : %s doesn't exists.",
output_dir)
exit(1)
if not os.path.exists(es_constants.base_tmp_dir):
os.mkdir(es_constants.base_tmp_dir)
if not os.path.exists(es_constants.processed_list_base_dir):
os.mkdir(es_constants.processed_list_base_dir)
if not os.path.exists(es_constants.processed_list_eum_dir):
os.mkdir(es_constants.processed_list_eum_dir)
while 1:
try:
time_sleep = user_def_sleep
logger.debug("Sleep time set to : %s.", time_sleep)
except:
logger.warning(
"Sleep time not defined. Setting to default=1min. Continue."
)
time_sleep = 60
# try:
logger.debug("Reading active EUMETCAST data sources from database")
eumetcast_sources_list = querydb.get_eumetcast_sources()
logger.debug("N. %i active EUMETCAST data sources found",
len(eumetcast_sources_list))
# Get the EUMETCast MESA_JRC files
try:
get_archives_eumetcast()
except:
logger.error(
"Error in executing get_archives_eumetcast. Continue")
# Loop over active triggers
for eumetcast_source in eumetcast_sources_list:
# Define a file_handler logger 'source-specific' (for GUI)
logger_spec = log.my_logger('apps.get_eumetcast.' +
eumetcast_source.eumetcast_id)
logger.info("Processing eumetcast source %s.",
eumetcast_source.eumetcast_id)
if sys.platform == 'win32': # Pierluigi
processed_list_filename = es_constants.get_eumetcast_processed_list_prefix + str(
eumetcast_source.eumetcast_id).replace(':',
'_') + '.list'
processed_info_filename = es_constants.get_eumetcast_processed_list_prefix + str(
eumetcast_source.eumetcast_id).replace(':',
'_') + '.info'
else:
processed_list_filename = es_constants.get_eumetcast_processed_list_prefix + str(
eumetcast_source.eumetcast_id) + '.list'
processed_info_filename = es_constants.get_eumetcast_processed_list_prefix + str(
eumetcast_source.eumetcast_id) + '.info'
# Create objects for list and info
processed_list = []
processed_info = {
'length_proc_list': 0,
'time_latest_exec': datetime.datetime.now(),
'time_latest_copy': datetime.datetime.now()
}
logger.debug("Loading the processed file list for source %s ",
eumetcast_source.eumetcast_id)
# Restore/Create List
processed_list = functions.restore_obj_from_pickle(
processed_list, processed_list_filename)
# Restore/Create Info
processed_info = functions.restore_obj_from_pickle(
processed_info, processed_info_filename)
# Update processing time (in case it is restored)
processed_info['time_latest_exec'] = datetime.datetime.now()
logger.debug(
"Create current list of file to process for trigger %s.",
eumetcast_source.eumetcast_id)
current_list = find_files(
input_dir, eumetcast_source.filter_expression_jrc)
#logger.debug("Number of files currently on PC1 for trigger %s is %i", eumetcast_source.eumetcast_id, len(current_list))
logger_spec.info(
"Number of files currently on PC1 for trigger %s is %i",
eumetcast_source.eumetcast_id, len(current_list))
if len(current_list) > 0:
#logger.debug("Number of files already copied for trigger %s is %i", eumetcast_source.eumetcast_id, len(processed_list))
logger_spec.debug(
"Number of files already copied for trigger %s is %i",
eumetcast_source.eumetcast_id, len(processed_list))
listtoprocess = []
listtoprocess = set(current_list) - set(processed_list)
#logger.debug("Number of files to be copied for trigger %s is %i", eumetcast_source.eumetcast_id, len(listtoprocess))
logger_spec.debug(
"Number of files to be copied for trigger %s is %i",
eumetcast_source.eumetcast_id, len(listtoprocess))
if listtoprocess != set([]):
logger_spec.debug("Loop on the found files.")
for filename in list(listtoprocess):
if os.path.isfile(os.path.join(
input_dir, filename)):
if os.stat(os.path.join(
input_dir, filename)).st_mtime < int(
time.time()):
logger_spec.debug(
"Processing file: " +
os.path.basename(filename))
if not dry_run:
if subprocess.getstatusoutput(
"cp " + filename + " " +
output_dir + os.sep +
os.path.basename(filename)
)[0] == 0:
logger_spec.info(
"File %s copied.", filename)
processed_list.append(filename)
# Update processing info
processed_info[
'time_latest_copy'] = datetime.datetime.now(
)
processed_info[
'length_proc_list'] = len(
processed_list)
else:
logger_spec.warning(
"Problem while copying file: %s.",
filename)
else:
logger_spec.info(
'Dry_run is set: do not get files')
else:
logger_spec.error(
"File %s removed by the system before being processed.",
filename)
else:
logger.debug(
"Nothing to process - go to next trigger.")
pass
for infile in processed_list:
if not os.path.exists(infile):
processed_list.remove(infile)
if not dry_run:
functions.dump_obj_to_pickle(processed_list,
processed_list_filename)
functions.dump_obj_to_pickle(processed_info,
processed_info_filename)
logger.info("End of Get EUMETCast loop. Sleep")
time.sleep(float(time_sleep))
exit(0)
def loop_eumetcast(dry_run=False):
global processed_list_filename, processed_list
global processed_info_filename, processed_info
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGILL, signal_handler)
logger.info("Starting retrieving EUMETCast data.")
while True:
logger.debug("Check if the EUMETCast input directory : %s exists.", input_dir)
if not os.path.exists(input_dir):
logger.error("The EUMETCast input directory : %s is not yet mounted.", input_dir)
logger.debug("Check if the Ingest Server input directory : %s exists.", output_dir)
if not os.path.exists(output_dir):
logger.fatal("The Ingest Server input directory : %s doesn't exists.", output_dir)
# TODO Jurvtk: Create the Ingest Server output directory if it doesn't exist!
exit(1)
if not os.path.exists(es_constants.base_tmp_dir):
os.mkdir(es_constants.base_tmp_dir)
if not os.path.exists(es_constants.processed_list_base_dir):
os.mkdir(es_constants.processed_list_base_dir)
if not os.path.exists(es_constants.processed_list_eum_dir):
os.mkdir(es_constants.processed_list_eum_dir)
while 1:
try:
time_sleep = user_def_sleep
logger.debug("Sleep time set to : %s.", time_sleep)
except:
logger.warning("Sleep time not defined. Setting to default=1min. Continue.")
time_sleep = 60
# try:
logger.debug("Reading active EUMETCAST data sources from database")
eumetcast_sources_list = querydb.get_eumetcast_sources(echo=echo_query)
logger.debug("N. %i active EUMETCAST data sources found", len(eumetcast_sources_list))
# Loop over active triggers
for eumetcast_source in eumetcast_sources_list:
# Define a file_handler logger 'source-specific' (for GUI)
logger_spec = log.my_logger('apps.get_eumetcast.'+eumetcast_source.eumetcast_id)
logger.debug("Processing eumetcast source %s.", eumetcast_source.eumetcast_id)
processed_list_filename = es_constants.get_eumetcast_processed_list_prefix+str(eumetcast_source.eumetcast_id)+'.list'
processed_info_filename = es_constants.get_eumetcast_processed_list_prefix+str(eumetcast_source.eumetcast_id)+'.info'
# Create objects for list and info
processed_list = []
processed_info = {'length_proc_list': 0,
'time_latest_exec': datetime.datetime.now(),
'time_latest_copy': datetime.datetime.now()}
logger.debug("Loading the processed file list for source %s ", eumetcast_source.eumetcast_id)
# Restore/Create List
processed_list=functions.restore_obj_from_pickle(processed_list, processed_list_filename)
# Restore/Create Info
processed_info=functions.restore_obj_from_pickle(processed_info, processed_info_filename)
# Update processing time (in case it is restored)
processed_info['time_latest_exec']=datetime.datetime.now()
logger.debug("Create current list of file to process for trigger %s.", eumetcast_source.eumetcast_id)
current_list = find_files(input_dir, eumetcast_source.filter_expression_jrc)
#logger.debug("Number of files currently on PC1 for trigger %s is %i", eumetcast_source.eumetcast_id, len(current_list))
logger_spec.debug("Number of files currently on PC1 for trigger %s is %i", eumetcast_source.eumetcast_id, len(current_list))
if len(current_list) > 0:
#logger.debug("Number of files already copied for trigger %s is %i", eumetcast_source.eumetcast_id, len(processed_list))
logger_spec.debug("Number of files already copied for trigger %s is %i", eumetcast_source.eumetcast_id, len(processed_list))
listtoprocess = []
listtoprocess = set(current_list) - set(processed_list)
#logger.debug("Number of files to be copied for trigger %s is %i", eumetcast_source.eumetcast_id, len(listtoprocess))
logger_spec.debug("Number of files to be copied for trigger %s is %i", eumetcast_source.eumetcast_id, len(listtoprocess))
if listtoprocess != set([]):
logger_spec.debug("Loop on the found files.")
for filename in list(listtoprocess):
if os.path.isfile(os.path.join(input_dir, filename)):
if os.stat(os.path.join(input_dir, filename)).st_mtime < int(time.time()):
logger_spec.debug("Processing file: "+os.path.basename(filename))
if not dry_run:
if commands.getstatusoutput("cp " + filename + " " + output_dir + os.sep + os.path.basename(filename))[0] == 0:
logger.info("File %s copied.", filename)
processed_list.append(filename)
# Update processing info
processed_info['time_latest_copy']=datetime.datetime.now()
processed_info['length_proc_list']=len(processed_list)
else:
logger_spec.warning("Problem while copying file: %s.", filename)
else:
logger_spec.info('Dry_run is set: do not get files')
else:
logger_spec.error("File %s removed by the system before being processed.", filename)
else:
logger.debug("Nothing to process - go to next trigger.")
pass
for infile in processed_list:
if not os.path.exists(infile):
processed_list.remove(infile)
if not dry_run:
functions.dump_obj_to_pickle(processed_list, processed_list_filename)
functions.dump_obj_to_pickle(processed_info, processed_info_filename)
time.sleep(float(10))
# except Exception, e:
# logger.fatal(str(e))
# exit(1)
exit(0)