def Test_get_active_internet_sources(self):
internet_sources = querydb.get_active_internet_sources()
logger.info("Internet sources are: %s", internet_sources)
for internet_source in internet_sources:
print internet_source.url
self.assertEqual(1, 1)
def Test_get_active_internet_sources(self):
internet_sources = querydb.get_active_internet_sources()
logger.info("Internet sources are: %s", internet_sources)
for internet_source in internet_sources:
print internet_source.url
self.assertEqual(1, 1)
def testLocal_EOS_JEODESK_OLCI(self):
source_active = False
list_internet_id = ['EOS:S3A:OLCI:WRR', 'EOS:S3B:OLCI:WRR']
#internet_id = 'EOS:S3A:OLCI:WRR'
start_date_dyn = -5
end_date_dyn = -1
for internet_id in list_internet_id:
internet_sources = querydb.get_active_internet_sources()
for s in internet_sources:
if s.internet_id == internet_id:
internet_source = s
source_active = True
if source_active:
my_source = SourceEOS(
internet_id=internet_id,
url=internet_source.url,
descriptive_name="OLCI WRR",
include_files_expression=internet_source.
include_files_expression,
pull_frequency=internet_source.pull_frequency,
user_name=internet_source.user_name,
password=internet_source.password,
start_date=start_date_dyn,
end_date=end_date_dyn,
frequency_id=internet_source.frequency_id,
type=internet_source.type,
files_filter_expression=internet_source.
files_filter_expression,
https_params=internet_source.https_params)
productcode = 'olci-wrr'
productversion = 'V02.0'
product = {
"productcode": productcode,
"version": productversion
}
result = get_internet.loop_get_internet(
test_one_source=internet_id)
self.assertEqual(0, 0)
def testLocal_EOS_JEODESK_SLSTR(self):
source_active = False
list_internet_id = ['EOS:S3A:SLSTR:WST', 'EOS:S3B:SLSTR:WST']
start_date_dyn = -5
end_date_dyn = -1
internet_sources = querydb.get_active_internet_sources()
for internet_id in list_internet_id:
for s in internet_sources:
if s.internet_id == internet_id:
internet_source = s
source_active = True
if source_active:
my_source = SourceEOS(
internet_id=internet_id,
url=internet_source.url,
descriptive_name='sentinel',
include_files_expression=internet_source.
include_files_expression,
pull_frequency=internet_source.pull_frequency,
user_name=internet_source.user_name,
password=internet_source.password,
start_date=start_date_dyn,
end_date=end_date_dyn,
frequency_id=internet_source.frequency_id,
type=internet_source.type,
files_filter_expression=internet_source.
files_filter_expression,
https_params=internet_source.https_params)
productcode = 'slstr-sst'
productversion = '1.0'
product = {
"productcode": productcode,
"version": productversion
}
# Test download (dynamic dates
result = get_internet.loop_get_internet(
test_one_source=internet_id)
self.assertEqual(0, 0)
def testLocal_EOS_JEODESK_SLSTR(self):
source_active = False
internet_id = 'EOS:S3A:SLSTR:WST'
start_date_fixed = 20200301
end_date_fixed = 20200310
start_date_dyn = -5
end_date_dyn = -3
file_to_check = '32e61b08-0bcb-4d0a-a06e-f3d499dfb5fc/S3A_SL_2_WST____20200310T073813_20200310T091913_20200311T185257_6059_056_006______MAR_O_NT_003'
internet_sources = querydb.get_active_internet_sources()
for s in internet_sources:
if s.internet_id == internet_id:
internet_source = s
source_active = True
if source_active:
my_source = Source(
internet_id=internet_id,
url=internet_source.url,
descriptive_name='sentinel',
include_files_expression=internet_source.include_files_expression,
pull_frequency=internet_source.pull_frequency,
user_name=internet_source.user_name,
password=internet_source.password,
start_date=start_date_dyn,
end_date=end_date_dyn,
frequency_id=internet_source.frequency_id,
type=internet_source.type,
files_filter_expression=internet_source.files_filter_expression,
https_params=internet_source.https_params)
productcode = 'slstr-sst'
productversion = '1.0'
product = {"productcode": productcode, "version": productversion}
# Test download (dynamic dates
if True:
result = loop_get_internet(test_one_source=internet_id,
my_source=my_source)
self.assertEqual(result, 0)
def testLocal_EOS_JEODESK_OLCI(self):
source_active = False
internet_id = 'EOS:S3A:OLCI:WRR'
start_date_fixed = 20200301
end_date_fixed = 20200310
start_date_dyn = -2
end_date_dyn = -1
file_to_check = '44c285d7-3809-4810-836e-510ee52f326a/S3A_OL_2_WRR____20200310T065044_20200310T073438_20200311T133228_2634_056_006______MAR_O_NT_002'
internet_sources = querydb.get_active_internet_sources()
for s in internet_sources:
if s.internet_id == internet_id:
internet_source = s
source_active = True
if source_active:
my_source = Source(
internet_id=internet_id,
url=internet_source.url,
descriptive_name="OLCI WRR",
include_files_expression=internet_source.include_files_expression,
pull_frequency=internet_source.pull_frequency,
user_name=internet_source.user_name,
password=internet_source.password,
start_date=start_date_dyn,
end_date=end_date_dyn,
frequency_id=internet_source.frequency_id,
type=internet_source.type,
files_filter_expression=internet_source.files_filter_expression,
https_params=internet_source.https_params)
productcode = 'olci-wrr'
productversion = 'V02.0'
product = {"productcode": productcode, "version": productversion}
if True:
result = loop_get_internet(test_one_source=internet_id,
my_source=my_source)
self.assertEqual(result, 0)
def loop_get_internet(dry_run=False):
global processed_list_filename, processed_list
global processed_info_filename, processed_info
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGILL, signal_handler)
logger.info("Starting retrieving data from INTERNET.")
while True:
output_dir = es_constants.ingest_dir
logger.debug("Check if the Ingest Server input directory : %s exists.", output_dir)
if not os.path.exists(output_dir):
logger.fatal("The Ingest Server input directory : %s doesn't exists.", output_dir)
exit(1)
if not os.path.exists(es_constants.processed_list_int_dir):
os.mkdir(es_constants.processed_list_int_dir)
while 1:
try:
time_sleep = user_def_sleep
logger.debug("Sleep time set to : %s.", time_sleep)
except:
logger.warning("Sleep time not defined. Setting to default=1min. Continue.")
time_sleep = 60
logger.debug("Reading active INTERNET data sources from database")
internet_sources_list = querydb.get_active_internet_sources(echo=echo_query)
# Loop over active triggers
try:
for internet_source in internet_sources_list:
logger.debug("Processing internet source %s.", internet_source.descriptive_name)
processed_list_filename = es_constants.get_internet_processed_list_prefix+str(internet_source.internet_id)+'.list'
processed_info_filename = es_constants.get_internet_processed_list_prefix+str(internet_source.internet_id)+'.info'
# Create objects for list and info
processed_list = []
processed_info = {'length_proc_list': 0,
'time_latest_exec': datetime.datetime.now(),
'time_latest_copy': datetime.datetime.now()}
# Restore/Create List
processed_list=functions.restore_obj_from_pickle(processed_list, processed_list_filename)
# Restore/Create Info
processed_info=functions.restore_obj_from_pickle(processed_info, processed_info_filename)
# Update processing time (in case it is restored)
processed_info['time_latest_exec']=datetime.datetime.now()
logger.debug("Create current list of file to process for source %s.", internet_source.internet_id)
if internet_source.user_name is None:
user_name = "anonymous"
else:
user_name = internet_source.user_name
if internet_source.password is None:
password = "******"
else:
password = internet_source.password
usr_pwd = str(user_name)+':'+str(password)
logger.debug(" Url is %s.", internet_source.url)
logger.debug(" usr/pwd is %s.", usr_pwd)
logger.debug(" regex is %s.", internet_source.include_files_expression)
internet_type = internet_source.type
if internet_type == 'ftp':
# Note that the following list might contain sub-dirs (it reflects full_regex)
current_list = get_list_matching_files_dir_ftp(str(internet_source.url), str(usr_pwd), str(internet_source.include_files_expression))
elif internet_type == 'http_tmpl':
# Manage the dates:start_date is mandatory .. end_date replaced by 'today' if missing/wrong
try:
if functions.is_date_yyyymmdd(str(internet_source.start_date), silent=True):
datetime_start=datetime.datetime.strptime(str(internet_source.start_date),'%Y%m%d')
else:
raise Exception("Start Date not valid")
except:
raise Exception("Start Date not valid")
try:
if functions.is_date_yyyymmdd(str(internet_source.end_date), silent=True):
datetime_end=datetime.datetime.strptime(str(internet_source.end_date),'%Y%m%d')
else:
datetime_end=datetime.datetime.today()
except:
pass
# Create the full filename from a 'template' which contains
try:
current_list = build_list_matching_for_http(str(internet_source.url),
str(internet_source.include_files_expression),
datetime_start,
datetime_end,
str(internet_source.frequency_id))
except:
logger.error("Error in creating date lists. Continue")
logger.debug("Number of files currently available for source %s is %i", internet_source.internet_id, len(current_list))
if len(current_list) > 0:
logger.debug("Number of files already copied for trigger %s is %i", internet_source.internet_id, len(processed_list))
listtoprocess = []
for current_file in current_list:
if len(processed_list) == 0:
listtoprocess.append(current_file)
else:
#if os.path.basename(current_file) not in processed_list: -> save in .list subdirs as well !!
if current_file not in processed_list:
listtoprocess.append(current_file)
logger.debug("Number of files to be copied for trigger %s is %i", internet_source.internet_id, len(listtoprocess))
if listtoprocess != set([]):
logger.debug("Loop on the found files.")
if not dry_run:
for filename in list(listtoprocess):
logger.debug("Processing file: "+str(internet_source.url)+os.path.sep+filename)
try:
result = get_file_from_url(str(internet_source.url)+os.path.sep+filename, target_file=os.path.basename(filename), target_dir=es_constants.ingest_dir, userpwd=str(usr_pwd))
if not result:
logger.info("File %s copied.", filename)
processed_list.append(filename)
except:
logger.warning("Problem while copying file: %s.", filename)
else:
logger.info('Dry_run is set: do not get files')
if not dry_run:
functions.dump_obj_to_pickle(processed_list, processed_list_filename)
functions.dump_obj_to_pickle(processed_info, processed_info_filename)
sleep(float(user_def_sleep))
# Loop over sources
except Exception as inst:
logger.error("Error while processing source %s. Continue" % internet_source.descriptive_name)
sleep(float(user_def_sleep))
exit(0)
def loop_get_internet(dry_run=False, test_one_source=False):
global processed_list_filename, processed_list
global processed_info_filename, processed_info
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGILL, signal_handler)
logger.info("Starting retrieving data from INTERNET.")
while True:
output_dir = es_constants.get_internet_output_dir
logger.debug("Check if the Ingest Server input directory : %s exists.",
output_dir)
if not os.path.exists(output_dir):
# ToDo: create output_dir - ingest directory
logger.fatal(
"The Ingest Server input directory : %s doesn't exists.",
output_dir)
exit(1)
if not os.path.exists(es_constants.processed_list_int_dir):
os.mkdir(es_constants.processed_list_int_dir)
while 1:
# Check internet connection (or continue)
if not functions.internet_on():
logger.error(
"The computer is not currently connected to the internet. Wait 1 minute."
)
time.sleep(60)
else:
try:
time_sleep = user_def_sleep
logger.debug("Sleep time set to : %s.", time_sleep)
except:
logger.warning(
"Sleep time not defined. Setting to default=1min. Continue."
)
time_sleep = 60
logger.info(
"Reading active INTERNET data sources from database")
internet_sources_list = querydb.get_active_internet_sources()
# Loop over active triggers
for internet_source in internet_sources_list:
try:
if test_one_source and (internet_source.internet_id !=
test_one_source):
logger.info(
"Running in test mode, and source is not %s. Continue.",
test_one_source)
continue
execute_trigger = True
# Get this from the pads database table (move from internet_source 'pull_frequency' to the pads table,
# so that it can be exploited by eumetcast triggers as well). It is in minute
pull_frequency = internet_source.pull_frequency
# Manage the case of files to be continuously downloaded (delay < 0)
if pull_frequency < 0:
do_not_consider_processed_list = True
delay_time_source_minutes = -pull_frequency
else:
do_not_consider_processed_list = False
delay_time_source_minutes = pull_frequency
if sys.platform == 'win32':
internet_id = str(
internet_source.internet_id).replace(':', '_')
else:
internet_id = str(internet_source.internet_id)
logger_spec = log.my_logger('apps.get_internet.' +
internet_id)
logger.info("Processing internet source %s.",
internet_source.descriptive_name)
# Create objects for list and info
processed_info_filename = es_constants.get_internet_processed_list_prefix + str(
internet_id) + '.info'
# Restore/Create Info
processed_info = None
processed_info = functions.restore_obj_from_pickle(
processed_info, processed_info_filename)
if processed_info is not None:
# Check the delay
current_delta = datetime.datetime.now(
) - processed_info['time_latest_exec']
current_delta_minutes = int(current_delta.seconds /
60)
if current_delta_minutes < delay_time_source_minutes:
logger.debug(
"Still waiting up to %i minute - since latest execution.",
delay_time_source_minutes)
execute_trigger = False
else:
# Create processed_info object
processed_info = {
'lenght_proc_list': 0,
'time_latest_exec': datetime.datetime.now(),
'time_latest_copy': datetime.datetime.now()
}
execute_trigger = True
if execute_trigger:
# Restore/Create List
processed_list = []
if not do_not_consider_processed_list:
processed_list_filename = es_constants.get_internet_processed_list_prefix + internet_id + '.list'
processed_list = functions.restore_obj_from_pickle(
processed_list, processed_list_filename)
processed_info[
'time_latest_exec'] = datetime.datetime.now()
logger.debug(
"Create current list of file to process for source %s.",
internet_source.internet_id)
if internet_source.user_name is None:
user_name = "anonymous"
else:
user_name = internet_source.user_name
if internet_source.password is None:
password = "******"
else:
password = internet_source.password
usr_pwd = str(user_name) + ':' + str(password)
logger_spec.debug(" Url is %s.",
internet_source.url)
logger_spec.debug(" usr/pwd is %s.",
usr_pwd)
logger_spec.debug(
" regex is %s.",
internet_source.include_files_expression)
internet_type = internet_source.type
if internet_type == 'ftp' or internet_type == 'http':
# Manage the end_date (added for MODIS_FIRMS)
if (internet_source.end_date != ''):
end_date = internet_source.end_date
else:
end_date = None
# Note that the following list might contain sub-dirs (it reflects full_regex)
try:
current_list = get_list_matching_files(
str(internet_source.url),
str(usr_pwd),
str(internet_source.
include_files_expression),
internet_type,
end_date=end_date)
except:
logger.error(
"Error in creating file lists. Continue"
)
continue
elif internet_type == 'http_tmpl':
# Create the full filename from a 'template' which contains
try:
current_list = build_list_matching_files_tmpl(
str(internet_source.url),
str(internet_source.
include_files_expression),
internet_source.start_date,
internet_source.end_date,
str(internet_source.frequency_id))
except:
logger.error(
"Error in creating date lists. Continue"
)
continue
elif internet_type == 'motu_client':
# Create the full filename from a 'template' which contains
try:
current_list = build_list_matching_files_motu(
str(internet_source.url),
str(internet_source.
include_files_expression),
internet_source.start_date,
internet_source.end_date,
str(internet_source.frequency_id),
str(internet_source.user_name),
str(internet_source.password),
str(internet_source.
files_filter_expression),
)
except:
logger.error(
"Error in creating motu_client lists. Continue"
)
continue
# elif internet_type == 'sentinel_sat':
# # Create the full filename from a 'template' which contains
# try:
# current_list = build_list_matching_files_sentinel_sat(str(internet_source.url),
# str(internet_source.include_files_expression),
# internet_source.start_date,
# internet_source.end_date,
# str(internet_source.frequency_id),
# str(internet_source.user_name),
# str(internet_source.password),
# #str(internet_source.files_filter_expression),
# )
#
# except:
# logger.error("Error in creating sentinel_sat lists. Continue")
# continue
elif internet_type == 'local':
logger.info(
"This internet source is meant to copy data on local filesystem"
)
try:
current_list = get_list_matching_files_dir_local(
str(internet_source.url),
str(internet_source.
include_files_expression))
except:
logger.error(
"Error in creating date lists. Continue"
)
continue
elif internet_type == 'offline':
logger.info(
"This internet source is meant to work offline (GoogleDrive)"
)
current_list = []
else:
logger.error(
"No correct type for this internet source type: %s"
% internet_type)
current_list = []
logger_spec.debug(
"Number of files currently available for source %s is %i",
internet_id, len(current_list))
if len(current_list) > 0:
logger_spec.debug(
"Number of files already copied for trigger %s is %i",
internet_id, len(processed_list))
listtoprocess = []
for current_file in current_list:
if len(processed_list) == 0:
listtoprocess.append(current_file)
else:
#if os.path.basename(current_file) not in processed_list: -> save in .list subdirs as well !!
if current_file not in processed_list:
listtoprocess.append(current_file)
logger_spec.debug(
"Number of files to be copied for trigger %s is %i",
internet_id, len(listtoprocess))
if listtoprocess != set([]):
# # Debug
# toprint=''
# for elem in listtoprocess:
# toprint+=elem+','
# logger_spec.info('List in get_list_matching_files: %s' % toprint)
logger_spec.debug(
"Loop on the found files.")
if not dry_run:
for filename in list(listtoprocess):
logger_spec.debug(
"Processing file: " +
str(internet_source.url) +
os.path.sep + filename)
try:
if internet_type == 'local':
shutil.copyfile(
str(internet_source[
'url']) +
os.path.sep + filename,
es_constants.ingest_dir
+ os.path.basename(
filename))
result = 0
elif internet_type == 'motu_client':
result = get_file_from_motu_command(
str(filename),
#target_file=internet_source.files_filter_expression,
target_dir=es_constants
.ingest_dir,
userpwd=str(usr_pwd))
# elif internet_type == 'sentinel_sat':
# result = get_file_from_sentinelsat_url(str(filename),
# target_dir=es_constants.ingest_dir)
else:
result = get_file_from_url(
str(internet_source.url
) + os.path.sep +
filename,
target_file=os.path.
basename(filename),
target_dir=es_constants
.ingest_dir,
userpwd=str(usr_pwd))
if not result:
logger_spec.info(
"File %s copied.",
filename)
processed_list.append(
filename)
else:
logger_spec.warning(
"File %s not copied: ",
filename)
except:
logger_spec.warning(
"Problem while copying file: %s.",
filename)
else:
logger_spec.info(
'Dry_run is set: do not get files')
if not dry_run:
functions.dump_obj_to_pickle(
processed_list, processed_list_filename)
functions.dump_obj_to_pickle(
processed_info, processed_info_filename)
sleep(float(user_def_sleep))
# Loop over sources
except Exception as inst:
logger.error(
"Error while processing source %s. Continue" %
internet_source.descriptive_name)
sleep(float(user_def_sleep))
exit(0)