def run(self): systemsettings = functions.getSystemSettings() # Special case for mesa-proc @ JRC if systemsettings['type_installation'] == 'Server': get_eumetcast.loop_eumetcast(dry_run=self.dry_run) else: get_eumetcast.loop_eumetcast_ftp(dry_run=self.dry_run)
def test_system_data_sync(self): source = es_constants.es2globals['processing_dir'] system_settings = functions.getSystemSettings() ip_target = system_settings['ip_pc2'] target = ip_target + '::products' + es_constants.es2globals['processing_dir'] status = es2system.system_data_sync(source, target) self.assertEqual(status, 0)
def test_setSystemSetting(self): setting = 'ingest_archive_eum' value = 'true' result = functions.setSystemSetting(setting, value) self.assertTrue(result) systemsettings = functions.getSystemSettings() if 'ingest_archive_eum' in systemsettings.keys(): self.assertEqual(systemsettings['ingest_archive_eum'], 'true') else: self.assertTrue(False)
def get_status_local_machine(): # Get info on the status of the local machine # logger.debug("Entering routine %s" % 'get_status_local_machine') # Get the local systems settings systemsettings = functions.getSystemSettings() # Get status of all services status_services = functions.getStatusAllServices() get_eumetcast_status = status_services['eumetcast'] get_internet_status = status_services['internet'] ingestion_status = status_services['ingest'] processing_status = status_services['process'] system_status = status_services['system'] # Get status of postgresql psql_status = functions.getStatusPostgreSQL() # Get internet connection internet_status = functions.internet_on() # ToDo: check disk status! status_local_machine = { 'get_eumetcast_status': get_eumetcast_status, 'get_internet_status': get_internet_status, 'ingestion_status': ingestion_status, 'processing_status': processing_status, 'system_status': system_status, 'system_execution_time': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), 'postgresql_status': str(psql_status).lower(), 'internet_connection_status': str(internet_status).lower(), 'active_version': systemsettings['active_version'], 'mode': systemsettings['mode'], 'disk_status': 'true' } return status_local_machine
def system_bucardo_config(): # Check if bucardo has been already configured, and if there are conditions to do so res_bucardo_config = 0 logger.debug("Entering routine %s" % 'system_bucardo_config') # Returns 0 if no any sync exists res_list_sync = os.system( 'bucardo list sync | grep "No syncs found" 1>/dev/null') # If no any sync exists, bucardo still to be configured if not res_list_sync: # Get relevant variables sysSettings = functions.getSystemSettings() role = sysSettings['role'] # Check the other computer is ready if role == 'PC2': other_pc = 'MESA-PC3' else: other_pc = 'MESA-PC2' command = '/usr/pgsql-9.3/bin/pg_isready -h ' + other_pc other_pc_not_ready = os.system(command) if not other_pc_not_ready: # Execute the configuration command = '/var/www/eStation2/config/install/bucardo_config.sh '+ role.lower() +\ ' 1>/var/log/bucardo/bucardo_config.log'+ ' 2>/var/log/bucardo/bucardo_config.err' res_bucardo_config = os.system(command) # Restart bucardo command = 'bucardo restart' res_bucardo_config += os.system(command) else: logger.error('The other computer ' + other_pc + ' is not ready. Exit.') # Exit return res_bucardo_config
def remove_old_files(productcode, subproductcode, version, mapsetcode, product_type, nmonths, logger=None): # Check logger if logger is None: logger = log.my_logger(__name__) # Get the existing dates for the dataset logger.info("Entering routine %s " % 'remove_old_files') # Check the installation type sysSettings = functions.getSystemSettings() if sysSettings['type_installation'] == 'Server': logger.info("File housekeeping not done on Server ") return prod_subdir = functions.set_path_sub_directory(productcode, subproductcode, product_type, version, mapsetcode) prod_dir = es_constants.es2globals[ 'processing_dir'] + os.path.sep + prod_subdir list_files = sorted(glob.glob(prod_dir + os.path.sep + '*.tif')) # Define the earliest date to be kept month_now = datetime.date.today().month year_now = datetime.date.today().year for my_file in list_files: # Extract the date date = functions.get_date_from_path_filename(os.path.basename(my_file)) date_yyyy = int(date[0:4]) date_month = int(date[4:6]) if date_yyyy < year_now or (date_month + nmonths) <= month_now: logger.debug("Deleting file %s " % my_file) os.remove(my_file)
class TestSystem(unittest.TestCase): systemsettings = functions.getSystemSettings() install_type = systemsettings['type_installation'].lower() def test_manage_lock(self): # Check the management of the lock files lock_id = 'test_operation' # Delete all existing locks status = es2system.system_manage_lock('All_locks', 'Delete') self.assertEquals(status, 0) # Check a (not existing) lock status = es2system.system_manage_lock(lock_id, 'Check') self.assertEquals(status, 0) # Create a lock status = es2system.system_manage_lock(lock_id, 'Create') self.assertEquals(status, 0) # Check an (existing) lock status = es2system.system_manage_lock(lock_id, 'Check') self.assertEquals(status, 1) def test_save_status(self): # Save machine status in /eStation/system status_system_file = es2system.save_status_local_machine() self.assertEqual(status_system_file, 0) @unittest.skipIf(install_type != 'full', "Test only on MESA Station - Full install") def test_change_ip_addresses_default(self): ip_pc1 = '192.168.0.11' ip_pc2 = '192.168.0.15' ip_pc3 = '192.168.0.16' ip_dns = '192.168.0.1' gateway = '192.168.0.1' ip_lan = '192.168.0.0/24' sudo_psw = 'mesadmin' command = es_constants.es2globals['base_dir'] + '/apps/es2system/network_config_1.0.sh ' + \ ip_pc1 + ' ' + \ ip_pc2 + ' ' + \ ip_pc3 + ' ' + \ ip_dns + ' ' + \ gateway + ' ' + \ ip_lan status = os.system('echo %s | sudo -S %s' % (sudo_psw, command)) self.assertEqual(status, 0) @unittest.skipIf(install_type != 'full', "Test only on MESA Station - Full install") def test_change_ip_addresses_Container(self): ip_pc1 = '10.191.231.11' # the machine does not actually exist .. ip_pc2 = '10.191.231.89' ip_pc3 = '10.191.231.90' ip_dns = '139.191.1.132' # Only primary ... secondary to be added gateway = '10.191.231.1' ip_lan = '10.191.231.0/24' # To be checked (?) sudo_psw = 'mesadmin' command = es_constants.es2globals['base_dir'] + '/apps/es2system/network_config_1.0.sh ' + \ ip_pc1 + ' ' + \ ip_pc2 + ' ' + \ ip_pc3 + ' ' + \ ip_dns + ' ' + \ gateway + ' ' + \ ip_lan status = os.system('echo %s | sudo -S %s' % (sudo_psw, command)) self.assertEqual(status, 0) # The following test enters an infinite loop (?) -> see test_system_loop below with dry_run = True # def test_system_service(self): # # status = es2system.loop_system(dry_run=False) # self.assertEqual(status, 0) @unittest.skipIf(True, "Not working - to be checked") def test_system_db_dump_docker(self): list_dump = ['products', 'analysis'] status = es2system.system_db_dump_docker(list_dump) self.assertEqual(status, b'') @unittest.skipIf(install_type != 'full', "Test only on MESA Station - Full install") def test_system_db_dump(self): list_dump = ['products', 'analysis'] status = es2system.system_db_dump(list_dump) self.assertEqual(status, 0) @unittest.skipIf(install_type != 'full', "Test only on MESA Station - Full install") def test_system_manage_dumps(self): status = es2system.system_manage_dumps() self.assertEqual(status, 0) @unittest.skipIf(install_type != 'full', "Test only on MESA Station - Full install") def test_system_data_sync(self): source = es_constants.es2globals['processing_dir'] system_settings = functions.getSystemSettings() ip_target = system_settings['ip_pc2'] target = ip_target + '::products' + es_constants.es2globals['processing_dir'] status = es2system.system_data_sync(source, target) self.assertEqual(status, 0) @unittest.skipIf(install_type != 'full', "Test only on MESA Station - Full install") def test_db_sync_full_from_PC2(self): list_syncs = ['sync_pc2_products_full', 'sync_pc2_analysis_full'] status = es2system.system_db_sync(list_syncs) @unittest.skipIf(install_type != 'full', "Test only on MESA Station - Full install") def test_db_sync_full_from_PC3(self): list_syncs = ['sync_pc3_analysis_full', 'sync_pc3_products_full'] status = es2system.system_db_sync(list_syncs) @unittest.skipIf(install_type != 'full', "Test only on MESA Station - Full install") def test_system_db_sync_full(self): # Should get here the role of my machine ... status = es2system.system_db_sync_full('pc2') self.assertTrue(status) @unittest.skipIf(install_type != 'full', "Test only on MESA Station - Full install") def test_system_bucardo_config(self): # Should get here the role of my machine ... status = es2system.system_bucardo_config() @unittest.skipIf(install_type != 'full', "Test only on MESA Station - Full install") def test_system_status_PC1(self): # Should get here the role of my machine ... status = es2system.get_status_PC1() # @unittest.skipIf(install_type != 'full',"Test only on MESA Station - Full install") def test_system_install_report(self): # Should get here the role of my machine ... repfile = es2system.system_install_report() self.assertTrue(os.path.isfile(repfile)) def test_system_create_report(self): # Create the report and check file axists repfile = es2system.system_create_report() self.assertTrue(os.path.isfile(repfile)) def test_clean_temp_dir(self): # Clean the temp dirs and check the status status = es2system.clean_temp_dir() self.assertEqual(status, 0) @unittest.skipIf(install_type != 'full', "Test only on MESA Station - Full install") def test_bucardo_service(self): # Should get here the role of my machine ... status = es2system.system_bucardo_service('stop') def test_system_loop(self): # Call the system loop in dry mode (exits after first iteration) status = es2system.loop_system(dry_run=True) self.assertEqual(status, 0) @unittest.skipIf(install_type != 'server', "Test only on JRC Server Installation") def test_push_ftp_aruba(self): try: import apps.es2system.test.aruba_credentials as ac except: return 1 # Masked=FALSE means the masked products are pushed. status = es2system.push_data_ftp(url=ac.url, user=ac.user, psw=ac.psw, trg_dir=ac.trg_dir, masked=False) self.assertEqual(status, 0) @unittest.skipIf(install_type != 'server', "Test only on JRC Server Installation") def test_push_ftp_jrc(self): # Execute w.o. arguments: they are read from config/server_ftp.py # The products/versions considered for sync are: the 'activated' ones - except the ones set as 'EXCLUDE' in server_ftp.py definitions # Masked=TRUE means the masked sub-products are not pushed (which is the default) status = es2system.push_data_ftp(masked=True) self.assertEqual(status, 0)
def loop_system(dry_run=False): # Driver of the system service # Reads configuration from the system setting file (system_settings.ini) # Perform the needed operations, according to the machine role/mode # Arguments: dry_run -> if > 0, only report what has to be done (no actions) # -> if = 0, do operations (default) logger.info("Entering routine %s" % 'loop_system') # Specific settings for the system operations delay_data_sync_minutes = es_constants.es2globals[ 'system_delay_data_sync_min'] time_for_db_dump = es_constants.es2globals['system_time_db_dump_hhmm'] time_for_spirits_conv = es_constants.es2globals['system_time_spirits_conv'] try: time_for_push_ftp = es_constants.es2globals['system_time_push_ftp'] except: logger.info("Parameter not defined in factory_settings: %s" % 'system_time_push_ftp') time_for_push_ftp = '24:01' do_bucardo_config = False # Restart bucardo command = 'bucardo restart' res_bucardo_restart = os.system(command) if res_bucardo_restart: logger.warning("Error in restarting bucardo") else: logger.info("Bucardo restarted correctly") # Loop to manage the 'cron-like' operations, i.e.: # 0. Check bucardo config # a. Data sync (not anymore, done by TPZF) # b. DB sync: bucardo # c. DB dump (create/manage) # d. Spirits conversion # e. Clean Temporary directory execute_loop = True while execute_loop: logger.info("") logger.info("Entering the System Service loop") # Read the relevant info from system settings system_settings = functions.getSystemSettings() logger.debug('System Settings Mode: %s ' % system_settings['mode']) # Initialize To Do flags do_data_sync = False schemas_db_sync = [] schemas_db_dump = [] do_convert_spirits = False do_clean_tmp = True do_push_ftp = False if system_settings['data_sync'] in [ 'True', 'true', '1', 't', 'y', 'Y', 'yes', 'Yes' ]: do_data_sync = True else: do_data_sync = False if system_settings['db_sync'] in [ 'True', 'true', '1', 't', 'y', 'Y', 'yes', 'Yes' ]: do_db_sync = True else: do_db_sync = False # Implement the logic of operations based on type/role/mode if system_settings['type_installation'] == 'Full': do_bucardo_config = True if system_settings['role'] == 'PC2': status_otherPC = functions.get_remote_system_status('mesa-pc3') if len(status_otherPC) != 0: mode_otherPC = status_otherPC['mode'] else: mode_otherPC = 'unreachable' # ip_target = system_settings['ip_pc3'] if system_settings['mode'] == 'nominal': if mode_otherPC == 'recovery': do_data_sync = False logger.info( "Do not do data_sync because other PC is in Recovery Mode" ) elif mode_otherPC == 'unreachable': do_data_sync = False logger.info( "Do not do data_sync because other PC is not reachable" ) schemas_db_sync = ['products'] schemas_db_dump = ['products', 'analysis'] do_convert_spirits = True bucardo_action = 'start' if system_settings['mode'] == 'recovery': schemas_db_sync = [] schemas_db_dump = ['products', 'analysis'] do_convert_spirits = True bucardo_action = 'stop' if system_settings['mode'] == 'maintenance': schemas_db_sync = [] schemas_db_dump = [] bucardo_action = 'stop' if system_settings['role'] == 'PC3': status_otherPC = functions.get_remote_system_status('mesa-pc2') if len(status_otherPC) != 0: mode_otherPC = status_otherPC['mode'] else: mode_otherPC = 'unreachable' # ip_target = system_settings['ip_pc2'] if system_settings['mode'] == 'nominal': if mode_otherPC == 'recovery': do_data_sync = False logger.info( "Do not do data_sync because other PC is in Recovery Mode" ) elif mode_otherPC == 'unreachable': do_data_sync = False logger.info( "Do not do data_sync because other PC is not reachable" ) schemas_db_sync = ['analysis'] schemas_db_dump = ['products', 'analysis'] bucardo_action = 'start' if system_settings['mode'] == 'recovery': schemas_db_sync = [] schemas_db_dump = ['products', 'analysis'] bucardo_action = 'stop' if system_settings['mode'] == 'maintenance': schemas_db_sync = [] schemas_db_dump = [] bucardo_action = 'stop' if system_settings['type_installation'] == 'SinglePC': do_data_sync = False schemas_db_sync = [] schemas_db_dump = ['products', 'analysis'] if system_settings['type_installation'] == 'Server': do_data_sync = False do_db_sync = False schemas_db_sync = [] schemas_db_dump = ['products', 'analysis'] do_convert_spirits = False do_push_ftp = True if es_constants.es2globals['do_spirits_conversion'] in [ 'True', 'true', '1', 't', 'y', 'Y', 'yes', 'Yes' ]: do_convert_spirits = True # Report on the actions to be done logger.info("") logger.info("* Schedule follows ") logger.info("* Do bucardo config: " + str(do_bucardo_config)) logger.info("* Do data-sync: " + str(do_data_sync)) logger.info("* Do db-sync: " + str(do_db_sync)) # logger.info("* Nr schema to dump: " + str(len(schemas_db_dump))) logger.info("* Do spirits convers: " + str(do_convert_spirits)) logger.info("* Do push to ftp: " + str(do_push_ftp)) logger.info("") # do_bucardo_config if do_bucardo_config: system_bucardo_config() # do_data_sync operation = 'data_sync' if do_data_sync: check_time = check_delay_time( operation, delay_minutes=delay_data_sync_minutes) logger.info("check_time: " + str(check_time)) if check_time: logger.info("Executing data synchronization") data_source = es_constants.es2globals['processing_dir'] # data_target = ip_target+'::products'+es_constants.es2globals['processing_dir'] data_target = 'PC3::products' system_data_sync(data_source, data_target) check_delay_time(operation, delay_minutes=delay_data_sync_minutes, write=True) # DB sync: execute every cycle if in system_setting (no delay) operation = 'db_sync' if len(schemas_db_sync) > 0: if do_db_sync: logger.info("Executing db synchronization") # Call the function system_bucardo_service(bucardo_action) # DB dump operation = 'db_dump' if len(schemas_db_dump) > 0: check_time = check_delay_time(operation, time=time_for_db_dump) if check_time: # Execute the dump of the schemas active on the machine - Correct Tuleap #10905 logger.info("Executing db dump") system_db_dump(schemas_db_dump) # Manage the file dumps (rotation) logger.info("Executing manage dumps") system_manage_dumps() # Convert to spirits format operation = 'convert_spirits' if do_convert_spirits: check_time = check_delay_time(operation, time=time_for_spirits_conv) if check_time: logger.info("Convert to SPIRITS format") output_dir = es_constants.es2globals['spirits_output_dir'] conv.convert_driver(output_dir) # Push data to ftp server operation = 'push_to_ftp' if do_push_ftp: check_time = check_delay_time(operation, time=time_for_push_ftp) if check_time: logger.info("Push data to remote ftp server") status = push_data_ftp() # Clean temporary directory operation = 'clean_temp' if do_clean_tmp: logger.info("Cleaning Temporary dirs") clean_temp_dir() # Exit in case of dry_run if dry_run: execute_loop = False # Sleep some time time.sleep(float(es_constants.es2globals['system_sleep_time_sec']))
from lib.python import functions from lib.python import es_logging as log from config import es_constants from apps.tools import ingest_historical_archives as iha from apps.es2system import convert_2_spirits as conv from database import querydb from apps.productmanagement.products import * from lib.python.daemon import DaemonDryRunnable _author__ = "Marco Clerici" logger = log.my_logger(__name__) data_dir = es_constants.es2globals['processing_dir'] # Get the local systems settings systemsettings = functions.getSystemSettings() def get_status_local_machine(): # Get info on the status of the local machine # logger.debug("Entering routine %s" % 'get_status_local_machine') # Get the local systems settings # systemsettings = functions.getSystemSettings() # Get status of all services status_services = functions.getStatusAllServices() get_eumetcast_status = status_services['eumetcast']
def loop_ingestion_drive(dry_run=False, test_one_product=None): echo_query = False # Get all active product ingestion records with a subproduct count. active_product_ingestions = querydb.get_ingestion_product(allrecs=True) for active_product_ingest in active_product_ingestions: productcode = active_product_ingest[0] productversion = active_product_ingest[1] # Verify the test-one-product case do_ingest_product = is_test_one_product(test_one_product, productcode) if do_ingest_product: logger.info("Ingestion active for product: [%s] subproduct N. %s" % (active_product_ingest[0], active_product_ingest[2])) # For the current active product ingestion: get all product = {"productcode": productcode, "version": productversion} logger.debug("Processing product: %s - version %s" % (productcode, productversion)) # Get the list of acquisition sources that are defined for this ingestion 'trigger' # (i.e. prod/version) # NOTE: the following implies there is 1 and only 1 '_native' subproduct associated to a 'product'; native_product = { "productcode": productcode, "subproductcode": productcode + "_native", "version": productversion } sources_list = querydb.get_product_sources(**native_product) logger.debug("For product [%s] N. %s source is/are found" % (productcode, len(sources_list))) systemsettings = functions.getSystemSettings() for source in sources_list: logger_spec = log.my_logger('apps.ingestion.' + productcode + '.' + productversion) logger.debug("Processing Source type [%s] with id [%s]" % (source.type, source.data_source_id)) # Re-initialize the datasource_descr # datasource_descr = None # Get datasource desctiption datasource_descr = querydb.get_datasource_descr( source_type=source.type, source_id=source.data_source_id) datasource_descr = datasource_descr[0] # TODO optimize this in order to get direct file filter expression my_filter_expr = get_filenaming_info(source, datasource_descr) files = get_files_matching_with_file_expression(my_filter_expr) # See ES2-204 logger_spec.debug( "Number of files found for product [%s] is: %s" % (active_product_ingest[0], len(files))) if len(files) > 0: # Get list of ingestions triggers [prod/subprod/mapset] ingestions = querydb.get_ingestion_subproduct( allrecs=False, **product) # Loop over ingestion triggers subproducts = list() for ingest in ingestions: # TODO if one ingest gives true and another false? dates_not_in_filename = is_date_not_in_filename( ingest.input_to_process_re) logger.debug(" --> processing subproduct: %s" % ingest.subproductcode) args = { "productcode": product['productcode'], "subproductcode": ingest.subproductcode, "datasource_descr_id": datasource_descr.datasource_descr_id, "version": product['version'] } product_in_info = querydb.get_product_in_info(**args) # TODO verify the approach Should we get subproduct from single query subproduct = get_subproduct( ingest, product_in_info, datasource_descr.datasource_descr_id) if subproduct is not None: subproducts.append(subproduct) if subproducts is None: #TODO what to do? logger.error( "For current active ingestion No subproducts for Product [%s] " % (productcode)) # Get the list of unique dates by extracting the date from all files. dates_list = get_list_unique_dates(datasource_descr, files, dates_not_in_filename, product_in_info, ingest.mapsetcode) # Loop over dates and get list of files for in_date in dates_list: date_fileslist = get_dates_file_list( dates_not_in_filename, files, my_filter_expr, in_date, logger_spec) # Pass list of files to ingestion routine if (not dry_run): try: result = ingestion(date_fileslist, in_date, product, subproducts, datasource_descr, logger_spec, echo_query=echo_query) except: logger.error( "Error in ingestion of file [%s] " % (functions.conv_list_2_string( date_fileslist))) else: # Result is None means we are still waiting for some files to be received. Keep files in /data/ingest # dates_not_in_filename means the input files contains many dates (e.g. GSOD precip) if result is not None and not dates_not_in_filename: if source.store_original_data or systemsettings[ 'type_installation'] == 'Server': store_native_files( product, date_fileslist, logger_spec) else: delete_files(date_fileslist, logger_spec) else: time.sleep(10)