def __init__(self): """ """ loader = config_loader.ConfigLoader() self.configuration = loader.get_config() loader = config_loader.ConfigLoader(self.configuration['options_file']) self.finance_configuration = loader.get_config() self.connection_pool = database_connection_pool.ConnectionPool() connection = self.connection_pool.get_connection(self.configuration) self.user_database = user_db.UserDatabase(connection) self.mail_agent = mailing.Mailing()
def initialize(self): """ Load config from file. Returns: True/False: 读取配置成功返回 True,否则返回 False """ config_loader_inst = config_loader.ConfigLoader(self.config_file_path) conf_load_res = config_loader_inst.initialize() if not conf_load_res: self.program_end('MiniSpider Load config failed!') return False self.url_list_file = config_loader_inst.get_url_list_file() self.output_dir = config_loader_inst.get_output_dir() self.max_depth = config_loader_inst.get_max_depth() self.crawl_interval = config_loader_inst.get_crawl_interval() self.crawl_timeout = config_loader_inst.get_crawl_timeout() self.target_url = config_loader_inst.get_target_url() self.thread_count = config_loader_inst.get_thread_count() self.tag_dict = config_loader_inst.get_tag_dict() self.url_pattern = re.compile( self.target_url) # 使用 re.complie 预先编译提升正则匹配性能 seedfile_is_exist = self.get_seed_urls() return seedfile_is_exist
def get_connection(self): """ Create and return a stomp connection into wanted host message broker. :return: """ loader = config_loader.ConfigLoader() configuration = loader.get_config() logger( "INFO", "message_broker_interface/connection : start to connecting to JMS") logger( "INFO", "message_broker_interface/connection : jms server host : " + configuration["check_point.message_server"]["host"]) logger( "INFO", "message_broker_interface/connection : jms server port : " + str(configuration["check_point.message_server"]["port"])) connection = stomp.Connection([ (configuration["check_point.message_server"]["host"], configuration["check_point.message_server"]["port"]) ]) connection.start() connection.connect( configuration["check_point.message_server"]["user_name"], configuration["check_point.message_server"]["password"], wait=True) return connection
def __init__(self, connection, connection_listener, topic, config_file="monit_engine.yml"): loader = config_loader.ConfigLoader() configuration = loader.get_config() self.connection = connection self.connection.set_listener("message_broker_interface_listener", connection_listener.Listener()) self.connection.subscribe( destination=topic, id=configuration["monitor_engine.message_server"]["id"], ack='auto') logger( "INFO", "message_broker/connection_handler : connection stablished successfully for topic : " + topic) thread = threading.Thread(target=self.__run__, args=()) thread.daemon = True thread.start()
def subscribe(self): """ Start to subscribing the check point into main service :param connection: :param config_file: :return: """ loader = config_loader.ConfigLoader() configuration = loader.get_config() # subscription message dictionary subscription_message_dictionary = { "type": "registry", "queue": configuration["check_point.message_server"]["subscription"], "id": configuration["check_point.message_server"]["id"], "time": date_time.get_date(), "location": configuration["check_point.specification"]["location"], "host": configuration["check_point.specification"]["host"] } # send data with sender module sender = send_module.SendModule() sender.send(destination=configuration["monit_engine.message_queue"] ["registry"], message=subscription_message_dictionary) logging.logger( "INFO", "subscriber/check_point_subscriber : subscription is complete")
def __init__(self): """ """ loader = config_loader.ConfigLoader() connection_pool = database_connection_pool.ConnectionPool() connection = connection_pool.get_connection(configuration=loader.get_config()) self.db = result_db.ResultDatabase(connection=connection)
def __init__(self): """ """ loader = config_loader.ConfigLoader() self.config = loader.get_config() self.validator = validator.SchemaValidator( schema_path=self.config["schema"]["handler_schema"]) self.url_service = url_service.UrlService()
def __init__(self): """ """ loader = config_loader.ConfigLoader() self.config = loader.get_config() connection_pool = database_connection_pool.ConnectionPool() connection = connection_pool.get_connection(self.config) database = connection[self.config["check_point.data_base"]["database_name"]] self.collection = database[self.config["check_point.data_base"]["job_collection"]]
def test_parse_name(self, conf_loader_mocker): # Verify class was mocked successfully assert conf_loader_mocker == config_loader.ConfigLoader mock_conf_loader = config_loader.ConfigLoader("test") scanner = FileScanner(mock_conf_loader) file = "V4.5_Create_schema.sql" version, name = scanner._parse_name(file) assert version == 'V4.5' assert name == "Create schema"
def get_date(config_file="monit_engine.yml"): """ generate and return the date :return: """ loader = config_loader.ConfigLoader() configuration = loader.get_config() return strftime( "%a, %d %b %Y %H:%M:%S " + str(configuration["monitor_engine.property"]["time_region"]), gmtime())
def __init__(self): """ this class created to manage and handle the url actions """ loader = config_loader.ConfigLoader() self.config = loader.get_config() self.user_service = user_service.UserService() self.check_point_service = check_point_service.CheckPointService() self.sender = send_module.SendModule() connection_pool = database_connection_pool.ConnectionPool() db_connection = connection_pool.get_connection(self.config) self.url_database = url_db.UrlDatabase(db_connection)
def main(**kwargs): counter = 0 long(counter) cfg = cfg_instance.ConfigLoader(args.config) while True: log.info( "======================== start iteration [%d] ===========================" % (counter)) monitor = OpenvpnMgmtInterface(cfg, **kwargs) time.sleep(interval) log.info( "======================== end iteration [%d] ===========================" % (counter)) counter = counter + 1
def __init__(self): """ initial starter kit """ loader = config_loader.ConfigLoader() self.configuration = loader.get_config() logger("INFO", "service_starter : start starter kit to initialization") # start all of the services self.__start_all__() logger( "INFO", "service_starter : starter service has completed initialization")
def main(): loader = config_loader.ConfigLoader() config = loader.get_config() connector = connection.Connection() general_connection = connector.get_connection() connection_handler.Handler( connection=general_connection, connection_listener=listener, topic=config["check_point.message_server"]["subscription"]) # subscribe the check point into monitoring engine Subscriber().subscribe() # load jobs into scheduler url_service.UrlService().load_urls_to_scheduler() # run the application for ever keep_alive()
def __init__(self, connection, config_file="monit_engine.yml"): """ pass """ loader = config_loader.ConfigLoader() self.configuration = loader.get_config() # select the correct database self.database = connection[ self.configuration["monitor_engine.data_base"]["database_name"]] self.collection = self.database[ self.configuration["monitor_engine.data_base"]["url_collection"]] logger( "INFO", "database_management_system/url_db : url database has initialized")
def __init__(self, connection): """ oops """ loader = config_loader.ConfigLoader() self.configuration = loader.get_config() # select the correct database self.database = connection[ self.configuration["monitor_engine.data_base"]["database_name"]] self.collection = self.database[self.configuration[ "monitor_engine.data_base"]["result_collection"]] logger( "INFO", "database_management_system/result_db : result database has initialized" )
def test_scan(self, conf_loader_mocker): conf_loader_mocker.return_value.get_migrations_path.return_value = { 'up': os.path.join(test_data_path, "up"), 'down': os.path.join(test_data_path, "down") } # Given mock_conf_loader = config_loader.ConfigLoader("test") scanner = FileScanner(mock_conf_loader) # When migrations_metadata = scanner.scan() # Then assert len(migrations_metadata) == 2 for meta in migrations_metadata: assert type(meta) is ScriptMetadata
def test_find_rollback(self, listdir_mocker, conf_loader_mocker): down_path = os.path.join(test_data_path, "down") conf_loader_mocker.return_value.get_migrations_path.return_value = { 'up': os.path.join(test_data_path, "up"), 'down': down_path } listdir_mocker.return_value = [ 'V1_Create_tables.sql', 'V1.1_Add_tables.sql', 'V2_Add_foreign_keys.sql', 'V3_Add_last_table.sql' ] mock_conf_loader = config_loader.ConfigLoader("test") scanner = FileScanner(mock_conf_loader) rollback_path = scanner._find_rollback("V1") assert rollback_path == os.path.abspath(os.path.join( down_path, "V1_Create_tables.sql" )) rollback_path = scanner._find_rollback("V3") assert rollback_path == os.path.abspath(os.path.join( down_path, "V3_Add_last_table.sql" )) rollback_path = scanner._find_rollback("V1.1") assert rollback_path == os.path.abspath(os.path.join( down_path, "V1.1_Add_tables.sql" )) rollback_path = scanner._find_rollback("V2") assert rollback_path == os.path.abspath(os.path.join( down_path, "V2_Add_foreign_keys.sql" )) rollback_path = scanner._find_rollback("V43") assert not rollback_path
def get_connection(self, config_file="monit_engine.yml"): """ Create and return a stomp connection into wanted host message broker. :return: """ loader = config_loader.ConfigLoader() configuration = loader.get_config() logger( "INFO", "message_broker_interface/connection : start to connecting to JMS") logger( "INFO", "message_broker_interface/connection : jms server host : " + configuration["monitor_engine.message_server"]["host"]) logger( "INFO", "message_broker_interface/connection : jms server port : " + str(configuration["monitor_engine.message_server"]["port"])) connection = stomp.Connection([ (configuration["monitor_engine.message_server"]["host"], configuration["monitor_engine.message_server"]["port"]) ]) connection.start() connection.connect( configuration["monitor_engine.message_server"]["user_name"], configuration["monitor_engine.message_server"]["password"], wait=True) # Don't subscribe into any queue just return connection itself. return connection
def setUp(self): self.conf_loader = config_loader.ConfigLoader('./spider.conf')
@Date : 10/july/2018 """ # todo : use app scheduler... # todo : minimize database access # todo : change database access to use g from datetime import timedelta from flask import Flask from flask_restful import Api import config_loader import service_starter from restful_interface import url, checkpoint, result, user from flask_cors import CORS from flask_jwt_extended import JWTManager loader = config_loader.ConfigLoader() configuration = loader.get_config() app = Flask(__name__) CORS(app) api = Api(app) app.config['JWT_SECRET_KEY'] = configuration["monitor_engine.property"][ "jtw_secret_key"] app.config['JWT_EXPIRATION_DELTA'] = timedelta( seconds=configuration["monitor_engine.property"]["jwt_expire_delta"]) jwt = JWTManager(app) blacklist = set() @jwt.token_in_blacklist_loader def check_if_token_in_blacklist(decrypted_token):
import csv import time import keras import numpy as np import config_loader import generate_training_data import training import xrrplots config = config_loader.ConfigLoader('organic.config') def main(): # load the model model_name = config.get_model_name() custom_object_dict = dict([('abs_err_%d' % (i + 1), training.y_absolute_error(i + 1)) for i in range(4)]) custom_object_dict['abs_err'] = training.y_absolute_error(0) model = keras.models.load_model(model_name, custom_objects=custom_object_dict) # import your test data path_test_data = config.get_test_data_file_name() # path_test_data = 'interpolated_log_reflectivity_SLS_RT.txt' test_data = np.loadtxt(path_test_data, delimiter='\t') q_vector = test_data[:, 0] * 1e10