def __init__(self):
"""
"""
loader = config_loader.ConfigLoader()
self.configuration = loader.get_config()
loader = config_loader.ConfigLoader(self.configuration['options_file'])
self.finance_configuration = loader.get_config()
self.connection_pool = database_connection_pool.ConnectionPool()
connection = self.connection_pool.get_connection(self.configuration)
self.user_database = user_db.UserDatabase(connection)
self.mail_agent = mailing.Mailing()
def initialize(self):
"""
Load config from file.
Returns:
True/False: 读取配置成功返回 True，否则返回 False
"""
config_loader_inst = config_loader.ConfigLoader(self.config_file_path)
conf_load_res = config_loader_inst.initialize()
if not conf_load_res:
self.program_end('MiniSpider Load config failed!')
return False
self.url_list_file = config_loader_inst.get_url_list_file()
self.output_dir = config_loader_inst.get_output_dir()
self.max_depth = config_loader_inst.get_max_depth()
self.crawl_interval = config_loader_inst.get_crawl_interval()
self.crawl_timeout = config_loader_inst.get_crawl_timeout()
self.target_url = config_loader_inst.get_target_url()
self.thread_count = config_loader_inst.get_thread_count()
self.tag_dict = config_loader_inst.get_tag_dict()
self.url_pattern = re.compile(
self.target_url) # 使用 re.complie 预先编译提升正则匹配性能
seedfile_is_exist = self.get_seed_urls()
return seedfile_is_exist
def get_connection(self):
"""
Create and return a stomp connection into
wanted host message broker.
:return:
"""
loader = config_loader.ConfigLoader()
configuration = loader.get_config()
logger(
"INFO",
"message_broker_interface/connection : start to connecting to JMS")
logger(
"INFO",
"message_broker_interface/connection : jms server host : " +
configuration["check_point.message_server"]["host"])
logger(
"INFO",
"message_broker_interface/connection : jms server port : " +
str(configuration["check_point.message_server"]["port"]))
connection = stomp.Connection([
(configuration["check_point.message_server"]["host"],
configuration["check_point.message_server"]["port"])
])
connection.start()
connection.connect(
configuration["check_point.message_server"]["user_name"],
configuration["check_point.message_server"]["password"],
wait=True)
return connection
def __init__(self,
connection,
connection_listener,
topic,
config_file="monit_engine.yml"):
loader = config_loader.ConfigLoader()
configuration = loader.get_config()
self.connection = connection
self.connection.set_listener("message_broker_interface_listener",
connection_listener.Listener())
self.connection.subscribe(
destination=topic,
id=configuration["monitor_engine.message_server"]["id"],
ack='auto')
logger(
"INFO",
"message_broker/connection_handler : connection stablished successfully for topic : "
+ topic)
thread = threading.Thread(target=self.__run__, args=())
thread.daemon = True
thread.start()
def subscribe(self):
"""
Start to subscribing the check point into main service
:param connection:
:param config_file:
:return:
"""
loader = config_loader.ConfigLoader()
configuration = loader.get_config()
# subscription message dictionary
subscription_message_dictionary = {
"type": "registry",
"queue":
configuration["check_point.message_server"]["subscription"],
"id": configuration["check_point.message_server"]["id"],
"time": date_time.get_date(),
"location": configuration["check_point.specification"]["location"],
"host": configuration["check_point.specification"]["host"]
}
# send data with sender module
sender = send_module.SendModule()
sender.send(destination=configuration["monit_engine.message_queue"]
["registry"],
message=subscription_message_dictionary)
logging.logger(
"INFO",
"subscriber/check_point_subscriber : subscription is complete")
def __init__(self):
"""
"""
loader = config_loader.ConfigLoader()
connection_pool = database_connection_pool.ConnectionPool()
connection = connection_pool.get_connection(configuration=loader.get_config())
self.db = result_db.ResultDatabase(connection=connection)
def __init__(self):
"""
"""
loader = config_loader.ConfigLoader()
self.config = loader.get_config()
self.validator = validator.SchemaValidator(
schema_path=self.config["schema"]["handler_schema"])
self.url_service = url_service.UrlService()
def __init__(self):
"""
"""
loader = config_loader.ConfigLoader()
self.config = loader.get_config()
connection_pool = database_connection_pool.ConnectionPool()
connection = connection_pool.get_connection(self.config)
database = connection[self.config["check_point.data_base"]["database_name"]]
self.collection = database[self.config["check_point.data_base"]["job_collection"]]
def test_parse_name(self, conf_loader_mocker):
# Verify class was mocked successfully
assert conf_loader_mocker == config_loader.ConfigLoader
mock_conf_loader = config_loader.ConfigLoader("test")
scanner = FileScanner(mock_conf_loader)
file = "V4.5_Create_schema.sql"
version, name = scanner._parse_name(file)
assert version == 'V4.5'
assert name == "Create schema"
def get_date(config_file="monit_engine.yml"):
"""
generate and return the date
:return:
"""
loader = config_loader.ConfigLoader()
configuration = loader.get_config()
return strftime(
"%a, %d %b %Y %H:%M:%S " +
str(configuration["monitor_engine.property"]["time_region"]), gmtime())
def __init__(self):
"""
this class created to manage and handle the url actions
"""
loader = config_loader.ConfigLoader()
self.config = loader.get_config()
self.user_service = user_service.UserService()
self.check_point_service = check_point_service.CheckPointService()
self.sender = send_module.SendModule()
connection_pool = database_connection_pool.ConnectionPool()
db_connection = connection_pool.get_connection(self.config)
self.url_database = url_db.UrlDatabase(db_connection)
def main(**kwargs):
counter = 0
long(counter)
cfg = cfg_instance.ConfigLoader(args.config)
while True:
log.info(
"======================== start iteration [%d] ==========================="
% (counter))
monitor = OpenvpnMgmtInterface(cfg, **kwargs)
time.sleep(interval)
log.info(
"======================== end iteration [%d] ==========================="
% (counter))
counter = counter + 1
def __init__(self):
"""
initial starter kit
"""
loader = config_loader.ConfigLoader()
self.configuration = loader.get_config()
logger("INFO", "service_starter : start starter kit to initialization")
# start all of the services
self.__start_all__()
logger(
"INFO",
"service_starter : starter service has completed initialization")
def main():
loader = config_loader.ConfigLoader()
config = loader.get_config()
connector = connection.Connection()
general_connection = connector.get_connection()
connection_handler.Handler(
connection=general_connection,
connection_listener=listener,
topic=config["check_point.message_server"]["subscription"])
# subscribe the check point into monitoring engine
Subscriber().subscribe()
# load jobs into scheduler
url_service.UrlService().load_urls_to_scheduler()
# run the application for ever
keep_alive()
def __init__(self, connection, config_file="monit_engine.yml"):
"""
pass
"""
loader = config_loader.ConfigLoader()
self.configuration = loader.get_config()
# select the correct database
self.database = connection[
self.configuration["monitor_engine.data_base"]["database_name"]]
self.collection = self.database[
self.configuration["monitor_engine.data_base"]["url_collection"]]
logger(
"INFO",
"database_management_system/url_db : url database has initialized")
def __init__(self, connection):
"""
oops
"""
loader = config_loader.ConfigLoader()
self.configuration = loader.get_config()
# select the correct database
self.database = connection[
self.configuration["monitor_engine.data_base"]["database_name"]]
self.collection = self.database[self.configuration[
"monitor_engine.data_base"]["result_collection"]]
logger(
"INFO",
"database_management_system/result_db : result database has initialized"
)
def test_scan(self, conf_loader_mocker):
conf_loader_mocker.return_value.get_migrations_path.return_value = {
'up': os.path.join(test_data_path, "up"),
'down': os.path.join(test_data_path, "down")
}
# Given
mock_conf_loader = config_loader.ConfigLoader("test")
scanner = FileScanner(mock_conf_loader)
# When
migrations_metadata = scanner.scan()
# Then
assert len(migrations_metadata) == 2
for meta in migrations_metadata:
assert type(meta) is ScriptMetadata
def test_find_rollback(self, listdir_mocker, conf_loader_mocker):
down_path = os.path.join(test_data_path, "down")
conf_loader_mocker.return_value.get_migrations_path.return_value = {
'up': os.path.join(test_data_path, "up"),
'down': down_path
}
listdir_mocker.return_value = [
'V1_Create_tables.sql',
'V1.1_Add_tables.sql',
'V2_Add_foreign_keys.sql',
'V3_Add_last_table.sql'
]
mock_conf_loader = config_loader.ConfigLoader("test")
scanner = FileScanner(mock_conf_loader)
rollback_path = scanner._find_rollback("V1")
assert rollback_path == os.path.abspath(os.path.join(
down_path,
"V1_Create_tables.sql"
))
rollback_path = scanner._find_rollback("V3")
assert rollback_path == os.path.abspath(os.path.join(
down_path,
"V3_Add_last_table.sql"
))
rollback_path = scanner._find_rollback("V1.1")
assert rollback_path == os.path.abspath(os.path.join(
down_path,
"V1.1_Add_tables.sql"
))
rollback_path = scanner._find_rollback("V2")
assert rollback_path == os.path.abspath(os.path.join(
down_path,
"V2_Add_foreign_keys.sql"
))
rollback_path = scanner._find_rollback("V43")
assert not rollback_path
def get_connection(self, config_file="monit_engine.yml"):
"""
Create and return a stomp connection into
wanted host message broker.
:return:
"""
loader = config_loader.ConfigLoader()
configuration = loader.get_config()
logger(
"INFO",
"message_broker_interface/connection : start to connecting to JMS")
logger(
"INFO",
"message_broker_interface/connection : jms server host : " +
configuration["monitor_engine.message_server"]["host"])
logger(
"INFO",
"message_broker_interface/connection : jms server port : " +
str(configuration["monitor_engine.message_server"]["port"]))
connection = stomp.Connection([
(configuration["monitor_engine.message_server"]["host"],
configuration["monitor_engine.message_server"]["port"])
])
connection.start()
connection.connect(
configuration["monitor_engine.message_server"]["user_name"],
configuration["monitor_engine.message_server"]["password"],
wait=True)
# Don't subscribe into any queue just return connection itself.
return connection
def setUp(self):
self.conf_loader = config_loader.ConfigLoader('./spider.conf')
@Date : 10/july/2018
"""
# todo : use app scheduler...
# todo : minimize database access
# todo : change database access to use g
from datetime import timedelta
from flask import Flask
from flask_restful import Api
import config_loader
import service_starter
from restful_interface import url, checkpoint, result, user
from flask_cors import CORS
from flask_jwt_extended import JWTManager
loader = config_loader.ConfigLoader()
configuration = loader.get_config()
app = Flask(__name__)
CORS(app)
api = Api(app)
app.config['JWT_SECRET_KEY'] = configuration["monitor_engine.property"][
"jtw_secret_key"]
app.config['JWT_EXPIRATION_DELTA'] = timedelta(
seconds=configuration["monitor_engine.property"]["jwt_expire_delta"])
jwt = JWTManager(app)
blacklist = set()
@jwt.token_in_blacklist_loader
def check_if_token_in_blacklist(decrypted_token):
import csv
import time
import keras
import numpy as np
import config_loader
import generate_training_data
import training
import xrrplots
config = config_loader.ConfigLoader('organic.config')
def main():
# load the model
model_name = config.get_model_name()
custom_object_dict = dict([('abs_err_%d' % (i + 1),
training.y_absolute_error(i + 1))
for i in range(4)])
custom_object_dict['abs_err'] = training.y_absolute_error(0)
model = keras.models.load_model(model_name,
custom_objects=custom_object_dict)
# import your test data
path_test_data = config.get_test_data_file_name()
# path_test_data = 'interpolated_log_reflectivity_SLS_RT.txt'
test_data = np.loadtxt(path_test_data, delimiter='\t')
q_vector = test_data[:, 0] * 1e10
