def load(self, apply_ignored=True):
"""
Load configurations from configuration repository at the given path.
:param bool apply_ignored: whether 'ignored' settings
should be taken into account
"""
self.config_loader = ConfigLoader(self.args.config, self.settings,
apply_ignored)
self.entities = self.config_loader.load()
if self.okta_users:
if self.args.action == 'check':
self.lg.info('Okta user loading not supported in test')
self.entities['user'] = {}
self.okta_groups = []
return
# only groups defined both in IPA & Okta are taken for Okta users
ipa_groups = self.entities.get('group', []).keys()
self.okta_loader = OktaLoader(self.settings, ipa_groups)
if self.entities.get('user'):
self.lg.warning(
'%d users parsed from Git but will be overwritten by Okta',
len(self.entities['user']))
self.entities['user'] = self.okta_loader.load()
# parse Okta groups to use for constructing diff
self.okta_groups = self.okta_loader.load_groups()
def main():
config = ConfigLoader('config.ini')
feed_fetcher = FeedFetcher(config)
# initial poll to get most recent alert id
print("Initial fetch to set references...")
feed_fetcher.poll()
mail = MailSender(config)
def job():
try:
new_results = feed_fetcher.poll_new()
new_results.reverse()
for result in new_results:
mail.mail_entry(result)
print(f"Mailing {len(new_results)} entries")
except FeedFetcher.Inaccessible:
print('Failed to load feed, deferring to next interval')
print("Scheduling...")
schedule.every(config.get_interval()).minutes.do(job)
print("Now looping!")
while True:
schedule.run_pending()
time.sleep(5)
def setup(
self, filepath: str, make_outputs: bool = True
):
"""
Load a configuration file and create output directories.
Parameters
----------
filepath : str
Path to the configuration file.
make_outputs: bool
If true, create output directories, but only when output_dir_key exists.
Returns
-------
config : Configuration
"""
config_loader = ConfigLoader()
self.cfg = config_loader.load(filepath)
self._validate_dic_keys()
if self.cfg["EXPERIMENT"]["USER"] != getpass.getuser():
raise WrongUserKeyError("USER value in config file does not match current user ")
if make_outputs:
self._create_output_dirs()
return self.cfg
def __init__(self):
self.conn = None
self.engine = None
cfg = ConfigLoader()
self.config = cfg.get_config()
self.mysql_host = self.config.get('mysql_host')
self.mysql_user = self.config.get('mysql_user')
self.mysql_pass = self.config.get('mysql_pass')
self.mysql_database = self.config.get('mysql_database')
self.logging = Logger()
conn_string = "mysql+mysqldb://{0}:{1}@{2}/{3}".format(
self.mysql_user, self.mysql_pass, self.mysql_host,
self.mysql_database)
self.engine = create_engine(conn_string)
meta = MetaData(bind=self.engine)
self.dbconn = None
### Recommendations Table ###
self.recommendations = Table(
'product_recommendations', meta,
Column('product_id', TEXT, nullable=False),
Column('related_product_id', TEXT, nullable=False),
Column('school_code', Integer, nullable=True),
Column('subject_code', Integer, nullable=True),
Column('class_year', Integer, nullable=True),
Column('tag_similarity', Integer, nullable=True),
Column('similarity', Integer, nullable=True),
Column('similarity_score', Integer, nullable=True))
def __init__(self):
cfg = ConfigLoader()
self.config = cfg.get_config()
host = self.config.get('redis_host')
port = self.config.get('redis_port')
db = self.config.get('redis_db')
self.ttl = self.config.get('redis_ttl')
self.db = redis.StrictRedis(host=host, port=port, db=db)
def configure(self, df, config_num):
"""
Configures NN inputs - selects config_num and creates train/test split
"""
self.config_num = config_num
CL = ConfigLoader(df, self.channel)
X_train, X_test, y_train, y_test = CL.configTrainTestData(self.config_num, self.binary)
return X_train, X_test, y_train, y_test
def __init__(self, config_loader: ConfigLoader, parser: Parser):
self.__high_frequency_threshold = config_loader.get_high_frequency_threshold(
)
self.__parser = parser
self.__sym_spell_filtered_file_path = config_loader.get_sym_spell_filtered_file_path(
)
self.__sym_spell = SymSpell(max_dictionary_edit_distance=2,
prefix_length=7)
self.__english_dictionary = set(words.words())
def load(self, apply_ignored=True):
"""
Load configurations from configuration repository at the given path.
:param bool apply_ignored: whether 'ignored' seetings
should be taken into account
"""
self.config_loader = ConfigLoader(self.args.config, self.settings,
apply_ignored)
self.entities = self.config_loader.load()
def __init__(self):
"""
this class created for handling and dealing with check points
"""
config_loader = ConfigLoader()
self.config = config_loader.get_config()
connection_pool = database_connection_pool.ConnectionPool()
connection = connection_pool.get_connection(self.config)
self.database = check_point_db.CheckPointDatabase(
connection=connection)
def test_init_no_user_configs_but_merge_specified(self):
kwargs = {"operator_merge_default": "eh"}
with(self.assertRaises(Exception)):
cl = ConfigLoader(**kwargs)
kwargs = {"token_merge_default": "eh"}
with(self.assertRaises(Exception)):
cl = ConfigLoader(**kwargs)
kwargs = {"parselet_merge_default": "eh"}
with(self.assertRaises(Exception)):
cl = ConfigLoader(**kwargs)
def __init__(self):
"""
the loaded config file will pass into this class
:param config:
"""
loader = ConfigLoader()
self.config = loader.get_config()
self.smtp_ssl_host = self.config["mail_service.mail_host"]["email_host"][0]
self.smtp_ssl_port = self.config["mail_service.mail_host"]["email_port"]
self.username = self.config["mail_service.mail_host"]["email_user_name"]
self.password = self.config["mail_service.mail_host"]["email_host_password"]
self.sender = self.config["mail_service.mail_host"]["email_user_name"]
class SQLiteDatabase:
def __init__(self, env):
self.config_loader = ConfigLoader(env)
self.db_filename = self.config_loader.get_db_filename()
logging.basicConfig(filename=self.config_loader.get_log_filename(), level=logging.INFO,
format="[%(levelname)s] %(asctime)s: %(message)s")
try:
# sqlite will create a new db if file doesn't exist, check before connecting
open(self.db_filename, "r").close()
except IOError:
raise IOError("Couldn't find database file.")
def connect(self):
self.connection = sqlite3.connect(self.db_filename)
self.connection.text_factory = str
self.cursor = self.connection.cursor()
logging.info("Database connection established.")
def close(self):
self.cursor.close()
self.connection.close()
logging.info("Database connection closed.")
def run_sql(self, sql, parameters):
self.cursor.execute(sql, parameters)
return self.cursor.fetchall()
def autocomplete_table(self, incomplete_name):
table_name_matches = self.run_sql("SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE ?",
[incomplete_name + "%"])
if len(table_name_matches) == 1:
return table_name_matches[0][0]
else:
raise ValueError("Found no exact match for the given table name fragment: %s" % incomplete_name)
def autocomplete_column_from_table(self, table_name, incomplete_name):
columns_in_table = self.run_sql("PRAGMA table_info(%s)" % table_name, "")
# result column header cid|name|type|notnull|dflt_value|pk
column_name_matches = [column_line[1] for column_line in columns_in_table
if column_line[1].startswith(incomplete_name)]
if len(column_name_matches) == 1:
return column_name_matches[0]
else:
raise ValueError("Found no exact match for the given column name fragment: %s" % incomplete_name)
def get_primary_key_for(self, table_name):
columns_in_table = self.run_sql("PRAGMA table_info(%s)" % table_name, "")
# result column header cid|name|type|notnull|dflt_value|pk
primary_key_line = next(column for column in columns_in_table if column[5] == 1)
return primary_key_line[1]
def get_table_list(self):
return [table_line[0] for table_line in self.run_sql("SELECT name FROM sqlite_master WHERE type = 'table'", "")]
def __init__(self, config_file_name, dataset_files):
"""Initialise the program.
:param config_file_name: Name of the config file.
:param dataset_files: List of dataset file names.
"""
self._config_file_path = path.join(self._WORKING_DIR, self._CONFIG_DIR,
config_file_name)
self._config_loader = ConfigLoader(self._config_file_path)
self._config_loader.read_config()
self._dataset_files = dataset_files
self._iscx2012_loader = ISCX2012IDS(dataset_files)
def __init__(self):
cfg = ConfigLoader()
self.config = cfg.load()
self.MONGO_HOST = self.config.get('mongo_host')
self.MONGO_PORT = self.config.get('mongo_port')
self.USER_LIMIT = self.config.get('data_user_limit')
self.DOC_LIMIT = self.config.get('data_doc_limit')
self.MIN_DOWNLOADS = self.config.get('data_min_downloads')
client = MongoClient(self.MONGO_HOST, self.MONGO_PORT)
self.db = client.mU
self.users = None
def __init__(self):
# logging.basicConfig(level=logging.DEBUG,
# format='[%(levelname)s] (%(threadName)-10s) %(message)s',
# )
self.logging = Logger()
self.repo = MongoRepository()
self.mysql = MysqlRepository()
cfg = ConfigLoader()
self.config = cfg.get_config()
self.CUTTOFF_DAYS = timedelta(
days=self.config.get('data_days_to_analyze'))
self.redis = RedisRepository()
self.ar_counter = 0
self.cb_counter = 0
def main():
loader = ConfigLoader("config.ini",['ubuntu','staging','development'])
print(loader.get('ftp'))
print(loader.get('ftp.name'))
print(loader.get('ftp.enabled'))
print(loader.get('poda'))
print(loader.get('common'))
print(loader.get('common.basic_size_limit'))
def __init__(self, contr):
# Load config
path_to_config = os.path.dirname(__file__) + "/config/"
self._config = ConfigLoader(path_to_config +
self._CONFIG_POLICIES_FILE_NAME,
path_to_config +
self._CONFIG_RULE_FILE_NAME)
# Set logging
logging_config = self._config.get_logging_config()
self._logging = logging.getLogger(__name__)
self._logging.setLevel(logging_config["min_lvl"])
self._logging.propagate = logging_config["propagate"]
self._logging.addHandler(logging_config["handler"])
# Set flow table numbers
self._table_id_blacklist = 0
self._table_id_whitelist = 1
self._table_id_next = 2
self._contr = contr
self._supported = self._verify_contr_handlers()
self._logging.info("Starting ACLSwitch...")
# Create objects to manage different features
self._flow_man = FlowManager(self, logging_config)
self._api = ACLSwitchAPI(logging_config, self._VERSION,
self._flow_man)
self._api.policy_create(self._POLICY_DEFAULT)
# Read config files
# TODO Command line argument for custom location for config file
policies, pd_assignments = self._config.load_policies()
rules = self._config.load_rules()
for pol in policies:
self._api.policy_create(pol)
for rule in rules:
self._api.acl_create_rule(rule)
for assignment in pd_assignments:
self._api.switch_register(assignment["switch_id"])
self._api.policy_assign_switch(assignment["switch_id"],
assignment["policy"],
from_file=True)
# Register REST WSGI through the controller app
self._contr.register_rest_wsgi(ACLSwitchREST, kwargs={
self._INSTANCE_NAME_ASW_API: self._api})
self._logging.info("ACLSwitch started successfully.")
def __init__(self):
""" Virtually private constructor. """
if Scheduler.__instance is not None:
raise Exception("This class is a singleton!")
else:
Scheduler.__instance = self
config_loader = ConfigLoader()
self.config = config_loader.get_config()
executors = {
'default': ThreadPoolExecutor(self.config['check_point.scheduler']['thread_pool_executor']),
'processpool': ProcessPoolExecutor(self.config['check_point.scheduler']['process_pool_executor'])
}
job_defaults = {
'coalesce': False,
'max_instances': 10
}
self.scheduler = BackgroundScheduler(executors=executors, job_defaults=job_defaults)
self.scheduler.start()
def __init__(self, env):
self.config_loader = ConfigLoader(env)
self.db_filename = self.config_loader.get_db_filename()
logging.basicConfig(filename=self.config_loader.get_log_filename(), level=logging.INFO,
format="[%(levelname)s] %(asctime)s: %(message)s")
try:
# sqlite will create a new db if file doesn't exist, check before connecting
open(self.db_filename, "r").close()
except IOError:
raise IOError("Couldn't find database file.")
def test_init_merge_defaults_when_use_default_not_present(self):
kwargs = {
"operator_merge_default": True,
"config_location": "eh"
}
cl = ConfigLoader(**kwargs)
self.assertTrue(cl.operator_merge_default)
kwargs = {
"parselet_merge_default": True,
"config_location": "eh"
}
cl = ConfigLoader(**kwargs)
self.assertTrue(cl.parselet_merge_default)
kwargs = {
"token_merge_default": True,
"config_location": "eh"
}
cl = ConfigLoader(**kwargs)
self.assertTrue(cl.token_merge_default)
def __init__(self, config_kwargs=None):
""" Given the configuration options specified, build the parser
environment.
Arguments:
config_kwargs (dict): options for configuration, see config_loader
"""
if not config_kwargs:
config_kwargs = {}
self.configurator = Configurator(ConfigLoader(**config_kwargs).load())
self.var_repo = VarRepo()
self.parser_instance = None
self.tokenizer = None
def test_init_no_merge_if_both_not_true(self):
kwargs = {
"operator_merge_default": True,
"config_location": "eh",
"operator_use_default": False
}
cl = ConfigLoader(**kwargs)
self.assertFalse(cl.operator_merge_default)
kwargs = {
"parselet_merge_default": True,
"config_location": "eh",
"parselet_use_default": False
}
cl = ConfigLoader(**kwargs)
self.assertFalse(cl.parselet_merge_default)
kwargs = {
"token_merge_default": True,
"config_location": "eh",
"token_use_default": False
}
cl = ConfigLoader(**kwargs)
self.assertFalse(cl.token_merge_default)
def __init__(self):
conf_loader = ConfigLoader()
dict_conf = conf_loader.dict_conf
dict_bm = conf_loader.dict_bookmark
self.conf_loader = conf_loader
self.bm_viewer = BmViewer(self, dict_bm['bookmarks'])
# init viewers
self.var_zsbook_search_viewer = ZSBookSearchViewer(self)
self.var_menu_viewer = MenuViewer(self)
self.var_book_viewer = EBookReader(self)
self.var_img_viewer = EImgReader(self)
self.reader_viewer = None
self.var_zsbook_loader = ZSBookLoader()
self.var_ebook_loader = EBookLoader(dict_conf)
self.var_eing_loader = EImgLoader(dict_conf)
self.var_loader = None
self.curr_type = None
bottom_view = HomeViewer(self).view
view = ui.NavigationView(bottom_view)
self.stack_views = [bottom_view] # 额外保存，用于读取navigation view栈内信息
view.navigation_bar_hidden = True
btn_item_pop = ui.ButtonItem(
image=ui.Image.named('iob:arrow_return_left_24'))
btn_item_pop.action = self.pop_1_view
btn_item_push_menu = ui.ButtonItem(
image=ui.Image.named('iob:navicon_round_24'))
btn_item_push_menu.action = self.push_menu_view
self.btn_items_pop = [btn_item_pop]
self.btn_items_push_menu = [btn_item_push_menu]
self.view = view
self.set_right_button_items(bottom_view.right_btns_desc)
self.queue_body = None
self.queue_index = None
self.is_req_body = False
self.is_req_index = False
def __init__(self, ELoader, ReaderViewer):
conf_loader = ConfigLoader()
dict_conf = conf_loader.dict_conf
dict_bm = conf_loader.dict_bookmark
self.reader_viewer = ReaderViewer(self)
self.bm_viewer = BmViewer(self, dict_bm['bookmarks'])
self.menu_viewer = MenuViewer(self)
self.var_eloader = ELoader(dict_conf)
self.navi_viewer = NaviViewer(
self, self.reader_viewer.var_ebody_viewer.reader_view)
self.conf_loader = conf_loader
self.queue_ebody = Queue()
self.queue_eindex = Queue()
self.has_sent_req = False
self.has_sent_eindex_req = False
def readConfig(self, env_var, path):
self.__config_loader = ConfigLoader(path)
self.ready = False
if env_var:
self.ready = self.__config_loader.readFromEnv()
else:
self.ready = self.__config_loader.readFromFile()
writers = self.__config_loader.writers
readers = self.__config_loader.readers
initial_collabs = self.__config_loader.initial_collabs
typing_speed = self.__config_loader.typing_speed
self.__clients_manager = ClientsManager(self,
readers,
writers,
initial_collabs,
typing_speed)
return self.ready
def load(config_path):
global CALIBRATION_STAGES, CALIBRATION_CONFIGS, TARGET_SOURCE_STAGES, \
TARGET_SOURCE_CONFIGS, MAIN_STAGES, PIPELINE_CONFIGS, \
GLOBAL_CONFIGS, CONFIG_PATH, CASA_CONFIGS, IMAGING_CONFIGS
CALIBRATION_STAGES = ConfigLoader().load(
config_path + "calibration.yml")['calibration_stages']
CALIBRATION_CONFIGS = ConfigLoader().load(config_path +
"calibration.yml")['calibration']
PIPELINE_CONFIGS = ConfigLoader().load(config_path + "pipeline.yml")
TARGET_SOURCE_STAGES = ConfigLoader().load(
config_path + "target_source.yml")['target_source_stages']
TARGET_SOURCE_CONFIGS = ConfigLoader().load(
config_path + "target_source.yml")['target_source']
IMAGING_CONFIGS = ConfigLoader().load(config_path + "imaging.yml")
CASA_CONFIGS = ConfigLoader().load(config_path + "casa.yml")
GLOBAL_CONFIGS = PIPELINE_CONFIGS['global']
MAIN_STAGES = PIPELINE_CONFIGS['stages']
CONFIG_PATH = config_path
def write_user(self, dict_new):
self.webhub.set_status(dict_new)
ConfigLoader().write_user(dict_new, self.user_id)
import os
from time import sleep
from GUI.ka_gui import ka_gui
from config_loader import ConfigLoader
from measurements_retriever import MeasurementsRetriever
from utils import check_limit
import datetime
import argparse
if __name__ == '__main__':
config_loader = ConfigLoader()
config_loader.parse_config()
monitors = config_loader.get_monitors()
refresh_period = config_loader.get_refresh_period()
parser = argparse.ArgumentParser(description="Wyswietla dane z monitorow")
parser.add_argument("--sort", help="Nazwa metryki po ktorej maja byc sortowane wyniki", type=str)
parser.add_argument("--monitor", help="Nazwa monitora dla ktorego maja byc wyswietlone wyniki", type=str)
parser.add_argument('--limit', help="Ilosc wypisywanych wynikow", type=check_limit)
args = parser.parse_args()
measurements_retriever = MeasurementsRetriever(monitors)
limit = 10
if args.limit:
limit = args.limit
gui = ka_gui(limit)
os.system('cls' if os.name == "nt" else "clear")
while 1:
from config_loader import ConfigLoader
from documentation_spider import DocumentationSpider
from scrapy.crawler import CrawlerProcess
from strategies.default_strategy import DefaultStrategy
from custom_middleware import CustomMiddleware
try:
# disable boto (S3 download)
from scrapy import optional_features
if 'boto' in optional_features:
optional_features.remove('boto')
except ImportError:
pass
CONFIG = ConfigLoader()
CustomMiddleware.driver = CONFIG.driver
DocumentationSpider.NB_INDEXED = 0
if CONFIG.use_anchors:
import scrapy_patch
STRATEGIES = {
'default': DefaultStrategy
}
CONFIG_STRATEGY = CONFIG.strategy
if CONFIG_STRATEGY not in STRATEGIES:
exit("Strategy '" + CONFIG_STRATEGY + "' does not exist")
STRATEGY = STRATEGIES[CONFIG_STRATEGY](CONFIG)
# command line arguments: lib_path conf_path log_path task
if __name__ == "__main__":
# check command line arguments
check_result = check_args(sys.argv)
if type(check_result) is str:
print(check_result)
exit(1)
else:
(lib_path, conf_path, log_path, task) = check_result
# non-builtin modules mustn't be imported before this statement
sys.path.insert(0, lib_path)
# load configurations
config = ConfigLoader.load(conf_path)
config["paths"] = dict()
config["paths"]["lib_path"] = lib_path
config["paths"]["conf_path"] = conf_path
config["paths"]["log_path"] = log_path
config["paths"]["job_db_path"] = os.path.join(log_path, "jobs.db")
config["paths"]["pid_path"] = os.path.join(log_path, "pid")
# check configurations
check_result = check_config(config)
if type(check_result) is str:
print(check_result)
exit(1)
else:
(job_manager_class, slave_class, uploader_class) = check_result
class Controller(object):
"""docstring for Manager"""
def __init__(self):
self.__exp_sarted = False
self.__exp_ended = False
self.ready = False
self.__beginning_time = 'Not yet started'
self.__persistence_manager = PersistenceManager(self)
def readConfig(self, env_var, path):
self.__config_loader = ConfigLoader(path)
self.ready = False
if env_var:
self.ready = self.__config_loader.readFromEnv()
else:
self.ready = self.__config_loader.readFromFile()
writers = self.__config_loader.writers
readers = self.__config_loader.readers
initial_collabs = self.__config_loader.initial_collabs
typing_speed = self.__config_loader.typing_speed
self.__clients_manager = ClientsManager(self,
readers,
writers,
initial_collabs,
typing_speed)
return self.ready
def addNode(self, node_id):
return self.__clients_manager.addNode(node_id)
def acknowledgeRegistration(self, node_id):
self.__clients_manager.acknowledgeRegistration(node_id)
def saveResults(self, node_id, results):
self.__persistence_manager.saveResults(node_id, results)
def startExp(self):
logger.debug("=== Experience will start NOW ===")
self.__exp_sarted = True
self.__beginning_time = time.strftime("%H:%M:%S")
self.__rmq_communication = RMQCommunication(self)
self.__rmq_communication.broadcastStartSignal()
def getReadyCollabsNmbr(self):
return self.__clients_manager.getReadyCollabsNmbr()
def getNoAckCollabsNmbr(self):
return self.__clients_manager.getNoAckCollabsNmbr()
def getStatus(self):
return {
'ready': self.getReadyCollabsNmbr(),
'no_ack': self.getNoAckCollabsNmbr(),
'is_exp_started': self.__exp_sarted,
'is_exp_ended': self.__exp_ended,
'beginning_time': self.__beginning_time,
}
def getConfig(self):
return self.__config_loader.getJSONConfig()
def __init__(self):
"""
"""
loader = ConfigLoader()
self.config = loader.get_config()
self.sender = send_module.SendModule()
def configure(self):
config = ConfigLoader(self.config_path + "settings.ini")
self.nick = config.get_nick()
self.network = config.get_network()
self.port = config.get_port()
self.channel = config.get_channel()
import bili_console
import threading
from cdn import Host
from super_user import SuperUser
if sys.platform == 'win32':
loop = asyncio.ProactorEventLoop()
asyncio.set_event_loop(loop)
else:
loop = asyncio.get_event_loop()
fileDir = os.path.dirname(os.path.realpath('__file__'))
file_color = f'{fileDir}/config/color.toml'
file_user = f'{fileDir}/config/user.toml'
file_bili = f'{fileDir}/config/bili.toml'
file_ip = f'{fileDir}/config/ips.toml'
cfg = ConfigLoader(file_color, file_user, file_bili, file_ip)
dict_user = cfg.read_user()
dict_bili = cfg.read_bili()
dict_color = cfg.read_color()
dict_ip = cfg.read_ip()
Printer(dict_color, dict_user['print_control']['danmu'],
dict_user['platform']['platform'])
task_control = dict_user['task_control']
if len(dict_user['users']) < 10:
users = [
User(i, user_info, dict_bili, task_control, False)
for i, user_info in enumerate(dict_user['users'])
]
else:
def test_init_use_defaults_by_default(self):
kwargs = {}
cl = ConfigLoader(**kwargs)
self.assertTrue(cl.operator_use_default)
self.assertTrue(cl.parselet_use_default)
self.assertTrue(cl.token_use_default)
class ACLSwitch(ABCRyuApp):
"""Main class for ACLSwitch.
"""
_APP_NAME = "ACLSwitch"
_CONFIG_POLICIES_FILE_NAME = "policy_domains.yaml"
_CONFIG_RULE_FILE_NAME = "acl_rules.yaml"
_EXPECTED_HANDLERS = (EventOFPSwitchFeatures.__name__, )
_INSTANCE_NAME_ASW_API = "asw_api"
# Default priority is defined to be in the middle (0x8000 in 1.3)
# Note that for a priority p, 0 <= p <= MAX (i.e. 65535)
_OFP_MAX_PRIORITY = ofproto_v1_3.OFP_DEFAULT_PRIORITY * 2 - 1
_POLICY_DEFAULT = "default"
_RULE_TCP = "tcp"
_RULE_UDP = "udp"
_RULE_WILDCARD = "*"
# TODO Table IDs should be obtained from an application with higher knowledge.
# An api call to the controller. ACLSwitch should have no idea what
# what other apps have what table IDS, just what table to forward
# entries onto.
_VERSION = "1.1.0"
def __init__(self, contr):
# Load config
path_to_config = os.path.dirname(__file__) + "/config/"
self._config = ConfigLoader(path_to_config +
self._CONFIG_POLICIES_FILE_NAME,
path_to_config +
self._CONFIG_RULE_FILE_NAME)
# Set logging
logging_config = self._config.get_logging_config()
self._logging = logging.getLogger(__name__)
self._logging.setLevel(logging_config["min_lvl"])
self._logging.propagate = logging_config["propagate"]
self._logging.addHandler(logging_config["handler"])
# Set flow table numbers
self._table_id_blacklist = 0
self._table_id_whitelist = 1
self._table_id_next = 2
self._contr = contr
self._supported = self._verify_contr_handlers()
self._logging.info("Starting ACLSwitch...")
# Create objects to manage different features
self._flow_man = FlowManager(self, logging_config)
self._api = ACLSwitchAPI(logging_config, self._VERSION,
self._flow_man)
self._api.policy_create(self._POLICY_DEFAULT)
# Read config files
# TODO Command line argument for custom location for config file
policies, pd_assignments = self._config.load_policies()
rules = self._config.load_rules()
for pol in policies:
self._api.policy_create(pol)
for rule in rules:
self._api.acl_create_rule(rule)
for assignment in pd_assignments:
self._api.switch_register(assignment["switch_id"])
self._api.policy_assign_switch(assignment["switch_id"],
assignment["policy"],
from_file=True)
# Register REST WSGI through the controller app
self._contr.register_rest_wsgi(ACLSwitchREST, kwargs={
self._INSTANCE_NAME_ASW_API: self._api})
self._logging.info("ACLSwitch started successfully.")
def add_acl_fte(self, switch_id, rule):
"""Add a rule to the flow table as a flow table entry.
:param switch_id: The switch to add an entry to.
:param rule: The rule to add.
"""
datapath = self._contr.switch_get_datapath(switch_id)
ofproto = datapath.ofproto
parser = datapath.ofproto_parser
priority = self._OFP_MAX_PRIORITY
if rule.action == "drop":
actions = []
inst = [parser.OFPInstructionActions(
ofproto.OFPIT_APPLY_ACTIONS, actions)]
table = self._table_id_blacklist
else:
inst = [parser.OFPInstructionGotoTable(self._table_id_next)]
table = self._table_id_whitelist
match = self._create_match(rule)
if rule.time_enforce == "N/A":
hard_timeout = 0
else:
hard_timeout = rule.time_enforce[1]
self._contr.add_flow(datapath, priority, match, inst,
hard_timeout, table)
def remove_acl_fte(self, switch_id, rule):
"""Remove a flow table entry.
:param switch_id: The switch to remove the entry from.
:param rule: The rule to remove.
"""
datapath = self._contr.switch_get_datapath(switch_id)
ofproto = datapath.ofproto
parser = datapath.ofproto_parser
table = self._table_id_blacklist if rule.action == "drop" else\
self._table_id_whitelist
remove_type = ofproto.OFPFC_DELETE_STRICT
priority = self._OFP_MAX_PRIORITY
match = self._create_match(rule)
out_port = ofproto.OFPP_ANY
out_group = ofproto.OFPG_ANY
self._contr.remove_flow(datapath, parser, table, remove_type,
priority, match, out_port, out_group)
def _create_match(self, rule):
"""Create an OFPMatch instance based on the contents of an
ACL_ENTRY.
:param rule: The rule entry to create an OFPMatch instance from.
:return: The OFPMatch instance.
"""
match = ofp13_parser.OFPMatch()
ip_version = self._return_ip_version(rule.ip_src, rule.ip_dst)
# Match IP layer (layer 3)
if ip_version == 4:
# Match IPv4
match.append_field(ofproto_v1_3.OXM_OF_ETH_TYPE,
ethernet.ether.ETH_TYPE_IP)
if rule.ip_src != self._RULE_WILDCARD:
match.append_field(ofproto_v1_3.OXM_OF_IPV4_SRC,
int(IPAddress(rule.ip_src)))
if rule.ip_dst != self._RULE_WILDCARD:
match.append_field(ofproto_v1_3.OXM_OF_IPV4_DST,
int(IPAddress(rule.ip_dst)))
else:
# Match IPv6
match.append_field(ofproto_v1_3.OXM_OF_ETH_TYPE,
ethernet.ether.ETH_TYPE_IPV6)
if rule.ip_src != self._RULE_WILDCARD:
match.append_field(ofproto_v1_3.OXM_OF_IPV6_SRC,
IPAddress(rule.ip_src).words)
if rule.ip_dst != self._RULE_WILDCARD:
match.append_field(ofproto_v1_3.OXM_OF_IPV6_DST,
IPAddress(rule.ip_dst).words)
# Match transport layer (layer 4)
if rule.tp_proto != self._RULE_WILDCARD:
if rule.tp_proto == self._RULE_TCP:
# Match TCP
match.append_field(ofproto_v1_3.OXM_OF_IP_PROTO,
ipv4.inet.IPPROTO_TCP) # covers IPv6
if rule.port_src != self._RULE_WILDCARD:
match.append_field(ofproto_v1_3.OXM_OF_TCP_SRC,
int(rule.port_src))
if rule.port_dst != self._RULE_WILDCARD:
match.append_field(ofproto_v1_3.OXM_OF_TCP_DST,
int(rule.port_dst))
elif rule.tp_proto == self._RULE_UDP:
# Match UDP
match.append_field(ofproto_v1_3.OXM_OF_IP_PROTO,
ipv4.inet.IPPROTO_UDP) # covers IPv6
if rule.port_src != self._RULE_WILDCARD:
match.append_field(ofproto_v1_3.OXM_OF_UDP_SRC,
int(rule.port_src))
if rule.port_dst != self._RULE_WILDCARD:
match.append_field(ofproto_v1_3.OXM_OF_UDP_DST,
int(rule.port_dst))
return match
def _return_ip_version(self, ip_src, ip_dst):
"""Return the IP version being used given the source and
destination addresses.
:param ip_src: the source IP address to check.
:param ip_dst: the destination IP address to check.
:return: the IP version being used.
"""
if self._RULE_WILDCARD not in ip_src:
return IPAddress(ip_src).version
else:
return IPAddress(ip_dst).version
def switch_features(self, event):
"""Process a switch features event from the controller.
:param event: The OpenFlow event.
"""
datapath = event.msg.datapath
datapath_id = event.msg.datapath_id
ofproto = datapath.ofproto
parser = datapath.ofproto_parser
# Establish the flow table pipeline used by ACLSwitch. The
# first flow table represents a blacklist for dropping flows
# and the second flow table represents a whitelist for
# allowing flows. We must set the appropriate table miss
# entries and allow ARP traffic through the whitelist.
# Table miss entries
match = parser.OFPMatch()
inst = [parser.OFPInstructionGotoTable(self._table_id_whitelist)]
self._contr.add_flow(datapath, 0, match, inst, 0,
self._table_id_blacklist)
actions = []
inst = [parser.OFPInstructionActions(
ofproto.OFPIT_APPLY_ACTIONS, actions)]
self._contr.add_flow(datapath, 0, match, inst, 0,
self._table_id_whitelist)
# ARP entries
inst = [parser.OFPInstructionGotoTable(self._table_id_next)]
match = parser.OFPMatch(arp_op=arp.ARP_REQUEST,
eth_type=ethernet.ether.ETH_TYPE_ARP)
self._contr.add_flow(datapath, 1, match, inst, 0,
self._table_id_whitelist)
match = parser.OFPMatch(arp_op=arp.ARP_REPLY,
eth_type=ethernet.ether.ETH_TYPE_ARP)
self._contr.add_flow(datapath, 1, match, inst, 0,
self._table_id_whitelist)
# Registered the switch, assign the default policy, then set
# to a connected state. Once connected, the rules will be sent
# to the switch.
self._api.switch_register(datapath_id)
self._api.policy_assign_switch(datapath_id, self._POLICY_DEFAULT)
self._api.switch_connect(datapath_id)
def get_app_name(self):
return self._APP_NAME
def get_expected_handlers(self):
return self._EXPECTED_HANDLERS
def is_supported(self):
return self._supported
def _verify_contr_handlers(self):
contr_handlers = self._contr.get_ofpe_handlers()
failures = ()
for expected_h in self._EXPECTED_HANDLERS:
if expected_h not in contr_handlers:
failures = failures + (expected_h,)
if not len(failures) == 0:
fail_msg = ("{0}: The following OpenFlow protocol events "
"are not supported by the controller:".format(
self._APP_NAME))
for f in failures:
fail_msg += "\n\t- {0}".format(f)
self._logging.critical(fail_msg)
return False
else:
return True
return cert_applied
def manage_ucp_certs():
logger.info("UPC certificate management started...")
cert_applied = False
if generate_ucp_certs() or config['certbot']['always_apply_certs']['ucp']:
cert_applied = True
logger.info("Applying UCP certificates")
apply_ucp_certs()
logger.info("UCP certificates applied")
logger.info("UCP certificate management complete")
return cert_applied
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("--conf_dir", help="Configuration directory to use")
args = parser.parse_args()
config = ConfigLoader(args.conf_dir).load()
logger = Logger(filename=__file__,
log_level=config['logging']['log_level'])
manage_certs()
class FreeIPAManager(FreeIPAManagerCore):
"""
Main runnable class responsible for coordinating module functionality.
"""
def __init__(self):
# parse_args is called by argcomplete; must be as fast as possible
self.args = utils.parse_args()
super(FreeIPAManager, self).__init__()
utils.init_logging(self.args.loglevel)
self._load_settings()
def run(self):
"""
Execute the task selected by arguments (check config, upload etc).
"""
try:
self._register_alerting()
{
'check': self.check,
'push': self.push,
'pull': self.pull,
'diff': self.diff,
'template': self.template,
'roundtrip': self.roundtrip
}[self.args.action]()
except ManagerError as e:
self.lg.error(e)
sys.exit(1)
finally:
for plugin in self.alerting_plugins:
plugin.dispatch()
def _register_alerting(self):
"""
For each alerting plugin listed in settings:
1. Instantiate the plugin based on config.
2. Add the plugin as a root logging handler.
3. Add the plugin to the `alerting_plugins` list attribute,
so that it dispatches the results in the end of the run.
"""
self.alerting_plugins = []
plugins_config = self.settings.get('alerting')
if not plugins_config:
self.lg.info('No alerting plugins configured in settings')
return
self.lg.debug('Registering %d alerting plugins', len(plugins_config))
root_logger = logging.getLogger()
for name, config in plugins_config.iteritems():
try:
module_path = 'ipamanager.alerting.%s' % (config['module'])
module = importlib.import_module(module_path)
plugin_config = config.get('config', {})
plugin = getattr(module, config['class'])(plugin_config)
root_logger.addHandler(plugin)
self.alerting_plugins.append(plugin)
self.lg.debug('Registered plugin %s', plugin)
except (AttributeError, ImportError, ManagerError) as e:
raise ManagerError(
'Could not register alerting plugin %s: %s' % (name, e))
self.lg.debug('Registered %d alerting plugins',
len(self.alerting_plugins))
def load(self, apply_ignored=True):
"""
Load configurations from configuration repository at the given path.
:param bool apply_ignored: whether 'ignored' seetings
should be taken into account
"""
self.config_loader = ConfigLoader(self.args.config, self.settings,
apply_ignored)
self.entities = self.config_loader.load()
def check(self):
"""
Run integrity check on the loaded configuration.
:raises ConfigError: in case of configuration syntax errors
:raises IntegrityError: in case of config entity integrity violations
"""
self.load()
self.integrity_checker = IntegrityChecker(self.entities, self.settings)
self.integrity_checker.check()
def push(self):
"""
Run upload of configuration to FreeIPA via API.
This can only be run locally on FreeIPA nodes.
Arguments to the IpaConnector instance
are passed from `self.args` in the `_api_connect` method.
:raises ConfigError: in case of configuration syntax errors
:raises IntegrityError: in case of config entity integrity violations
:raises ManagerError: in case of API connection error or update error
"""
self.check()
from ipa_connector import IpaUploader
utils.init_api_connection(self.args.loglevel)
self.uploader = IpaUploader(self.settings, self.entities,
self.args.threshold, self.args.force,
self.args.deletion)
self.uploader.push()
def pull(self):
"""
Run upload of configuration to FreeIPA via API.
This can only be run locally on FreeIPA nodes.
Arguments to the IpaConnector instance
are passed from `self.args` in the `_api_connect` method.
:raises ConfigError: in case of configuration syntax errors
:raises IntegrityError: in case of config entity integrity violations
:raises ManagerError: in case of API connection error or update error
"""
self.load()
from ipa_connector import IpaDownloader
utils.init_api_connection(self.args.loglevel)
self.downloader = IpaDownloader(self.settings, self.entities,
self.args.config, self.args.dry_run,
self.args.add_only,
self.args.pull_types)
self.downloader.pull()
def diff(self):
"""
Makes set-like difference between 2 dirs. Arguments to the diff are
passed from `self.args` in the `_api_connect` method.
:raises IntegrityError: in case the difference is not empty
"""
diff = FreeIPADifference(self.args.config, self.args.sub_path)
diff.run()
def template(self):
"""
Creates groups, hostgroups, and rules for a given subcluster according
to config defined in template.
:raises ConfigError: in case of wrong template file
"""
data = ConfigTemplateLoader(self.args.template).load_config()
for template in data:
for name, values in template.iteritems():
FreeIPATemplate(name, values, self.args.config,
self.args.dry_run).create()
def roundtrip(self):
"""
Run load, then save the configuration back into config files.
This is done to ensure a "normal" formatting when config files
are syntactically & logically correct but have a non-standard format
(e.g., unsorted membership list, larger or smaller indents etc).
:raises ConfigError: in case of configuration syntax errors
:raises IntegrityError: in case of config entity integrity violations
"""
if self.args.no_ignored:
self.lg.info('Loading ALL entities because of --no-ignored flag')
self.load(apply_ignored=not self.args.no_ignored)
for entity_type, entity_list in self.entities.iteritems():
self.lg.info('Re-writing %s entities to file', entity_type)
for e in entity_list.itervalues():
e.normalize()
e.write_to_file()
self.lg.info('Entity round-trip complete')
def _load_settings(self):
"""
Load the settings file. The file contains integrity check settings,
ignored entities configuration and other useful settings.
"""
self.lg.debug('Loading settings file from %s', self.args.settings)
try:
self.settings = utils.load_settings(self.args.settings)
except Exception as e:
raise ManagerError('Error loading settings: %s' % e)
self.lg.debug('Settings parsed: %s', self.settings)
salt1 = config["salt1"]
salt2 = config["salt2"]
# salt_random = b""
connection_id = 0
# keys = None
if __name__ == "__main__":
from logger import Logger
from keys import KeyManager
from policy import PolicyManager
from config_loader import ConfigLoader
from auth import Connector
from epoll import Epoll
from request_queue import RequestQueue
from server import Server
from sockets import ServerSocket
logger = Logger(verbosity=config["verbosity"])
keys = KeyManager(logger)
policy = PolicyManager(keys, logger)
loader = ConfigLoader(logger, config["path_to_users"], config["path_to_policies"], policy, keys)
loader.read()
logger(policy.dump_policies())
with Server(logger, keys, policy) as server:
epoll, queue = Epoll(server), RequestQueue(server)
server_socket = ServerSocket(server, Connector)
logger("Server started")
server.run()
bo.spawn_block()
#bo.bo_pos=[16*scale, 16*scale] #Legacy position
bo.bo_pos=[3*16*scale, 8*16*scale] #New position
bo.end_count_pos = [16*scale, (bo.visible_height+3)*16*scale]
bo.init_images(scale)
bo2 = board.board()
bo2.spawn_block()
#bo2.bo_pos=[16*17*scale, 16*scale] #Legacy position
bo2.bo_pos=[20*16*scale, 8*16*scale] #New position
bo2.end_count_pos = [16*scale, (bo2.visible_height+3)*16*scale]
bo2.init_images(scale)
#Configure keybindings
if ai == "1":
cfg = ConfigLoader(bo, bo, configfile)
elif ai == "2":
cfg = ConfigLoader(bo, bo2, configfile)
cfg.actions = {}
elif ai == "3" or ai == "4":
cfg = ConfigLoader(bo, bo, configfile)
if opponent_ip == None:
print("Opponent ip not specified")
exit(1)
if ai == "3":
bo2.add_send_connection(opponent_ip)
else:
bo.draw_matrix(win.screen, str_to_matrix("please\nwait\nfor the\nsecond\nplayer"),
[1*16*scale, 1*16*scale], bo.purple_block_image)
pygame.display.flip()
bo2.add_wait_connection(opponent_ip)
if __name__ == "__main__":
# event loop
loop = asyncio.get_event_loop()
# signal
signal.signal(signal.SIGINT, clash_handler)
# working directory
working_path = os.path.abspath(sys.argv[1])
print("working directory: {0}".format(working_path))
# make data directory
_data_directory = "{0}/data".format(working_path)
if os.path.exists(_data_directory):
shutil.rmtree(_data_directory)
print("remove old data folder")
os.mkdir(_data_directory)
print("make data folder")
# load configuration
config = Configuration("", [])
_config_path = "{0}/config.yml".format(working_path)
_config_loader = ConfigLoader(path=_config_path, config=config)
print("Listener:\n{0}\nSpeaker:\n{1}".format(config.listener_token,
config.speaker_token_list))
# lunch bot
bot_manager = BotManager(config=config,
loop=loop,
working_path=_data_directory)
# run loop
loop.run_forever()
def __init__(self, tip):
loader = ConfigLoader(commvals.CRAWLER_CONFIG_XML)
SITES = loader.sites
SITE = SITES[SITES.index(tip.upper())]
host_element = loader.get_element(SITE, 'host').get('element', [{}])[0]
video_element = loader.get_element(SITE, 'video').get('element', [{}])[0]
self.site = SITE
self.able = False if loader.get_site(SITE).get('able', 'true') == 'false' else True
self.host_url = loader.get_site(SITE).get('host_url')
self.url_format = deal_url_format(loader.get_site(SITE).get('url_format'))
self.snapshot = False if loader.get_site(SITE).get('snapshot', 'true') == 'false' else True
self.video_url_end = loader.get_site(SITE).get('video_url_end', '')
self.host = {
'onlyurl': True if host_element.get('onlyurl', 'false') == 'true' else False,
'request': host_element.get('request', ''),
'param': host_element.get('param', ''),
'children': tuple([child.tag for child in loader.get_children(SITE, 'host')]),
'filter': loader.get_element(SITE, 'filter').get('element')
}
self.video = {
'request': video_element.get('request', ''),
'param': video_element.get('param', ''),
'isre': True if video_element.get('isre', 'false') == 'true' else False,
'children': tuple([child.tag for child in loader.get_children(SITE, 'video')]),
'null': video_element.get('request', '') == '' and video_element.get('param', '') == ''
}
self.url = loader.get_element(SITE, 'url').get('element', [])
self.title = loader.get_element(SITE, 'title').get('element', [])
self.thumbnail = loader.get_element(SITE, 'thumbnail').get('element', [])
self.playcount = loader.get_element(SITE, 'playcount').get('element', [])
self.community = loader.get_element(SITE, 'community').get('element', [])
self.upcount = loader.get_element(SITE, 'upcount').get('element', [])
self.downcount = loader.get_element(SITE, 'downcount').get('element', [])
self.favorite = loader.get_element(SITE, 'favorite').get('element', [])
self.host_children = []
self.video_children = []
self.init_children()
