def __init__(self, slave = False, protocol = None, address = None, port = None, target_name = None):
self.config = Configurator()
self.is_connected = False
self.is_slave = slave
self.parameters = None
self.datastream_queue = None
self.datastream_poller = None
self.datastream_collector = None
self.signal_structure = None
# Sets basic configurations
if protocol:
self.config.setAttr('protocol', protocol)
if address:
self.config.setAttr('address', address)
if port:
self.config.setAttr('port', port)
if target_name:
self.config.setAttr('target_name', target_name)
# Initializes connection
self.connection = Connector(self.config)
def test_multiple_config(self):
C.initialize(["--config", "field.name=blah",
"--config", "other.prop=foo",
"--config", "bool.prop=True"])
assert_equals("foo", C.get("other", "prop"))
assert_equals("blah", C.get("field", "name"))
assert_equals(True, C.getboolean("bool", "prop"))
def __init__(self):
# Setting general utilities
configurator = Configurator()
self.config_parser, self.logger = configurator.set_configurator()
self.vectorizer = TextNormalizer.get_tfidf_vectorizer()
self.classifier = Classifier.get_random_forest_classifier()
def _load_options(self):
"""Load options. :py:class:`GenieConfSection` objects will be created from
sections with options set to the :py:class:`GenieConfSection` object's
attributes."""
# this is work-around for when config values are loaded as a list when
# execution is threaded (still need to look closer into why this is
# happening)
def unlist(value):
return value[0] if isinstance(value, list) else value
# explicitly set genie section object
default_username = os.environ.get('USER', DEFAULT_GENIE_USERNAME)
self.genie.set('url', unlist(C.get('genie.url', DEFAULT_GENIE_URL)))
self.genie.set('username',
unlist(C.get('genie.username', default_username)))
self.genie.set('version',
unlist(C.get('genie.version', DEFAULT_GENIE_VERSION)))
# create and set other section objects
for section in C.config.sections():
section_clean = section.replace(' ', '_').replace('.', '_')
gcs = self.genie if section_clean == 'genie' \
else GenieConfSection(name=section_clean)
for option in C.config.options(section):
option_clean = option.replace(' ', '_').replace('.', '_')
if section_clean != 'genie' \
or option_clean not in {'url', 'username', 'version'}:
gcs.set(option_clean, unlist(C.get(section, option)))
setattr(self, section_clean, gcs)
def __init__(self, parent=None):
QDialog.__init__(self, parent)
self.config = Configurator()
self.setupUi(self)
self.pushButtonLoading.setDisabled(True)
self.comboBoxDatabase.activated.connect(self.showFbPathButton)
self.pushButtonShemaList.clicked.connect(self.showTableShemaList)
self.toolButtonPath.clicked.connect(self.browse)
self.pushButtonLoading.clicked.connect(self.downloading)
self.lineEditVersionDownload.textChanged.connect(self.activatePushButtonLoading)
self.lineEditPath.textChanged.connect(self.activatePushButtonLoading)
self.lineEditReport.textChanged.connect(self.activatePushButtonLoading)
self.pushButtonReport.clicked.connect(self.viewReports)
self.pushButtonVersion.clicked.connect(self.viewVersion)
self.table = TreeShemaForm(self)
self.table.setTextToAzkFormLabel.connect(self.setTextToAzkFormLabel)
self.reportDialog = DialogReports(self)
self.reportDialog.setTextToLineEditReport.connect(self.setTextlineEditReport)
self.DialogLog = DialogLog(self)
self.versiondict = {}
self.reportdict = {}
self.pathReport1 = self.config.getoption('azkfin', 'report1')
self.pathReport2 = self.config.getoption('azkfin', 'report2')
self.pathVersion1 = self.config.getoption('azkfin', 'path1')
self.pathVersion2 = self.config.getoption('azkfin', 'path2')
self.extr = self.config.getoption('dialog', '7z')
def test_two_instances(self):
conf = Configurator(["instance1", "instance2", "--parameters", "inasubdir\\hello.txt"])
p = os.path.join(pysmac.__path__[0], "scenario.txt")
with open(p, "r") as f:
c = f.read()
c = c % {"parameters": "parameters.txt"}
self.assertEqual(conf.get_scenario_file(), c)
def _load_options(self):
"""Load options. :py:class:`GenieConfSection` objects will be created from
sections with options set to the :py:class:`GenieConfSection` object's
attributes."""
# this is work-around for when config values are loaded as a list when
# execution is threaded (still need to look closer into why this is
# happening)
def unlist(value):
return value[0] if isinstance(value, list) else value
# explicitly set genie section object
default_username = os.environ.get("USER", DEFAULT_GENIE_USERNAME)
self.genie.set("url", unlist(C.get("genie.url", DEFAULT_GENIE_URL)))
self.genie.set("username", unlist(C.get("genie.username", default_username)))
self.genie.set("version", unlist(C.get("genie.version", DEFAULT_GENIE_VERSION)))
# create and set other section objects
for section in C.config.sections():
section_clean = section.replace(" ", "_").replace(".", "_")
gcs = self.genie if section_clean == "genie" else GenieConfSection(name=section_clean)
for option in C.config.options(section):
option_clean = option.replace(" ", "_").replace(".", "_")
if section_clean != "genie" or option_clean not in {"url", "username", "version"}:
gcs.set(option_clean, unlist(C.get(section, option)))
setattr(self, section_clean, gcs)
def test_two_instances(self):
conf = Configurator(["instance1", "instance2", "--parameters", "parameters.txt"])
expected_result = (
'"instance1"\n'
'"instance2"'
)
self.assertEqual(conf.get_instance_file(), expected_result)
def test_env_override(self):
import os
os.environ["test_prop"] = "something"
os.environ["bool_prop"] = "False"
C.initialize(["--config", "test.prop=blah",
"--config", "bool.prop=True"])
assert_equals("blah", C.get("test", "prop"))
assert_equals(True, C.getboolean("bool", "prop"))
def test_two_instances(self):
conf = Configurator(
["instance1", "instance2", "--parameters", "inasubdir\\hello.txt"])
p = os.path.join(pysmac.__path__[0], "scenario.txt")
with open(p, "r") as f:
c = f.read()
c = c % {"parameters": "parameters.txt"}
self.assertEqual(conf.get_scenario_file(), c)
def test_env_override(self):
import os
os.environ["test_prop"] = "something"
os.environ["bool_prop"] = "False"
C.initialize(
["--config", "test.prop=blah", "--config", "bool.prop=True"])
assert_equals("blah", C.get("test", "prop"))
assert_equals(True, C.getboolean("bool", "prop"))
def wrapper(*args, **kwargs):
C.initialize(sys.argv)
for env_var in {'genie_username', 'USER'}:
try:
del os.environ[env_var]
except Exception:
pass
return func(*args, **kwargs)
def wrapper(*args, **kwargs):
C.initialize(sys.argv)
for env_var in {'genie_username', 'USER'}:
try:
del os.environ[env_var]
except Exception:
pass
return func(*args, **kwargs)
def test_make_operator_with_unknown_value(self):
operatordefs = {"=": "thingie"}
configuration = Configuration()
configuration.set_config("operator", operatordefs)
configurator = Configurator(configuration)
with (self.assertRaises(Exception)):
operator = configurator.make_operator("&")
def __init__(self, parent=None):
QMainWindow.__init__(self, parent)
self.config = Configurator()
self.setupUi(self)
self.AzkForm = AzkForm(self)
self.azkButton.clicked.connect(self.showAzkFinForm)
self.AzkForm.lineEditPath.setText(self.config.getoption('dialog', 'path'))
self.AzkForm.lineEditPort.setText(self.config.getoption('dialog', 'port'))
def test_make_operator(self):
operatordefs = {"=": "thingie"}
configuration = Configuration()
configuration.set_config("operator", operatordefs)
configurator = Configurator(configuration)
operator = configurator.make_operator("=")
self.assertEquals("thingie", operator)
def test_make_tokenizer(self):
tokendefs = [(r"[A-Z]", None)]
configuration = Configuration()
configuration.set_config("token", tokendefs)
configurator = Configurator(configuration)
tokenizer = configurator.make_tokenizer()
self.assertTrue(isinstance(tokenizer, Tokenizer))
def test_make_parser_with_weird_parselet_type(self):
parselet_map = {0: PrefixParselet, 1: basestring}
configuration = Configuration()
configuration.set_config("parselet", parselet_map)
configurator = Configurator(configuration)
with (self.assertRaises(Exception)):
parser = configurator.make_parser()
def test_multiple_config(self):
C.initialize([
"--config", "field.name=blah", "--config", "other.prop=foo",
"--config", "bool.prop=True"
])
assert_equals("foo", C.get("other", "prop"))
assert_equals("blah", C.get("field", "name"))
assert_equals(True, C.getboolean("bool", "prop"))
def run():
configuration = Configurator('QA')
database_path = configuration.get_database()
connector = Connector(database_path)
reporter = Reporter()
processor = Processor(configuration, connector, reporter)
processor.execute()
def __init__(self):
self.CNF = Configurator()
self.ConfData = self.CNF.getConfigData()
self.WL = WriterLog(self.ConfData['logfile']['filepath'],
self.ConfData['logfile']['filename'])
self.TP = Tape(self.ConfData['tape']['filepath'],
self.ConfData['tape']['filename'],
self.ConfData['tape']['separator'])
self.TP.readInput()
def test_valid_formats(self):
C.initialize([
"--config", "value.one=One",
"--config", "value.two=Two"
])
assert_equals("One", C.get('value', 'one', 'Three'), C.get('value.one', 'Two'))
assert_equals('Two', C.get('value.two'), C.get('value', 'two'))
assert_equals('Three', C.get('value.three', 'Three'), C.get('value', 'three', 'Three'))
assert_equals(None, C.get('value.four'), C.get('value', 'four'))
def test_make_parser(self):
parselet_map = {0: PrefixParselet, 1: InfixParselet, 2: InfixParselet}
configuration = Configuration()
configuration.set_config("parselet", parselet_map)
configurator = Configurator(configuration)
parser = configurator.make_parser()
self.assertEquals(1, len(parser.prefix_parselets))
self.assertEquals(2, len(parser.infix_parselets))
def main():
if scraper.args.redownload:
scraper.re_download()
elif scraper.args.configure:
config = Configurator()
config.menu()
else:
scraper.run()
logging.info('Logger stopped')
def test_load_config_file_order_env(self, load_config_file,
config_home_ini, config_env):
"""Test loading configuration file order with specified in environment variable."""
C.initialize()
config_env.return_value = self.config_file
config_home_ini.return_value = None
GenieConf()
load_config_file.assert_called_once_with(self.config_file)
def test_valid_formats(self):
C.initialize(
["--config", "value.one=One", "--config", "value.two=Two"])
assert_equals("One", C.get('value', 'one', 'Three'),
C.get('value.one', 'Two'))
assert_equals('Two', C.get('value.two'), C.get('value', 'two'))
assert_equals('Three', C.get('value.three', 'Three'),
C.get('value', 'three', 'Three'))
assert_equals(None, C.get('value.four'), C.get('value', 'four'))
def prepareData(path, process_number=1):
print "Analysing path: %s" % path
process = pexpect.spawn(
Configurator.getPath(Configurator.KEY_SPAWN_SCRIPT),
[
Configurator.getPath(Configurator.KEY_BASE_DIR) + "/config",
path, str(process_number),
"%d%s" % (Neo4jHelper.HEAP_SIZE[0],
Neo4jHelper.HEAP_SIZE[1])
],
None
)
if Configurator.isDebuggingEnabled():
process.logfile = sys.stdout
expectation = process.expect([
# "graph.db still exists",
"Remote interface ready",
"java.net.BindException",
"java.io.IOException: Unable to create directory path",
pexpect.EOF,
"ERROR: No write access to data/ directory",
(
"There is insufficient memory for the Java Runtime "
"Environment to continue."
)
])
if expectation == 1:
# BindException (port already taken?)
raise BindException()
elif expectation == 2:
# Unable to create directory path
raise PathException()
elif expectation == 3:
# EOF
# print process.before
raise BindException()
elif expectation == 4:
raise Exception(
"ERROR: No write access to neo4j data directory. "
"Check for sufficient write permissions in all neo4j "
"instances' data directory."
)
elif expectation == 5:
# Not enough space to allocate the specified amount of heap space.
raise HeapException()
return process
def test_load_config_file_order_env(self,
load_config_file,
config_home_ini,
config_env):
"""Test loading configuration file order with specified in environment variable."""
C.initialize()
config_env.return_value = self.config_file
config_home_ini.return_value = None
GenieConf()
load_config_file.assert_called_once_with(self.config_file)
def run():
config = Configurator('uat')
database_url = config.get_database_url()
connector = Connector(database_url)
logger = Result()
test_processor = TestProcessor(config, connector, logger)
test_processor.process()
logger.finish_test()
def run(data):
"""Собираем все вместе, реализован только первый тестовый случай, когда хотим подсчитать
число строк в таблицах"""
data_connect = Configurator(data)
if data_connect.get_test_case():
db = data_connect.get_database()
sv = data_connect.get_server()
connection = Connect(sv, db, data)
connection.queries_count()
x = Execute(data)
x.parse_test_files()
def test_list(self):
test_foo_value = ["a", "b"]
data = {"test": {"foo": test_foo_value}}
t = TestComponent()
c = Configurator()
c.register_component(t, "test")
cr = ConfigurationReader(c, data)
cr.config_components()
self.assertEquals(test_foo_value, t.get_foo())
def test_simple(self):
test_bla_value = 12
data = {"test": {"bla": test_bla_value}}
t = TestComponent()
c = Configurator()
c.register_component(t, "test")
cr = ConfigurationReader(c, data)
cr.config_components()
self.assertEquals(test_bla_value, t.get_bla())
class Environment:
""" Class responsible for holding the state of the parser and its various
tools.
Attributes:
configurator (Configurator): used to generate parser and other tools
var_repo (dict): variable store
parser_instance (Parser): built via configurator
tokenizer (Tokenizer): built via configurator
"""
def __init__(self, config_kwargs=None):
""" Given the configuration options specified, build the parser
environment.
Arguments:
config_kwargs (dict): options for configuration, see config_loader
"""
if not config_kwargs:
config_kwargs = {}
self.configurator = Configurator(ConfigLoader(**config_kwargs).load())
self.var_repo = VarRepo()
self.parser_instance = None
self.tokenizer = None
def config(self):
""" Uses configurator to build the parser and tokenizer.
"""
self.parser_instance = self.configurator.make_parser()
self.tokenizer = self.configurator.make_tokenizer()
return self
def parser(self, line):
""" Parses the supplied line using the parser and tokenizer.
Arguments:
line (str): line of text to parse
"""
if self.parser_instance is None:
self.config()
tokens = self.tokenizer.tokenize(line)
self.parser_instance.token_define(tokens)
return self.parser_instance
def operator(self, operator_str):
""" Creates an operator from the operator string supplied
Arguments:
operator_str (str): symbol etc corresponding to operator
"""
return self.configurator.make_operator(operator_str)
def get_var_repo(self):
""" Returns the current variable repo.
"""
return self.var_repo
def correct_misdiagnosis(df, nid, extract_type_id, code_system_id, adjust_id,
remove_decimal):
conf = Configurator('standard')
mc_process_dir = conf.get_directory('mc_process_data')
package_dir = conf.get_directory('rd_process_inputs') + "FILEPATH"
misdiagnosis_path = conf.get_resource('misdiagnosis_prob_path')
if adjust_id == 543:
misdiagnosis_version_id = 4
elif adjust_id == 544:
misdiagnosis_version_id = 3
elif adjust_id == 500:
misdiagnosis_version_id = 3
misdiagnosis_path = misdiagnosis_path.format(
adjust_id=adjust_id,
version_id=misdiagnosis_version_id,
code_system_id=code_system_id)
start_deaths = df['deaths'].sum()
start_deaths_target = df.loc[df.cause_id == adjust_id, 'deaths'].sum()
start_deaths_cc = df.loc[df.cause_id == 919, 'deaths'].sum()
df = df.loc[df.deaths > 0]
print_log_message("Adding packages")
df = add_packages(df, code_system_id, remove_decimal, package_dir)
print_log_message("Getting deaths to move")
move_df = get_deaths_to_move(df, adjust_id, misdiagnosis_path,
mc_process_dir, nid, extract_type_id,
code_system_id)
print_log_message("Jumbling up deaths")
df = death_jumble(df, move_df, adjust_id, code_system_id)
print_log_message("Checking deaths jumbled well")
end_deaths = df['deaths'].sum()
end_deaths_target = df.loc[df.cause_id == adjust_id, 'deaths'].sum()
end_deaths_cc = df.loc[df.cause_id == 919, 'deaths'].sum()
assert abs(int(start_deaths) - int(end_deaths)) <= 5, \
'Bad jumble - added/lost deaths ' \
'(started: {}, ended: {})'.format(str(int(start_deaths)),
str(int(end_deaths)))
print_log_message("Storing intermediate data")
store_intermediate_data(df, move_df, mc_process_dir, adjust_id, nid,
extract_type_id)
print_log_message('Deaths moved: ' + str(
int((end_deaths_target + end_deaths_cc) -
(start_deaths_target + start_deaths_cc))))
return df
def run_simulation(parameters_file, larval_capacity=None, **options):
# setup
configurator = Configurator(parameters_file)
config_lo = configurator.create_configuration(transmission='low',
**options)
config_me = configurator.create_configuration(transmission='medium',
**options)
config_hi = configurator.create_configuration(transmission='high',
**options)
# adjust larval capacity if one is specified (convenient for sensitivity analysis)
if larval_capacity:
config_lo.population.larval_capacity = larval_capacity
config_me.population.larval_capacity = larval_capacity
config_hi.population.larval_capacity = larval_capacity
model = lifecycle.create_model(config_lo.lifecycle)
# run a demographic simulation
selectors = _get_demographic_selectors(model, config_lo)
dem_results = run_demographic_simulation(
config_lo,
selectors) # for demographics, does not matter which config we use
# run malaria simulations (low and high transmission) based on the demographic results
feeder_names = _get_feeder_names(model)
feeder_timeseries = {x: dem_results.timeseries[x] for x in feeder_names}
exposure_history = {
x: get_exposure_history(x, model, config_lo.malaria.sporogony_days)
for x in feeder_names
}
exposed_fractions = {
x:
dem_results.end_model.get_compartment(x).get_attr('fraction_exposed')
for x in feeder_names
}
mal_results_lo = run_malaria_simulation(config_lo, feeder_timeseries,
exposure_history,
exposed_fractions)
mal_results_me = run_malaria_simulation(config_me, feeder_timeseries,
exposure_history,
exposed_fractions)
mal_results_hi = run_malaria_simulation(config_hi, feeder_timeseries,
exposure_history,
exposed_fractions)
# return results (+ additional information?)
return Bunch(population=dem_results,
malaria=Bunch(low=mal_results_lo,
medium=mal_results_me,
high=mal_results_hi))
def __init__(self, config_kwargs=None):
""" Given the configuration options specified, build the parser
environment.
Arguments:
config_kwargs (dict): options for configuration, see config_loader
"""
if not config_kwargs:
config_kwargs = {}
self.configurator = Configurator(ConfigLoader(**config_kwargs).load())
self.var_repo = VarRepo()
self.parser_instance = None
self.tokenizer = None
def getProcessedSet(self):
# TODO: refactor - same code is in scrapeforum, move out to a DB object
config = Configurator()
db = config.getDb()
connectionString = config.getDbConnectionString()
engine = create_engine(connectionString, echo=(db['echo'] == 'True'))
Session = sessionmaker(bind=engine)
session = Session()
# TODO - end
return session.query(LatestTopic).filter_by(batch_id=self.batch_id)
def run():
#tesing of commit to Git
# config = Configurator(sys.argv[1])
config = Configurator("dev")
database_url = config.get_database_url()
connector = Connector(database_url)
logger = Result()
test_processor = TestProcessor(config, connector, logger)
test_processor.process()
logger.finish_test()
def __init__(self, cause_set_version_id, pop_run_id,
distribution_set_version_id,
verbose=False, collect_diagnostics=False, id_cols=None,
value_column='deaths'):
self.cause_set_version_id = cause_set_version_id
self.pop_run_id = pop_run_id
self.distribution_set_version_id = distribution_set_version_id
self.conf = Configurator('standard')
self.verbose = verbose
self.collect_diagnostics = collect_diagnostics
self.value_column = value_column
if id_cols is not None:
self.id_cols = id_cols
def prepareData(path, process_number=1):
print "Analysing path: %s" % path
process = pexpect.spawn(
Configurator.getPath(Configurator.KEY_SPAWN_SCRIPT), [
Configurator.getPath(Configurator.KEY_BASE_DIR) + "/config",
path,
str(process_number),
"%d%s" % (Neo4jHelper.HEAP_SIZE[0], Neo4jHelper.HEAP_SIZE[1])
], None)
if Configurator.isDebuggingEnabled():
process.logfile = sys.stdout
expectation = process.expect([
# "graph.db still exists",
"Remote interface ready",
"java.net.BindException",
"java.io.IOException: Unable to create directory path",
pexpect.EOF,
"ERROR: No write access to data/ directory",
("There is insufficient memory for the Java Runtime "
"Environment to continue.")
])
if expectation == 1:
# BindException (port already taken?)
raise BindException()
elif expectation == 2:
# Unable to create directory path
raise PathException()
elif expectation == 3:
# EOF
# print process.before
raise BindException()
elif expectation == 4:
raise Exception(
"ERROR: No write access to neo4j data directory. "
"Check for sufficient write permissions in all neo4j "
"instances' data directory.")
elif expectation == 5:
# Not enough space to allocate the specified amount of heap space.
raise HeapException()
return process
def __init__(self, slave=False, protocol=None, address=None, port=None, target_name=None):
self.config = Configurator()
self.is_connected = False
self.is_slave = slave
self.parameters = None
self.datastream_queue = None
self.datastream_poller = None
self.datastream_collector = None
self.signal_structure = None
# Sets basic configurations
if protocol:
self.config.setAttr("protocol", protocol)
if address:
self.config.setAttr("address", address)
if port:
self.config.setAttr("port", port)
if target_name:
self.config.setAttr("target_name", target_name)
# Initializes connection
self.connection = Connector(self.config)
def __init__(self, debug=0, skip_auth=False):
self.__debug = debug
if self.__debug > 2:
print "initing engine"
self._repo = self._get_repo()
if not self._repo:
return
self._cw = Configurator(self._repo.config_writer())
self._cr = Configurator(self._repo.config_reader())
self._gh = None
if not skip_auth:
if self.__debug > 0:
print "Authorizing engine..."
if not self.do_auth():
print "Authorization failed! Exiting."
return
try:
self._gh_repo = self._gh.get_user().get_repo(self._cr.flowhub.structure.name)
except GithubException:
try:
self._get_repo = [x for x in self._gh.get_user().get_repos() if x.name == self._cr.flowhub.structure.name][0]
except IndexError:
raise ImproperlyConfigured("No repo with given name: {}".format(self._cr.flowhub.structure.name))
if self._gh.rate_limiting[0] < 100:
warnings.warn("You are close to exceeding your GitHub access "
"rate; {} left out of {} initially".format(*self._gh.rate_limiting))
else:
if self.__debug > 0:
print "Skipping auth - GitHub accesses will fail."
def load_config_file(self, config_file):
"""Load a configuration file (ini)."""
if config_file is not None and os.path.exists(config_file):
logger.debug("adding config file: %s", config_file)
self._config_files.append(config_file)
C.addOptionFile(config_file)
# options specified via cmd line should always override config files
sys_argv_configs = [sys.argv[i + 1] for i, arg in enumerate(sys.argv) if arg == "--config"]
for option in sys_argv_configs:
C.addOption(option)
self._load_options()
logger.debug("configuration:\n%s", self.to_json())
return self
def config_build( self, bind_dir, build_dir ):
cf_master = Configurator( db_cur = self.cur, filename ="named.conf.maintain", bind_dir=bind_dir, build_dir=build_dir)
cf_master.build_named_conf_master()
cf_master.build_named_conf_slave()
if self.test_zone_files:
cf_master.test_zone_files()
def __init__(self, type=None):
self.type = "Error" if type == constants.EMAIL_TYPE_ERROR else "Notification"
# Get configuration values from Configurator
configurator = Configurator()
# Get configuration groups for email account, sender and receivers
self.emailAccount = configurator.getEmailAccount()
self.emailSender = configurator.getEmailSender()
self.emailReceivers = configurator.getEmailReceivers(type)
# Configure sender and receivers
self.sender = self.emailSender['sender_email']
self.receiver_emails = self.emailReceivers['receiver_emails'].replace(" ", "").split(',')
self.receiver_names = self.emailReceivers['receiver_names'].replace(" ", "").split(',')
self.subjectPrefix = "[ERROR] " if type == constants.EMAIL_TYPE_ERROR else "[NOTIFY] "
def __init__(self):
try:
# TODO: refactor - same code is in scrapeforum, move out to a DB object
config = Configurator()
db = config.getDb()
connectionString = config.getDbConnectionString()
engine = create_engine(connectionString, echo=(db['echo'] == 'True'))
Session = sessionmaker(bind=engine)
session = Session()
# TODO - end
# TODO add filter for 'success = 1'
self.batch_id = session.query(func.max(LatestTopic.batch_id).label("max_batch_id")).one().max_batch_id
except:
# TODO: report error
raise
finally:
engine.dispose()
def startConsole(self, path, port=1):
"""
Import the php file/project AST from 'path' into the neo4j
database and start the neo4j console, using the 'SPAWN_SCRIPT' file.
"""
process = subprocess.call(
[
Configurator.getPath(Configurator.KEY_SPAWN_SCRIPT),
Configurator.getPath(Configurator.KEY_BASE_DIR) + "/config",
path, str(port),
"%d%s" % (Neo4jHelper.HEAP_SIZE[0],
Neo4jHelper.HEAP_SIZE[1])
],
preexec_fn=os.setsid
)
def signalHandler(signalnum, handler):
os.killpg(process.pid, signal.SIGINT)
signal.signal(signal.SIGINT, signalHandler)
signal.signal(signal.SIGTERM, signalHandler)
def get_cfg_vars(config_type, config_path, logger=None):
'''
Target:
- get all the variables stored in a config file with cfg extension.
Parameters:
- config_type: the type of config file which is going to be loaded,
to differ it from the other types which will have different
sections and variables.
- logger: a logger to show and log some messages.
Return:
- the parser which will contain all the config file variables.
'''
configurator = Configurator()
# In the case of logger being None, this is because a logger config
# file is being loaded to specify some settings
if logger:
configurator.load_cfg(config_type, config_path, logger)
else:
configurator.load_cfg(config_type, config_path)
return configurator.parser
def loadOracleShemasInfo(self):
self.treeWidgetShemaList.clear()
self.config = Configurator()
server, sid, user, password = self.config.getdbparam()
print(server, sid, user, password)
for num, cur in enumerate(getDbShemaList(server, sid, user, password)):
name = cur[0]
version = cur[1]
date = QDateTime(cur[2]).toString('yyyy-MM-dd')
item_0 = QtWidgets.QTreeWidgetItem(self.treeWidgetShemaList)
self.treeWidgetShemaList.insertTopLevelItem(num, item_0.setText(0, name))
self.treeWidgetShemaList.insertTopLevelItem(num, item_0.setText(1, version))
self.treeWidgetShemaList.insertTopLevelItem(num, item_0.setText(2, date))
self.treeWidgetShemaList.sortByColumn(0, Qt.AscendingOrder)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("configuration", help="Path to the configuration file")
parser.add_argument("strategies", help="Path to the strategy repository")
group = parser.add_mutually_exclusive_group()
group.add_argument("-e", "--event", help="Path to the semantic description for event to be configured")
group.add_argument("-a", "--application",
help="Path to the semantic description for the application to be configured")
parser.add_argument("-c", "--costmodel", help="Existing cost model to be used for configuration")
args = parser.parse_args()
configurator = Configurator(args.strategies)
if args.event:
configurator.configureEvent(args.event, args.costmodel)
sys.exit(0)
if args.application:
configurator.calculate_cost_model(args.application)
sys.exit(0)
else:
sys.exit(1)
class MainForm(QMainWindow, Ui_MainWindow):
def __init__(self, parent=None):
QMainWindow.__init__(self, parent)
self.config = Configurator()
self.setupUi(self)
self.AzkForm = AzkForm(self)
self.azkButton.clicked.connect(self.showAzkFinForm)
self.AzkForm.lineEditPath.setText(self.config.getoption('dialog', 'path'))
self.AzkForm.lineEditPort.setText(self.config.getoption('dialog', 'port'))
def showAzkFinForm(self):
self.AzkForm.show()
def exception_hook(self, type_, value, tb):
logging.error('Unhandled top level exception:\n%s', ''.join(traceback.format_exception(type_, value, tb)))
QMessageBox.critical(self, "Ошибка", "{}".format(''.join(traceback.format_exception(type_, value, tb))))
def __init__(self, port):
'''
Constructor
'''
self.j = JoernSteps()
self.j.setGraphDbURL('http://localhost:%d/db/data/' % (int(port)))
# self.j.addStepsDir(
# Configurator.getPath(Configurator.KEY_PYTHON_JOERN) +
# "/joern/phpjoernsteps"
# )
self.j.addStepsDir(
Configurator.getPath(Configurator.KEY_BASE_DIR) +
"/custom_gremlin_steps"
)
self.j.connectToDatabase()
class TreeShemaForm(QDialog, Ui_DialogShemaList):
setTextToAzkFormLabel = pyqtSignal()
def __init__(self, parent=None):
QDialog.__init__(self, parent)
self.setupUi(self)
self.treeWidgetShemaList.itemDoubleClicked.connect(self.setTextToAzkFormLabel)
def loadOracleShemasInfo(self):
self.treeWidgetShemaList.clear()
self.config = Configurator()
server, sid, user, password = self.config.getdbparam()
print(server, sid, user, password)
for num, cur in enumerate(getDbShemaList(server, sid, user, password)):
name = cur[0]
version = cur[1]
date = QDateTime(cur[2]).toString('yyyy-MM-dd')
item_0 = QtWidgets.QTreeWidgetItem(self.treeWidgetShemaList)
self.treeWidgetShemaList.insertTopLevelItem(num, item_0.setText(0, name))
self.treeWidgetShemaList.insertTopLevelItem(num, item_0.setText(1, version))
self.treeWidgetShemaList.insertTopLevelItem(num, item_0.setText(2, date))
self.treeWidgetShemaList.sortByColumn(0, Qt.AscendingOrder)
def init_menu(self):
self.conf = Configurator()
self.menu = self.conf.build_main_menu()
self.curr_choice = 0
#!/usr/bin/python
#
# Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
from configurator import Configurator
if __name__ == '__main__':
try:
configurator = Configurator()
configurator.update_deck_settings()
except (RuntimeError, IOError) as e:
sys.stderr.write(str(e) + '\n')
sys.exit(-1)
def startSearchMode(flow, continuous=False):
flow["in"] = os.path.abspath(flow["in"])
if not os.path.exists(flow["in"]):
print "Given path (-in) does not exist."
sys.exit()
level = 0
multithreads = 0
neo4j_helper = Neo4jHelper()
heap_size = Configurator.getHeapVal()
neo4j_helper.setHeapsize(heap_size[0], heap_size[1])
if continuous:
# Continuous mode was specified, so read the config file
try:
stats_path = getArg(flow, "s", "statsfile")
except:
stats_path = DEFAULT_STATS_PATH
lock = multiprocessing.Lock()
shared_data = SharedData(stats_path, lock, in_path=flow["in"])
in_path = shared_data.getInPath()
if in_path != flow["in"]:
print (
"The given path with \"-in\" is not the path, "
"which was used before."
)
_ = raw_input("Ctrl-C or Ctrl-D to abort. "
"Press any key to continue")
shared_data.setInPath(flow["in"])
Neo4jHelper.setStatisticsObj(shared_data)
try:
multithreads = int(getArg(flow, "m", "multithreading"))
except:
pass
code_path = getArg(flow, "q", "queries")
code = []
# Read given query.
if os.path.isfile(code_path):
with open(code_path, "r") as fh:
code.append(
QueryFile(code_path, fh.read())
)
elif os.path.isdir(code_path):
# Given path is a directory - get all files recursively inside the
# directory.
for path, _, files in os.walk(code_path):
for name in files:
file_path = os.path.join(path, name)
with open(file_path, "r") as fh:
code.append(
QueryFile(file_path, fh.read())
)
if not code:
# Did not find any file recursively inside 'code_path'.
print "Query-path (-q/--queries) does not contain any files."
sys.exit()
else:
# Path does not exist
print "Query-Path (-q/--queries) does not exist."
sys.exit()
try:
level = int(getArg(flow, "l", "level"))
except ArgException:
# Parameter "-l/--level" was not specified.
pass
if level == 0:
# Analyse specified file/files in specified directory with given
# gremlin query/queries.
Neo4jHelper.analyseData((
code, flow["in"], 1
))
else:
# Analyse folders 'level' levels under specified path.
try:
# Get the root directory of every project in a generator.
path_generator = getRootDirectories(flow["in"], level)
if continuous:
# Check if given in-path is the same as the one in the
# given stats file.
if in_path == flow["in"]:
# Skip generator elements if they were already
# analysed before.
projects_analysed_count = shared_data.getProjectsCount()
skipGeneratorElements(path_generator, projects_analysed_count)
except Exception as err:
print "An exception occured: %s" % err
sys.exit()
projects_analysed = 0
if multithreads > 1:
# Multithreading was specified.
process_number_generator = ProcessIdGenerator()
# Start a lazy pool of processes.
pool = LazyMP().poolImapUnordered(
analyseDataHelper, itertools.izip(
itertools.repeat(code), path_generator,
process_number_generator.getGenerator([1,2,3,4]),
),
multithreads,
process_number_generator
)
# And let them work.
try:
while True:
# Let multiprocessing pool process all arguments.
pool.next()
projects_analysed += 1
except Exception as err:
# Done
print err
pass
else:
# No multithreading.
for path in path_generator:
neo4j_helper.analyseData((
code, path, 1
))
projects_analysed += 1
if projects_analysed == 0:
print "No project analysed for path: '%s'" %(
flow["in"]
)
def main(argv):
# Setup command line arguments.
parser = ModeArgsParser()
setupArgs(parser)
try:
flow = parser.parseArgs(argv[1], argv[2:])
except:
parser.printHelp(argv[0])
sys.exit()
# Make config point to the absolute path.
full_path = os.path.abspath(argv[0])
last_slash_index = full_path.rfind("/")
base_dir = full_path[0:last_slash_index]
global CONFIG_PATH
CONFIG_PATH = base_dir + "/" + CONFIG_PATH
# Set debugging.
try:
getArg(flow, "d", "debug")
Configurator.setDebugging(True)
except ArgException as err:
# # Debugging was not specified.
pass
if flow[parser.KEY_MODE] != ARGS_CONFIG:
# Load config.
Configurator.load(CONFIG_PATH)
if flow[parser.KEY_MODE] == ARGS_HELP:
parser.printHelp(argv[0])
sys.exit()
elif flow[parser.KEY_MODE] == ARGS_COMBINE_STATISTICS:
combineStatsFiles(flow)
elif flow[parser.KEY_MODE] == ARGS_CONSOLE:
id = 1
try:
id = getArg(flow, "id")
except:
pass
heap_size = Configurator.getHeapVal()
Neo4jHelper().setHeapsize(heap_size[0], heap_size[1])
Neo4jHelper().startConsole(os.path.abspath(flow["in"]), str(id))
elif flow[parser.KEY_MODE] == ARGS_SEARCH:
startSearchMode(flow)
elif flow[parser.KEY_MODE] == ARGS_CONTINUOUS_SEARCH:
startSearchMode(flow, continuous=True)
elif flow[parser.KEY_MODE] == ARGS_CONFIG:
Configurator().setupConfig(
CONFIG_PATH, base_dir, getArg(flow, "p", "path")
)
elif flow[parser.KEY_MODE] == ARGS_PRINT_STATS:
printStats(flow)
elif flow[parser.KEY_MODE] == ARGS_PRINT_RESULTS:
printResults(flow)
else:
parser.printHelp(argv[0])
sys.exit()
