class IdentityList:
_ids = None
def __init__(self, model: Any, key: str):
self.collector = Collector(model)
self.model = model
self.key = key
def __repr__(self):
return f"{self.__class__.__name__}/{self.key}: {len(self)} ids"
def __len__(self):
return len(self.ids)
def __iter__(self):
for i in self.ids:
yield i
@property
def ids(self) -> List[str]:
if not self._ids:
try:
self._ids = self.model.objects(name=self.key)[0].ids
except Exception:
self._ids = []
return self._ids
@ids.setter
def ids(self, ids: Union[List[str], bytes], sep: str = "\n"):
if isinstance(ids, bytes):
self._set_ids_from_bytes(ids, sep=sep)
else:
self._set_ids(ids)
def _set_ids(self, ids: list):
if ids:
ids = [x for x in ids if x != ""] # drop trailing empties
self.collector.save({
"name": self.key,
"count": len(ids),
"ids": ids
})
logger.info("Saved %s ids to %s/%s", len(ids), self.model.__name__,
self.key)
def _set_ids_from_bytes(self, ids: bytes, sep: str = "\n"):
self._set_ids(ids=ids.decode().split(sep))
def delete(self):
self.model.objects(name=self.key).delete()
self.ids = []
def __init__(self, currency: str, should_use_localhost_db: bool = True):
# make sure currency is in lower-case format
if not currency.islower():
raise Exception("<currency> parameter should be a lower-cased symbol of the target currency!")
super().__init__()
# init db
mongodb_uri = Global.read_mongodb_uri(should_use_localhost_db)
db_client = MongoClient(mongodb_uri)
bithumb_db = db_client["bithumb"]
coinone_db = db_client["coinone"]
# korbit_db = db_client["korbit"]
gopax_db = db_client["gopax"]
okcoin_db = db_client["okcoin"]
# coinnest_db = db_client["coinnest"]
# init api
bithumb_api = BithumbApi.instance(True)
coinone_api = CoinoneApi.instance(True)
# korbit_api = KorbitApi.instance(True)
gopax_api = GopaxApi.instance(True)
okcoin_api = OkcoinApi.instance(True)
# coinnest_api = CoinnestApi.instance(True)
# init currency
bithumb_currency = BithumbCurrency[currency.upper()]
coinone_currency = CoinoneCurrency[currency.upper()]
# korbit_currency = KorbitCurrency[currency.upper()]
goapx_currency = GopaxCurrency[currency.upper()]
okcoin_currency = OkcoinCurrency[currency.upper()]
# coinnest_currency = CoinnestCurrency[currency.upper()]
# init collector
self.bt_collector = Collector(
bithumb_api, bithumb_currency, bithumb_db
)
self.co_collector = Collector(
coinone_api, coinone_currency, coinone_db
)
# self.kb_collector = Collector(
# korbit_api, korbit_currency, korbit_db
# )
self.go_collector = Collector(
gopax_api, goapx_currency, gopax_db
)
self.oc_collector = Collector(
okcoin_api, okcoin_currency, okcoin_db
)
def test_parse_error(self):
"""Test that an error retrieving the data is handled."""
mock_response = Mock()
mock_response.text = "1"
with patch("requests.get", return_value=mock_response):
response = Collector().get("tests", self.sources)
self.assertTrue(response["sources"][0]["parse_error"].startswith("Traceback"))
def test_missing_location(self):
"""Test that an exception is raised if the violation location is missing."""
mock_response = Mock()
mock_response.text = """<audit xmlns="http://xmlns.oracle.com/jdeveloper/1013/audit">
<violation-count>2</violation-count>
<models>
<model id="a">
<file>
<path>a</path>
</file>
</model>
<model id="b">
<file>
<path>b</path>
</file>
</model>
</models>
<construct>
<violation>
<message>a</message>
<values>
<value>medium</value>
</values>
</violation>
</construct>
</audit>"""
sources = dict(a=dict(type="ojaudit", parameters=dict(url="http://ojaudit.xml")))
with patch("requests.get", return_value=mock_response):
self.assertTrue(
"has no location element" in Collector().get("violations", sources)["sources"][0]["parse_error"])
def setUp(self):
"""Simple response fixture."""
Collector.RESPONSE_CACHE.clear()
mock_response = Mock()
mock_response.text = "<testsuite tests='2'></testsuite>"
sources = dict(a=dict(type="junit", parameters=dict(url="http://url")))
with patch("requests.get", return_value=mock_response):
self.response = Collector().get("tests", sources)
def test_tests(self):
"""Test that the number of tests is returned."""
mock_response = Mock()
mock_response.json = Mock(return_value=dict(passCount=4, failCount=2))
sources = dict(a=dict(type="jenkins_test_report",
parameters=dict(url="http://jenkins/job/job")))
with patch("requests.get", return_value=mock_response):
response = Collector().get("tests", sources)
self.assertEqual("6", response["sources"][0]["value"])
def test_tests(self):
"""Test that the number of tests is returned."""
mock_response = Mock()
mock_response.json.return_value = dict(
component=dict(measures=[dict(metric="tests", value="88")]))
sources = dict(
a=dict(type="sonarqube", parameters=dict(url="http://sonar", component="id")))
with patch("requests.get", return_value=mock_response):
response = Collector().get("tests", sources)
self.assertEqual("88", response["sources"][0]["value"])
def setUp(self):
"""Simple response fixture."""
Collector.RESPONSE_CACHE.clear()
mock_response = Mock()
mock_response.json.return_value = dict(
jobs=[dict(name="job", url="http://job", buildable=True)]) # Works for both Gitlab and Jenkins
self.sources = dict(
a=dict(type="jenkins", parameters=dict(url="http://jenkins")),
b=dict(type="gitlab", parameters=dict(url="http://gitlab", project="project", private_token="token")))
with patch("requests.get", return_value=mock_response):
self.response = Collector().get("jobs", self.sources)
def test_violations(self):
"""Test that the number of violations is returned."""
mock_response = Mock()
mock_response.text = """<audit xmlns="http://xmlns.oracle.com/jdeveloper/1013/audit">
<violation-count>2</violation-count>
<models>
<model id="a">
<file>
<path>a</path>
</file>
</model>
<model id="b">
<file>
<path>b</path>
</file>
</model>
</models>
<construct>
<children>
<construct>
<children>
<violation>
<message>a</message>
<location model="a">
<line-number>20</line-number>
<column-offset>4</column-offset>
</location>
<values>
<value>warning</value>
</values>
</violation>
</children>
</construct>
<violation>
<message>b</message>
<location model="b">
<line-number>10</line-number>
<column-offset>2</column-offset>
</location>
<values>
<value>exception</value>
</values>
</violation>
</children>
</construct>
</audit>"""
sources = dict(a=dict(type="ojaudit", parameters=dict(url="http://ojaudit.xml")))
with patch("requests.get", return_value=mock_response):
response = Collector().get("violations", sources)
self.assertEqual(
[dict(component="a:20:4", key="894756a0231a17f66b33d0ac18570daa193beea3", message="a", severity="warning"),
dict(component="b:10:2", key="2bdb532d49f0bf2252e85dc2d41e034c8c3e1af3", message="b", severity="exception")],
response["sources"][0]["units"])
self.assertEqual("2", response["sources"][0]["value"])
def test_covered_branches(self):
"""Test that the number of covered branches is returned."""
mock_response = Mock()
mock_response.json.return_value = dict(
component=dict(
measures=[
dict(metric="conditions_to_cover", value="100"), dict(metric="uncovered_conditions", value="10")]))
sources = dict(
a=dict(type="sonarqube", parameters=dict(url="http://sonar", component="id")))
with patch("requests.get", return_value=mock_response):
response = Collector().get("covered_branches", sources)
self.assertEqual("90", response["sources"][0]["value"])
def test_failed_tests(self):
"""Test that the number of failed tests is returned."""
mock_response = Mock()
mock_response.json = Mock(return_value=dict(
failCount=2,
suites=[
dict(cases=[
dict(status="FAILED", name="tc1", className="c1"),
dict(status="FAILED", name="tc2", className="c2")
])
]))
sources = dict(a=dict(type="jenkins_test_report",
parameters=dict(url="http://jenkins/job/job")))
with patch("requests.get", return_value=mock_response):
response = Collector().get("failed_tests", sources)
self.assertEqual("2", response["sources"][0]["value"])
self.assertEqual([
dict(class_name="c1", key="tc1", name="tc1"),
dict(class_name="c2", key="tc2", name="tc2")
], response["sources"][0]["units"])
def test_violations(self):
"""Test that the number of violations is returned."""
mock_response = Mock()
mock_response.json.return_value = dict(
total="2",
issues=[
dict(key="a", message="a", component="a", severity="INFO", type="BUG"),
dict(key="b", message="b", component="b", severity="MAJOR", type="CODE_SMELL")])
sources = dict(
a=dict(type="sonarqube", parameters=dict(url="http://sonar", component="id")))
with patch("requests.get", return_value=mock_response):
response = Collector().get("violations", sources)
self.assertEqual(
[
dict(component="a", key="a", message="a", severity="info", type="bug",
url="http://sonar/project/issues?id=id&open=a"),
dict(component="b", key="b", message="b", severity="major", type="code_smell",
url="http://sonar/project/issues?id=id&open=b")
],
response["sources"][0]["units"])
self.assertEqual("2", response["sources"][0]["value"])
def test_connection_error(self):
"""Test that an error retrieving the data is handled."""
with patch("requests.get", side_effect=Exception):
response = Collector().get("tests", self.sources)
self.assertTrue(response["sources"][0]["connection_error"].startswith("Traceback"))
from collector.collector import Collector
from decider.decider import Decider
logging.basicConfig(filename='actuator.log', level=logging.INFO)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.info("Reading configuration file")
CONFIG = ConfigParser.ConfigParser()
CONFIG.read('actuator.cfg')
REDIS_HOST = CONFIG.get("REDIS", "address")
REDIS_PORT = CONFIG.get("REDIS", "port")
TOLERANCE = CONFIG.get("ACTUATION", "tolerance")
MAX_SIZE = CONFIG.get("ACTUATION", "max_size")
INIT_SIZE = CONFIG.get("ACTUATION", "init_size")
CONTEXTS = CONFIG.get("FEDERATION", "contexts").split(",")
APPLICATION = CONFIG.get("FEDERATION", "application")
COLLECTOR = Collector(redis_host=REDIS_HOST, redis_port=REDIS_PORT)
DECIDERS = {}
for context in CONTEXTS:
DECIDERS[context] = Decider(tolerance=TOLERANCE,
max_size=MAX_SIZE,
context=context,
application=APPLICATION,
init_size=INIT_SIZE)
while True:
METRIC = COLLECTOR.collect_recent()
x = json.loads(METRIC[1].replace("'", '"'))
for context_metric in x:
DECIDERS[context_metric].process_metric(x[context_metric])
time.sleep(7)
def __init__(self, model: Any, key: str):
self.collector = Collector(model)
self.model = model
self.key = key
if __name__ == '__main__':
for dataset in datasets:
print('########### Start LSTM on Dataset ' + dataset + ' ###########')
config.init('LSTM_' + dataset)
checkpoint_name = config.path + FLAGS.checkpoint_name
(x_train, y_train), (x_test, y_test), (x_validate, y_validate) = (None, None), (None, None), (None, None)
collector = None
result_folder = config.path + FLAGS.result_folder
if dataset == 'BGL':
data_instances = config.BGL_data
(x_train, y_train), (x_test, y_test), (x_validate, y_validate) = load_BGL(data_instances, 0.35, 0.6)
collector = Collector(result_folder, (1, 1, 1, 1), False, config.BGL_col_header, 100)
if dataset == 'HDFS':
data_instances = config.HDFS_data
(x_train, y_train), (x_test, y_test), (x_validate, y_validate) = dataloader.load_HDFS(data_instances,
train_ratio=0.35,
is_data_instance=True,
test_ratio=0.6)
collector = Collector(result_folder, (1, 1, 1, 1), False, config.HDFS_col_header, 100)
assert FLAGS.h < FLAGS.plb
lstm_preprocessor = preprocessing.LstmPreprocessor(x_train, x_test, x_validate)
sym_count = len(lstm_preprocessor.vectors) - 1
print('Total symbols: %d' % sym_count)
print(lstm_preprocessor.syms)
def test_create_with_functions_override(self, endpoints):
endpoint = endpoints["no_mappings"]
temp = {"test_func": "test_value"}
c = Collector("test", functions=temp)
assert c.functions == temp
