class IdentityList: _ids = None def __init__(self, model: Any, key: str): self.collector = Collector(model) self.model = model self.key = key def __repr__(self): return f"{self.__class__.__name__}/{self.key}: {len(self)} ids" def __len__(self): return len(self.ids) def __iter__(self): for i in self.ids: yield i @property def ids(self) -> List[str]: if not self._ids: try: self._ids = self.model.objects(name=self.key)[0].ids except Exception: self._ids = [] return self._ids @ids.setter def ids(self, ids: Union[List[str], bytes], sep: str = "\n"): if isinstance(ids, bytes): self._set_ids_from_bytes(ids, sep=sep) else: self._set_ids(ids) def _set_ids(self, ids: list): if ids: ids = [x for x in ids if x != ""] # drop trailing empties self.collector.save({ "name": self.key, "count": len(ids), "ids": ids }) logger.info("Saved %s ids to %s/%s", len(ids), self.model.__name__, self.key) def _set_ids_from_bytes(self, ids: bytes, sep: str = "\n"): self._set_ids(ids=ids.decode().split(sep)) def delete(self): self.model.objects(name=self.key).delete() self.ids = []
def __init__(self, currency: str, should_use_localhost_db: bool = True): # make sure currency is in lower-case format if not currency.islower(): raise Exception("<currency> parameter should be a lower-cased symbol of the target currency!") super().__init__() # init db mongodb_uri = Global.read_mongodb_uri(should_use_localhost_db) db_client = MongoClient(mongodb_uri) bithumb_db = db_client["bithumb"] coinone_db = db_client["coinone"] # korbit_db = db_client["korbit"] gopax_db = db_client["gopax"] okcoin_db = db_client["okcoin"] # coinnest_db = db_client["coinnest"] # init api bithumb_api = BithumbApi.instance(True) coinone_api = CoinoneApi.instance(True) # korbit_api = KorbitApi.instance(True) gopax_api = GopaxApi.instance(True) okcoin_api = OkcoinApi.instance(True) # coinnest_api = CoinnestApi.instance(True) # init currency bithumb_currency = BithumbCurrency[currency.upper()] coinone_currency = CoinoneCurrency[currency.upper()] # korbit_currency = KorbitCurrency[currency.upper()] goapx_currency = GopaxCurrency[currency.upper()] okcoin_currency = OkcoinCurrency[currency.upper()] # coinnest_currency = CoinnestCurrency[currency.upper()] # init collector self.bt_collector = Collector( bithumb_api, bithumb_currency, bithumb_db ) self.co_collector = Collector( coinone_api, coinone_currency, coinone_db ) # self.kb_collector = Collector( # korbit_api, korbit_currency, korbit_db # ) self.go_collector = Collector( gopax_api, goapx_currency, gopax_db ) self.oc_collector = Collector( okcoin_api, okcoin_currency, okcoin_db )
def test_parse_error(self): """Test that an error retrieving the data is handled.""" mock_response = Mock() mock_response.text = "1" with patch("requests.get", return_value=mock_response): response = Collector().get("tests", self.sources) self.assertTrue(response["sources"][0]["parse_error"].startswith("Traceback"))
def test_missing_location(self): """Test that an exception is raised if the violation location is missing.""" mock_response = Mock() mock_response.text = """<audit xmlns="http://xmlns.oracle.com/jdeveloper/1013/audit"> <violation-count>2</violation-count> <models> <model id="a"> <file> <path>a</path> </file> </model> <model id="b"> <file> <path>b</path> </file> </model> </models> <construct> <violation> <message>a</message> <values> <value>medium</value> </values> </violation> </construct> </audit>""" sources = dict(a=dict(type="ojaudit", parameters=dict(url="http://ojaudit.xml"))) with patch("requests.get", return_value=mock_response): self.assertTrue( "has no location element" in Collector().get("violations", sources)["sources"][0]["parse_error"])
def setUp(self): """Simple response fixture.""" Collector.RESPONSE_CACHE.clear() mock_response = Mock() mock_response.text = "<testsuite tests='2'></testsuite>" sources = dict(a=dict(type="junit", parameters=dict(url="http://url"))) with patch("requests.get", return_value=mock_response): self.response = Collector().get("tests", sources)
def test_tests(self): """Test that the number of tests is returned.""" mock_response = Mock() mock_response.json = Mock(return_value=dict(passCount=4, failCount=2)) sources = dict(a=dict(type="jenkins_test_report", parameters=dict(url="http://jenkins/job/job"))) with patch("requests.get", return_value=mock_response): response = Collector().get("tests", sources) self.assertEqual("6", response["sources"][0]["value"])
def test_tests(self): """Test that the number of tests is returned.""" mock_response = Mock() mock_response.json.return_value = dict( component=dict(measures=[dict(metric="tests", value="88")])) sources = dict( a=dict(type="sonarqube", parameters=dict(url="http://sonar", component="id"))) with patch("requests.get", return_value=mock_response): response = Collector().get("tests", sources) self.assertEqual("88", response["sources"][0]["value"])
def setUp(self): """Simple response fixture.""" Collector.RESPONSE_CACHE.clear() mock_response = Mock() mock_response.json.return_value = dict( jobs=[dict(name="job", url="http://job", buildable=True)]) # Works for both Gitlab and Jenkins self.sources = dict( a=dict(type="jenkins", parameters=dict(url="http://jenkins")), b=dict(type="gitlab", parameters=dict(url="http://gitlab", project="project", private_token="token"))) with patch("requests.get", return_value=mock_response): self.response = Collector().get("jobs", self.sources)
def test_violations(self): """Test that the number of violations is returned.""" mock_response = Mock() mock_response.text = """<audit xmlns="http://xmlns.oracle.com/jdeveloper/1013/audit"> <violation-count>2</violation-count> <models> <model id="a"> <file> <path>a</path> </file> </model> <model id="b"> <file> <path>b</path> </file> </model> </models> <construct> <children> <construct> <children> <violation> <message>a</message> <location model="a"> <line-number>20</line-number> <column-offset>4</column-offset> </location> <values> <value>warning</value> </values> </violation> </children> </construct> <violation> <message>b</message> <location model="b"> <line-number>10</line-number> <column-offset>2</column-offset> </location> <values> <value>exception</value> </values> </violation> </children> </construct> </audit>""" sources = dict(a=dict(type="ojaudit", parameters=dict(url="http://ojaudit.xml"))) with patch("requests.get", return_value=mock_response): response = Collector().get("violations", sources) self.assertEqual( [dict(component="a:20:4", key="894756a0231a17f66b33d0ac18570daa193beea3", message="a", severity="warning"), dict(component="b:10:2", key="2bdb532d49f0bf2252e85dc2d41e034c8c3e1af3", message="b", severity="exception")], response["sources"][0]["units"]) self.assertEqual("2", response["sources"][0]["value"])
def test_covered_branches(self): """Test that the number of covered branches is returned.""" mock_response = Mock() mock_response.json.return_value = dict( component=dict( measures=[ dict(metric="conditions_to_cover", value="100"), dict(metric="uncovered_conditions", value="10")])) sources = dict( a=dict(type="sonarqube", parameters=dict(url="http://sonar", component="id"))) with patch("requests.get", return_value=mock_response): response = Collector().get("covered_branches", sources) self.assertEqual("90", response["sources"][0]["value"])
def test_failed_tests(self): """Test that the number of failed tests is returned.""" mock_response = Mock() mock_response.json = Mock(return_value=dict( failCount=2, suites=[ dict(cases=[ dict(status="FAILED", name="tc1", className="c1"), dict(status="FAILED", name="tc2", className="c2") ]) ])) sources = dict(a=dict(type="jenkins_test_report", parameters=dict(url="http://jenkins/job/job"))) with patch("requests.get", return_value=mock_response): response = Collector().get("failed_tests", sources) self.assertEqual("2", response["sources"][0]["value"]) self.assertEqual([ dict(class_name="c1", key="tc1", name="tc1"), dict(class_name="c2", key="tc2", name="tc2") ], response["sources"][0]["units"])
def test_violations(self): """Test that the number of violations is returned.""" mock_response = Mock() mock_response.json.return_value = dict( total="2", issues=[ dict(key="a", message="a", component="a", severity="INFO", type="BUG"), dict(key="b", message="b", component="b", severity="MAJOR", type="CODE_SMELL")]) sources = dict( a=dict(type="sonarqube", parameters=dict(url="http://sonar", component="id"))) with patch("requests.get", return_value=mock_response): response = Collector().get("violations", sources) self.assertEqual( [ dict(component="a", key="a", message="a", severity="info", type="bug", url="http://sonar/project/issues?id=id&open=a"), dict(component="b", key="b", message="b", severity="major", type="code_smell", url="http://sonar/project/issues?id=id&open=b") ], response["sources"][0]["units"]) self.assertEqual("2", response["sources"][0]["value"])
def test_connection_error(self): """Test that an error retrieving the data is handled.""" with patch("requests.get", side_effect=Exception): response = Collector().get("tests", self.sources) self.assertTrue(response["sources"][0]["connection_error"].startswith("Traceback"))
from collector.collector import Collector from decider.decider import Decider logging.basicConfig(filename='actuator.log', level=logging.INFO) logging.getLogger("requests").setLevel(logging.WARNING) logging.info("Reading configuration file") CONFIG = ConfigParser.ConfigParser() CONFIG.read('actuator.cfg') REDIS_HOST = CONFIG.get("REDIS", "address") REDIS_PORT = CONFIG.get("REDIS", "port") TOLERANCE = CONFIG.get("ACTUATION", "tolerance") MAX_SIZE = CONFIG.get("ACTUATION", "max_size") INIT_SIZE = CONFIG.get("ACTUATION", "init_size") CONTEXTS = CONFIG.get("FEDERATION", "contexts").split(",") APPLICATION = CONFIG.get("FEDERATION", "application") COLLECTOR = Collector(redis_host=REDIS_HOST, redis_port=REDIS_PORT) DECIDERS = {} for context in CONTEXTS: DECIDERS[context] = Decider(tolerance=TOLERANCE, max_size=MAX_SIZE, context=context, application=APPLICATION, init_size=INIT_SIZE) while True: METRIC = COLLECTOR.collect_recent() x = json.loads(METRIC[1].replace("'", '"')) for context_metric in x: DECIDERS[context_metric].process_metric(x[context_metric]) time.sleep(7)
def __init__(self, model: Any, key: str): self.collector = Collector(model) self.model = model self.key = key
if __name__ == '__main__': for dataset in datasets: print('########### Start LSTM on Dataset ' + dataset + ' ###########') config.init('LSTM_' + dataset) checkpoint_name = config.path + FLAGS.checkpoint_name (x_train, y_train), (x_test, y_test), (x_validate, y_validate) = (None, None), (None, None), (None, None) collector = None result_folder = config.path + FLAGS.result_folder if dataset == 'BGL': data_instances = config.BGL_data (x_train, y_train), (x_test, y_test), (x_validate, y_validate) = load_BGL(data_instances, 0.35, 0.6) collector = Collector(result_folder, (1, 1, 1, 1), False, config.BGL_col_header, 100) if dataset == 'HDFS': data_instances = config.HDFS_data (x_train, y_train), (x_test, y_test), (x_validate, y_validate) = dataloader.load_HDFS(data_instances, train_ratio=0.35, is_data_instance=True, test_ratio=0.6) collector = Collector(result_folder, (1, 1, 1, 1), False, config.HDFS_col_header, 100) assert FLAGS.h < FLAGS.plb lstm_preprocessor = preprocessing.LstmPreprocessor(x_train, x_test, x_validate) sym_count = len(lstm_preprocessor.vectors) - 1 print('Total symbols: %d' % sym_count) print(lstm_preprocessor.syms)
def test_create_with_functions_override(self, endpoints): endpoint = endpoints["no_mappings"] temp = {"test_func": "test_value"} c = Collector("test", functions=temp) assert c.functions == temp