def __init__(self, root_cert, urn_authority, url): self._url = url self._api_version = 2 self._slices = dict() self._agg = Aggregate() self._agg.add_resources([FakeVM(self._agg) for _ in range(3)]) self._cred_verifier = geni.CredentialVerifier(root_cert) self._urn_authority = urn_authority self._my_urn = publicid_to_urn("%s %s %s" % (self._urn_authority, 'authority', 'am')) self.max_lease = datetime.timedelta(days=REFAM_MAXLEASE_DAYS) self.logger = logging.getLogger('gcf.am2')
def test_big_example(self): # bigger agg example sa = Aggregate('my test', exp_en=np.ones(50), exp_premium=0, exp_lr=.75, exp_attachment=20, exp_limit=np.linspace(10, 500, 50), sev_name='lognorm', sev_mean=100, sev_cv=0.8, freq_name='poisson', freq_a=0) bs = 1 # 2 and 13 log2 = 13 N = 1 << log2 MAXL = N * bs xs = np.linspace(0, MAXL, N, endpoint=False) audit = sa.update(xs, 1, None, 'exact', sev_calc='discrete', discretization_calc='survival', verbose=True) # average of square errors is small: self.assertTrue( np.sum(np.abs(audit.iloc[0:-3, :]['rel sev err']) ** 2) ** .5 / (len(audit) - 3) < 1e-6) self.assertTrue( np.sum(np.abs(audit.iloc[0:-3, :]['abs sev err']) ** 2) ** .5 / (len(audit) - 3) < 1e-4)
def test_Aggregate(self): sig = .75 d1b1 = Aggregate('single', exp_en=10, sev_name='lognorm', sev_a=[sig, sig / 2, sig / 5], sev_scale=100 * np.exp(-sig ** 2 / 2), sev_wt=1, freq_name='gamma', freq_a=0.81) xs = np.linspace(0, 1023, 1024) d1b1.update(xs, verbose=False) self.assertTrue(d1b1.statistics_total_df.T.loc['el', :].sum() == 5146.408347357843) sig = 0.75 d1b2 = Aggregate('sublines', exp_en=[10, 10, 10, 10], exp_attachment=[0, 0, 100, 50], exp_limit=[np.inf, 100, np.inf, 100], sev_name='lognorm', sev_a=sig, sev_scale=100 * np.exp(-sig ** 2 / 2), sev_wt=1, freq_name='gamma', freq_a=0.81) self.assertTrue(d1b2.statistics_total_df.T.loc['el', :].sum() == 6152.9037739696396) d2b1 = Aggregate('mixed', exp_en=50, exp_attachment=[0, 50, 100, 150, 200], exp_limit=[250, 250, np.inf, 100, 5000], sev_name='lognorm', sev_mean=100, sev_cv=np.linspace(0.05, 2, 5), sev_wt=0.2 * np.ones(5), freq_name='gamma', freq_a=1.2) self.assertTrue(d2b1.statistics_total_df.T.loc['el', :].sum() == 42977.102931408575) fixed = Aggregate('chistogram', exp_en=1, sev_name='dhistogram', sev_xs=[0, 1, 2, 3, 4], sev_ps=.2, freq_name='fixed') self.assertTrue(fixed.statistics_df['sev_cv'].sum() + fixed.statistics_total_df['sev_cv'].sum()== 2.1213203435596428)
def test_aggregate_is_offline_true(): lun1 = Lun(name='TestLun1') paths1 = ['hostname_port1.L123', 'hostname_port2.L456'] lun1.set_paths(paths=paths1) plex1 = Plex(name='TestPlex1') plex1.add_lun(lun=lun1) lun2 = Lun(name='TestLun2') paths2 = ['hostname_port1.L123', 'hostname_port4.L456'] lun2.set_paths(paths=paths2) plex2 = Plex(name='TestPlex2') plex2.add_lun(lun=lun2) aggregate = Aggregate(name='TestAggr') aggregate.add_plex(plex=plex1) aggregate.add_plex(plex=plex2) assert aggregate.is_offline(down_ports=['port1', 'port2', 'port4']) is True
def create_root_aggregate(self, defaults): """ If a root aggregate doesn't already exist, create one """ try: aggr = self.get_root_aggregate() return except ValueError, valerr: try: root_aggr_name = defaults.get("vfiler", "default_root_aggregate") xmldata = """<aggregate type="root" name="%s"/>""" % root_aggr_name node = etree.fromstring(xmldata) aggr = Aggregate() aggr.configure_from_node(node, defaults, self) self.add_child(aggr) except (NoSectionError, NoOptionError), e: raise valerr
def create_root_aggregate(self, defaults): """ If a root aggregate doesn't already exist, create one """ try: aggr = self.get_root_aggregate() return except ValueError, valerr: try: root_aggr_name = defaults.get('vfiler', 'default_root_aggregate') xmldata = """<aggregate type="root" name="%s"/>""" % root_aggr_name node = etree.fromstring(xmldata) aggr = Aggregate() aggr.configure_from_node(node, defaults, self) self.add_child(aggr) except (NoSectionError, NoOptionError), e: raise valerr
def find_target_aggr(self, targetfiler, aggrname, defaults): """ Given a name, find the target aggregate in the config, or create a reference object if we just have a name """ log.debug("Finding target aggregates...") try: targetaggr = [ x for x in targetfiler.get_aggregates() if x.name == aggrname ][0] except IndexError: # An indexerror means the aggregate isn't defined in the # project XML, which is ok. We invent an aggregate and # add it to the target filer. xmldata = """<aggregate name="%s" />""" % aggrname node = etree.fromstring(xmldata) targetaggr = Aggregate() targetaggr.configure_from_node(node, defaults, targetfiler) targetfiler.add_child(targetaggr) pass log.debug("Added a placeholder aggregate to project: %s", targetaggr) log.debug("%s", targetfiler.get_aggregates()) return targetaggr
def test_aggregate_add_plex(): lun1 = Lun(name='TestLun1') paths1 = ['hostname_port1.L123', 'hostname_port2.L456'] lun1.set_paths(paths=paths1) plex1 = Plex(name='TestPlex1') plex1.add_lun(lun=lun1) lun2 = Lun(name='TestLun2') paths2 = ['hostname_port1.L123', 'hostname_port4.L456'] lun2.set_paths(paths=paths2) plex2 = Plex(name='TestPlex2') plex2.add_lun(lun=lun2) aggregate = Aggregate(name='TestAggr') aggregate.add_plex(plex=plex1) assert (plex1 in aggregate.plexes) is True assert (plex1.name in aggregate.plex_names) is True aggregate.add_plex(plex=plex2) assert (plex1 in aggregate.plexes) is True assert (plex1.name in aggregate.plex_names) is True assert (plex2 in aggregate.plexes) is True assert (plex2.name in aggregate.plex_names) is True
def do_aggregate(self,datacontainer,run_id): ag = Aggregate(self.root_path,self.aggregate_id,self.aggregate_to_file,self.name) for meth in self.aggregate_methods: getattr(ag,meth)()
class ReferenceAggregateManager(object): '''A reference Aggregate Manager that manages fake resources.''' # root_cert is a single cert or dir of multiple certs # that are trusted to sign credentials def __init__(self, root_cert, urn_authority, url): self._url = url self._api_version = 2 self._slices = dict() self._agg = Aggregate() self._agg.add_resources([FakeVM(self._agg) for _ in range(3)]) self._cred_verifier = geni.CredentialVerifier(root_cert) self._urn_authority = urn_authority self._my_urn = publicid_to_urn("%s %s %s" % (self._urn_authority, 'authority', 'am')) self.max_lease = datetime.timedelta(days=REFAM_MAXLEASE_DAYS) self.logger = logging.getLogger('gcf.am2') def GetVersion(self, options): '''Specify version information about this AM. That could include API version information, RSpec format and version information, etc. Return a dict.''' self.logger.info("Called GetVersion") reqver = [dict(type="geni", version="3", schema="http://www.geni.net/resources/rspec/3/request.xsd", namespace="http://www.geni.net/resources/rspec/3", extensions=[])] adver = [dict(type="geni", version="3", schema="http://www.geni.net/resources/rspec/3/ad.xsd", namespace="http://www.geni.net/resources/rspec/3", extensions=[])] api_versions = dict() api_versions[str(self._api_version)] = self._url versions = dict(geni_api=2, geni_api_versions=api_versions, geni_request_rspec_versions=reqver, geni_ad_rspec_versions=adver) return dict(geni_api=versions['geni_api'], code=dict(geni_code=0, am_type="gcf2", am_code=0), value=versions, output="") # The list of credentials are options - some single cred # must give the caller required permissions. # The semantics of the API are unclear on this point, so # this is just the current implementation def ListResources(self, credentials, options): '''Return an RSpec of resources managed at this AM. If a geni_slice_urn is given in the options, then only return resources assigned to that slice. If geni_available is specified in the options, then only report available resources. And if geni_compressed option is specified, then compress the result.''' self.logger.info('ListResources(%r)' % (options)) # Note this list of privileges is really the name of an operation # from the privilege_table in sfa/trust/rights.py # Credentials will specify a list of privileges, each of which # confers the right to perform a list of operations. # EG the 'info' privilege in a credential allows the operations # listslices, listnodes, policy # could require list or listnodes? privileges = () # Note that verify throws an exception on failure. # Use the client PEM format cert as retrieved # from the https connection by the SecureXMLRPCServer # to identify the caller. self._cred_verifier.verify_from_strings(self._server.pem_cert, credentials, None, privileges) # If we get here, the credentials give the caller # all needed privileges to act on the given target. if 'geni_rspec_version' not in options: # This is a required option, so error out with bad arguments. self.logger.error('No geni_rspec_version supplied to ListResources.') return self.errorResult(1, 'Bad Arguments: option geni_rspec_version was not supplied.') if 'type' not in options['geni_rspec_version']: self.logger.error('ListResources: geni_rspec_version does not contain a type field.') return self.errorResult(1, 'Bad Arguments: option geni_rspec_version does not have a type field.') if 'version' not in options['geni_rspec_version']: self.logger.error('ListResources: geni_rspec_version does not contain a version field.') return self.errorResult(1, 'Bad Arguments: option geni_rspec_version does not have a version field.') # Look to see what RSpec version the client requested # Error-check that the input value is supported. rspec_type = options['geni_rspec_version']['type'] if isinstance(rspec_type, str): rspec_type = rspec_type.lower().strip() rspec_version = options['geni_rspec_version']['version'] if rspec_type != 'geni': self.logger.error('ListResources: Unknown RSpec type %s requested', rspec_type) return self.errorResult(4, 'Bad Version: requested RSpec type %s is not a valid option.' % (rspec_type)) if rspec_version != '3': self.logger.error('ListResources: Unknown RSpec version %s requested', rspec_version) return self.errorResult(4, 'Bad Version: requested RSpec version %s is not a valid option.' % (rspec_type)) self.logger.info("ListResources requested RSpec %s (%s)", rspec_type, rspec_version) if 'geni_slice_urn' in options: slice_urn = options['geni_slice_urn'] if slice_urn in self._slices: result = self.manifest_rspec(slice_urn) else: # return an empty rspec return self._no_such_slice(slice_urn) else: all_resources = self._agg.catalog(None) available = 'geni_available' in options and options['geni_available'] resource_xml = "" for r in all_resources: if available and not r.available: continue resource_xml = resource_xml + self.advert_resource(r) result = self.advert_header() + resource_xml + self.advert_footer() self.logger.debug("Result is now \"%s\"", result) # Optionally compress the result if 'geni_compressed' in options and options['geni_compressed']: try: result = base64.b64encode(zlib.compress(result)) except Exception, exc: import traceback self.logger.error("Error compressing and encoding resource list: %s", traceback.format_exc()) raise Exception("Server error compressing resource list", exc) return dict(code=dict(geni_code=0, am_type="gcf2", am_code=0), value=result, output="")
def test_create_aggregate(): aggregate = Aggregate(name='TestAggr') assert aggregate.name == 'TestAggr' assert aggregate.plexes == [] assert aggregate.plex_names == []
irs_thrs = [2, 4, 6, 8, 10] irs_max = [12, 14, 16, 18, 20] if (False): print "Going to sweep over Maximum and threshold IRS values" for (i, irs_ths) in enumerate(irs_thrs): for (j, irs_mx) in enumerate(irs_max): config['model']['irs_threshold'] = irs_ths config['model']['max_irs_value'] = irs_mx aggregate_id = str(uuid.uuid4()) sim = Simulation(config, aggregate_id) sim.run() a = Aggregate(config['file_root'], aggregate_id, True) results[i, j] = a.default_distribution() print results path = orig_file_root + "%s.png" cbmax = int(np.max(results) + 0.5) #cbmax = int(5.2+0.5) heat_map(results, "Exponent heatmap", 'IrsThreshold', 'MaxIrsValue', irs_thrs, irs_max, range(0, cbmax), path, "IrsThs_IrsMax") min_alpha = np.min(results[np.where(results > 0)]) (xr, yr) = np.where(results == min_alpha) i = xr[0] j = yr[0]
def __init__(self, id = None, name = '””, password = “”, emailid = “” ): Aggregate.__init__(self) self._apply_changes(Userdetails(id, name, password, emailid))
class ReferenceAggregateManager(object): '''A reference Aggregate Manager that manages fake resources.''' # root_cert is a single cert or dir of multiple certs # that are trusted to sign credentials def __init__(self, root_cert, urn_authority, url): self._url = url self._api_version = 2 self._slices = dict() self._agg = Aggregate() self._agg.add_resources([FakeVM(self._agg) for _ in range(3)]) self._cred_verifier = geni.CredentialVerifier(root_cert) self._urn_authority = urn_authority self._my_urn = publicid_to_urn("%s %s %s" % (self._urn_authority, 'authority', 'am')) self.max_lease = datetime.timedelta(days=REFAM_MAXLEASE_DAYS) self.logger = logging.getLogger('gcf.am2') def GetVersion(self, options): '''Specify version information about this AM. That could include API version information, RSpec format and version information, etc. Return a dict.''' self.logger.info("Called GetVersion") reqver = [dict(type="geni", version="3", schema="http://www.geni.net/resources/rspec/3/request.xsd", namespace="http://www.geni.net/resources/rspec/3", extensions=[])] adver = [dict(type="geni", version="3", schema="http://www.geni.net/resources/rspec/3/ad.xsd", namespace="http://www.geni.net/resources/rspec/3", extensions=[])] api_versions = dict() api_versions[str(self._api_version)] = self._url versions = dict(geni_api=2, geni_api_versions=api_versions, geni_request_rspec_versions=reqver, geni_ad_rspec_versions=adver) return dict(geni_api=versions['geni_api'], code=dict(geni_code=0, am_type="gcf2", am_code=0), value=versions, output="") # The list of credentials are options - some single cred # must give the caller required permissions. # The semantics of the API are unclear on this point, so # this is just the current implementation def ListResources(self, credentials, options): '''Return an RSpec of resources managed at this AM. If a geni_slice_urn is given in the options, then only return resources assigned to that slice. If geni_available is specified in the options, then only report available resources. And if geni_compressed option is specified, then compress the result.''' self.logger.info('ListResources(%r)' % (options)) # Note this list of privileges is really the name of an operation # from the privilege_table in sfa/trust/rights.py # Credentials will specify a list of privileges, each of which # confers the right to perform a list of operations. # EG the 'info' privilege in a credential allows the operations # listslices, listnodes, policy # could require list or listnodes? privileges = () # Note that verify throws an exception on failure. # Use the client PEM format cert as retrieved # from the https connection by the SecureXMLRPCServer # to identify the caller. try: self._cred_verifier.verify_from_strings(self._server.pem_cert, credentials, None, privileges) except Exception, e: raise xmlrpclib.Fault('Insufficient privileges', str(e)) # If we get here, the credentials give the caller # all needed privileges to act on the given target. if 'geni_rspec_version' not in options: # This is a required option, so error out with bad arguments. self.logger.error('No geni_rspec_version supplied to ListResources.') return self.errorResult(1, 'Bad Arguments: option geni_rspec_version was not supplied.') if 'type' not in options['geni_rspec_version']: self.logger.error('ListResources: geni_rspec_version does not contain a type field.') return self.errorResult(1, 'Bad Arguments: option geni_rspec_version does not have a type field.') if 'version' not in options['geni_rspec_version']: self.logger.error('ListResources: geni_rspec_version does not contain a version field.') return self.errorResult(1, 'Bad Arguments: option geni_rspec_version does not have a version field.') # Look to see what RSpec version the client requested # Error-check that the input value is supported. rspec_type = options['geni_rspec_version']['type'] if isinstance(rspec_type, str): rspec_type = rspec_type.lower().strip() rspec_version = options['geni_rspec_version']['version'] if rspec_type != 'geni': self.logger.error('ListResources: Unknown RSpec type %s requested', rspec_type) return self.errorResult(4, 'Bad Version: requested RSpec type %s is not a valid option.' % (rspec_type)) if rspec_version != '3': self.logger.error('ListResources: Unknown RSpec version %s requested', rspec_version) return self.errorResult(4, 'Bad Version: requested RSpec version %s is not a valid option.' % (rspec_type)) self.logger.info("ListResources requested RSpec %s (%s)", rspec_type, rspec_version) if 'geni_slice_urn' in options: slice_urn = options['geni_slice_urn'] if slice_urn in self._slices: result = self.manifest_rspec(slice_urn) else: # return an empty rspec return self._no_such_slice(slice_urn) else: all_resources = self._agg.catalog(None) available = 'geni_available' in options and options['geni_available'] resource_xml = "" for r in all_resources: if available and not r.available: continue resource_xml = resource_xml + self.advert_resource(r) result = self.advert_header() + resource_xml + self.advert_footer() self.logger.debug("Result is now \"%s\"", result) # Optionally compress the result if 'geni_compressed' in options and options['geni_compressed']: try: result = base64.b64encode(zlib.compress(result)) except Exception, exc: import traceback self.logger.error("Error compressing and encoding resource list: %s", traceback.format_exc()) raise Exception("Server error compressing resource list", exc)
with conn.cursor() as cur: # print(cur.execute(""" # SELECT * from test; # """)) cur.execute(""" CREATE TABLE if NOT EXISTS public.test ( id SERIAL PRIMARY KEY, created TIMESTAMP DEFAULT now(), scrape json DEFAULT '{}' ); """) conn.commit() if __name__ == '__main__': print('Settings up database') create_tables() choice = input('Do you want to scrape data? y/N').lower() or 'N' if choice == 'y': print('Start scraping...') sts = SpeedTestScraper(conn) sts.scrape() choice = input('Do you want to aggregate data? y/N').lower() or 'N' if choice == 'y': print('aggregating...') agg = Aggregate(conn) path = agg.to_csv() print(f'Output file in {path}')
from collection import Collection from iterator_a import IteratorA from iterator_b import IteratorB from aggregate import Aggregate if __name__ == "__main__": aggregate = Aggregate() aggregate.add_item(1) aggregate.add_item(2) aggregate.add_item(3) print("Iterator A") it_a = IteratorA(aggregate) for el in it_a: print(el) print("Iterator B") it_b = IteratorB(aggregate) for el in it_b: print(el)
from aggregate import Aggregate import asyncio from aggregate_settings import CONFIG if __name__ == "__main__": asyncio.set_event_loop(asyncio.new_event_loop()) mainloop = asyncio.get_event_loop() aggregate_tasks = [] for aggregate in CONFIG["aggregates"]: new_aggregate = Aggregate( CONFIG["aggregates"][aggregate]["aggregate_stream_name"], CONFIG["aggregates"][aggregate]["collection_name"], CONFIG["aggregates"][aggregate]["subscription_name"], CONFIG["aggregates"][aggregate]["watched_stream_name"], ) aggregate_tasks.append(new_aggregate.aggregate_fn()) mainloop.run_until_complete(asyncio.wait(aggregate_tasks))
if __name__ == "__main__": app = QtGui.QApplication(sys.argv) cache = Cache() # inicjalizacja okna programu configuration = config.Config() # wczytanie konfiguracji database = db.Db(ip=configuration.db_ip, port=configuration.db_port, user=configuration.db_user, password=configuration.db_password, database=configuration.db_name, ui=cache.ui) # otwarcie połączenia z bazą danych database.prepare(tank_table=configuration.db_tank_table, nozzle_table=configuration.db_nozzle_table) # przygotowanie bazy danych tank_aggregate = Aggregate(values=database.tank_table_values, ui=cache.ui) # lista agregacyjna zbiornika paliwa nozzle_aggregate = Aggregate(values=database.nozzle_table_values, ui=cache.ui) # lista agregacyjna pistoletu paliwowego export = Export(ui=cache.ui) # inicjalizacja eksportu # przeliczenie list agregatów tank_aggregate.compile_tank() nozzle_aggregate.compile_nozzle() # aktualizacja tablic gui tank_aggregate.update_tank_table() nozzle_aggregate.update_nozzle_table() # automatyczne aktualizowanie tablic timer = QTimer()