def __init__(self, root_cert, urn_authority, url):
self._url = url
self._api_version = 2
self._slices = dict()
self._agg = Aggregate()
self._agg.add_resources([FakeVM(self._agg) for _ in range(3)])
self._cred_verifier = geni.CredentialVerifier(root_cert)
self._urn_authority = urn_authority
self._my_urn = publicid_to_urn("%s %s %s" % (self._urn_authority, 'authority', 'am'))
self.max_lease = datetime.timedelta(days=REFAM_MAXLEASE_DAYS)
self.logger = logging.getLogger('gcf.am2')
def test_big_example(self):
# bigger agg example
sa = Aggregate('my test', exp_en=np.ones(50), exp_premium=0, exp_lr=.75,
exp_attachment=20, exp_limit=np.linspace(10, 500, 50), sev_name='lognorm', sev_mean=100,
sev_cv=0.8, freq_name='poisson', freq_a=0)
bs = 1 # 2 and 13
log2 = 13
N = 1 << log2
MAXL = N * bs
xs = np.linspace(0, MAXL, N, endpoint=False)
audit = sa.update(xs, 1, None, 'exact', sev_calc='discrete', discretization_calc='survival', verbose=True)
# average of square errors is small:
self.assertTrue(
np.sum(np.abs(audit.iloc[0:-3, :]['rel sev err']) ** 2) ** .5 / (len(audit) - 3) < 1e-6)
self.assertTrue(
np.sum(np.abs(audit.iloc[0:-3, :]['abs sev err']) ** 2) ** .5 / (len(audit) - 3) < 1e-4)
def test_Aggregate(self):
sig = .75
d1b1 = Aggregate('single', exp_en=10, sev_name='lognorm', sev_a=[sig, sig / 2, sig / 5],
sev_scale=100 * np.exp(-sig ** 2 / 2), sev_wt=1, freq_name='gamma', freq_a=0.81)
xs = np.linspace(0, 1023, 1024)
d1b1.update(xs, verbose=False)
self.assertTrue(d1b1.statistics_total_df.T.loc['el', :].sum() == 5146.408347357843)
sig = 0.75
d1b2 = Aggregate('sublines', exp_en=[10, 10, 10, 10], exp_attachment=[0, 0, 100, 50],
exp_limit=[np.inf, 100, np.inf, 100], sev_name='lognorm', sev_a=sig,
sev_scale=100 * np.exp(-sig ** 2 / 2),
sev_wt=1, freq_name='gamma', freq_a=0.81)
self.assertTrue(d1b2.statistics_total_df.T.loc['el', :].sum() == 6152.9037739696396)
d2b1 = Aggregate('mixed', exp_en=50, exp_attachment=[0, 50, 100, 150, 200],
exp_limit=[250, 250, np.inf, 100, 5000],
sev_name='lognorm', sev_mean=100, sev_cv=np.linspace(0.05, 2, 5), sev_wt=0.2 * np.ones(5),
freq_name='gamma', freq_a=1.2)
self.assertTrue(d2b1.statistics_total_df.T.loc['el', :].sum() == 42977.102931408575)
fixed = Aggregate('chistogram', exp_en=1, sev_name='dhistogram', sev_xs=[0, 1, 2, 3, 4], sev_ps=.2,
freq_name='fixed')
self.assertTrue(fixed.statistics_df['sev_cv'].sum() +
fixed.statistics_total_df['sev_cv'].sum()== 2.1213203435596428)
def test_aggregate_is_offline_true():
lun1 = Lun(name='TestLun1')
paths1 = ['hostname_port1.L123', 'hostname_port2.L456']
lun1.set_paths(paths=paths1)
plex1 = Plex(name='TestPlex1')
plex1.add_lun(lun=lun1)
lun2 = Lun(name='TestLun2')
paths2 = ['hostname_port1.L123', 'hostname_port4.L456']
lun2.set_paths(paths=paths2)
plex2 = Plex(name='TestPlex2')
plex2.add_lun(lun=lun2)
aggregate = Aggregate(name='TestAggr')
aggregate.add_plex(plex=plex1)
aggregate.add_plex(plex=plex2)
assert aggregate.is_offline(down_ports=['port1', 'port2', 'port4']) is True
def create_root_aggregate(self, defaults):
"""
If a root aggregate doesn't already exist, create one
"""
try:
aggr = self.get_root_aggregate()
return
except ValueError, valerr:
try:
root_aggr_name = defaults.get("vfiler", "default_root_aggregate")
xmldata = """<aggregate type="root" name="%s"/>""" % root_aggr_name
node = etree.fromstring(xmldata)
aggr = Aggregate()
aggr.configure_from_node(node, defaults, self)
self.add_child(aggr)
except (NoSectionError, NoOptionError), e:
raise valerr
def create_root_aggregate(self, defaults):
"""
If a root aggregate doesn't already exist, create one
"""
try:
aggr = self.get_root_aggregate()
return
except ValueError, valerr:
try:
root_aggr_name = defaults.get('vfiler',
'default_root_aggregate')
xmldata = """<aggregate type="root" name="%s"/>""" % root_aggr_name
node = etree.fromstring(xmldata)
aggr = Aggregate()
aggr.configure_from_node(node, defaults, self)
self.add_child(aggr)
except (NoSectionError, NoOptionError), e:
raise valerr
def __init__(self, root_cert, urn_authority, url):
self._url = url
self._api_version = 2
self._slices = dict()
self._agg = Aggregate()
self._agg.add_resources([FakeVM(self._agg) for _ in range(3)])
self._cred_verifier = geni.CredentialVerifier(root_cert)
self._urn_authority = urn_authority
self._my_urn = publicid_to_urn("%s %s %s" % (self._urn_authority, 'authority', 'am'))
self.max_lease = datetime.timedelta(days=REFAM_MAXLEASE_DAYS)
self.logger = logging.getLogger('gcf.am2')
def find_target_aggr(self, targetfiler, aggrname, defaults):
"""
Given a name, find the target aggregate in the config,
or create a reference object if we just have a name
"""
log.debug("Finding target aggregates...")
try:
targetaggr = [ x for x in targetfiler.get_aggregates() if x.name == aggrname ][0]
except IndexError:
# An indexerror means the aggregate isn't defined in the
# project XML, which is ok. We invent an aggregate and
# add it to the target filer.
xmldata = """<aggregate name="%s" />""" % aggrname
node = etree.fromstring(xmldata)
targetaggr = Aggregate()
targetaggr.configure_from_node(node, defaults, targetfiler)
targetfiler.add_child(targetaggr)
pass
log.debug("Added a placeholder aggregate to project: %s", targetaggr)
log.debug("%s", targetfiler.get_aggregates())
return targetaggr
def find_target_aggr(self, targetfiler, aggrname, defaults):
"""
Given a name, find the target aggregate in the config,
or create a reference object if we just have a name
"""
log.debug("Finding target aggregates...")
try:
targetaggr = [
x for x in targetfiler.get_aggregates() if x.name == aggrname
][0]
except IndexError:
# An indexerror means the aggregate isn't defined in the
# project XML, which is ok. We invent an aggregate and
# add it to the target filer.
xmldata = """<aggregate name="%s" />""" % aggrname
node = etree.fromstring(xmldata)
targetaggr = Aggregate()
targetaggr.configure_from_node(node, defaults, targetfiler)
targetfiler.add_child(targetaggr)
pass
log.debug("Added a placeholder aggregate to project: %s", targetaggr)
log.debug("%s", targetfiler.get_aggregates())
return targetaggr
def test_aggregate_add_plex():
lun1 = Lun(name='TestLun1')
paths1 = ['hostname_port1.L123', 'hostname_port2.L456']
lun1.set_paths(paths=paths1)
plex1 = Plex(name='TestPlex1')
plex1.add_lun(lun=lun1)
lun2 = Lun(name='TestLun2')
paths2 = ['hostname_port1.L123', 'hostname_port4.L456']
lun2.set_paths(paths=paths2)
plex2 = Plex(name='TestPlex2')
plex2.add_lun(lun=lun2)
aggregate = Aggregate(name='TestAggr')
aggregate.add_plex(plex=plex1)
assert (plex1 in aggregate.plexes) is True
assert (plex1.name in aggregate.plex_names) is True
aggregate.add_plex(plex=plex2)
assert (plex1 in aggregate.plexes) is True
assert (plex1.name in aggregate.plex_names) is True
assert (plex2 in aggregate.plexes) is True
assert (plex2.name in aggregate.plex_names) is True
def do_aggregate(self,datacontainer,run_id):
ag = Aggregate(self.root_path,self.aggregate_id,self.aggregate_to_file,self.name)
for meth in self.aggregate_methods:
getattr(ag,meth)()
class ReferenceAggregateManager(object):
'''A reference Aggregate Manager that manages fake resources.'''
# root_cert is a single cert or dir of multiple certs
# that are trusted to sign credentials
def __init__(self, root_cert, urn_authority, url):
self._url = url
self._api_version = 2
self._slices = dict()
self._agg = Aggregate()
self._agg.add_resources([FakeVM(self._agg) for _ in range(3)])
self._cred_verifier = geni.CredentialVerifier(root_cert)
self._urn_authority = urn_authority
self._my_urn = publicid_to_urn("%s %s %s" % (self._urn_authority, 'authority', 'am'))
self.max_lease = datetime.timedelta(days=REFAM_MAXLEASE_DAYS)
self.logger = logging.getLogger('gcf.am2')
def GetVersion(self, options):
'''Specify version information about this AM. That could
include API version information, RSpec format and version
information, etc. Return a dict.'''
self.logger.info("Called GetVersion")
reqver = [dict(type="geni",
version="3",
schema="http://www.geni.net/resources/rspec/3/request.xsd",
namespace="http://www.geni.net/resources/rspec/3",
extensions=[])]
adver = [dict(type="geni",
version="3",
schema="http://www.geni.net/resources/rspec/3/ad.xsd",
namespace="http://www.geni.net/resources/rspec/3",
extensions=[])]
api_versions = dict()
api_versions[str(self._api_version)] = self._url
versions = dict(geni_api=2,
geni_api_versions=api_versions,
geni_request_rspec_versions=reqver,
geni_ad_rspec_versions=adver)
return dict(geni_api=versions['geni_api'],
code=dict(geni_code=0,
am_type="gcf2",
am_code=0),
value=versions,
output="")
# The list of credentials are options - some single cred
# must give the caller required permissions.
# The semantics of the API are unclear on this point, so
# this is just the current implementation
def ListResources(self, credentials, options):
'''Return an RSpec of resources managed at this AM.
If a geni_slice_urn
is given in the options, then only return resources assigned
to that slice. If geni_available is specified in the options,
then only report available resources. And if geni_compressed
option is specified, then compress the result.'''
self.logger.info('ListResources(%r)' % (options))
# Note this list of privileges is really the name of an operation
# from the privilege_table in sfa/trust/rights.py
# Credentials will specify a list of privileges, each of which
# confers the right to perform a list of operations.
# EG the 'info' privilege in a credential allows the operations
# listslices, listnodes, policy
# could require list or listnodes?
privileges = ()
# Note that verify throws an exception on failure.
# Use the client PEM format cert as retrieved
# from the https connection by the SecureXMLRPCServer
# to identify the caller.
self._cred_verifier.verify_from_strings(self._server.pem_cert,
credentials,
None,
privileges)
# If we get here, the credentials give the caller
# all needed privileges to act on the given target.
if 'geni_rspec_version' not in options:
# This is a required option, so error out with bad arguments.
self.logger.error('No geni_rspec_version supplied to ListResources.')
return self.errorResult(1, 'Bad Arguments: option geni_rspec_version was not supplied.')
if 'type' not in options['geni_rspec_version']:
self.logger.error('ListResources: geni_rspec_version does not contain a type field.')
return self.errorResult(1, 'Bad Arguments: option geni_rspec_version does not have a type field.')
if 'version' not in options['geni_rspec_version']:
self.logger.error('ListResources: geni_rspec_version does not contain a version field.')
return self.errorResult(1, 'Bad Arguments: option geni_rspec_version does not have a version field.')
# Look to see what RSpec version the client requested
# Error-check that the input value is supported.
rspec_type = options['geni_rspec_version']['type']
if isinstance(rspec_type, str):
rspec_type = rspec_type.lower().strip()
rspec_version = options['geni_rspec_version']['version']
if rspec_type != 'geni':
self.logger.error('ListResources: Unknown RSpec type %s requested', rspec_type)
return self.errorResult(4, 'Bad Version: requested RSpec type %s is not a valid option.' % (rspec_type))
if rspec_version != '3':
self.logger.error('ListResources: Unknown RSpec version %s requested', rspec_version)
return self.errorResult(4, 'Bad Version: requested RSpec version %s is not a valid option.' % (rspec_type))
self.logger.info("ListResources requested RSpec %s (%s)", rspec_type, rspec_version)
if 'geni_slice_urn' in options:
slice_urn = options['geni_slice_urn']
if slice_urn in self._slices:
result = self.manifest_rspec(slice_urn)
else:
# return an empty rspec
return self._no_such_slice(slice_urn)
else:
all_resources = self._agg.catalog(None)
available = 'geni_available' in options and options['geni_available']
resource_xml = ""
for r in all_resources:
if available and not r.available:
continue
resource_xml = resource_xml + self.advert_resource(r)
result = self.advert_header() + resource_xml + self.advert_footer()
self.logger.debug("Result is now \"%s\"", result)
# Optionally compress the result
if 'geni_compressed' in options and options['geni_compressed']:
try:
result = base64.b64encode(zlib.compress(result))
except Exception, exc:
import traceback
self.logger.error("Error compressing and encoding resource list: %s", traceback.format_exc())
raise Exception("Server error compressing resource list", exc)
return dict(code=dict(geni_code=0,
am_type="gcf2",
am_code=0),
value=result,
output="")
def test_create_aggregate():
aggregate = Aggregate(name='TestAggr')
assert aggregate.name == 'TestAggr'
assert aggregate.plexes == []
assert aggregate.plex_names == []
irs_thrs = [2, 4, 6, 8, 10]
irs_max = [12, 14, 16, 18, 20]
if (False):
print "Going to sweep over Maximum and threshold IRS values"
for (i, irs_ths) in enumerate(irs_thrs):
for (j, irs_mx) in enumerate(irs_max):
config['model']['irs_threshold'] = irs_ths
config['model']['max_irs_value'] = irs_mx
aggregate_id = str(uuid.uuid4())
sim = Simulation(config, aggregate_id)
sim.run()
a = Aggregate(config['file_root'], aggregate_id, True)
results[i, j] = a.default_distribution()
print results
path = orig_file_root + "%s.png"
cbmax = int(np.max(results) + 0.5)
#cbmax = int(5.2+0.5)
heat_map(results, "Exponent heatmap", 'IrsThreshold', 'MaxIrsValue',
irs_thrs, irs_max, range(0, cbmax), path, "IrsThs_IrsMax")
min_alpha = np.min(results[np.where(results > 0)])
(xr, yr) = np.where(results == min_alpha)
i = xr[0]
j = yr[0]
def __init__(self, id = None, name = '””, password = “”, emailid = “” ):
Aggregate.__init__(self)
self._apply_changes(Userdetails(id, name, password, emailid))
class ReferenceAggregateManager(object):
'''A reference Aggregate Manager that manages fake resources.'''
# root_cert is a single cert or dir of multiple certs
# that are trusted to sign credentials
def __init__(self, root_cert, urn_authority, url):
self._url = url
self._api_version = 2
self._slices = dict()
self._agg = Aggregate()
self._agg.add_resources([FakeVM(self._agg) for _ in range(3)])
self._cred_verifier = geni.CredentialVerifier(root_cert)
self._urn_authority = urn_authority
self._my_urn = publicid_to_urn("%s %s %s" % (self._urn_authority, 'authority', 'am'))
self.max_lease = datetime.timedelta(days=REFAM_MAXLEASE_DAYS)
self.logger = logging.getLogger('gcf.am2')
def GetVersion(self, options):
'''Specify version information about this AM. That could
include API version information, RSpec format and version
information, etc. Return a dict.'''
self.logger.info("Called GetVersion")
reqver = [dict(type="geni",
version="3",
schema="http://www.geni.net/resources/rspec/3/request.xsd",
namespace="http://www.geni.net/resources/rspec/3",
extensions=[])]
adver = [dict(type="geni",
version="3",
schema="http://www.geni.net/resources/rspec/3/ad.xsd",
namespace="http://www.geni.net/resources/rspec/3",
extensions=[])]
api_versions = dict()
api_versions[str(self._api_version)] = self._url
versions = dict(geni_api=2,
geni_api_versions=api_versions,
geni_request_rspec_versions=reqver,
geni_ad_rspec_versions=adver)
return dict(geni_api=versions['geni_api'],
code=dict(geni_code=0,
am_type="gcf2",
am_code=0),
value=versions,
output="")
# The list of credentials are options - some single cred
# must give the caller required permissions.
# The semantics of the API are unclear on this point, so
# this is just the current implementation
def ListResources(self, credentials, options):
'''Return an RSpec of resources managed at this AM.
If a geni_slice_urn
is given in the options, then only return resources assigned
to that slice. If geni_available is specified in the options,
then only report available resources. And if geni_compressed
option is specified, then compress the result.'''
self.logger.info('ListResources(%r)' % (options))
# Note this list of privileges is really the name of an operation
# from the privilege_table in sfa/trust/rights.py
# Credentials will specify a list of privileges, each of which
# confers the right to perform a list of operations.
# EG the 'info' privilege in a credential allows the operations
# listslices, listnodes, policy
# could require list or listnodes?
privileges = ()
# Note that verify throws an exception on failure.
# Use the client PEM format cert as retrieved
# from the https connection by the SecureXMLRPCServer
# to identify the caller.
try:
self._cred_verifier.verify_from_strings(self._server.pem_cert,
credentials,
None,
privileges)
except Exception, e:
raise xmlrpclib.Fault('Insufficient privileges', str(e))
# If we get here, the credentials give the caller
# all needed privileges to act on the given target.
if 'geni_rspec_version' not in options:
# This is a required option, so error out with bad arguments.
self.logger.error('No geni_rspec_version supplied to ListResources.')
return self.errorResult(1, 'Bad Arguments: option geni_rspec_version was not supplied.')
if 'type' not in options['geni_rspec_version']:
self.logger.error('ListResources: geni_rspec_version does not contain a type field.')
return self.errorResult(1, 'Bad Arguments: option geni_rspec_version does not have a type field.')
if 'version' not in options['geni_rspec_version']:
self.logger.error('ListResources: geni_rspec_version does not contain a version field.')
return self.errorResult(1, 'Bad Arguments: option geni_rspec_version does not have a version field.')
# Look to see what RSpec version the client requested
# Error-check that the input value is supported.
rspec_type = options['geni_rspec_version']['type']
if isinstance(rspec_type, str):
rspec_type = rspec_type.lower().strip()
rspec_version = options['geni_rspec_version']['version']
if rspec_type != 'geni':
self.logger.error('ListResources: Unknown RSpec type %s requested', rspec_type)
return self.errorResult(4, 'Bad Version: requested RSpec type %s is not a valid option.' % (rspec_type))
if rspec_version != '3':
self.logger.error('ListResources: Unknown RSpec version %s requested', rspec_version)
return self.errorResult(4, 'Bad Version: requested RSpec version %s is not a valid option.' % (rspec_type))
self.logger.info("ListResources requested RSpec %s (%s)", rspec_type, rspec_version)
if 'geni_slice_urn' in options:
slice_urn = options['geni_slice_urn']
if slice_urn in self._slices:
result = self.manifest_rspec(slice_urn)
else:
# return an empty rspec
return self._no_such_slice(slice_urn)
else:
all_resources = self._agg.catalog(None)
available = 'geni_available' in options and options['geni_available']
resource_xml = ""
for r in all_resources:
if available and not r.available:
continue
resource_xml = resource_xml + self.advert_resource(r)
result = self.advert_header() + resource_xml + self.advert_footer()
self.logger.debug("Result is now \"%s\"", result)
# Optionally compress the result
if 'geni_compressed' in options and options['geni_compressed']:
try:
result = base64.b64encode(zlib.compress(result))
except Exception, exc:
import traceback
self.logger.error("Error compressing and encoding resource list: %s", traceback.format_exc())
raise Exception("Server error compressing resource list", exc)
with conn.cursor() as cur:
# print(cur.execute("""
# SELECT * from test;
# """))
cur.execute("""
CREATE TABLE if NOT EXISTS public.test (
id SERIAL PRIMARY KEY,
created TIMESTAMP DEFAULT now(),
scrape json DEFAULT '{}'
);
""")
conn.commit()
if __name__ == '__main__':
print('Settings up database')
create_tables()
choice = input('Do you want to scrape data? y/N').lower() or 'N'
if choice == 'y':
print('Start scraping...')
sts = SpeedTestScraper(conn)
sts.scrape()
choice = input('Do you want to aggregate data? y/N').lower() or 'N'
if choice == 'y':
print('aggregating...')
agg = Aggregate(conn)
path = agg.to_csv()
print(f'Output file in {path}')
from collection import Collection
from iterator_a import IteratorA
from iterator_b import IteratorB
from aggregate import Aggregate
if __name__ == "__main__":
aggregate = Aggregate()
aggregate.add_item(1)
aggregate.add_item(2)
aggregate.add_item(3)
print("Iterator A")
it_a = IteratorA(aggregate)
for el in it_a:
print(el)
print("Iterator B")
it_b = IteratorB(aggregate)
for el in it_b:
print(el)
from aggregate import Aggregate
import asyncio
from aggregate_settings import CONFIG
if __name__ == "__main__":
asyncio.set_event_loop(asyncio.new_event_loop())
mainloop = asyncio.get_event_loop()
aggregate_tasks = []
for aggregate in CONFIG["aggregates"]:
new_aggregate = Aggregate(
CONFIG["aggregates"][aggregate]["aggregate_stream_name"],
CONFIG["aggregates"][aggregate]["collection_name"],
CONFIG["aggregates"][aggregate]["subscription_name"],
CONFIG["aggregates"][aggregate]["watched_stream_name"],
)
aggregate_tasks.append(new_aggregate.aggregate_fn())
mainloop.run_until_complete(asyncio.wait(aggregate_tasks))
if __name__ == "__main__":
app = QtGui.QApplication(sys.argv)
cache = Cache() # inicjalizacja okna programu
configuration = config.Config() # wczytanie konfiguracji
database = db.Db(ip=configuration.db_ip,
port=configuration.db_port,
user=configuration.db_user,
password=configuration.db_password,
database=configuration.db_name,
ui=cache.ui) # otwarcie połączenia z bazą danych
database.prepare(tank_table=configuration.db_tank_table,
nozzle_table=configuration.db_nozzle_table) # przygotowanie bazy danych
tank_aggregate = Aggregate(values=database.tank_table_values,
ui=cache.ui) # lista agregacyjna zbiornika paliwa
nozzle_aggregate = Aggregate(values=database.nozzle_table_values,
ui=cache.ui) # lista agregacyjna pistoletu paliwowego
export = Export(ui=cache.ui) # inicjalizacja eksportu
# przeliczenie list agregatów
tank_aggregate.compile_tank()
nozzle_aggregate.compile_nozzle()
# aktualizacja tablic gui
tank_aggregate.update_tank_table()
nozzle_aggregate.update_nozzle_table()
# automatyczne aktualizowanie tablic
timer = QTimer()
