def initialize(self):
"""Initialize a few parameters after they've been set by user"""
self.verbose = self.msg_svc.compare("WARNING") # if verbose level < "WARNING", verbose output!
# Use input filename to generate model name
if not self.model_name:
self.model_name = self.hep_data.split('/')[-1].split('.')[0]+'_'+self.date
# Store NN classes in custom object
self.classCollection = util.NNClassCollection()
for n,v in self.classes.iteritems():
nn_class = util.NNClass(n)
nn_class.value = v
self.classCollection.append(nn_class)
## -- Plotting framework
self.plotter = Empire() # class for plotting relevant NN information
self.plotter.backend = self.backend
self.plotter.output_dir = self.output_dir
self.plotter.image_format = 'pdf'
self.plotter.features = self.features
self.plotter.msg_svc = self.msg_svc
self.plotter.sample_labels = self.sample_labels
self.plotter.variable_labels = self.variable_labels
self.plotter.initialize(self.classCollection)
return
def resolve_empire(player_no, empire_filename) -> Empire:
cities_counter = 0
metropolis = None
empire_troops = {}
file = open(empire_filename)
for city in file.read().split('\n'):
if city == '':
continue
try:
city_name, troops = city.split(',')
except Exception:
raise ValueError('not enough values to unpack. {}'.format(city))
assert city_name not in empire_troops, 'City {} was already in the empire.'.format(
city_name)
if cities_counter == 0:
metropolis = city_name
empire_troops[city_name] = int(troops)
cities_counter += 1
empire = Empire(player_no, metropolis, empire_troops)
return empire
def createEmpires(self, **args):
countries = args['countries']
costs = []
for country in countries:
costs.append(country.getCost())
costs = np.array(costs)
sorted_indicates = np.argsort(costs)
new_countries = []
for i in sorted_indicates:
new_countries.append(self.countries[i])
new_countries = np.array(new_countries)
self.countries = new_countries
empires = self.countries[:Constant.IMPERIALIST_COUNT]
self.colonies = self.countries[Constant.IMPERIALIST_COUNT:]
for i in empires:
self.empires.append(Empire(i))
empires_costs = np.array(
[np.sum(empire.getCost()) for empire in self.empires])
P = np.absolute(np.divide(empires_costs, np.sum(empires_costs)))
for colony in self.colonies:
k = randomSelection(P)
self.empires[k].addColony(colony)
return self.empires
def create_fleet(ai_settings,screen,fleet,ship):
"""Create an empire fleet."""
empire_ship = Empire(ai_settings,screen)
# Calculate available Space
empire_ship_width = empire_ship.rect.width
# Calculate the number of empire ships in a screen row
nr_ships = get_number_ships(ai_settings,empire_ship_width)
nr_rows = get_number_rows(ai_settings,ship.rect.height,empire_ship.rect.height)
#Create a row of empire ships
for row_number in range(nr_rows):
for ship_nr in range(nr_ships):
empire_ship = Empire(ai_settings,screen)
empire_ship.x = empire_ship_width + 2 * empire_ship_width * ship_nr
empire_ship.rect.x = empire_ship.x
empire_ship.rect.y = empire_ship.rect.height + 2 * empire_ship.rect.height * row_number
fleet.add(empire_ship)
def createEmpires(self):
countries = self.countries
empires = countries[:Constant.EMPIRES_COUNT]
colonies = countries[Constant.EMPIRES_COUNT:]
for imperialist in empires:
self.empires.append(Empire(imperialist))
i = 0
while i < len(colonies):
colony = colonies[i]
imperialist = np.random.randint(0, Constant.EMPIRES_COUNT, 1)[0]
empire = self.empires[imperialist]
if empire.coloniesCount() < 9:
empire.addColony(colony)
i += 1
return self.empires
class Foundation(object):
"""Deep Learning base class"""
def __init__(self):
self.date = datetime.date.today().strftime('%d%b%Y')
## Handling NN objects and data -- set in the class
self.df = None # dataframe containing physics information
self.fpr = {} # ROC curve: false positive rate
self.tpr = {} # ROC curve: true positive rate
self.roc_auc = {} # ROC curve: Area under the curve
self.model = None # Keras model
self.history = None # model history
self.plotter = None # plotter module
self.scale = {} # scale for normalizing input features
self.offset = {} # offset for normalizing input features
self.metadata = {} # Metadata to store for accessing later, if necessary
self.features = [] # List of features used in the analysis
self.classes = {} # classes for neural network {"name":value}
self.classCollection = None # storing information about the different classes
self.sample_labels = {}
self.variable_labels = {}
## NN architecture & parameters -- set by config file
self.treename = 'features' # Name of TTree to access in ROOT file (via uproot)
self.lwtnn = True # export (& load model from) files for LWTNN
self.dnn_name = "dnn" # name to access in lwtnn ('variables.json')
self.hep_data = "" # Name for loading features (physics data) -- assumes all data in one file
self.model_name = "" # Name for saving/loading model
self.output_dir = 'data/dnn/' # directory for storing NN data
self.runDiagnostics = False # Make plots pre/post training
self.msg_svc = None
self.verbose = True
self.backend = 'uproot' # backend for making plots with hepPlotter
self.equal_statistics = True # Equal statistics for each class in the df
def initialize(self):
"""Initialize a few parameters after they've been set by user"""
self.verbose = self.msg_svc.compare("WARNING") # if verbose level < "WARNING", verbose output!
# Use input filename to generate model name
if not self.model_name:
self.model_name = self.hep_data.split('/')[-1].split('.')[0]+'_'+self.date
# Store NN classes in custom object
self.classCollection = util.NNClassCollection()
for n,v in self.classes.iteritems():
nn_class = util.NNClass(n)
nn_class.value = v
self.classCollection.append(nn_class)
## -- Plotting framework
self.plotter = Empire() # class for plotting relevant NN information
self.plotter.backend = self.backend
self.plotter.output_dir = self.output_dir
self.plotter.image_format = 'pdf'
self.plotter.features = self.features
self.plotter.msg_svc = self.msg_svc
self.plotter.sample_labels = self.sample_labels
self.plotter.variable_labels = self.variable_labels
self.plotter.initialize(self.classCollection)
return
def predict(self,data=None):
"""Return the prediction from a test sample"""
self.msg_svc.DEBUG("FOUNDATION : Get the DNN prediction")
if data is None:
self.msg_svc.ERROR("FOUNDATION : predict() given NoneType data. Returning -999.")
self.msg_svc.ERROR("FOUNDATION : Please check your configuration!")
return -999.
return self.model.predict( data )
def load_data(self,extra_variables=[]):
"""
Load the physics data (flat ntuple) for NN using uproot
Convert to DataFrame for easier slicing
@param extra_variables If there are extra variables to plot/analyze,
that aren't features of the NN, include them here
"""
self.msg_svc.INFO("FOUNDATION : Load HEP data")
file = uproot.open(self.hep_data)
data = file[self.treename]
self.df = data.pandas.df( self.features+extra_variables )
try:
self.metadata['metadata'] = file['metadata'] # names of samples, target values, etc.
except KeyError:
self.metadata['metadata'] = None
return
def preprocess_data(self,slices=[]):
"""Manipulate dataframe and keep only the necessary data
@param slices list of strings that contain arguments (separated by spaces) for slicing the dataframe
e.g., ['AK4_DeepCSVb >= 0','AK4_DeepCSVbb >= 0']
these selections are applied to the dataframe
"""
self.msg_svc.DEBUG("FOUNDATION : Preprocess data")
class_dfs = []
min_size = self.df.shape[0]
for k in self.classCollection:
tmp = self.df[ self.df.target==k.value ]
class_dfs.append(tmp)
if tmp.shape[0]<min_size:
min_size=tmp.shape[0]
# Make the dataset sizes equal for the different classes
if self.equal_statistics:
for td,cdf in enumerate(class_dfs):
# shuffle entries and select first events up to 'min_size'
if cdf.shape[0]>min_size:
class_dfs[td] = class_dfs[td].sample(frac=1)[0:min_size]
self.df = pd.concat( class_dfs ).sample(frac=1) # re-combine & shuffle entries
opts = {">":operator.gt,">=":operator.ge,
"<":operator.lt,"<=":operator.le,
"==":operator.eq,"!=":operator.ne}
for sl in slices:
arg,opt,val = sl.split(" ")
opt = opts[opt]
dtype = self.df[arg].dtype.name
val = getattr(np,dtype)(val)
self.df = self.df[ opt(self.df[arg],val) ]
return
def save_model(self):
"""Save the model for use later"""
self.msg_svc.DEBUG("FOUNDATION : Save model")
output = self.output_dir+'/'+self.model_name
if self.lwtnn:
## Save to format for LWTNN
with open(output+'_model.json', 'w') as outfile:
outfile.write(self.model.to_json()) # model architecture
self.model.save_weights(output+'_weights.h5') # model weights
else:
## Save to h5 format
self.model.save('{0}.h5'.format(output))
## Keep track of the different features used in this instance
featureKeysFile = self.output_dir+'/features.json'
try:
featureKeys = json.load(open(featureKeysFile))
except IOError:
featureKeys = {}
featureKey = -1
for key in featureKeys.keys():
if Counter(featureKeys[key])==Counter(config.features):
featureKey = int(key)
break
if featureKey<0:
keys = featureKeys.keys()
featureKey = max([int(i) for i in keys])+1 if keys else 0
featureKeys[str(featureKey)] = self.features
with open(featureKeysFile,'w') as outfile:
json.dump(featureKeys,outfile)
## Record different hyperparameters used in this instance
NN_parameters = ['epochs','batch_size','loss','optimizer','metrics',
'activations','nHiddenLayers','nNodes','input_dim']
outputFile = open(self.output_dir+'/ABOUT.txt','w')
outputFile.write(" * NN Setup * \n")
outputFile.write(" ------------ \n")
outputFile.write(" NN parameters: \n")
for NN_parameter in NN_parameters:
outputFile.write( NN_parameter+": "+str(getattr(self,NN_parameter))+"\n" )
outputFile.write( "\n NN Features: \n" )
for feature in self.features:
outputFile.write(" >> "+feature+"\n" )
outputFile.close()
return
def load_model(self):
"""Load existing model to make plots or predictions"""
self.msg_svc.DEBUG("FOUNDATION : Load model")
self.model = None
if self.lwtnn:
model_json = open(self.model_name+"_model.json",'r').read()
self.model = model_from_json(model_json)
self.model.load_weights(self.model_name+"_weights.h5")
self.model.compile(loss=self.loss,optimizer=self.optimizer,metrics=self.metrics)
else:
self.model = load_model('{0}.h5'.format(self.model_name))
return
def diagnostics(self,pre=False,post=False,**kwargs):
"""Diagnostic tests of the NN"""
self.msg_svc.DEBUG("FOUNDATION : Diagnostics")
# Plots to make pre-training
if pre:
# Use **kwargs to limit feature plots to 1D
ndims = kwargs.get("ndims",-1)
self.msg_svc.INFO("FOUNDATION : -- pre-training :: features")
self.plotter.feature(self.df,ndims=ndims) # compare features
self.msg_svc.INFO("FOUNDATION : -- pre-training :: correlations")
corrmats = {}
for c in self.classCollection:
t_ = self.df[self.df.target==c.value].drop('target',axis=1)
corrmats[c.name] = t_.corr()
self.plotter.correlation(corrmats) # correlations between features
self.msg_svc.INFO("FOUNDATION : -- pre-training :: separations")
self.plotter.separation() # separations between classes
# Plots to make post-training/testing
if post:
self.msg_svc.INFO("FOUNDATION : -- post-training :: ROC")
self.plotter.ROC(self.fpr,self.tpr,self.roc_auc) # Signal vs background eff
self.msg_svc.INFO("FOUNDATION : -- post-training :: History")
self.plotter.history(self.history) # loss vs epoch
return
parser.add_argument("num_empires",
help="number of empires to have in the simulation",
type=int)
parser.add_argument("size", help="height of the square map", type=int)
args = parser.parse_args()
root = Tk()
grid = Map(args.size)
app = App(root, grid)
Empire.APP = app
names = None
script_dir = os.path.dirname(__file__) # <-- absolute dir the script is in
rel_path = "empire_names.txt"
abs_file_path = os.path.join(script_dir, rel_path)
with open(abs_file_path) as f_name:
names = f_name.read().split("\n")
num_empires = args.num_empires
for _ in range(num_empires):
app.add_empire(Empire(random.choice(names), grid.rand_select()))
while True:
app.turn()
root.update()
root.destroy()
#!/usr/bin/python """ Run a handful of client commands. """ from empire import Empire import sys appkey = sys.argv[1] secrets_yaml = "empire_service_secrets.yaml" empire = Empire(appkey=appkey, enduser="******", secrets_yaml=secrets_yaml) empire.walkthrough()
from app import App
from cell import Cell
from empire import Empire
from map import Map
from tkinter import *
if __name__ == '__main__':
root = Tk()
size = 30
grid = Map(size)
app = App(root, grid)
Empire.APP = app
a = Empire("Good Morning", grid[29, 15])
a.color = "#00ff00"
app.add_empire(a)
b = Empire("NaNaNaNaNaNaN", grid[0, 15])
b.color = "#0000FF"
app.add_empire(b)
# radi = grid.getAdjacentLocs([(size/2, size/2), (16, 15), (17, 16)])
# for loc in radi:
# app[loc] = "#00FF00"
root.update()
while True:
for loc in b.getBeyondBorders():
app[loc.properties["Location"]] = "#00FF00"
root.update()
def setUp(self):
self.empire = Empire(appkey="MOCK_APPKEY", enduser="******", api_server='api.empire.co')
self.services = {}
class EmpireTest(unittest.TestCase):
json_headers = {'content-type': 'application/json'}
def setUp(self):
self.empire = Empire(appkey="MOCK_APPKEY", enduser="******", api_server='api.empire.co')
self.services = {}
def mock_response_200(self, request, data):
headers = {}
if not isinstance(data, basestring):
data = json.dumps(data)
headers = self.json_headers
response = httmock.response(200, data, headers, None, 5, request)
setattr(response, 'connection', FakeConnection())
return response
def mock_response_500(self, request, message):
response = httmock.response(500, message, {}, None, 5, request)
setattr(response, 'connection', FakeConnection())
return response
@httmock.urlmatch(netloc='api.empire.co', path='/empire/session/create')
def session_create_mock(self, url, request):
return self.mock_response_200(request, {'status': 'OK', 'sessionkey': 'TESTSESSION'})
@httmock.urlmatch(netloc='api.empire.co', path='/empire/services/salesforce/connect')
def connect_mock(self, url, request):
data = json.loads(request.body)
self.services['salesforce'] = data
return self.mock_response_200(request, {'status': 'OK'})
def test_connect(self):
sf_data = {
"access_token": "MOCK_ACCESS_TOKEN",
"client_id": "MOCK_CLIENT",
"refresh_token": "MOCK_REFRESH_TOKEN",
"endpoint": "https://na15.salesforce.com"
}
with httmock.HTTMock(self.session_create_mock):
with httmock.HTTMock(self.connect_mock):
self.empire.connect("salesforce", sf_data)
self.assertEqual(self.services['salesforce'], sf_data)
@httmock.urlmatch(netloc='api.empire.co', path='/empire/services')
def describe_all_mock(self, url, request):
return self.mock_response_200(request, {'status': 'OK', "name": "salesforce"})
def test_describe_all(self):
with httmock.HTTMock(self.session_create_mock):
with httmock.HTTMock(self.describe_all_mock):
services_data = self.empire.describe()
self.assertEqual(services_data, {'status': 'OK', "name": "salesforce"})
@httmock.urlmatch(netloc='api.empire.co', path='/empire/services/salesforce')
def describe_one_mock(self, url, request):
return self.mock_response_200(request, {'status': 'OK', "name": "salesforce", "tables": ["table1"]})
def test_describe_one(self):
with httmock.HTTMock(self.session_create_mock):
with httmock.HTTMock(self.describe_one_mock):
service_data = self.empire.describe("salesforce")
self.assertEqual(service_data, {'status': 'OK', "name": "salesforce", "tables": ["table1"]})
@httmock.urlmatch(netloc='api.empire.co', path='/empire/services/salesforce/table1')
def describe_table_mock(self, url, request):
return self.mock_response_200(request, {'status': 'OK', "name": "table1"})
def test_describe_table(self):
with httmock.HTTMock(self.session_create_mock):
with httmock.HTTMock(self.describe_table_mock):
service_data = self.empire.describe("salesforce", "table1")
self.assertEqual(service_data, {'status': 'OK', "name": "table1"})
def test_describe_table_without_service(self):
self.assertRaises(ValueError, self.empire.describe, None, "table1")
@httmock.urlmatch(netloc='api.empire.co', path='/empire/services/salesforce/table1')
def describe_table_broken_mock(self, url, request):
return self.mock_response_500(request, "Something is broken")
def test_describe_broken(self):
with httmock.HTTMock(self.session_create_mock):
with httmock.HTTMock(self.describe_table_broken_mock):
self.assertRaises(EmpireException, self.empire.describe, "salesforce", "table1")
@httmock.urlmatch(netloc='api.empire.co', path='/empire/query')
def query_mock(self, url, request):
return self.mock_response_200(request, json.dumps({'col1': 'val1'}) + '\n' + json.dumps({'col2': 'val2'}))
def test_query(self):
with httmock.HTTMock(self.session_create_mock):
with httmock.HTTMock(self.query_mock):
query_result = list(self.empire.query('SELECT * FROM salesforce.account'))
self.assertEqual(query_result, [{'col1': 'val1'}, {'col2': 'val2'}])
@httmock.urlmatch(netloc='api.empire.co', path='/empire/view/viewName')
def view_create_mock(self, url, request):
if request.method == 'PUT':
if json.loads(request.body) == {'query': 'SELECT QUERY'}:
self.view_created = True
return self.mock_response_200(request, {'status': 'OK'})
return self.mock_response_500(request, "Something is broken")
def test_materialize_view(self):
self.view_created = False
self.assertEqual(self.view_created, False)
with httmock.HTTMock(self.session_create_mock):
with httmock.HTTMock(self.view_create_mock):
r = self.empire.materialize_view(name='viewName', sql='SELECT QUERY')
self.assertEqual(r, {'status': 'OK'})
self.assertEqual(self.view_created, True)
@httmock.urlmatch(netloc='api.empire.co', path='/empire/view/viewName')
def view_delete_mock(self, url, request):
if request.method == 'DELETE':
self.view_deleted = True
return self.mock_response_200(request, {'status': 'OK'})
return self.mock_response_500(request, "Something is broken")
def test_drop_view(self):
self.view_deleted = False
self.assertEqual(self.view_deleted, False)
with httmock.HTTMock(self.session_create_mock):
with httmock.HTTMock(self.view_delete_mock):
r = self.empire.drop_view(name='viewName')
self.assertEqual(r, {'status': 'OK'})
self.assertEqual(self.view_deleted, True)