def __init__(self, **kwords):
self.attributes = Attributes(**kwords)
dimensions = attr_init(kwords, "dimensions", [])
storage_precision = attr_init(kwords, "storage_precision", None)
init_data = attr_init(kwords, "init_data", None)
self.table = create_multi_dim_array(dimensions, init_data=init_data)
self.dimensions = dimensions
self.storage_precision = storage_precision
def __init__(self, config):
'''
Constructor
'''
logging.info(' Network_selecter: Constructor')
self.config = config
self.network = Network_User(config)
self.attributes = Attributes(config)
self.attrs_0 = None
return
def __init__(self, name, atk_mod=0, dmg_mod=0):
Attributes.__init__(self)
self.name = name
# Modifiers
self.atk = intOrZero(atk_mod)
if type(dmg_mod) is int:
self.dmg_roll = DamageRoll(None, None, dmg_mod)
elif isinstance(dmg_mod, DamageRoll):
self.dmg_roll = dmg_mod
elif isinstance(dmg_mod, basestring):
self.dmg_roll = DamageRoll(None, None, intOrZero(dmg_mod))
else:
raise ValueError("Unrecognized damage modifier %s" % dmg_mod)
# UI Details
Buff._last_id += 1
self.id = self._last_id
self.ui_id = ''
def _convert_attributes_to_binary(self):
# convert attributes with domain > 2
self.binary_attributes = Attributes()
self.new_examples = self.examples
for i, attribute in enumerate(self.attributes):
if isinstance(attribute, NominalAttribute) and len(attribute.domain) > 2:
self._convert_attribute_to_binary(attribute, i)
else:
self.binary_attributes.add(attribute)
class ExchangeRates(object):
def __init__(self):
from attributes import Attributes
self.currencies = Attributes("Currency")
@transactional
def updateToday(self, session, currency_ids=[1, 2, 3]):
xrates = urlopen("http://www.x-rates.com/calculator.html").read()
data = '\s*=\s*new\s+Array\s*\(([\w",.\s]+)\)'
currency_list = list(eval(re.compile("currency" + data).search(xrates).groups()[0]))
rate_list = list(eval(re.compile("rate" + data).search(xrates).groups()[0]))
for id in currency_ids:
cur = self.currencies.getCode(id)
rate = float(rate_list[currency_list.index(cur)])
exchange_rate = ExchangeRate(rate_date=Date(), currency_id=id, rate=rate)
session.merge(exchange_rate)
@transactional
def updateLast120(self, session, currency_ids=[1, 2, 3]):
" scrapes rates for last 120 days from www.x-rates.com "
from sqlalchemy.sql import update
extractor = re.compile(
"<tr[^>]*>\n"
'<td><font face="Verdana" size="-1">([\d-]+)</font></td>\n'
'<td><font face="Verdana" size="-1">.*?</font></td>\n'
'<td><font face="Verdana" size="-1">([\d.]+)</font></td>\n'
'<td><font face="Verdana" size="-1">.*?</font></td>\n'
"</tr>"
)
for id in currency_ids:
cur = self.currencies.getCode(id)
last120 = urlopen("http://www.x-rates.com/d/%s/USD/data120.html" % cur).read()
for date, rate in extractor.findall(last120):
exchange_rate = ExchangeRate(rate_date=date, currency_id=id, rate=rate)
session.merge(exchange_rate)
@transactional
def getToday(self, session):
self.updateToday(session)
return session.query(ExchangeRate).filter_by(rate_date=Date()).all()
def authnRequest(self):
'''
Builds the AuthnRequest
'''
doc = Document()
authnRequest = self._createElementNS(doc,'urn:oasis:names:tc:SAML:2.0:protocol', 'saml2p:AuthnRequest')
authnRequest.setAttribute("Destination", self.setting.samlSsoEndpointUrl)
authnRequest.setAttribute("Provider-Name", "SuisseID Service Provider AG")
authnRequest.setAttribute("ForceAuthn", "true")
authnRequest.setAttribute("ID", str(uuid.uuid4()))
authnRequest.setAttribute("Version", "2.0")
authnRequest.setAttribute("IssueInstant", "2012-12-18T09:00:00")
authnRequest.setAttribute("AssertionConsumerServiceURL", self.setting.returnUrl)
issuer = self._createElementNS(doc,'urn:oasis:names:tc:SAML:2.0:assertion', 'saml2:Issuer')
issuer.appendChild(doc.createTextNode(self.setting.spName))
authnRequest.appendChild(issuer)
extensions = self._createElementNS(doc, 'urn:oasis:names:tc:SAML:2.0:protocol', 'saml2p:Extensions')
attributes = Attributes()
attributes.enable()
attributes.append(extensions)
privacyNotice = self._createElementNS(doc, 'http://schemas.xmlsoap.org/ws/2005/05/identity', 'ic:PrivacyNotice')
privacyNotice.setAttribute("Version", "1")
privacyNotice.appendChild(doc.createTextNode('http://localhost:8888/auth/privacy'))
extensions.appendChild(privacyNotice)
authnRequest.appendChild(extensions)
doc.appendChild(authnRequest)
#print doc.toprettyxml(" ", "\n")
#request = """
#<saml2p:AuthnRequest xmlns:saml2p="urn:oasis:names:tc:SAML:2.0:protocol" AssertionConsumerServiceURL="%s" Destination="https://idp.signdemo.com/suisseid/SSOPOST/metaAlias/swisssign.net/idp_v15" ForceAuthn="true" ID="_529a99cc-9aa3-4bef-899c-fd2587478740" IsPassive="false" IssueInstant="2012-08-31T16:18:17.878Z" ProviderName="Demo Service Provider" Version="2.0">
# <saml2:Issuer xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion">Service Provider</saml2:Issuer>
#</saml2p:AuthnRequest>
# """ % (self.setting.returnUrl)
#return base64.b64encode(str(request))
return base64.b64encode(str(doc.toxml()))
def build_correlaions(self):
dataset = pd.read_csv(self.data_path)
dataset = dataset.drop(labels='date', axis=1)
attr = Attributes()
data = dataset[attr.ALL_FEATURES]
relation = data.corr()
masking = np.zeros_like(relation, dtype=np.bool)
masking[np.triu_indices_from(masking)] = True
sns.heatmap(relation, annot=True, fmt=".2f", mask=masking)
plt.xticks(range(len(relation.columns)), relation.columns)
plt.yticks(range(len(relation.columns)), relation.columns)
plt.show()
print(self.__abs_correlations(data, 40))
def output(self):
'''Instantiates the Attributes and Correction classes,
assigns associated instance variables, and returns a plot
of the terrain correction array.
'''
self.attributes = Attributes(self.elevation_array,
resolution=self.elevation_array.shape[0],
projection=c.PROJECTION,
side_len=c.SIDE_LEN)
self.slope, self.aspect = self.attributes.calc_attributes()
self.correct = Correction(attribute_grids=(self.slope, self.aspect),
local_timezone=c.TIMEZONE,
date_str=self.DEM[:8],
lat_lon=c.LAT_LON)
self.param.time.bounds = (0, self.correct.sunposition_df.shape[0] - 1)
print(self.param.time.bounds)
self.correct_array = self.correct.calc_correction_onetime(self.time)
return self._imshow(array=self.correct_array,
cmap='magma',
opt='correction')
def getNewAttributes(self, attributes=None, args=None):
u"""
The ``getAttributes`` method answers ``attributes`` attribute or a new instance of
``Attributes``. The attribute set holds the @attributes of the ``self`` node. Note that the
attribute set is stored “as such” without making a copy. So any change to the attribute set when manipulating
the tree, will also reflect in the original attribute set.<br/>
If the ``args`` dictionary is defined, then the these values are added over the the key-value of new
create ``Attributes`` instance. This is a separate method to allow inheriting node classes to redefine
the class of the attribute set. The ``args`` attribute can be either a dictionary or a list of
dictionaries.
"""
if isinstance(attributes, dict):
attributes = Attributes(**attributes)
if attributes is None:
attributes = Attributes()
if args is not None:
if not isinstance(args, (list, tuple)):
args = (args,)
for arg in args:
if arg is not None:
attributes.update(arg)
return attributes
def authnRequest(self):
'''
Builds the AuthnRequest
'''
doc = Document()
authnRequest = self._createElementNS(
doc, 'urn:oasis:names:tc:SAML:2.0:protocol', 'saml2p:AuthnRequest')
authnRequest.setAttribute("Destination",
self.setting.samlSsoEndpointUrl)
authnRequest.setAttribute("Provider-Name",
"SuisseID Service Provider AG")
authnRequest.setAttribute("ForceAuthn", "true")
authnRequest.setAttribute("ID", str(uuid.uuid4()))
authnRequest.setAttribute("Version", "2.0")
authnRequest.setAttribute("IssueInstant", "2012-12-18T09:00:00")
authnRequest.setAttribute("AssertionConsumerServiceURL",
self.setting.returnUrl)
issuer = self._createElementNS(
doc, 'urn:oasis:names:tc:SAML:2.0:assertion', 'saml2:Issuer')
issuer.appendChild(doc.createTextNode(self.setting.spName))
authnRequest.appendChild(issuer)
extensions = self._createElementNS(
doc, 'urn:oasis:names:tc:SAML:2.0:protocol', 'saml2p:Extensions')
attributes = Attributes()
attributes.enable()
attributes.append(extensions)
privacyNotice = self._createElementNS(
doc, 'http://schemas.xmlsoap.org/ws/2005/05/identity',
'ic:PrivacyNotice')
privacyNotice.setAttribute("Version", "1")
privacyNotice.appendChild(
doc.createTextNode('http://localhost:8888/auth/privacy'))
extensions.appendChild(privacyNotice)
authnRequest.appendChild(extensions)
doc.appendChild(authnRequest)
#print doc.toprettyxml(" ", "\n")
#request = """
#<saml2p:AuthnRequest xmlns:saml2p="urn:oasis:names:tc:SAML:2.0:protocol" AssertionConsumerServiceURL="%s" Destination="https://idp.signdemo.com/suisseid/SSOPOST/metaAlias/swisssign.net/idp_v15" ForceAuthn="true" ID="_529a99cc-9aa3-4bef-899c-fd2587478740" IsPassive="false" IssueInstant="2012-08-31T16:18:17.878Z" ProviderName="Demo Service Provider" Version="2.0">
# <saml2:Issuer xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion">Service Provider</saml2:Issuer>
#</saml2p:AuthnRequest>
# """ % (self.setting.returnUrl)
#return base64.b64encode(str(request))
return base64.b64encode(str(doc.toxml()))
def getNewAttributes(self, attributes=None, args=None):
u"""
The <code>getAttributes</code> method answers <attr>attributes</attr> attribute or a new instance of
<code>Attributes</code>. The attribute set holds the @attributes of the <attr>self</attr> node. Note that the
attribute set is stored “as such” without making a copy. So any change to the attribute set when manipulating
the tree, will also reflect in the original attribute set.<br/>
If the <attr>args</attr> dictionary is defined, then the these values are added over the the key-value of new
create <code>Attributes</code> instance. This is a separate method to allow inheriting node classes to redefine
the class of the attribute set. The <attr>args</attr> attribute can be either a dictionary or a list of
dictionaries.
"""
if isinstance(attributes, dict):
attributes = Attributes(**attributes)
if attributes is None:
attributes = Attributes()
if args is not None:
if not isinstance(args, (list, tuple)):
args = (args,)
for arg in args:
if arg is not None:
attributes.update(arg)
return attributes
from attributes import Attributes
if __name__ == '__main__':
a = Attributes(3, 3, 3)
print(f'Цена за покупку аттрибутов {a.full_cost}')
print(a.print_attributes)
class ML_Table(ML_LeafNode):
""" Metalibm Table object """
def __init__(self, **kwords):
self.attributes = Attributes(**kwords)
dimensions = attr_init(kwords, "dimensions", [])
storage_precision = attr_init(kwords, "storage_precision", None)
init_data = attr_init(kwords, "init_data", None)
self.table = create_multi_dim_array(dimensions, init_data=init_data)
self.dimensions = dimensions
self.storage_precision = storage_precision
def __setitem__(self, key, value):
self.table[key] = value
def __getitem__(self, key):
return self.table[key]
def get_storage_precision(self):
return self.storage_precision
def get_precision(self):
return self.get_storage_precision()
def get_tag(self):
return self.attributes.get_tag()
def get_subset_interval(self, index_function, range_set):
# init bound values
low_bound = None
high_bound = None
# going through the selected valued list
# to build the range interval
for indexes in range_set:
value = index_function(self, indexes)
if low_bound is None or low_bound > value: low_bound = value
if high_bound is None or high_bound < value: high_bound = value
return Interval(low_bound, high_bound)
def get_definition(self, table_name, final=";", language=C_Code):
precision_name = self.get_storage_precision().get_name(
language=language)
return "%s %s[%s]" % (precision_name, table_name, "][".join(
[str(dim) for dim in self.dimensions]))
def get_content_init(self, language=C_Code):
return get_table_content(self.table,
self.dimensions,
self.get_storage_precision(),
language=language)
def get_str(self,
depth=None,
display_precision=False,
tab_level=0,
memoization_map={},
display_attribute=False,
display_id=False):
id_str = ("[id=%x]" % id(self)) if display_id else ""
attribute_str = "" if not display_attribute else self.attributes.get_str(
tab_level=tab_level)
precision_str = "" if not display_precision else "[%s]" % str(
self.get_storage_precision())
return " " * tab_level + "Table[%s]%s%s%s\n" % ("][".join([
str(dim) for dim in self.dimensions
]), precision_str, id_str, attribute_str)
def __init__(self, name, attack):
Attributes.__init__(self)
self.name = name
assert isinstance(attack, Attack)
self._attack = attack
def __init__(self, name):
Attributes.__init__(self)
self.name = name
def p_attributes_definition_recursive(p):
'attributes_definition : attribute_value attributes_definition'
p[0] = Attributes(p[1], p[2].attributes)
# AI Abby Virtual Assistant [Version 1.19-22alpha]
# (c) 2020, AI Abby Virtual Assistant. All Rights Reserved.
# Author: Johndev4
# Created: September 19-22, 2020
print("Initializing the program...")
from os import system
system('title AI Abby Virtual Assistant')
from attributes import Attributes
from voice_system import VoiceSystem
from get_started import get_username
import operations
if __name__ == "__main__":
attr = Attributes()
voice_Sys = VoiceSystem(attr.AI_ASSISTANT, attr.ME)
system('cls')
print(f"{attr.HEAD}\r\n\r\n")
voice_Sys.speak(f"Hello, I'm {attr.AI_ASSISTANT}, your virtual assistant.")
attr.ME = get_username(attr)
voice_Sys.greet(attr.ME)
voice_Sys.speak(f"What can I help you today?")
while True:
query = voice_Sys.listen()
operations.func(attr, query)
# leaf_font_size=6)
# pylab.savefig(ps.name + ".png")
#print Z
clusters_values = fcluster(Z, 0.75)
clusters_dict = {}
for i in range(len(clusters_values)):
c_id = clusters_values[i]
old_v = clusters_dict.get(c_id, [])
old_v.append(i)
clusters_dict[c_id] = old_v
clusters = []
for v in clusters_dict.values():
clusters.append(v)
attr = Attributes(ps, v, classifier)
print attr.extract_attributes()
res = t.get_results_purity_inverse_purity(ps.name, clusters)
res2 = t.get_results_bcubed(ps.name, clusters)
results[0] += res[0]
results[1] += res[1]
results2[0] += res2[0]
results2[1] += res2[1]
print res
print res2
print len(clusters)
#print clusters
#print len(set(clusters))
def transform(self, df):
### CLEAN AND DROP
attribs = Attributes()
# drop customer segmentation info (3) #tinker
segment_cols = attribs.get_segment_cols()
df.drop(columns=segment_cols, inplace=True)
# drop cols with data leakage (2)
leak_cols = attribs.get_leak_cols()
df.drop(columns=leak_cols, inplace=True)
# drop rows with leakage
df.drop(df[df.year_built == 2017].index, inplace=True)
df.drop(df[df.effective_year_built == 2017].index, inplace=True)
# drop cols with too many nulls (28)
null_cols = attribs.get_null_cols()
df.drop(columns=null_cols, inplace=True)
# drop redundant features (74)
redundant_cols = attribs.get_redundant_cols()
df.drop(columns=redundant_cols, inplace=True)
# drop irrelevant features (18)
irrelevant_cols = attribs.get_irrelevant_cols()
df.drop(columns=irrelevant_cols, inplace=True)
# drop 1050 rows without sale_date or sale_price (same set)
df.dropna(subset=['last_sale_price', 'last_sale_date'], inplace=True)
# remap buidling_condition (misspelling intentional)
df.replace({'buidling_condition':{
'LOW':1,
'FAIR':2,
'AVERAGE':3,
'AVERAGE +':4,
'AVERAGE ++':5,
'GOOD':6,
'GOOD +':7,
'GOOD ++':8,
'VERY GOOD':9,
'VERY GOOD +':10,
'VERY GOOD ++':11,
'EXCELLENT':12,
'EXCELLENT +':13,
'EXCELLENT++':14,
'EXCEPTIONAL 1':15}
}, inplace=True)
# convert true/false to 1/0
df['nrel_attached_garage'].astype(int, copy=False)
# combine full and half baths
df['num_baths'] = df['full_bath_count'] + (0.5 * df['half_bath_count'])
df.drop(columns=['full_bath_count', 'half_bath_count'], inplace=True)
### FEATURE ENGINEER
# Spatial clustering
#TODO won't work in production b/c engineering off of labels.
df['num_upgrades_parcel'] = \
df['labels'].groupby(df['parcel_id']).transform('sum')
df.drop(columns=['parcel_id', 'subdivision', 'zip'], inplace=True)
# Days since last sale
df['update_date'] = pd.to_datetime(df['update_date'])
df['last_sale_date'] = pd.to_datetime(df['last_sale_date'])
df['time_since_sale'] = \
(df['update_date'] - df['last_sale_date']).dt.days
df.drop(columns=['update_date', 'last_sale_date'], inplace=True)
# Handle sparse permits data #TODO improve method
#Quick: total permits ever
permit_cols = attribs.get_permit_cols()
df['num_permits_since_purchase'] = (df[permit_cols].notnull()).sum(1)
df.drop(columns=permit_cols, inplace=True)
### IMPUTATION
# Fill median (numerical)
df['acres'].fillna(df['acres'].median(), inplace=True)
df['census_income_median'].fillna(df['census_income_median'].median(),\
inplace=True)
# Fill mode (numerical)
df['pv_potential_kwhr_yr'].fillna(df['pv_potential_kwhr_yr'].mode()[0],\
inplace=True)
# Fill 'Unknown'
df.replace({'zillow_neighborhood': np.nan}, \
{'zillow_neighborhood': 'Unknown'}, inplace=True)
# Fill mode (categorical)
cols = ['ac_type', 'exterior_wall_type', 'frame_type', 'heating_type', \
'interior_wall_type', 'land_use', 'roof_cover_type']
for col in cols:
mode = df[col].mode()[0]
df[col].fillna(mode, inplace=True)
# DUMMYTIZE
dummy_cols = Attributes().get_dummy_cols()
df = pd.get_dummies(df, columns=dummy_cols, drop_first=True)
processed = df
return processed
def __init__(self, data_path, ablation):
self.data_path = data_path
self.attr = Attributes()
self.ablation = ablation
class Interact(param.Parameterized):
'''
'''
def __init__(self):
super(Interact, self).__init__()
self.datapath = name + '/data'
self.filelist = os.listdir(self.datapath)
self.filelist.sort()
# TODO: improve default setting
DEM = param.Selector(default='20190623_NNR300S20.npy')
time = param.Integer(0, bounds=(0, 100))
@staticmethod
def _format_imshow(fig,
ax,
title,
m=0.02,
bgc='#292929',
axc='#eee',
lblc='#fff'):
ax.margins(m)
ax.set_aspect(aspect=1)
ax.set_xlabel('Easting')
ax.set_ylabel('Northing')
ax.set_title(title, color=axc, loc='left', pad=20)
ax.xaxis.label.set_color(lblc)
ax.yaxis.label.set_color(lblc)
ax.tick_params(axis='x', colors=lblc)
ax.tick_params(axis='y', colors=lblc)
ax.spines['bottom'].set_color(axc)
ax.spines['top'].set_color(axc)
ax.spines['right'].set_color(axc)
ax.spines['left'].set_color(axc)
ax.set_facecolor(bgc)
fig.patch.set_facecolor(bgc)
@staticmethod
def _format_polar(fig, ax, bgc='#292929', axc='#eee', lblc='#fff'):
tks = [np.deg2rad(a) for a in np.linspace(0, 360, 8, endpoint=False)]
xlbls = np.array(['N', '45', 'E', '135', 'S', '225', 'W', '315'])
ax.set_theta_zero_location('N')
ax.set_xticks((tks))
ax.set_xticklabels(xlbls, rotation="vertical", size=12)
ax.tick_params(axis='x', pad=0.5)
ax.set_theta_direction(-1)
ax.set_rmin(0)
ax.set_rmax(90)
ax.set_rlabel_position(90)
ax.set_facecolor(bgc)
ax.spines['polar'].set_color(axc)
ax.xaxis.label.set_color(lblc)
ax.yaxis.label.set_color(lblc)
ax.tick_params(axis='x', colors=axc)
ax.tick_params(axis='y', colors=axc)
fig.patch.set_facecolor(bgc)
@staticmethod
def _set_title(fn, opt):
'''Assigns titles for self._imshow().
'''
date = f'{fn[:4]}-{fn[4:6]}-{fn[6:8]}'
if opt == 'elevation':
addendum = ': Interpolation'
elif opt == 'slope':
addendum = ': Slope (radians)'
elif opt == 'aspect':
addendum = ': Aspect (degrees)'
elif opt == 'correction':
addendum = ': Terrain Correction'
return date + addendum
def _imshow(self, array, cmap, opt):
'''Generalized method for calling plt.imshow()
'''
fig, ax = plt.subplots(1)
ax.imshow(
array,
origin='lower',
cmap=cmap,
)
title = self._set_title(fn=self.DEM, opt=opt)
self._format_imshow(fig=fig, ax=ax, title=title)
plt.close('all')
return fig
def _looped_correction(self):
'''
'''
for i in range(self.correct.sunposition_df.shape[0]):
if i == 0:
correct_stack = self.correct.calc_correction_onetime(i)
else:
correct_stack = np.dstack(
(correct_stack, self.correct.calc_correction_onetime(i)))
return correct_stack
def _ufunc_correction(self):
'''
'''
correct_stack = self.correct.calc_correction_fullday()
return correct_stack
@param.depends('DEM')
def input(self):
'''Assigns the self.filename and self.elevation_array
instance variables. Returns as plot of self.elevation_array.
'''
self.param.DEM.default = self.filelist[0]
self.param.DEM.objects = self.filelist
self.elevation_array = np.load(f'{self.datapath}/{self.DEM}')
return self._imshow(array=self.elevation_array,
cmap='viridis',
opt='elevation')
@param.depends('time')
def output(self):
'''Instantiates the Attributes and Correction classes,
assigns associated instance variables, and returns a plot
of the terrain correction array.
'''
self.attributes = Attributes(self.elevation_array,
resolution=self.elevation_array.shape[0],
projection=c.PROJECTION,
side_len=c.SIDE_LEN)
self.slope, self.aspect = self.attributes.calc_attributes()
self.correct = Correction(attribute_grids=(self.slope, self.aspect),
local_timezone=c.TIMEZONE,
date_str=self.DEM[:8],
lat_lon=c.LAT_LON)
self.param.time.bounds = (0, self.correct.sunposition_df.shape[0] - 1)
print(self.param.time.bounds)
self.correct_array = self.correct.calc_correction_onetime(self.time)
return self._imshow(array=self.correct_array,
cmap='magma',
opt='correction')
def export(self):
'''
'''
correct_stack = self._looped_correction()
outfile = TemporaryFile()
np.savez(
outfile,
elevation=self.elevation_array,
slope=self.slope,
aspect=self.aspect,
sunposition=self.correct.sunposition_df.to_numpy(),
correction_stack=correct_stack,
)
_ = outfile.seek(0)
name = f'{self.DEM[:-4]}_correction.npz'
return pn.widgets.FileDownload(file=outfile, filename=name)
def plot_slope(self):
'''Return a plot of the self.slope array
'''
return self._imshow(array=self.slope, cmap='YlOrBr', opt='slope')
def plot_aspect(self):
'''Return a plot of the self.aspect array
'''
return self._imshow(array=self.aspect, cmap='hsv', opt='aspect')
def plot_sun(self):
'''Return a plot of sun position
'''
xs = np.deg2rad(self.correct.sunposition_df['azimuth'])
ys = self.correct.sunposition_df['altitude']
fig = plt.figure()
ax = fig.add_subplot(111, projection='polar')
ax.scatter(xs, ys, s=10, c='orange', alpha=0.5)
self._format_polar(fig=fig, ax=ax)
plt.close('all')
return fig
def p_attributes_definition_final(p):
'attributes_definition : attribute_value'
p[0] = Attributes(p[1], [])
def __str__(self):
return '%s (%s)' % (self.name, Attributes.__str__(self))
def __init__(self, name, atk_mod=0, dmg_mod=0):
Attributes.__init__(self)
self.name = name
def __init__(self):
Attributes.__init__(self)
self.hit_die = 0
activation='relu',
recurrent_dropout=0.2,
input_shape=(X_Train.shape[1], X_Train.shape[2]))))
model.add(Dropout(0.2))
model.add(Dense(1))
model.compile(loss='mse',
optimizer='rmsprop',
metrics=[self.__r_square, self.__rmse])
model.fit(X_Train,
Y_Train,
epochs=100,
batch_size=10,
validation_data=(X_Test, Y_Test),
verbose=2,
shuffle=True)
test_pred = model.predict(X_Test)
train_pred = model.predict(X_Train)
print("The R2 score on the Train set is:\t{:0.3f}".format(
self.__r_square(Y_Train, train_pred)))
print("The R2 score on the Test set is:\t{:0.3f}".format(
self.__r_square(Y_Test, test_pred)))
if __name__ == '__main__':
attr = Attributes()
reg = RegressionModel(data_path=r'data/data.csv', ablation=False)
reg.run(attr.BEST_FIT)
reg.build_correlaions()
def __str__(self):
return self.name + ' ' + Attributes.__str__(self)
class DataSet(object):
"""Implements a class for a data set for machine-learning methods."""
def __init__(self, attributes=None, examples=None, name=None):
"""Constructor, default attributes argument=None."""
self._attributes = attributes
self._examples = Examples(self._attributes)
self._name = name
self.add(examples=examples)
def __str__(self):
"""Return string representation of dataset as would look in file."""
string = "@dataset %s\n\n" % self._name
string = string + str(self.attributes) + "\n"
string = string + "@examples\n\n"
string = string + str(self.examples)
return string
@property
def attributes(self):
"""Getter for _attributes."""
return self._attributes
@attributes.setter
def attributes(self, val):
self._attributes = val
@property
def examples(self):
"""Return self._examples."""
return self._examples
@examples.setter
def examples(self, val):
"""Set self._example."""
self._examples = val
@property
def attributes_size(self):
"""Return number of attributes."""
return self._attributes.size
@property
def examples_size(self):
"""Return number of examples."""
return self._examples.size
@property
def has_nominal_attributes(self): # noqa
return self._attributes.has_nominal_attributes
def has_numeric_attributes(self): # noqa
return self._attributes.has_numeric_attributes
def add(self, example=None, dataset=None, examples=None):
"""Add example or dataset to current dataset."""
if dataset:
self._add_dataset(dataset)
if example:
self._add_example(example)
if examples:
if isinstance(examples, list):
for e in examples:
self.add(example=e)
def _add_dataset(self, dataset):
"""Add dataset to current dataset."""
for example in dataset.examples:
self._examples.append(example)
for attribute in dataset.attributes:
self.attributes.add(attribute)
def _add_example(self, example):
"""Add example to current dataset."""
self._examples.append(example)
def nominal_to_linear(self):
"""Return dataset with linear encoded attributes."""
for i, attribute in enumerate(self.attributes):
# Use linear encoding for bpp
if isinstance(attribute, NominalAttribute) and len(attribute.domain) <= 2:
continue
mean = self.examples.mean(i)
stdev = self.examples.stdev(i)
for k, e in enumerate(self.examples):
self.examples[k].values[i] = (e.values[i] - mean) / stdev
def normalize_attributes(self):
"""Convert all attributes to binary or standardize numeric."""
self._convert_numeric_attributes()
self._convert_attributes_to_binary()
self.examples.attributes = self.binary_attributes
self._attributes = self.binary_attributes
# print(self.binary_attributes)
def _convert_numeric_attributes(self):
for i, attribute in enumerate(self.attributes):
if isinstance(attribute, NumericAttribute):
mean = self.examples.mean(i)
stdev = self.examples.stdev(i)
for k, e in enumerate(self.examples):
self.examples[k].values[i] = float(e.values[i] - mean) / stdev
def _convert_attributes_to_binary(self):
# convert attributes with domain > 2
self.binary_attributes = Attributes()
self.new_examples = self.examples
for i, attribute in enumerate(self.attributes):
if isinstance(attribute, NominalAttribute) and len(attribute.domain) > 2:
self._convert_attribute_to_binary(attribute, i)
else:
self.binary_attributes.add(attribute)
def _convert_attribute_to_binary(self, attribute, attributeindex):
"""Convert one attribute to binary."""
# conversion_dict = {}
if len(attribute.domain) < 2:
self.binary_attributes.add(attribute)
else:
for i, v in enumerate(attribute.domain):
a = NominalAttribute(name=attribute.name + str(i))
a.domain = [0.0, 1.0]
a.place = i
a.value = v
self.binary_attributes.add(a)
for example in self.examples:
if v != attribute.domain[i]:
example.values.insert((attributeindex + 1 + i), 0.0)
else:
example.values.insert((attributeindex + 1 + i), 1.0)
for i, example in enumerate(self.examples):
del example.values[attributeindex]
self.examples._examples[i] = example
for i, e in enumerate(self.examples):
self.examples[i].values[attributeindex]
# def _convert_examples_to_binary(self):
# """Convert examples to binary."""
# self.binary_examples = Examples(self.binary_attributes)
# for e in self.examples.to_binary(self.binary_attributes):
# self.binary_examples.add(e)
################### noqa
# ID3 Methods # noqa
################### noqa
def is_homogenous(self): # noqa
"""Return True if all classlabels are the same."""
classlabel_freq = self.attribute_frequency(self.attributes.classindex)
return len(classlabel_freq.keys()) == 1
def get_best_split_attribute(self):
"""Return best split attribute."""
best_gain = 0.0
best_attributeindex = None
for attributeindex in xrange(self.attributes.size):
if attributeindex == self.attributes.classindex:
continue
current_gain = self.info_gain(attributeindex)
if best_gain < current_gain:
best_gain = current_gain
best_attributeindex = attributeindex
return best_attributeindex
def split_on_attribute(self, attributeindex):
"""Split on best attribute."""
splits = []
for val in xrange(len(self.attributes[attributeindex].domain)):
examples = [e for e in self.examples if e[attributeindex] == val]
dataset = DataSet(examples=examples, attributes=self.attributes)
splits.append((dataset, val))
return splits
def info_gain(self, attributeindex, examples=None):
"""Calculate the gained from an attribute for the class."""
if examples is None: # base case
examples = self.examples
gain = 0.0
frequency = self.attribute_frequency(attributeindex)
n = sum(frequency.values())
for value, count in frequency.iteritems():
probability = count / n
# examples where all homogenous values for current value
homogenous_on_attr = [e for e in examples if e[attributeindex] == value]
# calc entropy for examples where the current attribute values
# are similar w/ respect ot the classinde
current_entropy = self.entropy(self.attributes.classindex, homogenous_on_attr)
gain += probability * current_entropy
gain = self.entropy(attributeindex, examples) - gain
return gain
def entropy(self, attributeindex, examples):
"""Entropy of dataset for the attribute param."""
frequency = self.attribute_frequency(attributeindex)
entropy = 0.0
for val in frequency.values():
entropy += (-val / examples.__len__()) * log(val / examples.__len__(), 2)
return entropy
def attribute_frequency(self, attributeindex):
"""Return frequency of attribute in data."""
frequency = defaultdict(lambda: 0.0)
for example in self.examples:
frequency[example[attributeindex]] += 1.0
return frequency
def highest_freq_classlabel(self):
"""Return classlabel attribute value w/ the highest frequency."""
frequency = self.attribute_frequency(self.attributes.classindex)
classlabel = max(frequency.iteritems(), key=itemgetter(1))[0]
return classlabel
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.ensemble import RandomForestClassifier
if __name__ == '__main__':
# Load and label the data
df = pd.read_csv('../data/city.csv', low_memory=False)
df['assessor_id'] = df['assessor_id'].str[1:]
df = add_labels(df)
# Clean, drop, and engineer features. Impute missing values.
clean = Preprocessing()
df = clean.transform(df)
# Scale numerical features
cols_to_scale = Attributes().get_num_attribs()
scaler = RobustScaler()
df[cols_to_scale] = scaler.fit_transform(df[cols_to_scale])
#Split the data
y = df.pop('labels')
X = df
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30)
X_train_res, y_train_res, idx_res = balance(X_train, y_train, method='downsample')
# Save lat/lon for map visual, but remove before training
loc_X_train = X_train_res[:, 12:14]
X_test = X_test.values
loc_X_test = X_test[:, 12:14]
class Modus_Selecter(object):
'''
classdocs
'''
def __init__(self, config):
'''
Constructor
'''
logging.info(' Network_selecter: Constructor')
self.config = config
self.network = Network_User(config)
self.attributes = Attributes(config)
self.attrs_0 = None
return
def save(self,
acc_test,
f1_weighted_test,
f1_mean_test,
ea_iter,
type_simple='training'):
xml_file_path = self.config['folder_exp'] + self.config['file_suffix']
xml_root = ET.Element("Experiment_{}".format(
self.config["name_counter"]))
child_network = ET.SubElement(xml_root,
"network",
dataset=str(self.config['network']))
child_dataset = ET.SubElement(child_network,
"dataset",
dataset=str(self.config['dataset']))
child = ET.SubElement(child_dataset,
"usage_modus",
usage_modus=str(self.config['usage_modus']))
child = ET.SubElement(child_dataset,
"dataset_finetuning",
dataset_finetuning=str(
self.config['dataset_finetuning']))
child = ET.SubElement(child_dataset,
"percentages",
percentages_names=str(
self.config['proportions']))
child = ET.SubElement(child_dataset,
"type_simple",
type_simple=str(type_simple))
child = ET.SubElement(child_dataset,
"output",
output=str(self.config['output']))
child = ET.SubElement(child_dataset, "lr", lr=str(self.config['lr']))
child = ET.SubElement(child_dataset,
"epochs",
epochs=str(self.config['epochs']))
child = ET.SubElement(child_dataset,
"reshape_input",
reshape_input=str(self.config["reshape_input"]))
child = ET.SubElement(child_dataset, "ea_iter", ea_iter=str(ea_iter))
child = ET.SubElement(child_dataset,
"freeze_options",
freeze_options=str(
self.config['freeze_options']))
for exp in range(len(acc_test)):
child = ET.SubElement(child_dataset,
"metrics",
acc_test=str(acc_test[exp]),
f1_weighted_test=str(f1_weighted_test[exp]),
f1_mean_test=str(f1_mean_test[exp]))
child = ET.SubElement(child_dataset,
"metrics_mean",
acc_test_mean=str(np.mean(acc_test)),
f1_weighted_test_mean=str(
np.mean(f1_weighted_test)),
f1_mean_test_mean=str(np.mean(f1_mean_test)))
child = ET.SubElement(child_dataset,
"metrics_std",
acc_test_mean=str(np.std(acc_test)),
f1_weighted_test_mean=str(
np.std(f1_weighted_test)),
f1_mean_test_mean=str(np.std(f1_mean_test)))
xmlstr = minidom.parseString(
ET.tostring(xml_root)).toprettyxml(indent=" ")
with open(xml_file_path, "a") as f:
f.write(xmlstr)
print(xmlstr)
return
def train(self, itera=1, testing=True):
logging.info(' Network_selecter: Train')
start_time_test = time.time()
#There will be only one iteration
#As there is not evolution
acc_train_ac = []
f1_weighted_train_ac = []
f1_mean_train_ac = []
if testing:
acc_test_ac = []
f1_weighted_test_ac = []
f1_mean_test_ac = []
for iter_evl in range(itera):
logging.info(
' Network_selecter: Train iter {}'.format(itera))
acc_train, f1_weighted_train, f1_mean_train = self.network.evolution_evaluation(
ea_iter=iter_evl)
acc_train_ac.append(acc_train)
f1_weighted_train_ac.append(f1_weighted_train)
f1_mean_train_ac.append(f1_mean_train)
elapsed_time_test = time.time() - start_time_test
logging.info(
' Network_selecter: Train: elapsed time {} acc {}, '
'f1_weighted {}, f1_mean {}'.format(elapsed_time_test,
acc_train,
f1_weighted_train,
f1_mean_train))
self.save(acc_train_ac,
f1_weighted_train_ac,
f1_mean_train_ac,
ea_iter=iter_evl)
if testing:
acc_test, f1_weighted_test, f1_mean_test = self.test(
testing=True)
acc_test_ac.append(acc_test)
f1_weighted_test_ac.append(f1_weighted_test)
f1_mean_test_ac.append(f1_mean_test)
self.save(acc_test_ac,
f1_weighted_test_ac,
f1_mean_test_ac,
ea_iter=iter_evl,
type_simple='testing')
if testing:
self.save(acc_test_ac,
f1_weighted_test_ac,
f1_mean_test_ac,
ea_iter=iter_evl,
type_simple='testing')
return
def test(self, testing=False):
start_time_test = time.time()
acc_test, f1_weighted_test, f1_mean_test = self.network.evolution_evaluation(
ea_iter=0, testing=testing)
elapsed_time_test = time.time() - start_time_test
logging.info(' Network_selecter: Train: elapsed time {} acc {}, '
'f1_weighted {}, f1_mean {}'.format(
elapsed_time_test, acc_test, f1_weighted_test,
f1_mean_test))
if not testing:
self.save([acc_test], [f1_weighted_test], [f1_mean_test],
ea_iter=0,
type_simple='testing')
return
return acc_test, f1_weighted_test, f1_mean_test
def evolution(self):
logging.info(' Network_selecter: Evolution')
# Setting attribute population
if os.path.isfile('../' + self.config['folder_exp'] + '/iters.txt'):
best_attrs = self.attributes.load_attrs(0, name_file='best_attrs')
self.attrs_0 = best_attrs[0]['attrs']
self.network.set_attrs(self.attrs_0)
init_iter = self.load_iters() + 1
logging.info(
' Network_selecter: Loading previous training in iters {}...'
.format(init_iter))
else:
self.attrs_0 = self.attributes.creating_init_population()
init_iter = 0
self.network.set_attrs(self.attrs_0)
logging.info(
' Network_selecter: No Loading training in iters {}...'.
format(init_iter))
start_time_test = time.time()
# initial evaluation of the population number 0
acc_test, f1_weighted_test, f1_mean_test = self.network.evolution_evaluation(
ea_iter=0)
elapsed_time_test = time.time() - start_time_test
logging.info(
' Network_selecter: EA: elapsed time {} acc {}, f1_weighted {}, f1_mean {}'
.format(elapsed_time_test, acc_test, f1_weighted_test,
f1_mean_test))
#Save validation results
self.save(acc_test, f1_weighted_test, f1_mean_test, ea_iter=0)
self.attributes.save_attrs(self.attrs_0,
f1_weighted_test,
init_iter,
name_file='attrs')
#Setting up the fitness
best_fitness = f1_weighted_test
best_attr = np.copy(self.attrs_0)
fitness = []
all_fitness = []
all_acc = []
iters = []
fitness.append(f1_weighted_test)
all_fitness.append(f1_weighted_test)
all_acc.append(acc_test)
iters.append(init_iter)
np.savetxt(self.config["folder_exp"] + 'fitness.txt',
fitness,
fmt='%.5f')
np.savetxt(self.config["folder_exp"] + 'iters.txt', iters, fmt='%d')
np.savetxt(self.config["folder_exp"] + 'best_attributes.txt',
best_attr,
fmt='%d')
np.savetxt(self.config["folder_exp"] + 'all_fitness.txt',
all_fitness,
fmt='%.5f')
np.savetxt(self.config["folder_exp"] + 'all_accuracies.txt',
all_acc,
fmt='%.5f')
# Starting the evolution
epochs_training = self.config["epochs"]
for ea_iter in range(1, self.config["evolution_iter"]):
logging.info(
' Network_selecter: EA: iter {} from {} with epochs {}...'
.format(ea_iter, self.config["evolution_iter"],
epochs_training))
#Mutating the attributes
# attr_new = self.mutation_nonlocal_percentage(best_attr, best_percentage, number_K = 8)
# attr_new = self.mutation_local(best_attr)
# attr_new = self.mutation_nonlocal(best_attr, number_K = 4)
attr_new = self.attributes.mutation_global(best_attr)
#Setting the new attributes to the network
self.network.set_attrs(attr_new)
#training and validating the network
acc_test, f1_weighted_test, f1_mean_test = self.network.evolution_evaluation(
ea_iter=ea_iter)
logging.info(
' Network_selecter: EA: elapsed time {} acc {}, f1_weighted {}, f1_mean {}'
.format(elapsed_time_test, acc_test, f1_weighted_test,
f1_mean_test))
#Store the fitness
all_fitness.append(f1_weighted_test)
np.savetxt(self.config["folder_exp"] + 'all_fitness.txt',
all_fitness,
fmt='%.5f')
self.save(acc_test,
f1_weighted_test,
f1_mean_test,
ea_iter=ea_iter)
all_acc.append(acc_test)
np.savetxt(self.config["folder_exp"] + 'all_accuracies.txt',
all_acc,
fmt='%.5f')
#save the attributes
self.attributes.save_attrs(attr_new,
f1_weighted_test,
ea_iter,
protocol_file='ab')
#select if fitness improved, if so, update the fitness and save the network and attributes
if f1_weighted_test > best_fitness:
logging.info(
' Network_selecter: EA: Got best attrs with f1{}...'
.format(f1_weighted_test))
best_fitness = f1_weighted_test
best_attr = np.copy(attr_new)
fitness.append(f1_weighted_test)
iters.append(ea_iter)
#Saving the best attributes and its network
self.attributes.save_attrs(attr_new,
f1_weighted_test,
ea_iter,
name_file='best_attrs')
self.network.save_network(ea_iter)
np.savetxt(self.config["folder_exp"] + 'fitness.txt',
fitness,
fmt='%.5f')
np.savetxt(self.config["folder_exp"] + 'iters.txt',
iters,
fmt='%d')
np.savetxt(self.config["folder_exp"] + 'best_attributes.txt',
best_attr,
fmt='%d')
return
def net_modus(self):
logging.info(' Network_selecter: Net modus: {}'.format(
self.config['usage_modus']))
if self.config['usage_modus'] == 'train':
self.train(itera=3, testing=True)
elif self.config['usage_modus'] == 'test':
self.test()
elif self.config['usage_modus'] == 'evolution':
self.evolution()
elif self.config['usage_modus'] == 'train_final':
self.train(itera=1, testing=True)
elif self.config['usage_modus'] == 'fine_tuning':
self.train(itera=4, testing=True)
return
def get_attributes(asin_list):
'''
Retrieve certain attributes for all active ASINS from goodstuff database
Args:
asin_list(list of lists):
Each internal list contains 10 ASIN. This because the mws function
allows to submit an api request for 10 ASIN at a time.
Returns:
master(list): A list of attributes for each ASIN.
The attributes are:
identifier: ASIN number
brand: The manufacturers brand
pkg_length: The lenght of the package containing the item
pkg_width: The width of the item.
pkg_height: The height of the item:
salesrank: The selling rate. lower the numbers the sale fater.
manufacturer: The company that makes the item
max_age: The suggested maximum age the item is for (months)
min_age: The minimum age the item is for (months)
product_group: Theretail category the item belongs.
'''
master = []
for asins in asin_list:
try:
response = mws.get_matching_product(MarketplaceId=marketplaceid,
ASINList=asins)
except AttributeError:
print('AttributeError')
time.sleep(20)
response = mws.get_matching_product(MarketplaceId=marketplaceid,
ASINList=asins)
except BotoServerError:
print('BotoServerError')
time.sleep(20)
response = mws.get_matching_product(MarketplaceId=marketplaceid,
ASINList=asins)
except:
print('Exception')
raise
for result in response._result:
product = result.Product
identifier = result['ASIN']
try:
attributes = product.AttributeSets.ItemAttributes[0]
except AttributeError:
continue
dimensions = attributes.PackageDimensions
a = Attributes(product, attributes, dimensions)
master.append([
identifier,
a.get_brand(),
a.get_length(),
a.get_width(),
a.get_height(),
a.get_salesrank(),
a.get_manufacturer(),
a.get_max_age(),
a.get_min_age(),
a.get_product_group()
])
return master
def __init__(self,
skills = Skills(), attributes = Attributes(), health = Health()
):
self.skills = skills
self.attributes = attributes
self.health = health
def __init__(self):
from attributes import Attributes
self.currencies = Attributes("Currency")
def __init__(self,
name="prisoner",
inventory=None,
equipped=None,
attributes=None,
skills=None,
level=1,
health=100,
fatigue=100,
magicka=10,
race="imperial",
encumbrance=0,
sneak=False,
effects=None,
spells=None,
sex="male",
actions=None,
gold=0,
quests=None,
notoriety=0,
bounty=0,
fame=0):
self.name = name
self.level = level
self.health = health
self.fatigue = fatigue
self.magicka = magicka
self.race = race
self.encumbrance = encumbrance
self.sneak = sneak
self.notoriety = notoriety
self.bounty = bounty
self.gold = gold
self.fame = fame
self.sex = sex
if inventory:
self.inventory = inventory
else:
self.inventory = Inventory()
if equipped:
self.equipped = equipped
else:
self.equipped = []
if attributes:
self.attributes = attributes
else:
self.attributes = Attributes()
if skills:
self.skills = skills
else:
self.skills = Skills()
if effects:
self.effects = effects
else:
self.effects = []
if spells:
self.spells = spells
else:
self.spells = []
if actions:
self.actions = actions
else:
self.actions = []
if quests:
self.quests = quests
else:
self.quests = []
