def __init__(self, **kwords): self.attributes = Attributes(**kwords) dimensions = attr_init(kwords, "dimensions", []) storage_precision = attr_init(kwords, "storage_precision", None) init_data = attr_init(kwords, "init_data", None) self.table = create_multi_dim_array(dimensions, init_data=init_data) self.dimensions = dimensions self.storage_precision = storage_precision
def __init__(self, config): ''' Constructor ''' logging.info(' Network_selecter: Constructor') self.config = config self.network = Network_User(config) self.attributes = Attributes(config) self.attrs_0 = None return
def __init__(self, name, atk_mod=0, dmg_mod=0): Attributes.__init__(self) self.name = name # Modifiers self.atk = intOrZero(atk_mod) if type(dmg_mod) is int: self.dmg_roll = DamageRoll(None, None, dmg_mod) elif isinstance(dmg_mod, DamageRoll): self.dmg_roll = dmg_mod elif isinstance(dmg_mod, basestring): self.dmg_roll = DamageRoll(None, None, intOrZero(dmg_mod)) else: raise ValueError("Unrecognized damage modifier %s" % dmg_mod) # UI Details Buff._last_id += 1 self.id = self._last_id self.ui_id = ''
def _convert_attributes_to_binary(self): # convert attributes with domain > 2 self.binary_attributes = Attributes() self.new_examples = self.examples for i, attribute in enumerate(self.attributes): if isinstance(attribute, NominalAttribute) and len(attribute.domain) > 2: self._convert_attribute_to_binary(attribute, i) else: self.binary_attributes.add(attribute)
class ExchangeRates(object): def __init__(self): from attributes import Attributes self.currencies = Attributes("Currency") @transactional def updateToday(self, session, currency_ids=[1, 2, 3]): xrates = urlopen("http://www.x-rates.com/calculator.html").read() data = '\s*=\s*new\s+Array\s*\(([\w",.\s]+)\)' currency_list = list(eval(re.compile("currency" + data).search(xrates).groups()[0])) rate_list = list(eval(re.compile("rate" + data).search(xrates).groups()[0])) for id in currency_ids: cur = self.currencies.getCode(id) rate = float(rate_list[currency_list.index(cur)]) exchange_rate = ExchangeRate(rate_date=Date(), currency_id=id, rate=rate) session.merge(exchange_rate) @transactional def updateLast120(self, session, currency_ids=[1, 2, 3]): " scrapes rates for last 120 days from www.x-rates.com " from sqlalchemy.sql import update extractor = re.compile( "<tr[^>]*>\n" '<td><font face="Verdana" size="-1">([\d-]+)</font></td>\n' '<td><font face="Verdana" size="-1">.*?</font></td>\n' '<td><font face="Verdana" size="-1">([\d.]+)</font></td>\n' '<td><font face="Verdana" size="-1">.*?</font></td>\n' "</tr>" ) for id in currency_ids: cur = self.currencies.getCode(id) last120 = urlopen("http://www.x-rates.com/d/%s/USD/data120.html" % cur).read() for date, rate in extractor.findall(last120): exchange_rate = ExchangeRate(rate_date=date, currency_id=id, rate=rate) session.merge(exchange_rate) @transactional def getToday(self, session): self.updateToday(session) return session.query(ExchangeRate).filter_by(rate_date=Date()).all()
def authnRequest(self): ''' Builds the AuthnRequest ''' doc = Document() authnRequest = self._createElementNS(doc,'urn:oasis:names:tc:SAML:2.0:protocol', 'saml2p:AuthnRequest') authnRequest.setAttribute("Destination", self.setting.samlSsoEndpointUrl) authnRequest.setAttribute("Provider-Name", "SuisseID Service Provider AG") authnRequest.setAttribute("ForceAuthn", "true") authnRequest.setAttribute("ID", str(uuid.uuid4())) authnRequest.setAttribute("Version", "2.0") authnRequest.setAttribute("IssueInstant", "2012-12-18T09:00:00") authnRequest.setAttribute("AssertionConsumerServiceURL", self.setting.returnUrl) issuer = self._createElementNS(doc,'urn:oasis:names:tc:SAML:2.0:assertion', 'saml2:Issuer') issuer.appendChild(doc.createTextNode(self.setting.spName)) authnRequest.appendChild(issuer) extensions = self._createElementNS(doc, 'urn:oasis:names:tc:SAML:2.0:protocol', 'saml2p:Extensions') attributes = Attributes() attributes.enable() attributes.append(extensions) privacyNotice = self._createElementNS(doc, 'http://schemas.xmlsoap.org/ws/2005/05/identity', 'ic:PrivacyNotice') privacyNotice.setAttribute("Version", "1") privacyNotice.appendChild(doc.createTextNode('http://localhost:8888/auth/privacy')) extensions.appendChild(privacyNotice) authnRequest.appendChild(extensions) doc.appendChild(authnRequest) #print doc.toprettyxml(" ", "\n") #request = """ #<saml2p:AuthnRequest xmlns:saml2p="urn:oasis:names:tc:SAML:2.0:protocol" AssertionConsumerServiceURL="%s" Destination="https://idp.signdemo.com/suisseid/SSOPOST/metaAlias/swisssign.net/idp_v15" ForceAuthn="true" ID="_529a99cc-9aa3-4bef-899c-fd2587478740" IsPassive="false" IssueInstant="2012-08-31T16:18:17.878Z" ProviderName="Demo Service Provider" Version="2.0"> # <saml2:Issuer xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion">Service Provider</saml2:Issuer> #</saml2p:AuthnRequest> # """ % (self.setting.returnUrl) #return base64.b64encode(str(request)) return base64.b64encode(str(doc.toxml()))
def build_correlaions(self): dataset = pd.read_csv(self.data_path) dataset = dataset.drop(labels='date', axis=1) attr = Attributes() data = dataset[attr.ALL_FEATURES] relation = data.corr() masking = np.zeros_like(relation, dtype=np.bool) masking[np.triu_indices_from(masking)] = True sns.heatmap(relation, annot=True, fmt=".2f", mask=masking) plt.xticks(range(len(relation.columns)), relation.columns) plt.yticks(range(len(relation.columns)), relation.columns) plt.show() print(self.__abs_correlations(data, 40))
def output(self): '''Instantiates the Attributes and Correction classes, assigns associated instance variables, and returns a plot of the terrain correction array. ''' self.attributes = Attributes(self.elevation_array, resolution=self.elevation_array.shape[0], projection=c.PROJECTION, side_len=c.SIDE_LEN) self.slope, self.aspect = self.attributes.calc_attributes() self.correct = Correction(attribute_grids=(self.slope, self.aspect), local_timezone=c.TIMEZONE, date_str=self.DEM[:8], lat_lon=c.LAT_LON) self.param.time.bounds = (0, self.correct.sunposition_df.shape[0] - 1) print(self.param.time.bounds) self.correct_array = self.correct.calc_correction_onetime(self.time) return self._imshow(array=self.correct_array, cmap='magma', opt='correction')
def getNewAttributes(self, attributes=None, args=None): u""" The ``getAttributes`` method answers ``attributes`` attribute or a new instance of ``Attributes``. The attribute set holds the @attributes of the ``self`` node. Note that the attribute set is stored “as such” without making a copy. So any change to the attribute set when manipulating the tree, will also reflect in the original attribute set.<br/> If the ``args`` dictionary is defined, then the these values are added over the the key-value of new create ``Attributes`` instance. This is a separate method to allow inheriting node classes to redefine the class of the attribute set. The ``args`` attribute can be either a dictionary or a list of dictionaries. """ if isinstance(attributes, dict): attributes = Attributes(**attributes) if attributes is None: attributes = Attributes() if args is not None: if not isinstance(args, (list, tuple)): args = (args,) for arg in args: if arg is not None: attributes.update(arg) return attributes
def authnRequest(self): ''' Builds the AuthnRequest ''' doc = Document() authnRequest = self._createElementNS( doc, 'urn:oasis:names:tc:SAML:2.0:protocol', 'saml2p:AuthnRequest') authnRequest.setAttribute("Destination", self.setting.samlSsoEndpointUrl) authnRequest.setAttribute("Provider-Name", "SuisseID Service Provider AG") authnRequest.setAttribute("ForceAuthn", "true") authnRequest.setAttribute("ID", str(uuid.uuid4())) authnRequest.setAttribute("Version", "2.0") authnRequest.setAttribute("IssueInstant", "2012-12-18T09:00:00") authnRequest.setAttribute("AssertionConsumerServiceURL", self.setting.returnUrl) issuer = self._createElementNS( doc, 'urn:oasis:names:tc:SAML:2.0:assertion', 'saml2:Issuer') issuer.appendChild(doc.createTextNode(self.setting.spName)) authnRequest.appendChild(issuer) extensions = self._createElementNS( doc, 'urn:oasis:names:tc:SAML:2.0:protocol', 'saml2p:Extensions') attributes = Attributes() attributes.enable() attributes.append(extensions) privacyNotice = self._createElementNS( doc, 'http://schemas.xmlsoap.org/ws/2005/05/identity', 'ic:PrivacyNotice') privacyNotice.setAttribute("Version", "1") privacyNotice.appendChild( doc.createTextNode('http://localhost:8888/auth/privacy')) extensions.appendChild(privacyNotice) authnRequest.appendChild(extensions) doc.appendChild(authnRequest) #print doc.toprettyxml(" ", "\n") #request = """ #<saml2p:AuthnRequest xmlns:saml2p="urn:oasis:names:tc:SAML:2.0:protocol" AssertionConsumerServiceURL="%s" Destination="https://idp.signdemo.com/suisseid/SSOPOST/metaAlias/swisssign.net/idp_v15" ForceAuthn="true" ID="_529a99cc-9aa3-4bef-899c-fd2587478740" IsPassive="false" IssueInstant="2012-08-31T16:18:17.878Z" ProviderName="Demo Service Provider" Version="2.0"> # <saml2:Issuer xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion">Service Provider</saml2:Issuer> #</saml2p:AuthnRequest> # """ % (self.setting.returnUrl) #return base64.b64encode(str(request)) return base64.b64encode(str(doc.toxml()))
def getNewAttributes(self, attributes=None, args=None): u""" The <code>getAttributes</code> method answers <attr>attributes</attr> attribute or a new instance of <code>Attributes</code>. The attribute set holds the @attributes of the <attr>self</attr> node. Note that the attribute set is stored “as such” without making a copy. So any change to the attribute set when manipulating the tree, will also reflect in the original attribute set.<br/> If the <attr>args</attr> dictionary is defined, then the these values are added over the the key-value of new create <code>Attributes</code> instance. This is a separate method to allow inheriting node classes to redefine the class of the attribute set. The <attr>args</attr> attribute can be either a dictionary or a list of dictionaries. """ if isinstance(attributes, dict): attributes = Attributes(**attributes) if attributes is None: attributes = Attributes() if args is not None: if not isinstance(args, (list, tuple)): args = (args,) for arg in args: if arg is not None: attributes.update(arg) return attributes
from attributes import Attributes if __name__ == '__main__': a = Attributes(3, 3, 3) print(f'Цена за покупку аттрибутов {a.full_cost}') print(a.print_attributes)
class ML_Table(ML_LeafNode): """ Metalibm Table object """ def __init__(self, **kwords): self.attributes = Attributes(**kwords) dimensions = attr_init(kwords, "dimensions", []) storage_precision = attr_init(kwords, "storage_precision", None) init_data = attr_init(kwords, "init_data", None) self.table = create_multi_dim_array(dimensions, init_data=init_data) self.dimensions = dimensions self.storage_precision = storage_precision def __setitem__(self, key, value): self.table[key] = value def __getitem__(self, key): return self.table[key] def get_storage_precision(self): return self.storage_precision def get_precision(self): return self.get_storage_precision() def get_tag(self): return self.attributes.get_tag() def get_subset_interval(self, index_function, range_set): # init bound values low_bound = None high_bound = None # going through the selected valued list # to build the range interval for indexes in range_set: value = index_function(self, indexes) if low_bound is None or low_bound > value: low_bound = value if high_bound is None or high_bound < value: high_bound = value return Interval(low_bound, high_bound) def get_definition(self, table_name, final=";", language=C_Code): precision_name = self.get_storage_precision().get_name( language=language) return "%s %s[%s]" % (precision_name, table_name, "][".join( [str(dim) for dim in self.dimensions])) def get_content_init(self, language=C_Code): return get_table_content(self.table, self.dimensions, self.get_storage_precision(), language=language) def get_str(self, depth=None, display_precision=False, tab_level=0, memoization_map={}, display_attribute=False, display_id=False): id_str = ("[id=%x]" % id(self)) if display_id else "" attribute_str = "" if not display_attribute else self.attributes.get_str( tab_level=tab_level) precision_str = "" if not display_precision else "[%s]" % str( self.get_storage_precision()) return " " * tab_level + "Table[%s]%s%s%s\n" % ("][".join([ str(dim) for dim in self.dimensions ]), precision_str, id_str, attribute_str)
def __init__(self, name, attack): Attributes.__init__(self) self.name = name assert isinstance(attack, Attack) self._attack = attack
def __init__(self, name): Attributes.__init__(self) self.name = name
def p_attributes_definition_recursive(p): 'attributes_definition : attribute_value attributes_definition' p[0] = Attributes(p[1], p[2].attributes)
# AI Abby Virtual Assistant [Version 1.19-22alpha] # (c) 2020, AI Abby Virtual Assistant. All Rights Reserved. # Author: Johndev4 # Created: September 19-22, 2020 print("Initializing the program...") from os import system system('title AI Abby Virtual Assistant') from attributes import Attributes from voice_system import VoiceSystem from get_started import get_username import operations if __name__ == "__main__": attr = Attributes() voice_Sys = VoiceSystem(attr.AI_ASSISTANT, attr.ME) system('cls') print(f"{attr.HEAD}\r\n\r\n") voice_Sys.speak(f"Hello, I'm {attr.AI_ASSISTANT}, your virtual assistant.") attr.ME = get_username(attr) voice_Sys.greet(attr.ME) voice_Sys.speak(f"What can I help you today?") while True: query = voice_Sys.listen() operations.func(attr, query)
# leaf_font_size=6) # pylab.savefig(ps.name + ".png") #print Z clusters_values = fcluster(Z, 0.75) clusters_dict = {} for i in range(len(clusters_values)): c_id = clusters_values[i] old_v = clusters_dict.get(c_id, []) old_v.append(i) clusters_dict[c_id] = old_v clusters = [] for v in clusters_dict.values(): clusters.append(v) attr = Attributes(ps, v, classifier) print attr.extract_attributes() res = t.get_results_purity_inverse_purity(ps.name, clusters) res2 = t.get_results_bcubed(ps.name, clusters) results[0] += res[0] results[1] += res[1] results2[0] += res2[0] results2[1] += res2[1] print res print res2 print len(clusters) #print clusters #print len(set(clusters))
def transform(self, df): ### CLEAN AND DROP attribs = Attributes() # drop customer segmentation info (3) #tinker segment_cols = attribs.get_segment_cols() df.drop(columns=segment_cols, inplace=True) # drop cols with data leakage (2) leak_cols = attribs.get_leak_cols() df.drop(columns=leak_cols, inplace=True) # drop rows with leakage df.drop(df[df.year_built == 2017].index, inplace=True) df.drop(df[df.effective_year_built == 2017].index, inplace=True) # drop cols with too many nulls (28) null_cols = attribs.get_null_cols() df.drop(columns=null_cols, inplace=True) # drop redundant features (74) redundant_cols = attribs.get_redundant_cols() df.drop(columns=redundant_cols, inplace=True) # drop irrelevant features (18) irrelevant_cols = attribs.get_irrelevant_cols() df.drop(columns=irrelevant_cols, inplace=True) # drop 1050 rows without sale_date or sale_price (same set) df.dropna(subset=['last_sale_price', 'last_sale_date'], inplace=True) # remap buidling_condition (misspelling intentional) df.replace({'buidling_condition':{ 'LOW':1, 'FAIR':2, 'AVERAGE':3, 'AVERAGE +':4, 'AVERAGE ++':5, 'GOOD':6, 'GOOD +':7, 'GOOD ++':8, 'VERY GOOD':9, 'VERY GOOD +':10, 'VERY GOOD ++':11, 'EXCELLENT':12, 'EXCELLENT +':13, 'EXCELLENT++':14, 'EXCEPTIONAL 1':15} }, inplace=True) # convert true/false to 1/0 df['nrel_attached_garage'].astype(int, copy=False) # combine full and half baths df['num_baths'] = df['full_bath_count'] + (0.5 * df['half_bath_count']) df.drop(columns=['full_bath_count', 'half_bath_count'], inplace=True) ### FEATURE ENGINEER # Spatial clustering #TODO won't work in production b/c engineering off of labels. df['num_upgrades_parcel'] = \ df['labels'].groupby(df['parcel_id']).transform('sum') df.drop(columns=['parcel_id', 'subdivision', 'zip'], inplace=True) # Days since last sale df['update_date'] = pd.to_datetime(df['update_date']) df['last_sale_date'] = pd.to_datetime(df['last_sale_date']) df['time_since_sale'] = \ (df['update_date'] - df['last_sale_date']).dt.days df.drop(columns=['update_date', 'last_sale_date'], inplace=True) # Handle sparse permits data #TODO improve method #Quick: total permits ever permit_cols = attribs.get_permit_cols() df['num_permits_since_purchase'] = (df[permit_cols].notnull()).sum(1) df.drop(columns=permit_cols, inplace=True) ### IMPUTATION # Fill median (numerical) df['acres'].fillna(df['acres'].median(), inplace=True) df['census_income_median'].fillna(df['census_income_median'].median(),\ inplace=True) # Fill mode (numerical) df['pv_potential_kwhr_yr'].fillna(df['pv_potential_kwhr_yr'].mode()[0],\ inplace=True) # Fill 'Unknown' df.replace({'zillow_neighborhood': np.nan}, \ {'zillow_neighborhood': 'Unknown'}, inplace=True) # Fill mode (categorical) cols = ['ac_type', 'exterior_wall_type', 'frame_type', 'heating_type', \ 'interior_wall_type', 'land_use', 'roof_cover_type'] for col in cols: mode = df[col].mode()[0] df[col].fillna(mode, inplace=True) # DUMMYTIZE dummy_cols = Attributes().get_dummy_cols() df = pd.get_dummies(df, columns=dummy_cols, drop_first=True) processed = df return processed
def __init__(self, data_path, ablation): self.data_path = data_path self.attr = Attributes() self.ablation = ablation
class Interact(param.Parameterized): ''' ''' def __init__(self): super(Interact, self).__init__() self.datapath = name + '/data' self.filelist = os.listdir(self.datapath) self.filelist.sort() # TODO: improve default setting DEM = param.Selector(default='20190623_NNR300S20.npy') time = param.Integer(0, bounds=(0, 100)) @staticmethod def _format_imshow(fig, ax, title, m=0.02, bgc='#292929', axc='#eee', lblc='#fff'): ax.margins(m) ax.set_aspect(aspect=1) ax.set_xlabel('Easting') ax.set_ylabel('Northing') ax.set_title(title, color=axc, loc='left', pad=20) ax.xaxis.label.set_color(lblc) ax.yaxis.label.set_color(lblc) ax.tick_params(axis='x', colors=lblc) ax.tick_params(axis='y', colors=lblc) ax.spines['bottom'].set_color(axc) ax.spines['top'].set_color(axc) ax.spines['right'].set_color(axc) ax.spines['left'].set_color(axc) ax.set_facecolor(bgc) fig.patch.set_facecolor(bgc) @staticmethod def _format_polar(fig, ax, bgc='#292929', axc='#eee', lblc='#fff'): tks = [np.deg2rad(a) for a in np.linspace(0, 360, 8, endpoint=False)] xlbls = np.array(['N', '45', 'E', '135', 'S', '225', 'W', '315']) ax.set_theta_zero_location('N') ax.set_xticks((tks)) ax.set_xticklabels(xlbls, rotation="vertical", size=12) ax.tick_params(axis='x', pad=0.5) ax.set_theta_direction(-1) ax.set_rmin(0) ax.set_rmax(90) ax.set_rlabel_position(90) ax.set_facecolor(bgc) ax.spines['polar'].set_color(axc) ax.xaxis.label.set_color(lblc) ax.yaxis.label.set_color(lblc) ax.tick_params(axis='x', colors=axc) ax.tick_params(axis='y', colors=axc) fig.patch.set_facecolor(bgc) @staticmethod def _set_title(fn, opt): '''Assigns titles for self._imshow(). ''' date = f'{fn[:4]}-{fn[4:6]}-{fn[6:8]}' if opt == 'elevation': addendum = ': Interpolation' elif opt == 'slope': addendum = ': Slope (radians)' elif opt == 'aspect': addendum = ': Aspect (degrees)' elif opt == 'correction': addendum = ': Terrain Correction' return date + addendum def _imshow(self, array, cmap, opt): '''Generalized method for calling plt.imshow() ''' fig, ax = plt.subplots(1) ax.imshow( array, origin='lower', cmap=cmap, ) title = self._set_title(fn=self.DEM, opt=opt) self._format_imshow(fig=fig, ax=ax, title=title) plt.close('all') return fig def _looped_correction(self): ''' ''' for i in range(self.correct.sunposition_df.shape[0]): if i == 0: correct_stack = self.correct.calc_correction_onetime(i) else: correct_stack = np.dstack( (correct_stack, self.correct.calc_correction_onetime(i))) return correct_stack def _ufunc_correction(self): ''' ''' correct_stack = self.correct.calc_correction_fullday() return correct_stack @param.depends('DEM') def input(self): '''Assigns the self.filename and self.elevation_array instance variables. Returns as plot of self.elevation_array. ''' self.param.DEM.default = self.filelist[0] self.param.DEM.objects = self.filelist self.elevation_array = np.load(f'{self.datapath}/{self.DEM}') return self._imshow(array=self.elevation_array, cmap='viridis', opt='elevation') @param.depends('time') def output(self): '''Instantiates the Attributes and Correction classes, assigns associated instance variables, and returns a plot of the terrain correction array. ''' self.attributes = Attributes(self.elevation_array, resolution=self.elevation_array.shape[0], projection=c.PROJECTION, side_len=c.SIDE_LEN) self.slope, self.aspect = self.attributes.calc_attributes() self.correct = Correction(attribute_grids=(self.slope, self.aspect), local_timezone=c.TIMEZONE, date_str=self.DEM[:8], lat_lon=c.LAT_LON) self.param.time.bounds = (0, self.correct.sunposition_df.shape[0] - 1) print(self.param.time.bounds) self.correct_array = self.correct.calc_correction_onetime(self.time) return self._imshow(array=self.correct_array, cmap='magma', opt='correction') def export(self): ''' ''' correct_stack = self._looped_correction() outfile = TemporaryFile() np.savez( outfile, elevation=self.elevation_array, slope=self.slope, aspect=self.aspect, sunposition=self.correct.sunposition_df.to_numpy(), correction_stack=correct_stack, ) _ = outfile.seek(0) name = f'{self.DEM[:-4]}_correction.npz' return pn.widgets.FileDownload(file=outfile, filename=name) def plot_slope(self): '''Return a plot of the self.slope array ''' return self._imshow(array=self.slope, cmap='YlOrBr', opt='slope') def plot_aspect(self): '''Return a plot of the self.aspect array ''' return self._imshow(array=self.aspect, cmap='hsv', opt='aspect') def plot_sun(self): '''Return a plot of sun position ''' xs = np.deg2rad(self.correct.sunposition_df['azimuth']) ys = self.correct.sunposition_df['altitude'] fig = plt.figure() ax = fig.add_subplot(111, projection='polar') ax.scatter(xs, ys, s=10, c='orange', alpha=0.5) self._format_polar(fig=fig, ax=ax) plt.close('all') return fig
def p_attributes_definition_final(p): 'attributes_definition : attribute_value' p[0] = Attributes(p[1], [])
def __str__(self): return '%s (%s)' % (self.name, Attributes.__str__(self))
def __init__(self, name, atk_mod=0, dmg_mod=0): Attributes.__init__(self) self.name = name
def __init__(self): Attributes.__init__(self) self.hit_die = 0
activation='relu', recurrent_dropout=0.2, input_shape=(X_Train.shape[1], X_Train.shape[2])))) model.add(Dropout(0.2)) model.add(Dense(1)) model.compile(loss='mse', optimizer='rmsprop', metrics=[self.__r_square, self.__rmse]) model.fit(X_Train, Y_Train, epochs=100, batch_size=10, validation_data=(X_Test, Y_Test), verbose=2, shuffle=True) test_pred = model.predict(X_Test) train_pred = model.predict(X_Train) print("The R2 score on the Train set is:\t{:0.3f}".format( self.__r_square(Y_Train, train_pred))) print("The R2 score on the Test set is:\t{:0.3f}".format( self.__r_square(Y_Test, test_pred))) if __name__ == '__main__': attr = Attributes() reg = RegressionModel(data_path=r'data/data.csv', ablation=False) reg.run(attr.BEST_FIT) reg.build_correlaions()
def __str__(self): return self.name + ' ' + Attributes.__str__(self)
class DataSet(object): """Implements a class for a data set for machine-learning methods.""" def __init__(self, attributes=None, examples=None, name=None): """Constructor, default attributes argument=None.""" self._attributes = attributes self._examples = Examples(self._attributes) self._name = name self.add(examples=examples) def __str__(self): """Return string representation of dataset as would look in file.""" string = "@dataset %s\n\n" % self._name string = string + str(self.attributes) + "\n" string = string + "@examples\n\n" string = string + str(self.examples) return string @property def attributes(self): """Getter for _attributes.""" return self._attributes @attributes.setter def attributes(self, val): self._attributes = val @property def examples(self): """Return self._examples.""" return self._examples @examples.setter def examples(self, val): """Set self._example.""" self._examples = val @property def attributes_size(self): """Return number of attributes.""" return self._attributes.size @property def examples_size(self): """Return number of examples.""" return self._examples.size @property def has_nominal_attributes(self): # noqa return self._attributes.has_nominal_attributes def has_numeric_attributes(self): # noqa return self._attributes.has_numeric_attributes def add(self, example=None, dataset=None, examples=None): """Add example or dataset to current dataset.""" if dataset: self._add_dataset(dataset) if example: self._add_example(example) if examples: if isinstance(examples, list): for e in examples: self.add(example=e) def _add_dataset(self, dataset): """Add dataset to current dataset.""" for example in dataset.examples: self._examples.append(example) for attribute in dataset.attributes: self.attributes.add(attribute) def _add_example(self, example): """Add example to current dataset.""" self._examples.append(example) def nominal_to_linear(self): """Return dataset with linear encoded attributes.""" for i, attribute in enumerate(self.attributes): # Use linear encoding for bpp if isinstance(attribute, NominalAttribute) and len(attribute.domain) <= 2: continue mean = self.examples.mean(i) stdev = self.examples.stdev(i) for k, e in enumerate(self.examples): self.examples[k].values[i] = (e.values[i] - mean) / stdev def normalize_attributes(self): """Convert all attributes to binary or standardize numeric.""" self._convert_numeric_attributes() self._convert_attributes_to_binary() self.examples.attributes = self.binary_attributes self._attributes = self.binary_attributes # print(self.binary_attributes) def _convert_numeric_attributes(self): for i, attribute in enumerate(self.attributes): if isinstance(attribute, NumericAttribute): mean = self.examples.mean(i) stdev = self.examples.stdev(i) for k, e in enumerate(self.examples): self.examples[k].values[i] = float(e.values[i] - mean) / stdev def _convert_attributes_to_binary(self): # convert attributes with domain > 2 self.binary_attributes = Attributes() self.new_examples = self.examples for i, attribute in enumerate(self.attributes): if isinstance(attribute, NominalAttribute) and len(attribute.domain) > 2: self._convert_attribute_to_binary(attribute, i) else: self.binary_attributes.add(attribute) def _convert_attribute_to_binary(self, attribute, attributeindex): """Convert one attribute to binary.""" # conversion_dict = {} if len(attribute.domain) < 2: self.binary_attributes.add(attribute) else: for i, v in enumerate(attribute.domain): a = NominalAttribute(name=attribute.name + str(i)) a.domain = [0.0, 1.0] a.place = i a.value = v self.binary_attributes.add(a) for example in self.examples: if v != attribute.domain[i]: example.values.insert((attributeindex + 1 + i), 0.0) else: example.values.insert((attributeindex + 1 + i), 1.0) for i, example in enumerate(self.examples): del example.values[attributeindex] self.examples._examples[i] = example for i, e in enumerate(self.examples): self.examples[i].values[attributeindex] # def _convert_examples_to_binary(self): # """Convert examples to binary.""" # self.binary_examples = Examples(self.binary_attributes) # for e in self.examples.to_binary(self.binary_attributes): # self.binary_examples.add(e) ################### noqa # ID3 Methods # noqa ################### noqa def is_homogenous(self): # noqa """Return True if all classlabels are the same.""" classlabel_freq = self.attribute_frequency(self.attributes.classindex) return len(classlabel_freq.keys()) == 1 def get_best_split_attribute(self): """Return best split attribute.""" best_gain = 0.0 best_attributeindex = None for attributeindex in xrange(self.attributes.size): if attributeindex == self.attributes.classindex: continue current_gain = self.info_gain(attributeindex) if best_gain < current_gain: best_gain = current_gain best_attributeindex = attributeindex return best_attributeindex def split_on_attribute(self, attributeindex): """Split on best attribute.""" splits = [] for val in xrange(len(self.attributes[attributeindex].domain)): examples = [e for e in self.examples if e[attributeindex] == val] dataset = DataSet(examples=examples, attributes=self.attributes) splits.append((dataset, val)) return splits def info_gain(self, attributeindex, examples=None): """Calculate the gained from an attribute for the class.""" if examples is None: # base case examples = self.examples gain = 0.0 frequency = self.attribute_frequency(attributeindex) n = sum(frequency.values()) for value, count in frequency.iteritems(): probability = count / n # examples where all homogenous values for current value homogenous_on_attr = [e for e in examples if e[attributeindex] == value] # calc entropy for examples where the current attribute values # are similar w/ respect ot the classinde current_entropy = self.entropy(self.attributes.classindex, homogenous_on_attr) gain += probability * current_entropy gain = self.entropy(attributeindex, examples) - gain return gain def entropy(self, attributeindex, examples): """Entropy of dataset for the attribute param.""" frequency = self.attribute_frequency(attributeindex) entropy = 0.0 for val in frequency.values(): entropy += (-val / examples.__len__()) * log(val / examples.__len__(), 2) return entropy def attribute_frequency(self, attributeindex): """Return frequency of attribute in data.""" frequency = defaultdict(lambda: 0.0) for example in self.examples: frequency[example[attributeindex]] += 1.0 return frequency def highest_freq_classlabel(self): """Return classlabel attribute value w/ the highest frequency.""" frequency = self.attribute_frequency(self.attributes.classindex) classlabel = max(frequency.iteritems(), key=itemgetter(1))[0] return classlabel
from sklearn.model_selection import train_test_split from sklearn.metrics import confusion_matrix, classification_report from sklearn.ensemble import RandomForestClassifier if __name__ == '__main__': # Load and label the data df = pd.read_csv('../data/city.csv', low_memory=False) df['assessor_id'] = df['assessor_id'].str[1:] df = add_labels(df) # Clean, drop, and engineer features. Impute missing values. clean = Preprocessing() df = clean.transform(df) # Scale numerical features cols_to_scale = Attributes().get_num_attribs() scaler = RobustScaler() df[cols_to_scale] = scaler.fit_transform(df[cols_to_scale]) #Split the data y = df.pop('labels') X = df X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30) X_train_res, y_train_res, idx_res = balance(X_train, y_train, method='downsample') # Save lat/lon for map visual, but remove before training loc_X_train = X_train_res[:, 12:14] X_test = X_test.values loc_X_test = X_test[:, 12:14]
class Modus_Selecter(object): ''' classdocs ''' def __init__(self, config): ''' Constructor ''' logging.info(' Network_selecter: Constructor') self.config = config self.network = Network_User(config) self.attributes = Attributes(config) self.attrs_0 = None return def save(self, acc_test, f1_weighted_test, f1_mean_test, ea_iter, type_simple='training'): xml_file_path = self.config['folder_exp'] + self.config['file_suffix'] xml_root = ET.Element("Experiment_{}".format( self.config["name_counter"])) child_network = ET.SubElement(xml_root, "network", dataset=str(self.config['network'])) child_dataset = ET.SubElement(child_network, "dataset", dataset=str(self.config['dataset'])) child = ET.SubElement(child_dataset, "usage_modus", usage_modus=str(self.config['usage_modus'])) child = ET.SubElement(child_dataset, "dataset_finetuning", dataset_finetuning=str( self.config['dataset_finetuning'])) child = ET.SubElement(child_dataset, "percentages", percentages_names=str( self.config['proportions'])) child = ET.SubElement(child_dataset, "type_simple", type_simple=str(type_simple)) child = ET.SubElement(child_dataset, "output", output=str(self.config['output'])) child = ET.SubElement(child_dataset, "lr", lr=str(self.config['lr'])) child = ET.SubElement(child_dataset, "epochs", epochs=str(self.config['epochs'])) child = ET.SubElement(child_dataset, "reshape_input", reshape_input=str(self.config["reshape_input"])) child = ET.SubElement(child_dataset, "ea_iter", ea_iter=str(ea_iter)) child = ET.SubElement(child_dataset, "freeze_options", freeze_options=str( self.config['freeze_options'])) for exp in range(len(acc_test)): child = ET.SubElement(child_dataset, "metrics", acc_test=str(acc_test[exp]), f1_weighted_test=str(f1_weighted_test[exp]), f1_mean_test=str(f1_mean_test[exp])) child = ET.SubElement(child_dataset, "metrics_mean", acc_test_mean=str(np.mean(acc_test)), f1_weighted_test_mean=str( np.mean(f1_weighted_test)), f1_mean_test_mean=str(np.mean(f1_mean_test))) child = ET.SubElement(child_dataset, "metrics_std", acc_test_mean=str(np.std(acc_test)), f1_weighted_test_mean=str( np.std(f1_weighted_test)), f1_mean_test_mean=str(np.std(f1_mean_test))) xmlstr = minidom.parseString( ET.tostring(xml_root)).toprettyxml(indent=" ") with open(xml_file_path, "a") as f: f.write(xmlstr) print(xmlstr) return def train(self, itera=1, testing=True): logging.info(' Network_selecter: Train') start_time_test = time.time() #There will be only one iteration #As there is not evolution acc_train_ac = [] f1_weighted_train_ac = [] f1_mean_train_ac = [] if testing: acc_test_ac = [] f1_weighted_test_ac = [] f1_mean_test_ac = [] for iter_evl in range(itera): logging.info( ' Network_selecter: Train iter {}'.format(itera)) acc_train, f1_weighted_train, f1_mean_train = self.network.evolution_evaluation( ea_iter=iter_evl) acc_train_ac.append(acc_train) f1_weighted_train_ac.append(f1_weighted_train) f1_mean_train_ac.append(f1_mean_train) elapsed_time_test = time.time() - start_time_test logging.info( ' Network_selecter: Train: elapsed time {} acc {}, ' 'f1_weighted {}, f1_mean {}'.format(elapsed_time_test, acc_train, f1_weighted_train, f1_mean_train)) self.save(acc_train_ac, f1_weighted_train_ac, f1_mean_train_ac, ea_iter=iter_evl) if testing: acc_test, f1_weighted_test, f1_mean_test = self.test( testing=True) acc_test_ac.append(acc_test) f1_weighted_test_ac.append(f1_weighted_test) f1_mean_test_ac.append(f1_mean_test) self.save(acc_test_ac, f1_weighted_test_ac, f1_mean_test_ac, ea_iter=iter_evl, type_simple='testing') if testing: self.save(acc_test_ac, f1_weighted_test_ac, f1_mean_test_ac, ea_iter=iter_evl, type_simple='testing') return def test(self, testing=False): start_time_test = time.time() acc_test, f1_weighted_test, f1_mean_test = self.network.evolution_evaluation( ea_iter=0, testing=testing) elapsed_time_test = time.time() - start_time_test logging.info(' Network_selecter: Train: elapsed time {} acc {}, ' 'f1_weighted {}, f1_mean {}'.format( elapsed_time_test, acc_test, f1_weighted_test, f1_mean_test)) if not testing: self.save([acc_test], [f1_weighted_test], [f1_mean_test], ea_iter=0, type_simple='testing') return return acc_test, f1_weighted_test, f1_mean_test def evolution(self): logging.info(' Network_selecter: Evolution') # Setting attribute population if os.path.isfile('../' + self.config['folder_exp'] + '/iters.txt'): best_attrs = self.attributes.load_attrs(0, name_file='best_attrs') self.attrs_0 = best_attrs[0]['attrs'] self.network.set_attrs(self.attrs_0) init_iter = self.load_iters() + 1 logging.info( ' Network_selecter: Loading previous training in iters {}...' .format(init_iter)) else: self.attrs_0 = self.attributes.creating_init_population() init_iter = 0 self.network.set_attrs(self.attrs_0) logging.info( ' Network_selecter: No Loading training in iters {}...'. format(init_iter)) start_time_test = time.time() # initial evaluation of the population number 0 acc_test, f1_weighted_test, f1_mean_test = self.network.evolution_evaluation( ea_iter=0) elapsed_time_test = time.time() - start_time_test logging.info( ' Network_selecter: EA: elapsed time {} acc {}, f1_weighted {}, f1_mean {}' .format(elapsed_time_test, acc_test, f1_weighted_test, f1_mean_test)) #Save validation results self.save(acc_test, f1_weighted_test, f1_mean_test, ea_iter=0) self.attributes.save_attrs(self.attrs_0, f1_weighted_test, init_iter, name_file='attrs') #Setting up the fitness best_fitness = f1_weighted_test best_attr = np.copy(self.attrs_0) fitness = [] all_fitness = [] all_acc = [] iters = [] fitness.append(f1_weighted_test) all_fitness.append(f1_weighted_test) all_acc.append(acc_test) iters.append(init_iter) np.savetxt(self.config["folder_exp"] + 'fitness.txt', fitness, fmt='%.5f') np.savetxt(self.config["folder_exp"] + 'iters.txt', iters, fmt='%d') np.savetxt(self.config["folder_exp"] + 'best_attributes.txt', best_attr, fmt='%d') np.savetxt(self.config["folder_exp"] + 'all_fitness.txt', all_fitness, fmt='%.5f') np.savetxt(self.config["folder_exp"] + 'all_accuracies.txt', all_acc, fmt='%.5f') # Starting the evolution epochs_training = self.config["epochs"] for ea_iter in range(1, self.config["evolution_iter"]): logging.info( ' Network_selecter: EA: iter {} from {} with epochs {}...' .format(ea_iter, self.config["evolution_iter"], epochs_training)) #Mutating the attributes # attr_new = self.mutation_nonlocal_percentage(best_attr, best_percentage, number_K = 8) # attr_new = self.mutation_local(best_attr) # attr_new = self.mutation_nonlocal(best_attr, number_K = 4) attr_new = self.attributes.mutation_global(best_attr) #Setting the new attributes to the network self.network.set_attrs(attr_new) #training and validating the network acc_test, f1_weighted_test, f1_mean_test = self.network.evolution_evaluation( ea_iter=ea_iter) logging.info( ' Network_selecter: EA: elapsed time {} acc {}, f1_weighted {}, f1_mean {}' .format(elapsed_time_test, acc_test, f1_weighted_test, f1_mean_test)) #Store the fitness all_fitness.append(f1_weighted_test) np.savetxt(self.config["folder_exp"] + 'all_fitness.txt', all_fitness, fmt='%.5f') self.save(acc_test, f1_weighted_test, f1_mean_test, ea_iter=ea_iter) all_acc.append(acc_test) np.savetxt(self.config["folder_exp"] + 'all_accuracies.txt', all_acc, fmt='%.5f') #save the attributes self.attributes.save_attrs(attr_new, f1_weighted_test, ea_iter, protocol_file='ab') #select if fitness improved, if so, update the fitness and save the network and attributes if f1_weighted_test > best_fitness: logging.info( ' Network_selecter: EA: Got best attrs with f1{}...' .format(f1_weighted_test)) best_fitness = f1_weighted_test best_attr = np.copy(attr_new) fitness.append(f1_weighted_test) iters.append(ea_iter) #Saving the best attributes and its network self.attributes.save_attrs(attr_new, f1_weighted_test, ea_iter, name_file='best_attrs') self.network.save_network(ea_iter) np.savetxt(self.config["folder_exp"] + 'fitness.txt', fitness, fmt='%.5f') np.savetxt(self.config["folder_exp"] + 'iters.txt', iters, fmt='%d') np.savetxt(self.config["folder_exp"] + 'best_attributes.txt', best_attr, fmt='%d') return def net_modus(self): logging.info(' Network_selecter: Net modus: {}'.format( self.config['usage_modus'])) if self.config['usage_modus'] == 'train': self.train(itera=3, testing=True) elif self.config['usage_modus'] == 'test': self.test() elif self.config['usage_modus'] == 'evolution': self.evolution() elif self.config['usage_modus'] == 'train_final': self.train(itera=1, testing=True) elif self.config['usage_modus'] == 'fine_tuning': self.train(itera=4, testing=True) return
def get_attributes(asin_list): ''' Retrieve certain attributes for all active ASINS from goodstuff database Args: asin_list(list of lists): Each internal list contains 10 ASIN. This because the mws function allows to submit an api request for 10 ASIN at a time. Returns: master(list): A list of attributes for each ASIN. The attributes are: identifier: ASIN number brand: The manufacturers brand pkg_length: The lenght of the package containing the item pkg_width: The width of the item. pkg_height: The height of the item: salesrank: The selling rate. lower the numbers the sale fater. manufacturer: The company that makes the item max_age: The suggested maximum age the item is for (months) min_age: The minimum age the item is for (months) product_group: Theretail category the item belongs. ''' master = [] for asins in asin_list: try: response = mws.get_matching_product(MarketplaceId=marketplaceid, ASINList=asins) except AttributeError: print('AttributeError') time.sleep(20) response = mws.get_matching_product(MarketplaceId=marketplaceid, ASINList=asins) except BotoServerError: print('BotoServerError') time.sleep(20) response = mws.get_matching_product(MarketplaceId=marketplaceid, ASINList=asins) except: print('Exception') raise for result in response._result: product = result.Product identifier = result['ASIN'] try: attributes = product.AttributeSets.ItemAttributes[0] except AttributeError: continue dimensions = attributes.PackageDimensions a = Attributes(product, attributes, dimensions) master.append([ identifier, a.get_brand(), a.get_length(), a.get_width(), a.get_height(), a.get_salesrank(), a.get_manufacturer(), a.get_max_age(), a.get_min_age(), a.get_product_group() ]) return master
def __init__(self, skills = Skills(), attributes = Attributes(), health = Health() ): self.skills = skills self.attributes = attributes self.health = health
def __init__(self): from attributes import Attributes self.currencies = Attributes("Currency")
def __init__(self, name="prisoner", inventory=None, equipped=None, attributes=None, skills=None, level=1, health=100, fatigue=100, magicka=10, race="imperial", encumbrance=0, sneak=False, effects=None, spells=None, sex="male", actions=None, gold=0, quests=None, notoriety=0, bounty=0, fame=0): self.name = name self.level = level self.health = health self.fatigue = fatigue self.magicka = magicka self.race = race self.encumbrance = encumbrance self.sneak = sneak self.notoriety = notoriety self.bounty = bounty self.gold = gold self.fame = fame self.sex = sex if inventory: self.inventory = inventory else: self.inventory = Inventory() if equipped: self.equipped = equipped else: self.equipped = [] if attributes: self.attributes = attributes else: self.attributes = Attributes() if skills: self.skills = skills else: self.skills = Skills() if effects: self.effects = effects else: self.effects = [] if spells: self.spells = spells else: self.spells = [] if actions: self.actions = actions else: self.actions = [] if quests: self.quests = quests else: self.quests = []