def get_codescriptors(): """ looks for all codescriptors of grounds (as COLD and SNOWY as) """ import sys sys.path.insert(1, '../') data = Data() with open('../res/wiki/alland.txt', 'r') as f: for line in f: words = line.split(" ") for ground in data.grounds.keys(): other = "" if words[1] == ground: other = words[3] elif words[3] == ground: other = words[1] if wn.synsets(other, wn.ADJ): print "adj " + other g = data.get_ground(ground) g.add_codescriptor(other) elif other != "": print "no adj " + other data.save()
def execute(self, startDate, endDate, today=None): data = Data() # Allow modifying today for unit tests to work consistently if self.today: today = self.today else: today = datetime.datetime.today().date() # Ensure we're looking into the future if startDate < today: raise RuntimeError("Engine only supports calculating future dates.") # Record the dates the data is for data.startDate = startDate data.endDate = endDate # Calculate our daily balance currentBalance = sum([account.getTotalBalance() for account in self.accounts]) for currentDate in daterange(today, data.endDate + datetime.timedelta(1)): # Apply our balance modifiers for modifier in self.balanceModifiers: if modifier.recurrence.isOnDate(currentDate): currentBalance += modifier.amount # Save the new balance for the current date if (currentDate >= startDate): data.balances[currentDate] = currentBalance return data
def test_abs(): # |-1| == 0 equal(Data.abs(d_min_ones).values, np.ones((100,100))) # |1| == 1 equal(Data.abs(d_ones).values, np.ones((100,100))) # |0| == 0 equal(Data.abs(d_zeros).values, np.zeros((100,100)))
def test_neg(): # -(-1) == 1 equal(Data.neg(d_min_ones).values, np.ones((100,100))) # -(0) == 0 equal(Data.neg(d_zeros).values, np.zeros((100,100))) # -(1) == -1 equal(Data.neg(d_ones).values, -np.ones((100,100)))
def main(): """ Initiates all components that are needed after arduino-yun's linuino has started. """ threads = [] # initiate all needed threads data_thread = Data() socket_thread = Websocket( Config.get_socket_host(), Config.get_socket_port(), data_thread ) threads.append(data_thread) threads.append(socket_thread) # start all needed threads for thread in threads: thread.daemon = True thread.start() # add observers to data-object logger = Logger("default") data_thread.add_observer(logger) data_thread.add_observer(socket_thread) # keep main() alive to prevent an exit of our # deamonized threads while True: time.sleep(1)
def request_file_deletion(self, file_name): data = Data(self.db_path) modifications=data.get_file_modifications(file_name) for modification in modifications: file_id=modification[1] message="DELETE "+str(file_id)+CRLF+CRLF self.mc.sendto(message, (self.mc_address, self.mc_port))
def __init__(self, k=10, n=100, m=50): Data.__init__(self) self.k = k self.m = m self.means = rnd.uniform(mean_low, mean_high, (self.k, self.m)) self.divs = rnd.uniform(div_low, div_high, (self.k, self.m)) self.data, self.labels = self.generate_data(n)
def testLogisticError(): k = 5 data = Data(k, 0, 0) data.importDataFromMat() data.normalize() lg = LogisticLinearClassifier(0.03, 0.03, 576, k, data) err_train, miss_train, err_val, miss_val = lg.train(30) mis_fig = plt.figure() ax2 = mis_fig.add_subplot(111) ax2.plot(err_val, label="error (validation)") ax2.plot(err_train, label="error (training)") title = "std(val)=%f std(err)=%f" % (sp.std(err_val), sp.std(err_train)) mis_fig.suptitle(title) ax2.set_ylabel("error") ax2.set_xlabel("epoch") plt.legend() mis_fig = plt.figure() ax2 = mis_fig.add_subplot(111) ax2.plot(miss_val, label="misclassification ratio (validation)") ax2.plot(miss_train, label="misclassification ratio (training)") mis_fig.suptitle(title) ax2.set_ylabel("misclassification ratio") ax2.set_xlabel("epoch") plt.legend() results, cat = lg.classify(data.test_left, data.test_right) lg.confusion_matrix(cat, data.test_cat.argmax(axis=0)) err = Error() err, misclass = err.norm_total_error(results.T, data.test_cat, k) print "Error on the test set " + str(err) print "Misclassification ratio on the test set " + str(misclass)
def compareParameters(): k = 5 data = Data(k, 0, 0) data.importDataFromMat() data.normalize() train = TrainerValidator(k, 40, 80, 60, 0.001, 0.1, 1, data) train.trainAndClassify() train2 = TrainerValidator(k, 40, 80, 60, 0.04, 0.1, 1, data) train2.trainAndClassify() train3 = TrainerValidator(k, 40, 80, 60, 0.1, 0.1, 1, data) train3.trainAndClassify() error_fig = plt.figure() ax1 = error_fig.add_subplot(111) ax1.plot(train.validation_error, label="validation error mu=0.1 nu=0.001") ax1.plot(train.training_error, label="training error mu=0.1 nu=0.001") ax1.plot(train2.validation_error, label="validation error mu=0.1 nu=0.04") ax1.plot(train2.training_error, label="training error mu=0.1 nu=0.04") ax1.plot(train3.validation_error, label="validation error mu=0.1 nu=0.1") ax1.plot(train3.training_error, label="training error mu=0.1 nu=0.1") ax1.set_ylabel("error") ax1.set_xlabel("epoch") title = "Validation and training errors k=5 H1=80 H2=60 batchsize=1" error_fig.suptitle(title) plt.legend()
def yelp_geo_search_and_elastic_search_put(self, latitude, longitude): param = self.get_search_parameters(latitude, longitude) yelp_businesses = self.request_yelp_search(param)['businesses'] for business in yelp_businesses: if not business.has_key('snippet_image_url'): business['snippet_image_url'] = 'http://rearviewcamera.net/crv/images/noimage.gif' if not business.has_key('url'): business['url'] = 'URL NOT FOUND' data = Data( instance_id=1, name=business['name'], geo=[ float(business['location']['coordinate']['latitude']), float(business['location']['coordinate']['longitude']) ], reference_url=business['url'], reference_picture=business['snippet_image_url'], probability=float(business['review_count']) / 10000.0, reason='float(n_comments) / 10000.0', genre='Yelp Restaurant' ) data.put()
def test_get_dtype(self): """ Test the _get_dtype function. """ d = Data() self.assertRaises(TypeError, d._get_dtype, np.int) self.assertRaises(TypeError, d._get_dtype, 'abc') for dtype in self.valid_dtypes: d._get_dtype(dtype)
def convert(self): conv_cls1, conv_cls2 = self.__converter_cls_list # parser parser = conv_cls1() file_dict = dict([(d.format, d.file) for d in self.__input_data_list]) parser.set_file_dict( file_dict ) # formatter formatter = conv_cls2() formatter.set_ndo( parser.get_ndo() ) file_dict = formatter.get_file_dict() odata_list = [] for out_fmt in self.__output_format_list: odata = Data( inpd.project, type=inpd.type) odata.format = out_fmt file = file_dict[out_fn] if file: odata.file = file else: raise FormatterError() odata_list.append( odata ) self.__output_data_list = odata_list
def main(): s = '{"coord":{"lon":138.93,"lat":34.97},"weather":[{"id":802,"main":"Clouds","description":"scattered clouds","icon":"03n"}],"base":"cmc stations","main":{"temp":288.655,"pressure":1028.6,"humidity":100,"temp_min":288.655,"temp_max":288.655,"sea_level":1038.23,"grnd_level":1028.6},"wind":{"speed":3.12,"deg":222.5},"clouds":{"all":44},"dt":1461155698,"sys":{"message":0.0038,"country":"JP","sunrise":1461096333,"sunset":1461144068},"id":1851632,"name":"Shuzenji","cod":200}' d = Data(s) d.parse_json() print d.get_needed_data()
def main(): # Data generation settting data_size = 100 class_size = 3 gauss_mean = [ (2.0, 2.0), (-2.0, 1.5), (-0.1, -2.5)] # DPMM setting alpha = 0.5 niter = 120 # Generate data Data.gen_data(data_size, class_size, gauss_mean) # Random partition data CRP.alpha = alpha CRP.init_partition() # Iterations for _it in range(niter): print "[ iteration %d ] ================" % (_it) CRP.gibbs_sampling() Evalue.dovalue() print "[ evalueation %d ] ==============" % (_it) print " cluster number : %d " % (len(Data.get_all_class_id())) print " diff_class : %f" % (Evalue.diff_c[_it]) print " M_dis1 : %f" % (Evalue.M_dis1[_it]) print " M_dis2 : %f" % (Evalue.M_dis2[_it]) Evalue.show_evaluation()
def main(config): net = NN(config) model = config.get("Input", "loadmodel") if model: net.loadmodel(model) else: dataTrain = Data(config.get("Input","train"), config.get("Input","format"), net.verbosity) try: trpr = net.train( dataTrain ) except KeyboardInterrupt: sys.stderr.write("Aborting the training procedure...\n") pass ctest = config.get("Input","test") if ctest: dataTest = Data(ctest, config.get("Input","format"), net.verbosity) conf, err, tepr = net.test( dataTest ) output = net.metrics.obtain( dataTest, tepr, conf, err ) print print "Test statistics:" print conf print "\n".join(map(string.strip,filter(len,output.values()))) ftr = config.get("Output", "probstrain") fte = config.get("Output", "probstest") if ftr: Data.writeProbs(trpr, ftr) if fte: Data.writeProbs(tepr, fte) model = config.get("Output", "savemodel") if model: net.savemodel(model)
def getData(self, url): data = Data() try: req = urllib2.Request(url, headers={'User-Agent' : "Magic Browser"}) request = urllib2.urlopen(req) mime = request.info().getheader('Content-Type') code = request.code print(colored('[' + mime + '] ' + url, 'yellow')) if code is 200: if 'text/html' in mime: html = request.read() data = self.parse(html, url) else: #ANALYSIS TYPE data.url = url data.type = mime elif code is 400: data.broke = True except UnicodeEncodeError as e : print(colored(e, 'red')) data.broke = True return data
def main(): data = Data() training_data = data.load_text('data\\100\\train\\') test_data = data.load_text('data\\100\\test\\') classifier = NaiveBayes(training_data, get_all_words) evaluate = EvaluateClassifier() evaluate.print_evaluation(classifier, test_data)
def test_get_user(self): '''Testing mfc1.Data.get_user(key).''' data = Data() r = data.get_user('interval') self.failUnlessEqual(first=r, second=30) r = data.get_user('text') self.failUnlessEqual(first=r, second='Text ..')
def request_cluster(argv): """ only request cluster on GCE, and output all configuration information :param argv: sys.argv :return: None """ if len(argv) < 7: print_help() exit(1) cluster_name = argv[2] ambari_agent_vm_num = int(argv[3]) docker_num = int(argv[4]) service_server_num = int(argv[5]) with_ambari_server = False ambari_server_num = int(argv[6]) if ambari_server_num > 0: with_ambari_server = True cluster = Cluster() cluster.request_gce_cluster(ambari_agent_vm_num, docker_num, service_server_num, with_ambari_server, cluster_name) time_to_wait = Config.ATTRIBUTES["gce_boot_time"] print "wait ", str(time_to_wait), " seconds for the cluster to boot ... ..." time.sleep(int(time_to_wait)) data = Data() data.add_new_cluster(cluster) print "complete"
def list_cluster(): """ list the cluster creation history :return: None """ data = Data() data.print_cluster_summary_list()
def __init__(self, query=None, router=None): self._events = {} self._results = {} self._query = query if not query: self._shadow = True manager = Manager() self._edge = Edge(core=manager.core) self._vrtx = Vrtx(core=manager.core) self._attr = Attr(core=manager.core) self._data = Data(self._vrtx, self._edge, self._attr, core=manager.core) from lib.link import Uplink self._link = Uplink(manager) else: manager = None self._shadow = False self._edge = Edge(router=router) self._vrtx = Vrtx(router=router) self._attr = Attr(router=router, rdonly=False) self._data = Data(self._vrtx, self._edge, self._attr, router=router, rdonly=False) from lib.link import Downlink link = Downlink(query) self._query.link = link self._link = link self._manager = manager self._lock = NamedLock() if manager: manager.start()
def process(novel): edb = EntityDatabase() edb.load(ENTITIES_FILENAME) data = Data() dialogue_stats = [] def process_chapter(chapter): global dialogue_stats dialogue_stats = [] def process_chapter_done(chapter): global dialogue_stats for i, (speaker, words) in enumerate(dialogue_stats): if speaker == data.NARRATOR: continue before, after = get_surrounding_speakers(i, dialogue_stats) score = len(words) if before and after and before != after: # with different persons before and after this speech # assume half of it was meant for the last person, half # of it for the next score //= 2 if before: data.add_talked_to(speaker, before, score) if after: data.add_talked_to(speaker, after, score) def process_chunk(chunk): global dialogue_stats if chunk.is_direct(): if not chunk.speaker: # print("unknowns speaker: {}".format(chunk.data['data'])) pass speaker = chunk.speaker or Data.UNKNOWN else: speaker = Data.NARRATOR words = [] for word in WORD_SPLIT.split(chunk.get_data()): word = word.lower().strip() if not word: continue words.append(word) data.add_word(word, speaker) dialogue_stats.append((speaker, words)) novel.for_each(chapter=process_chapter, chapter_done=process_chapter_done, chunk=process_chunk) data.save(args.input + '.stats')
def convert(self): conv_obj = self.__converter_cls() input_dict = dict( [(d.format, d.file) for d in self.__input_data_list] ) output_tup = zip(self.__output_format_list, self.output_filename_list) conv_obj.set_input_dict( input_dict ) conv_obj.set_output_dict( dict(output_tup) ) # run converter command and get files fn_dict = conv_obj.run_convert() odata_list = [] for fmt, fn in output_tup: odata = Data( inpd.project, type=inpd.type) odata.format = fmt if os.path.exists(fn): odata.filename = fn else: raise ConverterCommandException() odata_list.append( odata ) self.__output_data_list = odata_list
def findNuMuLinearClass(self, batchsize, k): k=5 data = Data(k, 0, 0) data.importDataFromMat() data.normalize() error_fig = plt.figure() ax1 = error_fig.add_subplot(111) for nu in [0.05, 0.1] : for mu in [0.05, 0.1] : lg = LogisticLinearClassifier(nu, mu, 576, k, data) err_train, miss_train, err_val, miss_val = lg.train(30) label1 = "validation error mu="+str(mu)+" nu="+str(nu) label2 = "training error mu="+str(mu)+" nu="+str(nu) ax1.plot(err_train, label=label1) ax1.plot(err_val, label=label2) ax1.set_ylabel('error') ax1.set_xlabel('epoch') title = "Validation and training errors" error_fig.suptitle(title) plt.legend()
def main(): data = loadCSVToList('../data/141022/squid_pattern_HM1508-2.csv', ',') xlist, y1list, y2list, temp = zip(*data) uSquid = Data.fromLists(xlist, y1list) uRef = Data.fromLists(xlist, y2list) uRef.multiplyY(0.1) c = TCanvas('c', '', 1280, 720) gSquid = uSquid.makeGraph('gSquid', 'Zeit t / s', 'Spannung U / V') gSquid.SetMarkerStyle(1) gRef = uRef.makeGraph('gRef') gRef.SetMarkerStyle(1) gRef.SetMarkerColor(2) gSquid.GetXaxis().SetLimits(0, 0.2) gSquid.Draw('AP') gRef.Draw('P') l = TLegend(0.725, 0.85, 0.99, 0.99) l.SetTextSize(0.03) l.AddEntry(gSquid, 'Spannung am SQUID', 'p') l.AddEntry(gRef, 'Spannung am Funktions-', 'p') l.AddEntry(0, 'generator (mit Faktor 0.1)', '') l.Draw() c.Update() c.Print('../img/pattern.pdf', 'pdf')
class Device(object): cloud_id = 0 fps_to_render_time = { 120: 8.3, 90: 11.1, 60: 16.6 } def __init__(self, fps, ticksPerEvent, location): self.rendering_time = Device.fps_to_render_time[fps] self.ticksPerEvent = ticksPerEvent self.firstEventOffset = randint(0, ticksPerEvent - 1) self.location = location self.data = Data() self.time = 0 def step(self): self.time += 1 if self.time % self.ticksPerEvent == self.firstEventOffset: packet = Packet(self.time, Device.cloud_id, self.id) id = packet.packet_id self.data.putSendTime(id, self.time) return [packet] else: return [] def receivePacket(self, packet): id = packet.packet_id finalTime = packet.arriveTime() + self.rendering_time self.data.putFinalTime(id, finalTime)
def test_greedy(self): print 'testing greedy function' #my sample data my_data = Data() test_data = 'test.json' my_data.extract_data(test_data) uni_set = my_data.get_all_stores() set_sub = my_data.get_zips() #sorted_set = sorted(set_sub.values(), key=operator.attrgetter('num_nearby_stores'), reverse = True) #for zip in sorted_set: # zip.print_nearby_stores() min_zip_check = greedy(uni_set, set_sub) #test on given input file real_data = Data() data_file = 'gistfile1.json' real_data.extract_data(data_file) uni_set = real_data.get_all_stores() set_sub = real_data.get_zips() min_zip_check = greedy(uni_set, set_sub)
def test_log(): kwargs = {'Subtract offset':False,'New min':0} # log(-1) == NaN equal(Data.log(d_min_ones, **kwargs).values, np.nan*np.ones((100, 100))) # log(0) == -inf equal(Data.log(d_zeros, **kwargs).values, -np.inf*np.ones((100, 100))) # log(1) == log(1) equal(Data.log(d_ones, **kwargs).values, np.log(np.ones((100, 100))))
def genrandom(length=6): a = Data() unique = False while not unique: val = ''.join(random.sample(string.digits, length)) if not a.getkey(val): unique = True return val
def convert(self): inp_d_list = self.__input_data_list out_f_list = self.__output_filename_list for d, file in zip(inp_d_list, out_f_list): odata = Data( d.project, type=d.type) odata.format = d.format odata.file = file self.__output_data_list.append(odata)
def main(): """ This is the main function that ties all other components together: """ parser = argparse.ArgumentParser() parser.add_argument('--debug', action="store_true", help="Puts Health Checker into debug logging mode.") args = parser.parse_args() logging.basicConfig( format= '%(asctime)s %(name)-12s %(levelname)-8s %(filename)s %(funcName)s %(message)s', datefmt='%m-%d %H:%M:%S', level=logging.INFO, filename="amp_health_checker_log.log") logging.warning("AMP Health Checker logging level is %s", \ logging.getLevelName(logging.getLogger().level)) logging.debug("%s: Starting Health Checker", time.ctime()) x_count = 0 button_size = (20, 1) layout = [ [sg.Text("AMP Version: ", tooltip="The current AMP version running on the system."), \ sg.Text("Loading...", key='_version')], [sg.Text("CPU Usage: ", tooltip="The current amount of CPU utilized by AMP executables."), \ sg.Text("0", key='_cpu', size=(5, 1))], [sg.Text("AMP Uptime: ", size=(10, 1)), sg.Text("", size=(27, 1), key="_uptime", \ tooltip="Time since AMP was last stopped")], [sg.Text("Isolation: ", tooltip="Shows if the connector is Isolated or Not Isolated. \ Refresh with Refresh button." ), sg.Text("", size=(12, 1), key="_isolated"), sg.Text("", tooltip="If Isolated, shows the unlock code. Requires valid API Credentials \ ." , size=(17, 1), key="_unlock_code")], [sg.Text('_'*50)], [sg.Text("TETRA Version: ", size=(11, 1)), sg.Text("", size=(8, 1), key="_tetra_version", \ tooltip="Shows the local TETRA version.\nGreen if up to date.\nYellow if not within \ last 5 or connectivity error to API.\nRed if TETRA is not enabled." ), sg.Button('Check TETRA Version', size=button_size, button_color=('black', '#F0F0F0'), \ key='_tetra_version_button', tooltip="Checks the API to see if TETRA is up to date. \ Requires Valid API Credentials." ), sg.Text("", key="_latest_tetra_version", \ size=(8, 1))], [sg.Text("Policy Serial: ", size=(11, 1)), sg.Text("", size=(8, 1), \ key="_policy_version", tooltip="Shows the current policy serial number.\nGreen \ if this matches the cloud version.\nYellow if there is a connectivity issue or \ invalid API Credentials.\nRed if the local policy doesn't match the cloud \ version. Try syncing policy." ), sg.Button("Check Policy Version", size=button_size, button_color=('black', '#F0F0F0'), \ key='_policy_version_button', tooltip="Checks the API to see if the policy is up \ to date." ), sg.Text("", key="_latest_policy_version", size=(8, 1))], [sg.Text("API Credentials: ", size=(13, 1), tooltip='Shows if the currently stored API \ Credentials are valid. Can read from text file named "apiCreds.txt" in the local \ directory.\nMust be in this format:\nclient_id="abcdabcdabcdabcdabcd"\napi_key= \ "abcd1234-abcd-1234-abcd-abcd1234abcd"' ), sg.Text("", size=(6, 1), \ key="_api_cred_valid"), sg.Button("Get API Credentials", button_color=('black', '#F0F0F0'), size=button_size, \ tooltip="Allows user to manually input API Credentials.")], [sg.Text('_'*50)], [sg.Button("Live Debugging", button_color=('black', '#F0F0F0'), size=button_size, \ tooltip="Live analysis used for determining potential exclusions."), \ sg.Button("Run Analysis", button_color=('black', '#F0F0F0'), size=button_size, \ tooltip="Runs analysis on the sfc.exe.log file to provide information on potential \ exclusions." )], [sg.Button("Live Top Processes", button_color=('black', '#F0F0F0'), size=button_size, \ tooltip="Shows the top processes seen on the system in a live view."), \ sg.Button("Top IPs", button_color=('black', '#F0F0F0'), size=button_size, \ tooltip="Shows the top IP addresses seen on the system in a live view.")], [sg.Button("Connectivity Test", button_color=('black', '#F0F0F0'), size=button_size, \ key="_connectivity_test", tooltip="Test connection to the required servers for \ AMP operations." ), sg.Button("Check Engines", button_color=('black', '#F0F0F0'), \ size=button_size, tooltip="Provides a quick view of which AMP engines \ are enabled on the system." )], [sg.Button("View Exclusions", button_color=('black', '#F0F0F0'), size=button_size, \ tooltip="Shows the file and process exclusions from the local policy."), \ sg.Button("Manual SFC Analysis", button_color=('black', '#F0F0F0'), \ size=button_size, tooltip="Allows importing external sfc.exe.log \ files for analysis." )], [sg.Button("Generate Diagnostic", button_color=('black', '#F0F0F0'), size=button_size, \ tooltip="Generate AMP diagnostic bundle with AMP Health Checker log. Both files \ will be on the desktop." )], [sg.Text('Log Level: ', tooltip="Select higher log level if requested by the \ tool developers." ), sg.Button('INFO', button_color=('white', 'green'), \ key='_INFO'), sg.Button('WARNING', button_color=('black', '#F0F0F0'), \ key="_WARNING"), sg.Button('DEBUG', button_color=('black', '#F0F0F0'), \ key="_DEBUG")], [sg.Text('Region: ', tooltip="Shows which region you have selected. Change this \ if using EU or APJC cloud for your deployment." ), sg.Button('NAM', \ button_color=('white', 'green'), key='_NAM'), sg.Button('EU', \ button_color=('black', '#F0F0F0'), key="_EU"), sg.Button('APJC', \ button_color=('black', '#F0F0F0'), key="_APJC"), sg.Text('', size=(8, 1)), sg.Button("Refresh", size=(7, 1), button_color=('black', \ '#F0F0F0'), tooltip="Refreshes calculated data, including Isolation Status."), \ sg.Button("Cancel", button_color=('black', '#F0F0F0'), \ tooltip="Exits the program.")] ] logging.debug('test') window = sg.Window("AMP Health Check", layout) is_first = True while True: if is_first: event, values = window.Read(timeout=0) logging.debug('Event - %s : Values - %s', event, values) d_instance = Data() is_first = False else: event, values = window.Read(timeout=5000) if x_count < 10: x_count += 1 else: if d_instance.api_cred_valid: d_instance.update_api_calls() x_count = 0 d_instance.update() logging.debug('Self Scan Count = %s', d_instance.internal_health_check) window.FindElement('_version').Update(d_instance.version) window.FindElement('_cpu').Update(d_instance.current_cpu) window.FindElement('_uptime').Update(d_instance.converted_uptime) window.FindElement('_tetra_version').Update( d_instance.tetra_version_display) window.FindElement('_policy_version').Update( d_instance.policy_dict['policy_sn']) window.FindElement('_api_cred_valid').Update('Valid' if d_instance.api_cred_valid \ else 'Invalid') window.FindElement('_isolated').Update(d_instance.isolated) window.FindElement('_unlock_code').Update(d_instance.unlock_code) if event in (None, "Cancel"): break elif event == "_NAM": d_instance.region = 'NAM' window.FindElement('_NAM').Update(button_color=('white', 'green')) window.FindElement('_EU').Update(button_color=('black', '#F0F0F0')) window.FindElement('_APJC').Update(button_color=('black', '#F0F0F0')) d_instance.verify_api_creds() window.Refresh() elif event == '_EU': d_instance.region = 'EU' window.FindElement('_NAM').Update(button_color=('black', '#F0F0F0')) window.FindElement('_EU').Update(button_color=('white', 'green')) window.FindElement('_APJC').Update(button_color=('black', '#F0F0F0')) d_instance.verify_api_creds() window.Refresh() elif event == '_APJC': d_instance.region = 'APJC' window.FindElement('_NAM').Update(button_color=('black', '#F0F0F0')) window.FindElement('_EU').Update(button_color=('black', '#F0F0F0')) window.FindElement('_APJC').Update(button_color=('white', 'green')) d_instance.verify_api_creds() window.Refresh() elif event == "_INFO": logging.getLogger().setLevel(logging.INFO) logging.info('Log level changed to %s', logging.getLevelName( \ logging.getLogger().level)) window.FindElement('_INFO').Update(button_color=('white', 'green')) window.FindElement('_WARNING').Update(button_color=('black', '#F0F0F0')) window.FindElement('_DEBUG').Update(button_color=('black', '#F0F0F0')) window.Refresh() elif event == '_WARNING': logging.getLogger().setLevel(logging.WARNING) logging.warning('Log level changed to %s', logging.getLevelName( \ logging.getLogger().level)) window.FindElement('_INFO').Update(button_color=('black', '#F0F0F0')) window.FindElement('_WARNING').Update(button_color=('white', 'green')) window.FindElement('_DEBUG').Update(button_color=('black', '#F0F0F0')) d_instance.verify_api_creds() window.Refresh() elif event == '_DEBUG': logging.getLogger().setLevel(logging.DEBUG) logging.debug('Log level changed to %s', logging.getLevelName( \ logging.getLogger().level)) window.FindElement('_INFO').Update(button_color=('black', '#F0F0F0')) window.FindElement('_WARNING').Update(button_color=('black', '#F0F0F0')) window.FindElement('_DEBUG').Update(button_color=('white', 'green')) d_instance.verify_api_creds() window.Refresh() elif event == "Live Debugging": popups.lpap(d_instance) elif event == "Live Top Processes": popups.just_process(d_instance) elif event == "_tetra_version_button": popups.check_latest_tetra(d_instance, window) elif event == "_policy_version_button": popups.check_latest_policy(d_instance, window) elif event == "_connectivity_test": popups.connectivity(d_instance) elif event == "Check Engines": popups.engines_enabled(d_instance) elif event == "View Exclusions": popups.view_exclusions(d_instance) elif event == "Run Analysis": popups.analysis(d_instance) elif event == "Top IPs": popups.topips(d_instance) elif event == "Refresh": d_instance.reset_data() window.Refresh() elif event == "Get API Credentials": popups.get_api_credentials(d_instance) elif event == "Manual SFC Analysis": popups.manual_sfc(d_instance) elif event == "Generate Diagnostic": d_instance.generate_diagnostic() if d_instance.diag_failed: popups.diag_failed_popup() if args.debug == True: logging.getLogger().setLevel(logging.DEBUG) logging.debug('Log level changed to %s', logging.getLevelName(logging.getLogger().level)) window.FindElement('_INFO').Update(button_color=('black', '#F0F0F0')) window.FindElement('_WARNING').Update(button_color=('black', '#F0F0F0')) window.FindElement('_DEBUG').Update(button_color=('white', 'green')) d_instance.verify_api_creds() window.Refresh() if d_instance.enabled_debug: d_instance.disable_debug() window.close()
# This model is the same with deep_model # except the input part replaced by placeholder for feed Tensor on prediction. # The origin model during training have tf.Example node in graph # which is not approriate for online prediction because can't feed Tensor. # config flags = tf.app.flags FLAGS = flags.FLAGS flags.DEFINE_string('dict', './libfm.dict', 'field feature dict') flags.DEFINE_string("model_dir", "./model", "model dirctory") flags.DEFINE_string('sparse_fields', '', 'sparse fields. example 0,1,2') flags.DEFINE_string('hidden_layer', '100,100,50', 'hidden size for eacy layer') flags.DEFINE_integer('embedding_size', 10, 'embedding size') # data iter data = Data(FLAGS.dict, FLAGS.sparse_fields) label, sparse_id, sparse_val = data.ReadBatchPlaceholder() # define model model = Model(FLAGS.embedding_size, data.Dict(), FLAGS.sparse_fields, FLAGS.hidden_layer) # define loss logits, all_parameter = model.forward(sparse_id, sparse_val) train_label = tf.to_int64(label) cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=logits, labels=train_label, name='cross_entropy') # save graph sess = tf.Session() tf.train.write_graph(sess.graph.as_graph_def(),
def __init__(self, tna, clock, menu, tbicon): # Data object. self.__data = Data() # Package directory self.__dir = self.determine_path() # Internationalisation self.set_in18() # Load saved datas. self.config_load() # Get frame title, frame size and icon. title = self.__data.get_sys('frame_title') size = self.__data.get_user('frame_size') icon = os.path.join(self.__dir, self.__data.get_sys('icon_name')) # Subclass wx.Frame.__init__(self, parent=None, id=wx.ID_ANY, title=title, size=size) # Icon if icon.endswith('.png'): self.SetIcon(wx.Icon(name=icon, type=wx.BITMAP_TYPE_PNG)) # GUI border style and distance between widgets. bstyl = self.__data.get_sys('gui_borderstyle') guiborders = { 'simple': wx.SIMPLE_BORDER, 'raised': wx.RAISED_BORDER, 'sunken': wx.SUNKEN_BORDER, 'no': wx.NO_BORDER } if bstyl in guiborders: self.__bstyl = guiborders[bstyl] else: self.__bstyl = wx.SIMPLE_BORDER self.__bdist = self.__data.get_sys('gui_borderdist') if not self.__bdist: self.__bdist = 5 # Set attributes for time interval and sound file. self.__interval = self.__data.get_user('interval') s = self.__data.get_user('sound') if s: self.__sound = os.path.join(self.__dir, s) else: self.__sound = '' # time interval. intervalbox = self.init_interval() # Text notification. textbox = self.init_text() # Sound notification. soundbox = self.init_sound() # Clock control. controlbox = self.init_buttons() # Timer self.__timer = wx.Timer(self, 1) self.Bind(event=wx.EVT_TIMER, handler=self.on_timer, source=self.__timer) # Clock status self.__clockstatus = False # Lock file userid = wx.GetUserId() wxpath = wx.StandardPaths.Get() userdir = wxpath.GetDocumentsDir() self.__lock = Lock(path=userdir, userid=userid) self.__lockstate = '' if self.__lock.one_instance('mindfulclock1'): # One instance. self.__lock.write_lock() self.__lockstate = 'written' else: # More than one instance. if self.start_question(): # Start the clock. self.__lock.write_lock() self.__lockstate = 'written' else: # Exit the program. self.__lockstate = 'exit' # Exit bindings. self.Bind(event=wx.EVT_CLOSE, handler=self.on_system_close) # type of taskbar notification. # 'appind' = application indicator # 'tbicon' = taskbaricon # '' = no taskbar started. self.__tbtype = '' # Exit when tbicon is False and __lockstate is 'exit' if not tbicon and self.__lockstate == 'exit': self.Close() # Layout vbox = wx.BoxSizer(wx.VERTICAL) vbox.Add(item=intervalbox, proportion=0, flag=wx.EXPAND | wx.ALL, border=self.__bdist) vbox.Add(item=textbox, proportion=1, flag=wx.EXPAND | wx.ALL, border=self.__bdist) vbox.Add(item=soundbox, proportion=0, flag=wx.EXPAND | wx.ALL, border=self.__bdist) vbox.Add(item=controlbox, proportion=0, flag=wx.EXPAND | wx.ALL, border=self.__bdist) self.SetSizer(vbox) # Disable stop & pause button self.__btnstop.Enable(False) self.__btnpause.Enable(False) # Clock not paused. self.__pausetime = -99 # System tray from command line options if tna: # Start in the taskbar. self.__tna = True else: if self.__miniopt.GetValue(): # Checkbox is set, start in the taskbar. self.__tna = True else: # Start normal. self.__tna = False if tbicon: # Use wx.TaskBarIcon self.init_tbicon() else: # Use Appindicator if menu: # Show the time not beside the indicator. self.__menutime = True else: # Show the time beside. self.__menutime = False wx.FutureCall(100, self.init_appind) # Centre window, show window. self.Center() # Exit when tbicon is True and __lockstate is 'exit' if tbicon and self.__lockstate == 'exit': self.Close() # Check autostart if clock: # Start clock automatically. wx.FutureCall(200, self.clock_start) else: if self.__autostart.GetValue(): # Checkbox is set, start clock automatically. wx.FutureCall(200, self.clock_start) if self.__tna: # Start in the system tray. self.Hide() else: self.Show()
def test_verifyTasks(self): data = Data() task = { "title": "", "description": "", "triggers": { "trigger": { "type": "or", "parent": "triggers", "operation": ["1594745648295", "1594746507144"] }, "1594745648295": { "type": "d2b", "parent": "1594745594679", "operation": "light001:on:=:true" }, "1594746507144": { "type": "d2b", "parent": "1594745594679", "operation": "light002:on:=:true" } }, "target": [{ "device": "light003", "param": "on", "value": True }] } data.createTask(task) device = { "attributes": { "commandOnlyOnOff": True, "queryOnlyOnOff": True, "commandOnlyBrightness": True }, "deviceInfo": { "hwVersion": "1.0", "swVersion": "1.0", "manufacturer": "Homeware", "model": "Homeware Lamp 2.0" }, "id": "light001", "name": { "defaultNames": ["Lamp"], "nicknames": ["Lamp"], "name": "Test Lamp" }, "traits": [ "action.devices.traits.OnOff", "action.devices.traits.Brightness" ], "type": "action.devices.types.LIGHT" } data.createDevice({"device": device, "status": {"on": True}}) device['id'] = "light002" data.createDevice({"device": device, "status": {"on": True}}) device['id'] = "light003" data.createDevice({"device": device, "status": {"on": False}}) self.assertFalse(data.getStatus()['light003']['on']) verifyTasks() self.assertTrue(data.getStatus()['light003']['on'])
def data_page(self, detail, origin): browser = self.star_chr() try: url = detail["detail_url"] browser.get(url) html = browser.page_source if 'robots' in html or '未连接到互联网' in html: time.sleep(60) raise Exception("robots") elif '404' in html: raise Exception("lost") wait = WebDriverWait(browser, 20) wait.until( EC.presence_of_element_located( (By.XPATH, '//ul[@class="media-stream"]/li/picture'))) source = etree.HTML(html) print("url--------" + url) price = detail["price"] if "price" in detail.keys( ) else self.get_price(source) bedroom = detail["bedrooms"] if "bedrooms" in detail.keys( ) else self.get_bedroom(source) bathroom = detail["bathrooms"] if "bathrooms" in detail.keys( ) else self.get_bathroom(source) street = detail["streetAddress"] if "streetAddress" in detail.keys( ) else self.get_street(source) deal_type = self.get_dealtype(source) img_url = self.get_imgurl(source) living_sqft = detail["livingArea"] if "livingArea" in detail.keys( ) else self.get_livingsqft(source) comments = self.get_desc(source).replace("\n", "") agent = self.get_agent(source) # house_type = self.get_info_by_keyword(source, 'Type:') house_type = self.get_info_by_keyword(source, 'Type:') _housetype = house_type if house_type else source.xpath( '//div[@class="home-facts-at-a-glance-section"]//div[contains(text(), "Type")]/following-sibling::div/text()' ) house_type = detail["homeType"] if "homeType" in detail.keys( ) else _housetype heating = self.get_heating(source) cooling = self.get_cooling(source) price_sqft = self.get_pricesqft(source) # Year built year_build = str( detail["yearBuilt"]) if "yearBuilt" in detail.keys( ) else self.get_info_by_keyword(source, 'Year built:') if not year_build or year_build == "-1": year_build = self.get_yearbuild(source) if not year_build: year_build = source.xpath( '//div[@class="home-facts-at-a-glance-section"]//div[contains(text(), "Year Built")]/following-sibling::div/text()' ) parking = self.get_parking(source) lot_sqft = detail["lotSize"] if "lotSize" in detail.keys( ) else self.get_lotsqft(source) hoa_fee = self.get_hoafee(source) mls = self.get_mls(source) apn = self.get_apn(source) garage = self.get_garage(source, 'Parking:') deposit = self.get_info_by_keyword(source, 'Deposit & fees:') contact_phone = self.get_contactphone(source) # time_on_zillow = detail["timeOnZillow"] if "timeOnZillow" in detail.keys() else 0 contact_name = self.get_contactname(source) data = Data(price, bedroom, bathroom, street, deal_type, img_url, living_sqft, comments, agent, house_type, heating, cooling, price_sqft, year_build, parking, lot_sqft, hoa_fee, contact_phone, contact_name, "", url, mls, apn, garage, deposit, detail["zipcode"], detail["latitude"], detail["longitude"], detail["city"], detail["state"]) self.re_queue.sadd("data", data.dict2str()) # to_mysql.to_sql(product) except Exception as e: info = e.args[0] if info == 'robots': self.detail_queue.sadd("content", origin) else: if self.re_queue.sismember("proxy_ip", self.ip): # 删除当前对应的无效IP self.re_queue.srem("proxy_ip", self.ip) self.get_ip() finally: browser.quit() return "OK"
def handle_accept(json: dict, address: tuple, *args, **kwargs): from UI import openDialog Data.add_following(address[0]) alert_message = f"user {address[0]} followed successfuly" openDialog(alert_message)
def handle_leave(json: dict, address: tuple, *args, **kwargs): from UI import openDialog Data.leave(address[0]) openDialog(f'{address[0]} left the network!')
def load_history_func_eval(self, data: Data, problem: Problem, Igiven: np.ndarray): """ Init history database JSON file """ if (self.tuning_problem_name is not None): json_data_path = self.history_db_path + "/" + self.tuning_problem_name + ".json" if os.path.exists(json_data_path): print("[HistoryDB] Found a history database file") if self.file_synchronization_method == 'filelock': with FileLock(json_data_path + ".lock"): with open(json_data_path, "r") as f_in: history_data = json.load(f_in) elif self.file_synchronization_method == 'rsync': temp_path = json_data_path + "." + self.process_uid + ".temp" os.system("rsync -a " + json_data_path + " " + temp_path) with open(temp_path, "r") as f_in: history_data = json.load(f_in) os.system("rm " + temp_path) else: with open(json_data_path, "r") as f_in: history_data = json.load(f_in) num_tasks = len(Igiven) num_loaded_data = 0 PS_history = [[] for i in range(num_tasks)] OS_history = [[] for i in range(num_tasks)] for func_eval in history_data["func_eval"]: if (self.check_load_deps(func_eval)): task_id = self.search_func_eval_task_id( func_eval, problem, Igiven) if (task_id != -1): # # current policy: skip loading the func eval result # # if the same parameter data has been loaded once (duplicated) # # YL: only need to search in PS_history[task_id], not PS_history # if self.is_parameter_duplication(problem, PS_history[task_id], func_eval["tuning_parameter"]): # current policy: allow duplicated samples # YL: This makes RCI-based multi-armed bandit much easier to implement, maybe we can add an option for changing this behavior if False: # self.is_parameter_duplication(problem, PS_history[task_id], func_eval["tuning_parameter"]): continue else: parameter_arr = [] for k in range(len(problem.PS)): if type(problem.PS[k] ).__name__ == "Categoricalnorm": parameter_arr.append( str(func_eval["tuning_parameter"][ problem.PS[k].name])) elif type(problem.PS[k] ).__name__ == "Integer": parameter_arr.append( int(func_eval["tuning_parameter"][ problem.PS[k].name])) elif type( problem.PS[k]).__name__ == "Real": parameter_arr.append( float(func_eval["tuning_parameter"] [problem.PS[k].name])) else: parameter_arr.append( func_eval["tuning_parameter"][ problem.PS[k].name]) PS_history[task_id].append(parameter_arr) OS_history[task_id].append(\ [func_eval["evaluation_result"][problem.OS[k].name] \ for k in range(len(problem.OS))]) num_loaded_data += 1 if (num_loaded_data > 0): data.I = Igiven #IS_history data.P = PS_history data.O = [] # YL: OS is a list of 2D numpy arrays for i in range(len(OS_history)): if (len(OS_history[i]) == 0): data.O.append(np.empty(shape=(0, problem.DO))) else: data.O.append(np.array(OS_history[i])) if (any(ele == [None] for ele in OS_history[i])): print( "history data contains null function values" ) exit() # print ("data.I: " + str(data.I)) # print ("data.P: " + str(data.P)) # print ("data.O: " + str(OS_history)) else: print("no history data has been loaded") else: print("[HistoryDB] Create a JSON file at " + json_data_path) if self.file_synchronization_method == 'filelock': with FileLock(json_data_path + ".lock"): with open(json_data_path, "w") as f_out: json_data = { "tuning_problem_name": self.tuning_problem_name, "model_data": [], "func_eval": [] } json.dump(json_data, f_out, indent=2) elif self.file_synchronization_method == 'rsync': temp_path = json_data_path + "." + self.process_uid + ".temp" with open(temp_path, "w") as f_out: json_data = { "tuning_problem_name": self.tuning_problem_name, "model_data": [], "func_eval": [] } json.dump(json_data, f_out, indent=2) os.system("rsync -u " + temp_path + " " + json_data_path) os.system("rm " + temp_path) else: with open(json_data_path, "w") as f_out: json_data = { "tuning_problem_name": self.tuning_problem_name, "model_data": [], "func_eval": [] } json.dump(json_data, f_out, indent=2)
def __init__(self): super().__init__() self.rarities = Data.getRarities() self.nations = Data.getNations() self.ship_types = Data.getShipTypes() self.setupUi()
class Controller: def __init__(self): self.d = None try: self.d = pickle.load(open("inventory_pickled.p", "rb")) except pickle.UnpicklingError: self.d = Data() except FileNotFoundError: self.d = Data() ### STUFF TO SEND TO GUI ### """returns left sidebar information on inventory status in format : (caloric_warnings, exp_warning)""" def inventory_status(self): cal_warning = self.d.get_caloric_warnings() exp_warning = self.d.get_expiration_warnings() return (cal_warning, exp_warning) """The profile's name and nutrition needs format : (name, caloric need, protein need, fat need""" def personal_needs(self): name = self.d.profile.name return name, self.d.get_protein_need(),\ self.d.get_caloric_need(), self.d.get_fat_need() """information about nutrients consumed today format : (calories, proteins, fats)""" def nutrient_info(self): return self.d.get_intake() ### Warnings ### """warnings in no particular order, (but there could be alot)""" def get_all_warnings(self): warnings_list = [] warnings_list.append(self.d.get_expiration_warnings()) warnings_list.append(self.d.get_caloric_warnings()) warnings_list.append(self.d.cals_days_remaining()) warnings_list.append(self.d.fats_days_remaining()) warnings_list.append(self.d.protein_days_remaining()) warnings_list.append("{} item(s) are expired!".format( str(len(self.d.inventory.get_expired_food())))) warnings_list.extend(self.d.inventory.get_expiration_warnings_1day()) return warnings_list ####### RECEIVING FROM GUI ###### ### Food added/eaten/trashed ### def add_food(self, name, weight, expiration_date): exp_date = datetime.strptime(expiration_date, '%Y-%m-%d') self.d.add_food(name, weight, exp_date.date()) pickle.dump(self.d, open("inventory_pickled.p", "wb")) def eat_food(self, name, weight): self.d.eat_food(name, weight) pickle.dump(self.d, open("inventory_pickled.p", "wb")) def trash_food(self, name, weight): self.d.throw_out_food(name, weight) pickle.dump(self.d, open("inventory_pickled.p", "wb")) ### Personal info to add to profile ### def enter_personal_info(self, info): self.d.set_profile(info[0], info[1], info[2], info[3], info[4], info[5]) self.d.update_consumptions() pickle.dump(self.d, open("inventory_pickled.p", "wb")) ### Summation information about all food items ### def get_summation_strings(self): return self.d.summation()
day = int(raw_input()) print 'Select hour:' hour = int(raw_input()) 'Gets the type of graph the user wants' print 'top 10 accesses(t), standard(s), all(a), top 10 files(f),' print 'bad distribution small(bs), bad distribution large(bl)' stype = raw_input() ctime = datetime(year, month, day, hour, 0, 0) done = True except: print 'please enter a valid date' data = Data() data.getDataFromDatabase(ctime) if stype == 's': data.sortByStandard() if stype == 't': data.sortByTop10Accesses() if stype == 'f': data.sortByTop10Files() if stype == 'bs': data.sortByBad(True) if stype == 'bl': data.sortByBad(False) data.showGraph()
args = parser.parse_args() # set arguments l_r = args.l_r batch_size = args.batch_size pickle_dir = args.pickle_dir max_seq = args.max_seq epochs = args.epochs is_reuse = args.is_reuse load_path = args.load_path save_path = args.save_path multi_gpu = args.multi_gpu # load data dataset = Data('dataset/processed') print(dataset) # load model learning_rate = criterion.CustomSchedule(par.embedding_dim) opt = Adam(l_r, beta_1=0.9, beta_2=0.98, epsilon=1e-9) strategy = tf.distribute.MirroredStrategy() # define model with strategy.scope(): mt = MusicTransformer(embedding_dim=256, vocab_size=par.vocab_size, num_layer=6, max_seq=max_seq, dropout=0.2,
import time from datetime import datetime import json import requests from base64 import b64encode from data import Data #Init the data managment object data_conector = Data() def verifyTasks(): tasks = data_conector.getTasks() status = data_conector.getStatus() for taskData in tasks: triggers = taskData['triggers'] try: execution_value = operationExecutor('trigger', triggers, status) if execution_value: for target in taskData['target']: value = target['value'] if target['value'] == 'true': value = True elif target['value'] == 'false': value = False elif target['param'] == 'color': value = {"spectrumRGB": value, "spectrumRgb": value} data_conector.updateParamStatus(target['device'], target['param'], value) except Exception as e: data_conector.log('Alert', 'Catch an error in execution of ' + taskData['title'] + 'task' + str(e)) def ddnsUpdater():
def main(params): # load data N_SENT = params["number_sentences"] data = Data(params["data_file"]) data.fetch(n_sentences=N_SENT) train_size = len(data.X_train) test_size = len(data.X_test) print("train_size:", train_size, " test_size:", test_size, " total:", train_size+test_size) if N_SENT is None: N_SENT = 47959 assert N_SENT == train_size + test_size # generate sequences sequence_list = SequenceList(LabelDictionary(data.word_to_pos), LabelDictionary(data.tag_to_pos)) for i in tqdm(range(train_size)): x,y = data.X_train[i], data.y_train[i] sequence_list.add_sequence(x,y, LabelDictionary(data.word_to_pos), LabelDictionary(data.tag_to_pos)) # generate features ex_feature_mapper = skseq.sequences.extended_feature.ExtendedFeatures(sequence_list) ex_feature_mapper.build_features() feature_mapper = ex_feature_mapper #print("Number of features:", len(feature_mapper.feature_dict), len(feature_mapper.feature_list)) features = set([x.split(":")[0] for x in feature_mapper.feature_dict.keys()]) print("Features:", features) # build SP model corpus = skseq.readers.pos_corpus.PostagCorpus() sp = spc.StructuredPerceptron(data.word_to_pos, data.tag_to_pos, feature_mapper) # train SP model num_epochs = params["number_epochs"] sp.fit(feature_mapper.dataset, num_epochs, tolerance=0.0005) print() # model convergence print('Model convergence...') plot_acc(sp.acc_per_epoch, filename='Acc'+params['output_sufix']) print() # METRICS list_classes = np.array(list(data.pos_to_tag.values())) print('Metrics for TRAIN SET:') y_train_true_tags = tags_to_ids(data, data.y_train) y_train_true_tags_flatten = flatten(y_train_true_tags) y_train_pred_tags = [list(sp.predict_tags_given_sentence(sentence)[0].y) for sentence in data.X_train] # predict on train set y_train_pred_tags_flatten = flatten(y_train_pred_tags) wrong_sentences_ids_train = metrics_to_report( y_train_true_tags_flatten, y_train_pred_tags_flatten, y_train_true_tags, y_train_pred_tags, list_classes, params['output_sufix'] ) print() print('Metrics for TEST SET:') y_test_true_tags = tags_to_ids(data, data.y_test) y_test_true_tags_flatten = flatten(y_test_true_tags) y_test_pred_tags = [list(sp.predict_tags_given_sentence(sentence)[0].y) for sentence in data.X_test] # predict on test set y_test_pred_tags_flatten = flatten(y_test_pred_tags) wrong_sentences_ids_test = metrics_to_report( y_test_true_tags_flatten, y_test_pred_tags_flatten, y_test_true_tags, y_test_pred_tags, list_classes, params['output_sufix'] ) print() if params["max_wrong_samples"] is not None: print("Some wrong predictions in the test set:\n") for id in wrong_sentences_ids_test[:params["max_wrong_samples"]]: sentence = data.X_test[id] true = data.y_test[id] pred = [data.pos_to_tag[tag] for tag in y_test_pred_tags[id]] compare_true_predict(sentence, true, pred) print() return
def train_convlstm_model(kernel_size, unit_list, kernel_sizes, output_channels): tf.reset_default_graph() sess = tf.Session(config=config) re = lambda x: x.replace(" ", "").replace("[", "_").replace( "]", "_").replace(",", "_") NAME = "word_convlstm_" + re(str(unit_list)) + "_conv_channel_" + re( str(output_channels)) + "_kernels_" + re(str(kernel_sizes)) store = Data(FLAGS.data_dir + "word/", FLAGS.truncate) x = tf.placeholder(tf.float32, shape=(None, FLAGS.truncate, store.vsize), name="input_x") y = tf.placeholder(tf.int32, shape=(None, FLAGS.truncate, 1), name="input_y") labels = y rnn = model.stacked_fully_conv_bi_lstm(x, kernel_size, unit_list, store.vsize) logits = model.convolutional_output(rnn, output_channels, kernel_sizes) predictions = tf.nn.sigmoid(logits, name='output_probs') valid_chars_in_batch = tf.reduce_sum(x) all_chars_in_batch = tf.size(x) / store.vsize valid_ratio = valid_chars_in_batch / tf.cast(all_chars_in_batch, tf.float32) loss = tools.loss(logits, labels) path = FLAGS.log_dir + NAME writer = tf.summary.FileWriter(path, graph=tf.get_default_graph()) saver = tf.train.Saver(max_to_keep=20) tf.summary.scalar("batch_loss", loss) precision, recall, accuracy, f1, predicted = tools.metrics( predictions, labels, x) tf.summary.scalar("accuracy", accuracy) tf.summary.scalar("precision", precision) tf.summary.scalar("recall", recall) tf.summary.scalar("f1", f1) summary_op = tf.summary.merge_all() if FLAGS.optimizer == "ADAM": opt = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate) elif FLAGS.optimizer == "RMS": opt = tf.train.RMSPropOptimizer(learning_rate=FLAGS.learning_rate) elif FLAGS.optimizer == "SGD": opt = tf.train.GradientDescentOptimizer( learning_rate=FLAGS.learning_rate) else: raise NotImplemented train_op = opt.minimize(loss) tf.global_variables_initializer() validation_loss_placeholder = tf.placeholder(tf.float32, name="validation") validation_loss_summary = tf.summary.scalar('validation_loss', validation_loss_placeholder) validation_accuracy_placeholder = tf.placeholder( tf.float32, name="validation_accuracy") validation_accuracy_summary = tf.summary.scalar( 'validation_accuracy', validation_accuracy_placeholder) validation_f1_placeholder = tf.placeholder(tf.float32, name="validation_f1") validation_f1_summary = tf.summary.scalar('validation_f1', validation_f1_placeholder) test_loss_placeholder = tf.placeholder(tf.float32, name="test") test_loss_summary = tf.summary.scalar('test_loss', test_loss_placeholder) test_f1_placeholder = tf.placeholder(tf.float32, name="test_f1") test_f1_summary = tf.summary.scalar('test_f1', test_f1_placeholder) test_accuracy_placeholder = tf.placeholder(tf.float32, name="test_accuracy") test_accuracy_summary = tf.summary.scalar('test_accuracy', test_accuracy_placeholder) def get_metrics_on_dataset(dataset, train_step): losses = [] accs = [] recalls = [] f1s = [] good_words = 0 step = int(store.size[dataset] / FLAGS.batch_size) for i in tqdm.trange(step): x_, y_ = store.get_padded_batch(dataset) feed = {x: x_, y: y_} batch_loss, rec, f, preds = sess.run([loss, recall, f1, predicted], feed_dict=feed) losses.append(batch_loss) accs.append( tools.char_accuracy_on_padded(x_, y_, preds, store.vsize)) recalls.append(rec) f1s.append(f) good_words += sum([ np.all(y_[i, :] == preds[i, :]) for i in range(FLAGS.batch_size) ]) l, a, r, f = np.average(losses), np.average(accs), np.average( recalls), np.average(f1s) word_acc = good_words / store.size[dataset] * 100 if dataset == "validation": feed = { validation_loss_placeholder: l, validation_accuracy_placeholder: float(a), validation_f1_placeholder: f } vl, va, vf = sess.run([ validation_loss_summary, validation_accuracy_summary, validation_f1_summary ], feed_dict=feed) writer.add_summary(vl, train_step) writer.add_summary(va, train_step) writer.add_summary(vf, train_step) elif dataset == "test": feed = { test_loss_placeholder: l, test_accuracy_placeholder: float(a), test_f1_placeholder: f } vl, va, vf = sess.run( [test_loss_summary, test_accuracy_summary, test_f1_summary], feed_dict=feed) writer.add_summary(vl, train_step) writer.add_summary(va, train_step) writer.add_summary(vf, train_step) writer.flush() return l, a, r, f, word_acc early = tools.Stopper(30) steps = FLAGS.epochs * int(store.size["train"] / FLAGS.batch_size) # run training sess.run(tf.global_variables_initializer()) for i in tqdm.trange(steps, unit="batches"): b_data, b_label = store.get_padded_batch("train") _, batch_loss, summary = sess.run([train_op, loss, summary_op], { x: b_data.astype(float), y: b_label }) assert not np.isnan(batch_loss) if i % 20 == 0: writer.add_summary(summary, i) if i % int(steps / FLAGS.epochs / 2) == 0: l, a, r, f, w = get_metrics_on_dataset("validation", i) print("loss: ", l, " accuracy: ", a, "% recall: ", r, "fscore", f, " word_acc: ", w, "%") if early.add(l): break if i % int(steps / FLAGS.epochs / 2) == 0: save_path = saver.save(sess, path + "/model.ckpt", global_step=i) print("Testing...") l, a, r, f, w = get_metrics_on_dataset("test", steps) print("loss: ", l, " accuracy: ", a, "% recall: ", r, "fscore", f, " word_acc: ", w, "%") # log test losses: with open(FLAGS.log_dir + "hyper.log", "a") as myfile: myfile.write("\n" + FLAGS.log_dir + NAME + "\t" + str(l) + "\t" + str(a) + "\t" + str(r) + "\t" + str(f) + "\t" + str(w)) writer.flush() sess.close() return l, a, r, f, w
def get_data(): """ Gets a data packet from peripheral """ _print("Getting Data.", DEBUG) return Data()
def handle_follow(json: dict, address: tuple, *args, **kwargs): from UI import openDialog Data.add_follower(address[0]) openDialog(f'{address[0]} followed you!') send_accept(address)
def mpi_run(custom_command=""): """ Launch a simple MPI run, with no communication of covariance matrix Each process will make sure to initialise the folder if needed. Then and only then, it will send the signal to its next in line to proceed. This allows for initialisation over an arbitrary cluster geometry (you can have a single node with many cores, and all the chains living there, or many nodes with few cores). The speed loss due to the time spend checking if the folder is created should be negligible when running decently sized chains. Each process will send the number that it found to be the first available to its friends, so that the gathering of information post-run is made easier. If a chain number is specified, this will be used as the first number, and then incremented afterwards with the rank of the process. """ from mpi4py import MPI comm = MPI.COMM_WORLD nprocs = comm.Get_size() rank = comm.Get_rank() success = True folder = '' # If the process is not the zeroth one, then wait for a signal from your # n-1 before initializing the folder if rank != 0: status = comm.recv(source=rank - 1, tag=1) folder = comm.recv(source=rank - 1, tag=2) if status == 'failed': success = False else: number = status if success: if not custom_command: custom_command = " ".join(sys.argv[1:]) if rank != 0: custom_command += " --chain-number %s" % str(int(number) + 1) # First check if the folder is there already_sent = False if rank != 0 and rank < nprocs - 1: status = int(number) + 1 if Data.folder_is_initialised(folder): comm.send(status, dest=rank + 1, tag=1) comm.send(folder, dest=rank + 1, tag=2) already_sent = True # Then, properly initialise cosmo1, cosmo2, data, command_line, success = safe_initialisation( custom_command, comm, nprocs) # The first initialisation should check a few more things if rank == 0: # Check that the run asked is compatible with mpirun and prepare. if command_line.subparser_name == 'info': warnings.warn("Analyzing the chains is not supported in mpirun" " so this will run on one core only.") status = 'failed' elif command_line.method == "MH": regexp = re.match(".*__(\w*).txt", data.out_name) suffix = regexp.groups()[0] status = suffix elif command_line.method == "NS": status = 1 elif command_line.method == "PC": status = 1 else: warnings.warn("The method '%s' is not supported" % (command_line.method) + " in mpirun so this will run on one core only.") status = 'failed' folder = data.out_name elif rank < nprocs - 1: status = int(number) + 1 # Send an "OK" signal to the next in line, giving the its own chain # number for the other to add 1 if rank < nprocs - 1: if not already_sent: comm.send(status, dest=rank + 1, tag=1) comm.send(folder, dest=rank + 1, tag=2) else: if rank < nprocs - 1: comm.send('failed', dest=rank + 1, tag=1) comm.send('', dest=rank + 1, tag=2) if success: import sampler sampler.run(cosmo1, cosmo2, data, command_line) return
from sklearn.preprocessing import FunctionTransformer def func(df, book_maker, type_of_bet): cols = ["{}_{}".format(book_maker, type_of_bet)] df_ = df[cols] return df_.values.tolist() betting_encoder = FunctionTransformer(func=func, kw_args={ "book_maker": "tw", "type_of_bet": "diff" }) if __name__ == "__main__": from data import Data data = Data(alliance="中華職棒") df1 = data.get_train(book_maker="tw", type_of_bet="diff") # print(df1[["game_time", "away_team", "home_team", "away_score", "home_score", # "is_back_to_back", ]]) print(betting_encoder.fit_transform(df1)) # print(game_count_encoder.steps[1][1].categories_)
class Train(): def __init__(self, difficulty): self.data_path = "../data" self.model_path = "../models" self.output_path = "../outputs" self.difficulty = difficulty self.timestamp = str(int(time.time())) self.model_name = "naive_regression_" + self.difficulty self.data = Data(difficulty=self.difficulty, data_path=self.data_path) (self.img_features, self.w2i, self.i2w, self.nwords, self.UNK, self.PAD) = self.data() self.train = list(self.data.get_train_data()) self.dev = list(self.data.get_validation_data()) self.test = list(self.data.get_test_data()) self.image_feature_size = 2048 self.output_vector_size = 10 def __call__(self, number_of_iterations=2, learning_rate=0.005, embedding_size=300): print("Starting 'Image Retrieval' in 'Naive Regression' mode with '" + self.difficulty + "' data") self.model_full_path = self.model_path + "/" + self.model_name + "_" + self.timestamp + "_" + str( learning_rate) + "_" + str(embedding_size) + ".pty" self.output_file_name = self.output_path + "/" + self.model_name + "_" + self.timestamp + "_" + str( learning_rate) + "_" + str(embedding_size) + ".csv" self.number_of_iterations = number_of_iterations self.learning_rate = learning_rate self.embedding_size = embedding_size self.model = Regression(self.nwords, self.embedding_size, self.image_feature_size, self.output_vector_size) self.criterion = nn.MSELoss() self.evaluate = Evaluate(self.model, self.img_features, self.minibatch, self.preprocess, self.image_feature_size) print(self.model) self.optimizer = optim.Adam(self.model.parameters(), lr=self.learning_rate) self.train_loss_values = [] self.dev_loss_values = [] self.test_loss_values = [] self.magic() self.save_model() self.save_data() def minibatch(self, data, batch_size=50): for i in range(0, len(data), batch_size): yield data[i:i + batch_size] def preprocess(self, batch): """Helper function for functional batches""" correct_indexes = [observation[2] for observation in batch] img_ids = [observation[1] for observation in batch] text_features = [observation[0] for observation in batch] #Add Padding to max len of sentence in batch max_length = max(map(len, text_features)) text_features = [ txt + [self.PAD] * (max_length - len(txt)) for txt in text_features ] #return in "stacked" format return text_features, img_ids, correct_indexes def magic(self): for ITER in range(self.number_of_iterations): random.shuffle(self.train) train_loss = 0.0 start = time.time() for iteration, batch in enumerate(self.minibatch(self.train)): #Outputs matrices of batch size text_features, h5_ids, correct_index = self.preprocess(batch) lookup_text_tensor = Variable(torch.LongTensor([text_features ])).squeeze() target = np.empty([len(batch), self.image_feature_size]) for obs, img_ids in enumerate(h5_ids): target[obs] = self.img_features[img_ids[ correct_index[obs]]] target = Variable( torch.from_numpy(target).type(torch.FloatTensor)) #Run model and calculate loss prediction = self.model(lookup_text_tensor) loss = self.criterion(prediction, target) train_loss += loss.data[0] self.optimizer.zero_grad() self.model.zero_grad() loss.backward() self.optimizer.step() #if iteration % verbosity_interval == 0: # print("ITERATION %r: %r: train loss/sent=%.4f, time=%.2fs" % (ITER+1, iteration, train_loss/(iteration + 1), time.time() - start)) print( "ITERATION %r: train loss/sent=%.4f, time=%.2fs" % (ITER + 1, train_loss / len(self.train), time.time() - start)) #print("Score on training", evaluate(train)) #print("Score on development", evaluate(dev)) self.train_loss_values.append(train_loss / len(self.train)) self.dev_loss_values.append(self.evaluate.calculate_loss(self.dev)) self.test_loss_values.append( self.evaluate.calculate_loss(self.test)) def save_model(self): #Save model torch.save(self.model, self.model_full_path) print("Saved model has test score", self.evaluate(self.test)) def plot(self): plt.plot(self.train_loss_values, label="Train loss") plt.plot(self.dev_loss_values, label="Validation loss") plt.plot(self.test_loss_values, label="Test loss") plt.legend(loc='best') plt.xlabel("Epochs") plt.ylabel("Loss") plt.title(self.model_name + " - has loss with lr = %.4f, embedding size = %r" % (self.learning_rate, self.embedding_size)) plt.show() def save_data(self): file = open(self.output_file_name, "w") file.write(", ".join(map(str, self.train_loss_values))) file.write("\n") file.write(", ".join(map(str, self.dev_loss_values))) file.write("\n") file.write(", ".join(map(str, self.test_loss_values))) file.write("\n") file.write(str(self.evaluate(self.test))) file.write("\n") file.close()
print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPU") except RuntimeError as e: # Visible devices must be set before GPUs have been initialized print(e) print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU'))) # Define multigpu strategy devices = ['/device:GPU:' + str(i) for i in gpu_ids] train_strategy = tf.distribute.MirroredStrategy(devices=devices) ################################################################################################# # Загрузка данных ################################################################################################# # Loading TRAIN data data = Data() vocab = data.vocab full_dataset = data.get_all_data() article = full_dataset['article_text'] extended_input_tokens = full_dataset['extended_article_tokens'] summary = full_dataset['summary_text'] extended_gt_tokens = full_dataset['extended_summary_tokens'] index = full_dataset['index'] oovs = full_dataset['oovs'] loss_mask = full_dataset['summary_loss_points'] with tf.device('CPU'): train_tf_dataset = tf.data.Dataset.from_tensor_slices( (extended_input_tokens, extended_gt_tokens, loss_mask, index)).batch(global_batch_size) train_tf_dataset = train_tf_dataset.shuffle(32)
def train(config, args): assert (config.use_adversary is False), 'To use adversarial training, run `adv_train.py`' start_time = time.time() global_step, n_checkpoints, v_auc_best, v_cvm = 0, 0, 0., 10. ckpt = tf.train.get_checkpoint_state(directories.checkpoints) print('Reading data ...') if args.input is None: input_file = directories.train test_file = directories.test else: input_file = args.input test_file = args.test features, labels, pivots = Data.load_data(input_file, parquet=args.parquet) test_features, test_labels, test_pivots = Data.load_data( test_file, parquet=args.parquet) # Build graph model = Model(config, features=features, labels=labels, args=args) saver = tf.train.Saver() with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)) as sess: sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) train_handle = sess.run(model.train_iterator.string_handle()) test_handle = sess.run(model.test_iterator.string_handle()) if args.restore_last and ckpt.model_checkpoint_path: # Continue training saved model saver.restore(sess, ckpt.model_checkpoint_path) print('{} restored.'.format(ckpt.model_checkpoint_path)) start_epoch = args.restart_epoch else: if args.restore_path: new_saver = tf.train.import_meta_graph('{}.meta'.format( args.restore_path)) new_saver.restore(sess, args.restore_path) print('{} restored.'.format(args.restore_path)) start_epoch = args.restart_epoch else: start_epoch = 0 sess.run(model.test_iterator.initializer, feed_dict={ model.test_features_placeholder: test_features, model.test_pivots_placeholder: test_pivots, model.pivots_placeholder: test_pivots, model.test_labels_placeholder: test_labels }) for epoch in range(start_epoch, config.num_epochs): sess.run(model.train_iterator.initializer, feed_dict={ model.features_placeholder: features, model.labels_placeholder: labels, model.pivots_placeholder: pivots }) v_auc_best = Utils.run_diagnostics(model, config, directories, sess, saver, train_handle, test_handle, start_time, v_auc_best, epoch, global_step, args.name, v_cvm) if epoch > 0: save_path = saver.save( sess, os.path.join( directories.checkpoints, 'MI_reg_{}_epoch{}_step{}.ckpt'.format( args.name, epoch, global_step)), global_step=epoch) print('Starting epoch {}, Weights saved to file: {}'.format( epoch, save_path)) while True: try: # Update weights global_step, *ops = sess.run([ model.global_step, model.opt_op, model.MINE_labels_train_op, model.auc_op, model.update_accuracy ], feed_dict={ model.training_phase: True, model.handle: train_handle }) if args.mutual_information_penalty: for _ in range(args.MI_iterations): sess.run(model.MINE_train_op, feed_dict={ model.training_phase: True, model.handle: test_handle }) if global_step % 1000 == 0: # Periodically show diagnostics v_MI_kraskov, v_MI_MINE, v_pred, v_labels, v_pivots, v_conf = sess.run( [ model.MI_logits_theta_kraskov, model.MI_logits_theta, model.pred, model.labels, model.pivots[:, 0], model.softmax ], feed_dict={ model.training_phase: False, model.handle: test_handle }) v_cvm = Utils.cvm_z(v_pivots, v_pred, v_labels, confidence=v_conf, selection_fraction=0.05) v_auc_best = Utils.run_diagnostics( model, config_train, directories, sess, saver, train_handle, test_handle, start_time, v_auc_best, epoch, global_step, args.name, v_cvm) if global_step % 1e5 == 0: save_path = saver.save( sess, os.path.join( directories.checkpoints, 'MI_reg_{}_epoch{}_step{}.ckpt'.format( args.name, epoch, global_step)), global_step=epoch) print('Weights saved to file: {}'.format(save_path)) except tf.errors.OutOfRangeError: print('End of epoch!') break except KeyboardInterrupt: save_path = saver.save( sess, os.path.join(directories.checkpoints, 'MI_reg_{}_last.ckpt'.format(args.name)), global_step=epoch) print('Interrupted, model saved to: ', save_path) sys.exit() save_path = saver.save(sess, os.path.join( directories.checkpoints, 'MI_reg_{}_end.ckpt'.format(args.name)), global_step=epoch) print("Training Complete. Model saved to file: {} Time elapsed: {:.3f} s". format(save_path, time.time() - start_time))
def __init__(self, config, paths, dataset, name='gan_compression', evaluate=False): # Build the computational graph print('Building computational graph ...') self.G_global_step = tf.Variable(0, trainable=False) self.D_global_step = tf.Variable(0, trainable=False) self.handle = tf.placeholder(tf.string, shape=[]) self.training_phase = tf.placeholder(tf.bool) # >>> Data handling self.path_placeholder = tf.placeholder(paths.dtype, paths.shape) self.test_path_placeholder = tf.placeholder(paths.dtype) train_dataset = Data.load_dataset(self.path_placeholder, config.batch_size, augment=False, multiscale=config.multiscale, training_dataset=dataset) test_dataset = Data.load_dataset(self.test_path_placeholder, config.batch_size, augment=False, multiscale=config.multiscale, training_dataset=dataset, test=True) self.iterator = tf.data.Iterator.from_string_handle( self.handle, train_dataset.output_types, train_dataset.output_shapes) self.train_iterator = train_dataset.make_initializable_iterator() self.test_iterator = test_dataset.make_initializable_iterator() if config.multiscale: self.example, self.example_downscaled2, self.example_downscaled4 = self.iterator.get_next( ) else: self.example = self.iterator.get_next() # Global generator: Encode -> quantize -> reconstruct # =======================================================================================================>>> with tf.variable_scope('generator'): self.feature_map = Network.encoder(self.example, config, self.training_phase, config.channel_bottleneck) self.w_hat = Network.quantizer(self.feature_map, config) if config.sample_noise is True: print('Sampling noise...') # noise_prior = tf.contrib.distributions.Uniform(-1., 1.) # self.noise_sample = noise_prior.sample([tf.shape(self.example)[0], config.noise_dim]) noise_prior = tf.contrib.distributions.MultivariateNormalDiag( loc=tf.zeros([config.noise_dim]), scale_diag=tf.ones([config.noise_dim])) v = noise_prior.sample(tf.shape(self.example)[0]) Gv = Network.dcgan_generator(v, config, self.training_phase, C=config.channel_bottleneck, upsample_dim=config.upsample_dim) self.z = tf.concat([self.w_hat, Gv], axis=-1) else: self.z = self.w_hat self.reconstruction = Network.decoder(self.z, config, self.training_phase, C=config.channel_bottleneck) print('Real image shape:', self.example.get_shape().as_list()) print('Reconstruction shape:', self.reconstruction.get_shape().as_list()) # Pass generated, real images to discriminator # =======================================================================================================>>> if config.multiscale: D_x, D_x2, D_x4, *Dk_x = Network.multiscale_discriminator( self.example, self.example_downscaled2, self.example_downscaled4, self.reconstruction, config, self.training_phase, use_sigmoid=config.use_vanilla_GAN, mode='real') D_Gz, D_Gz2, D_Gz4, *Dk_Gz = Network.multiscale_discriminator( self.example, self.example_downscaled2, self.example_downscaled4, self.reconstruction, config, self.training_phase, use_sigmoid=config.use_vanilla_GAN, mode='reconstructed', reuse=True) else: D_x = Network.discriminator(self.example, config, self.training_phase, use_sigmoid=config.use_vanilla_GAN) D_Gz = Network.discriminator(self.reconstruction, config, self.training_phase, use_sigmoid=config.use_vanilla_GAN, reuse=True) # Loss terms # =======================================================================================================>>> if config.use_vanilla_GAN is True: # Minimize JS divergence D_loss_real = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=D_x, labels=tf.ones_like(D_x))) D_loss_gen = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=D_Gz, labels=tf.zeros_like(D_Gz))) self.D_loss = D_loss_real + D_loss_gen # G_loss = max log D(G(z)) self.G_loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=D_Gz, labels=tf.ones_like(D_Gz))) else: # Minimize $\chi^2$ divergence self.D_loss = tf.reduce_mean(tf.square(D_x - 1.)) + tf.reduce_mean( tf.square(D_Gz)) self.G_loss = tf.reduce_mean(tf.square(D_Gz - 1.)) if config.multiscale: self.D_loss += tf.reduce_mean( tf.square(D_x2 - 1.)) + tf.reduce_mean( tf.square(D_x4 - 1.)) self.D_loss += tf.reduce_mean( tf.square(D_Gz2)) + tf.reduce_mean(tf.square(D_Gz4)) distortion_penalty = config.lambda_X * tf.losses.mean_squared_error( self.example, self.reconstruction) self.G_loss += distortion_penalty if config.use_feature_matching_loss: # feature extractor for generator D_x_layers, D_Gz_layers = [j for i in Dk_x for j in i ], [j for i in Dk_Gz for j in i] feature_matching_loss = tf.reduce_sum([ tf.reduce_mean(tf.abs(Dkx - Dkz)) for Dkx, Dkz in zip(D_x_layers, D_Gz_layers) ]) self.G_loss += config.feature_matching_weight * feature_matching_loss # Optimization # =======================================================================================================>>> G_opt = tf.train.AdamOptimizer(learning_rate=config.G_learning_rate, beta1=0.5) D_opt = tf.train.AdamOptimizer(learning_rate=config.D_learning_rate, beta1=0.5) theta_G = Utils.scope_variables('generator') theta_D = Utils.scope_variables('discriminator') print('Generator parameters:', theta_G) print('Discriminator parameters:', theta_D) G_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope='generator') D_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope='discriminator') # Execute the update_ops before performing the train_step with tf.control_dependencies(G_update_ops): self.G_opt_op = G_opt.minimize(self.G_loss, name='G_opt', global_step=self.G_global_step, var_list=theta_G) with tf.control_dependencies(D_update_ops): self.D_opt_op = D_opt.minimize(self.D_loss, name='D_opt', global_step=self.D_global_step, var_list=theta_D) G_ema = tf.train.ExponentialMovingAverage( decay=config.ema_decay, num_updates=self.G_global_step) G_maintain_averages_op = G_ema.apply(theta_G) D_ema = tf.train.ExponentialMovingAverage( decay=config.ema_decay, num_updates=self.D_global_step) D_maintain_averages_op = D_ema.apply(theta_D) with tf.control_dependencies(G_update_ops + [self.G_opt_op]): self.G_train_op = tf.group(G_maintain_averages_op) with tf.control_dependencies(D_update_ops + [self.D_opt_op]): self.D_train_op = tf.group(D_maintain_averages_op) # >>> Monitoring # tf.summary.scalar('learning_rate', learning_rate) tf.summary.scalar('generator_loss', self.G_loss) tf.summary.scalar('discriminator_loss', self.D_loss) tf.summary.scalar('distortion_penalty', distortion_penalty) tf.summary.scalar('feature_matching_loss', feature_matching_loss) tf.summary.scalar('G_global_step', self.G_global_step) tf.summary.scalar('D_global_step', self.D_global_step) tf.summary.image('real_images', self.example, max_outputs=4) tf.summary.image('compressed_images', self.reconstruction, max_outputs=4) self.merge_op = tf.summary.merge_all() self.train_writer = tf.summary.FileWriter(os.path.join( directories.tensorboard, '{}_train_{}'.format(name, time.strftime('%d-%m_%I:%M'))), graph=tf.get_default_graph()) self.test_writer = tf.summary.FileWriter( os.path.join( directories.tensorboard, '{}_test_{}'.format(name, time.strftime('%d-%m_%I:%M'))))
print( '[Without Re-Ranking] mAP: {:.4f} rank1: {:.4f} rank3: {:.4f} rank5: {:.4f} rank10: {:.4f}' .format(m_ap, r[0], r[2], r[4], r[9])) def test_my(self): self.model.eval() features = extract_feature(self.model, tqdm(self.mydata_loader)).numpy() dist = cdist(features, features) embed() if __name__ == '__main__': data = Data() model = MGN() loss = Loss() main = Main(model, loss, data) start_epoch = 1 if opt.weight: model.load_state_dict(torch.load(opt.weight)) start_epoch = 1 + int(opt.weight.split('_')[-1][:-3]) if opt.test_mine: print('=> Test my photos:') main.test_my() elif opt.mode == 'train':
class GUI(wx.Frame): '''mfc1.GUI(tna, clock, menu, tbicon) GUI of the MindfulClock. tna = True, False, start in the taskbar notification area. clock = True, False, start clock automatically. menu = True, False, show the time not beside te indicator. tbicon = True, False, use instead of appindicator wx.TaskBarIcon. ''' def __init__(self, tna, clock, menu, tbicon): # Data object. self.__data = Data() # Package directory self.__dir = self.determine_path() # Internationalisation self.set_in18() # Load saved datas. self.config_load() # Get frame title, frame size and icon. title = self.__data.get_sys('frame_title') size = self.__data.get_user('frame_size') icon = os.path.join(self.__dir, self.__data.get_sys('icon_name')) # Subclass wx.Frame.__init__(self, parent=None, id=wx.ID_ANY, title=title, size=size) # Icon if icon.endswith('.png'): self.SetIcon(wx.Icon(name=icon, type=wx.BITMAP_TYPE_PNG)) # GUI border style and distance between widgets. bstyl = self.__data.get_sys('gui_borderstyle') guiborders = { 'simple': wx.SIMPLE_BORDER, 'raised': wx.RAISED_BORDER, 'sunken': wx.SUNKEN_BORDER, 'no': wx.NO_BORDER } if bstyl in guiborders: self.__bstyl = guiborders[bstyl] else: self.__bstyl = wx.SIMPLE_BORDER self.__bdist = self.__data.get_sys('gui_borderdist') if not self.__bdist: self.__bdist = 5 # Set attributes for time interval and sound file. self.__interval = self.__data.get_user('interval') s = self.__data.get_user('sound') if s: self.__sound = os.path.join(self.__dir, s) else: self.__sound = '' # time interval. intervalbox = self.init_interval() # Text notification. textbox = self.init_text() # Sound notification. soundbox = self.init_sound() # Clock control. controlbox = self.init_buttons() # Timer self.__timer = wx.Timer(self, 1) self.Bind(event=wx.EVT_TIMER, handler=self.on_timer, source=self.__timer) # Clock status self.__clockstatus = False # Lock file userid = wx.GetUserId() wxpath = wx.StandardPaths.Get() userdir = wxpath.GetDocumentsDir() self.__lock = Lock(path=userdir, userid=userid) self.__lockstate = '' if self.__lock.one_instance('mindfulclock1'): # One instance. self.__lock.write_lock() self.__lockstate = 'written' else: # More than one instance. if self.start_question(): # Start the clock. self.__lock.write_lock() self.__lockstate = 'written' else: # Exit the program. self.__lockstate = 'exit' # Exit bindings. self.Bind(event=wx.EVT_CLOSE, handler=self.on_system_close) # type of taskbar notification. # 'appind' = application indicator # 'tbicon' = taskbaricon # '' = no taskbar started. self.__tbtype = '' # Exit when tbicon is False and __lockstate is 'exit' if not tbicon and self.__lockstate == 'exit': self.Close() # Layout vbox = wx.BoxSizer(wx.VERTICAL) vbox.Add(item=intervalbox, proportion=0, flag=wx.EXPAND | wx.ALL, border=self.__bdist) vbox.Add(item=textbox, proportion=1, flag=wx.EXPAND | wx.ALL, border=self.__bdist) vbox.Add(item=soundbox, proportion=0, flag=wx.EXPAND | wx.ALL, border=self.__bdist) vbox.Add(item=controlbox, proportion=0, flag=wx.EXPAND | wx.ALL, border=self.__bdist) self.SetSizer(vbox) # Disable stop & pause button self.__btnstop.Enable(False) self.__btnpause.Enable(False) # Clock not paused. self.__pausetime = -99 # System tray from command line options if tna: # Start in the taskbar. self.__tna = True else: if self.__miniopt.GetValue(): # Checkbox is set, start in the taskbar. self.__tna = True else: # Start normal. self.__tna = False if tbicon: # Use wx.TaskBarIcon self.init_tbicon() else: # Use Appindicator if menu: # Show the time not beside the indicator. self.__menutime = True else: # Show the time beside. self.__menutime = False wx.FutureCall(100, self.init_appind) # Centre window, show window. self.Center() # Exit when tbicon is True and __lockstate is 'exit' if tbicon and self.__lockstate == 'exit': self.Close() # Check autostart if clock: # Start clock automatically. wx.FutureCall(200, self.clock_start) else: if self.__autostart.GetValue(): # Checkbox is set, start clock automatically. wx.FutureCall(200, self.clock_start) if self.__tna: # Start in the system tray. self.Hide() else: self.Show() def clock_pause(self): '''Pause the clock.''' # stop timer self.__timer.Stop() # Show start icon, hide stop & pause icon self.__btnstart.Enable(True) self.__btnstop.Enable(True) self.__btnpause.Enable(False) self.__clockstatus = True # Read current time, set elapsed seconds timenow = int(time.time()) self.__pausetime = timenow - self.__start # Remaining minutes as text. remain = self.get_text_minutes(self.__end - timenow) remain = '%s (pause)' % remain # Taskbar if self.__tbtype == 'appind': # Application indicator self.__ind.set_menu_stop() self.__ind.set_menu_continue() self.__ind.set_remain_time(remain) elif self.__tbtype == 'tbicon': # TaskBarIcon self.__tbicon.set_menu_stop() self.__tbicon.set_menu_continue() self.__tbicon.set_remain_time(remain) def clock_start(self): '''Start the clock.''' # Clock paused ? if self.__pausetime == -99: # No pause, start clock, read interval interval = self.get_integer_interval() if interval != 'dev': # Time interval in seconds self.__seconds = interval * 60.0 # Start and end time, UTC in seconds self.__start = int(time.time()) self.__end = self.__start + self.__seconds else: self.__seconds = 5.0 self.__start = int(time.time()) self.__end = self.__start + self.__seconds # Status text notification & sound. self.__textnotif = 'clear' self.__soundplay = 'clear' else: # Clock paused, continue. timenow = int(time.time()) self.__start = timenow - self.__pausetime self.__end = self.__start + self.__seconds self.__pausetime = -99 # Start timer self.__timer.Start(self.__data.get_sys('wxtimer')) # Hide start icon, show stop & pause icon self.__btnstart.Enable(False) self.__btnstop.Enable(True) self.__btnpause.Enable(True) self.__clockstatus = True # Taskbar if self.__tbtype == 'appind': # Application indicator self.__ind.set_menu_stop() self.__ind.set_menu_pause() elif self.__tbtype == 'tbicon': # TaskBarIcon self.__tbicon.set_menu_stop() self.__tbicon.set_menu_pause() def clock_stop(self): '''Stop the clock.''' # stop timer self.__timer.Stop() # Show start icon, hide stop & pause icon self.__btnstart.Enable(True) self.__btnstop.Enable(False) self.__btnpause.Enable(False) self.__clockstatus = False # No pause self.__pausetime = -99 # Gauge self.__gauge.SetValue(0) # Taskbar if self.__tbtype == 'appind': # Application indicator self.__ind.set_menu_start() self.__ind.set_menu_pause_clear() self.__ind.set_remain_time('--:--') elif self.__tbtype == 'tbicon': # TaskBarIcon self.__tbicon.set_menu_start() self.__tbicon.set_menu_pause_clear() self.__tbicon.set_remain_time('--:--') def config_load(self): '''Load the settings with wx.config.''' # Config file config = wx.Config(self.__data.get_sys('config_file')) # Get the default dictionairy as text textdic = self.__data.get_user_textdic() # Read text, textdic as default. newdic = config.Read(key='dic', defaultVal=textdic) # Set text as new dictionairy. self.__data.set_user_textdic(newdic) def config_save(self): '''Save the settings with wx.config.''' # Config file config = wx.Config(self.__data.get_sys('config_file')) # Set text notification. self.__data.set_user('text', self.__msgtext.GetValue()) # Set sound notification. self.__data.set_user('sound', self.__sound) # Set time interval. if self.__interval != 'dev': self.__data.set_user('interval', self.__interval) # Set frame size. size = self.GetSize() self.__data.set_user('frame_size', (size[0], size[1])) # Set checkbox values. self.__data.set_user('mini_opt', self.__miniopt.GetValue()) self.__data.set_user('autostart', self.__autostart.GetValue()) # Get data dictionariy as text. textdic = self.__data.get_user_textdic() # Write text. config.Write(key='dic', value=textdic) def determine_path(self): '''Borrowed from wxglade.py, get the package directory.''' try: root = __file__ if os.path.islink(root): root = os.path.realpath(root) return os.path.dirname(os.path.abspath(root)) except: print 'I am sorry, but something is wrong. There is no ' print '__file__ variable. Please contact the author.' def get_clock(self): '''Get the status of the clock, True of False.''' return (self.__clockstatus) def get_integer_interval(self): '''Convert time interval as text to a integer value.''' # Get text from entry. text = self.__txtinterval.GetValue() # Error handling. try: if text != 'dev': interval = int(text) else: interval = text except ValueError: interval = self.__interval # Return integer. return (interval) def get_pause(self): '''Is the clock paused, True of False.''' if self.__pausetime == -99: # Clock not paused. status = False else: # Clock paused. status = True return (status) def get_text_minutes(self, seconds): '''Get the seconds in minutes as text, 'mm:ss'.''' try: mins = int(seconds // 60) secs = int(seconds % 60) except ValueError: mins, secs = 0, 0 return ('%#02d:%#02d' % (mins, secs)) def init_appind(self): '''Create the application indicator.''' # status of the indicator. self.__tbtype = 'appind' # Application indicator. icon = self.__data.get_sys('indicator_icon') path = os.path.join(self.__dir, self.__data.get_sys('indicator_path')) self.__ind = AppIndicator(frame=self, icon=icon, path=path, textdic={ 'start': _(u'Start'), 'stop': _(u'Stop'), 'show': _(u'Show'), 'hide': _(u'Hide'), 'exit': _(u'Exit'), 'pause': _(u'Pause'), 'cont': _(u'Continue') }, menutime=self.__menutime) if self.__tna: # Start in the system tray. self.__ind.set_menu_show() self.__ind.main() def init_buttons(self): '''Create the control buttons.''' # Title t = _(u'Clock control') label = wx.StaticText(parent=self, label=t) # Start bitmap button. icon = os.path.join(self.__dir, self.__data.get_sys('icon_start')) self.__btnstart = wx.BitmapButton(parent=self, bitmap=wx.Bitmap(icon)) self.__btnstart.SetToolTip(wx.ToolTip(_(u'Start Clock'))) # Stop bitmap button. icon = os.path.join(self.__dir, self.__data.get_sys('icon_stop')) self.__btnstop = wx.BitmapButton(parent=self, bitmap=wx.Bitmap(icon)) self.__btnstop.SetToolTip(wx.ToolTip(_(u'Clock stop'))) # Pause bitmap button. icon = os.path.join(self.__dir, self.__data.get_sys('icon_pause')) self.__btnpause = wx.BitmapButton(parent=self, bitmap=wx.Bitmap(icon)) self.__btnpause.SetToolTip(wx.ToolTip(_(u'Clock pause'))) # Minimize bitmap button. icon = os.path.join(self.__dir, self.__data.get_sys('icon_minimize')) minimize = wx.BitmapButton(parent=self, bitmap=wx.Bitmap(icon)) t = _(u'Minimize to Taskbar Notification Area') minimize.SetToolTip(wx.ToolTip(t)) # Exit bitmap button. icon = os.path.join(self.__dir, self.__data.get_sys('icon_exit')) exit_ = wx.BitmapButton(parent=self, bitmap=wx.Bitmap(icon)) exit_.SetToolTip(wx.ToolTip(_(u'Exit Clock'))) # Gauge to show the process. self.__gaugerange = self.__data.get_sys('gauge') self.__gauge = wx.Gauge(parent=self, range=self.__gaugerange) # Checkbox options t = _(u'Start in the Taskbar Notification Area') self.__miniopt = wx.CheckBox(parent=self, label=t) t = _(u'Automatically start the clock on program startup') self.__autostart = wx.CheckBox(parent=self, label=t) # Set value from user datas. if self.__data.get_user('mini_opt'): self.__miniopt.SetValue(True) else: self.__miniopt.SetValue(False) if self.__data.get_user('autostart'): self.__autostart.SetValue(True) else: self.__autostart.SetValue(False) # Bindings. self.__btnstart.Bind(event=wx.EVT_BUTTON, handler=self.on_start) self.__btnstop.Bind(event=wx.EVT_BUTTON, handler=self.on_stop) self.__btnpause.Bind(event=wx.EVT_BUTTON, handler=self.on_pause) minimize.Bind(event=wx.EVT_BUTTON, handler=self.on_minimize) exit_.Bind(event=wx.EVT_BUTTON, handler=self.on_exit) # Layout. vsiz = wx.BoxSizer(wx.VERTICAL) vsiz.Add(item=label, flag=wx.EXPAND | wx.LEFT, border=self.__bdist) hsiz = wx.BoxSizer(wx.HORIZONTAL) hsiz.Add(item=self.__btnstart, flag=wx.EXPAND | wx.ALL, border=self.__bdist) hsiz.Add(item=self.__btnpause, flag=wx.EXPAND | wx.ALL, border=self.__bdist) hsiz.Add(item=self.__btnstop, flag=wx.EXPAND | wx.ALL, border=self.__bdist) hsiz.AddStretchSpacer() hsiz.Add(item=minimize, flag=wx.EXPAND | wx.ALL, border=self.__bdist) hsiz.Add(item=exit_, flag=wx.EXPAND | wx.ALL, border=self.__bdist) vsiz.Add(item=hsiz, proportion=1, flag=wx.EXPAND) vsiz.Add(item=self.__gauge, flag=wx.EXPAND | wx.ALL, border=self.__bdist) vsiz.Add(item=self.__miniopt, flag=wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, border=self.__bdist) vsiz.Add(item=self.__autostart, flag=wx.EXPAND | wx.LEFT | wx.RIGHT | wx.BOTTOM, border=self.__bdist) return (vsiz) def init_interval(self): '''Create the time interval widgets.''' # Title t = _(u'Time interval in minutes') label = wx.StaticText(parent=self, label=t) # Text entry: Read default, create entry. self.__txtinterval = wx.TextCtrl(parent=self, value=str(self.__interval)) # Increase bitmap button. icon = os.path.join(self.__dir, self.__data.get_sys('icon_increase')) increase = wx.BitmapButton(parent=self, bitmap=wx.Bitmap(icon)) increase.SetToolTip(wx.ToolTip(_(u'Increase time'))) # Decrease bitmap button. icon = os.path.join(self.__dir, self.__data.get_sys('icon_decrease')) decrease = wx.BitmapButton(parent=self, bitmap=wx.Bitmap(icon)) decrease.SetToolTip(wx.ToolTip(_(u'Decrease time'))) # Bindings. increase.Bind(event=wx.EVT_BUTTON, handler=self.on_increase) decrease.Bind(event=wx.EVT_BUTTON, handler=self.on_decrease) self.__txtinterval.Bind(event=wx.EVT_KILL_FOCUS, handler=self.on_interval) # Layout. vsiz = wx.BoxSizer(wx.VERTICAL) hsiz = wx.BoxSizer(wx.HORIZONTAL) vsiz.Add(item=label, flag=wx.EXPAND | wx.LEFT, border=self.__bdist) hsiz.Add(item=self.__txtinterval, proportion=1, flag=wx.EXPAND | wx.ALL, border=self.__bdist) hsiz.Add(item=increase, flag=wx.EXPAND | wx.ALL, border=self.__bdist) hsiz.Add(item=decrease, flag=wx.EXPAND | wx.ALL, border=self.__bdist) vsiz.Add(item=hsiz, proportion=1, flag=wx.EXPAND) return (vsiz) def init_sound(self): '''Create the sound notification widgets.''' # Title t = _(u'Sound notification') label = wx.StaticText(parent=self, label=t) # Sound change button icon = os.path.join(self.__dir, self.__data.get_sys('icon_change')) change = wx.BitmapButton(parent=self, bitmap=wx.Bitmap(icon)) change.SetToolTip(wx.ToolTip(_(u'Change sound file'))) # Sound preview button icon = os.path.join(self.__dir, self.__data.get_sys('icon_preview')) preview = wx.BitmapButton(parent=self, bitmap=wx.Bitmap(icon)) preview.SetToolTip(wx.ToolTip(_(u'Preview sound'))) # Text entry, read defaults self.__msgsound = wx.TextCtrl(parent=self, value=self.__sound) # Bindings change.Bind(event=wx.EVT_BUTTON, handler=self.on_change) preview.Bind(event=wx.EVT_BUTTON, handler=self.on_preview) self.__msgsound.Bind(event=wx.EVT_KILL_FOCUS, handler=self.on_msgsound) # Layout vsiz = wx.BoxSizer(wx.VERTICAL) hsiz = wx.BoxSizer(wx.HORIZONTAL) vsiz.Add(item=label, flag=wx.EXPAND | wx.LEFT, border=self.__bdist) hsiz.Add(item=self.__msgsound, proportion=1, flag=wx.EXPAND | wx.ALL, border=self.__bdist) hsiz.Add(item=change, flag=wx.EXPAND | wx.ALL, border=self.__bdist) hsiz.Add(item=preview, flag=wx.EXPAND | wx.ALL, border=self.__bdist) vsiz.Add(item=hsiz, proportion=1, flag=wx.EXPAND) return (vsiz) def init_tbicon(self): '''Create the wx.TaskBarIcon.''' self.__tbtype = 'tbicon' # TaskbarIcon icon = os.path.join(self.__dir, self.__data.get_sys('tna_icon')) title = self.__data.get_sys('frame_title') self.__tbicon = TBIcon(frame=self, icon=icon, title=title, textdic={ 'start': _(u'Start'), 'stop': _(u'Stop'), 'show': _(u'Show'), 'hide': _(u'Hide'), 'exit': _(u'Exit'), 'pause': _(u'Pause'), 'cont': _(u'Continue') }) if self.__tna: # Start in the system tray. self.__tbicon.set_menu_show() def init_text(self): '''Create the text notification widgets.''' # Title t = _(u'Text notification') label = wx.StaticText(parent=self, label=t) # Text entry, read default, create entry. t = self.__data.get_user('text') self.__msgtext = wx.TextCtrl(parent=self, value=t, style=wx.TE_MULTILINE) # Layout. vsiz = wx.BoxSizer(wx.VERTICAL) vsiz.Add(item=label, flag=wx.EXPAND | wx.LEFT, border=self.__bdist) vsiz.Add(item=self.__msgtext, proportion=1, flag=wx.EXPAND | wx.ALL, border=self.__bdist) return (vsiz) def on_change(self, event): '''Event for button, change sound file.''' # Set filename, directory path, wildcards and title. sfile = os.path.basename(self.__sound) sdir = os.path.dirname(self.__sound) t = _(u'file') w1 = 'OGG- %s (*.ogg)|*.ogg' % (t) w2 = 'MP3- %s (*.mp3)|*.mp3' % (t) w3 = 'WAV- %s (*.wav)|*.wav' % (t) wcard = '%s|%s|%s' % (w1, w2, w3) t = _(u'Select the sound file') # Show open dialog, get user datas. dlg = wx.FileDialog(parent=self, message=t, defaultDir=sdir, defaultFile=sfile, wildcard=wcard, style=wx.FD_OPEN) if dlg.ShowModal() == wx.ID_OK: # Clicked ok, set path, destroy dialog. path = dlg.GetPath() dlg.Destroy() # Set path to text entry and sound attribute. self.__sound = path self.__msgsound.SetValue(path) self.__msgsound.SetInsertionPointEnd() else: # Not clicked ok, destroy dialog. dlg.Destroy() def on_decrease(self, event): '''Event for bitmap button, decrease time interval.''' # Get interval as integer. interval = self.get_integer_interval() # decrease interval and set it to the text entry. self.set_integer_interval(interval - 1) def on_exit(self, event): '''Event for button, exit program.''' self.Close() def on_increase(self, event): '''Event for bitmap button, increase time interval.''' # Get interval as integer. interval = self.get_integer_interval() # decrease interval and set it to the text entry. self.set_integer_interval(interval + 1) def on_interval(self, event): '''Event for text control, check time interval.''' # Get interval as integer. interval = self.get_integer_interval() # Set interval to the text entry. self.set_integer_interval(interval) def on_minimize(self, event): '''Event for button, minimize frame.''' if self.IsShown(): self.Hide() if self.__tbtype == 'appind': # Application indicator self.__ind.set_menu_show() elif self.__tbtype == 'tbicon': # wx.TaskBarIcon self.__tbicon.set_menu_show() def on_msgsound(self, event): '''Event for text control, check path to sound file.''' # Get text from entry. text = self.__msgsound.GetValue() if text: # Text is set, check path if not os.path.exists(text): self.__msgsound.SetValue(self.__sound) else: self.__sound = text else: # Text is not set. self.__sound = '' def on_pause(self, event): '''Event for button, pause the clock.''' self.clock_pause() def on_preview(self, event): '''Event for button, preview sound file.''' self.pygame_sound() def on_start(self, event): '''Event for button, start the clock.''' self.config_save() self.clock_start() def on_stop(self, event): '''Event for button, stop the clock.''' self.clock_stop() def on_system_close(self, event): '''Event before close the frame.''' if self.__lockstate == 'written': # Normal program start, normal program end, # delete lock file. self.__lock.delete_lock() self.__lockstate = 'deleted' # Close the taksbar or the indicator. if self.__tbtype == 'appind': # Application indicator self.__ind.quit() elif self.__tbtype == 'tbicon': # TaskBarIcon self.__tbicon.Destroy() # Save the settings self.config_save() self.Destroy() def on_timer(self, event): '''Event for timer, the MindfulClock.''' timenow = int(time.time()) if self.__textnotif == 'close' and self.__soundplay == 'close': # Text message & sound closed, start new interval. self.__start = timenow self.__end = self.__start + self.__seconds self.__gauge.SetValue(0) # Remaining minutes as text. remain = self.get_text_minutes(self.__seconds) # Set text notification & sound clear. self.__textnotif = 'clear' self.__soundplay = 'clear' elif self.__textnotif == 'clear' and \ self.__soundplay == 'clear' and timenow < self.__end: # End is not reached. progress = timenow - self.__start + 1 value = (self.__gaugerange / self.__seconds) * progress self.__gauge.SetValue(value) # Remaining minutes as text. remain = self.get_text_minutes(self.__end - timenow) elif self.__textnotif == 'clear' and \ self.__soundplay == 'clear' and timenow >= self.__end: # Play sound. self.pygame_sound() # Show text. self.show_popup() if self.__textnotif == 'clear' and self.__soundplay == 'clear': # Taskbar if self.__tbtype == 'appind': # Application indicator self.__ind.set_remain_time(remain) elif self.__tbtype == 'tbicon': # wx.TaskBarIcon self.__tbicon.set_remain_time(remain) elif self.__textnotif == 'show' or \ self.__soundplay.startswith('play'): # Taskbar if self.__tbtype == 'appind': # Application indicator self.__ind.set_remain_time('..') elif self.__tbtype == 'tbicon': # wx.TaskBarIcon self.__tbicon.set_remain_time('..') # Check sound. if self.__soundplay == 'play-sound': # wav or ogg if not pygame.mixer.get_busy(): self.__soundplay = 'close' elif self.__soundplay == 'play-music': # mp3 if not pygame.mixer.music.get_busy(): self.__soundplay = 'close' def pygame_sound(self): '''Play the 'soundfile' with Pygame.''' if self.__sound: # Soundfile is set, play sound. # Values of __playsound: # 'clear' = since the clock is running no sound played, # 'close' = sound finished., # 'play-sound' = play wav or ogg # 'play-music' = play mp3 if self.__sound.endswith('.wav') or \ self.__sound.endswith('.ogg'): mixer = pygame.mixer.Sound(self.__sound) mixer.play() self.__soundplay = 'play-sound' elif self.__sound.endswith('.mp3'): pygame.mixer.music.load(self.__sound) pygame.mixer.music.play() self.__soundplay = 'play-music' else: # No sound, set __soundplay 'close'. self.__soundplay = 'close' def set_integer_interval(self, interval): '''Control value of time interval and set it to the entry.''' # Check interval if interval != 'dev': minimum = self.__data.get_sys('min_interval') maximum = self.__data.get_sys('max_interval') if interval < minimum: interval = minimum elif interval > maximum: interval = maximum self.__txtinterval.SetValue(str(interval)) # Set current value as new default value. self.__interval = interval def start_question(self): '''Show Question, run mfc again?, return True or False.''' t1 = _(u'It seems the MindfulClock is running.') t2 = _(u'Do you want to start the clock anyway?') style = wx.YES_NO | wx.NO_DEFAULT | wx.ICON_QUESTION dlg = wx.MessageDialog(parent=None, message='%s %s' % (t1, t2), caption=_(u'Start control'), style=style) answer = dlg.ShowModal() if answer == wx.ID_NO: # Do not overwrite, return None status = False else: # Overwrite, return 'path' status = True return (status) def set_in18(self): '''Set the internationalization.''' # Get directory dir_ = os.path.join(self.__dir, 'locale') # Locale, set default language. self.__wxloc = wx.Locale(wx.LANGUAGE_DEFAULT) self.__wxloc.AddCatalogLookupPathPrefix(dir_) self.__wxloc.AddCatalog('mfc1') def set_textnotif(self, value): '''Set the value of __textnotif.''' # Values = 'show' message on the screen, 'close' message closed, # 'clear' since the clock is running no message showed. if value in ('show', 'close', 'clear'): self.__textnotif = value else: self.__textnotif = None def show_popup(self): '''Show the text notification popup.''' text = self.__msgtext.GetValue() if text: # Text is set, show dialog. Status of text notification. self.__textnotif = 'show' font = self.__data.get_sys('msg_font') colors = self.__data.get_sys('popup_colors') icon = os.path.join(self.__dir, self.__data.get_sys('icon_close')) popup = Popup(parent=self, style=self.__bdist, text=text, font=font, colors=colors, icon=icon) popw, poph = popup.get_size() dispw, disph = wx.GetDisplaySize() offx = (dispw - popw) / 2 offy = (disph - poph) / 2 popup.Position(ptOrigin=(0, 0), size=(offx, offy)) popup.Popup() else: # No text, no Popup, set __textnotif 'close'. self.__textnotif = 'close'
f = outputs[0].data.cpu() #print('f:', f) ff = ff + f #print('ff:', ff) #print() #print() fnorm = torch.norm(ff, p=2, dim=1, keepdim=True) ff = ff.div(fnorm.expand_as(ff)) features = torch.cat((features, ff), 0) return features if __name__ == '__main__': loader = Data() model = MGN() #print(torch.cuda.device_count()) model = torch.nn.DataParallel(model, output_device=1) loss = Loss() reid = Main(model, loss, loader) #print(opt.data_path) data_dir = opt.data_path.split('/')[-1] #print(data_dir) #print(opt.lr_scheduler) sch = '' for i in opt.lr_scheduler: sch = sch + str(i) + '_' model_dir = 'weights/' + data_dir + '/' + opt.train_name #print(model_dir)
def init(self, out: PathModel) -> PathModel: out = out.add(self.name) if Data.create(out): out.temp(space='').create() return out
def initialise(custom_command=''): """ Initialisation routine This function recovers the input from the command line arguments, from :mod:`parser_mp`, the parameter files. It then extracts the path of the used Monte Python code, and proceeds to initialise a :class:`data` instance, a cosmological code instance. Parameters ---------- custom_command: str allows for testing the code """ # Parsing line argument command_line = parser_mp.parse(custom_command) # Recovering the local configuration path = recover_local_path(command_line) # check for MPI try: from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() except ImportError: # set all chains to master if no MPI rank = 0 # Recover Monte Python's version number version_path = os.path.join(path['root'], 'VERSION') with open(version_path, 'r') as version_file: version = version_file.readline() if not command_line.silent and not rank: print('Running Monte Python v%s' % version) # If the info flag was used, read a potential chain (or set of chains) to # be analysed with default procedure. If the argument is a .info file, then # it will extract information from it (plots to compute, chains to analyse, # etc...) if command_line.subparser_name == "info": from analyze import analyze # only invoked when analyzing analyze(command_line) return None, None, command_line, False # Fill in data, starting from parameter file. If output folder already # exists, the input parameter file was automatically replaced by the # existing log.param. This prevents you to run different things in a same # folder. else: data = Data(command_line, path) # Overwrite arguments from parameter file with the command line if command_line.N is None: try: command_line.N = data.N except AttributeError: raise io_mp.ConfigurationError( "You did not provide a number of steps, neither via " + "command line, nor in %s" % command_line.param) # Loading up the cosmological backbone. For the moment, only CLASS has been # wrapped. cosmo = recover_cosmological_module(data) # Initialising the sampler # MH: Creating the file that will contain the chain if command_line.method == 'MH': io_mp.create_output_files(command_line, data) # NS: Creating the NS subfolder and the MultiNest arguments elif command_line.method == 'NS': from nested_sampling import initialise as initialise_ns initialise_ns(cosmo, data, command_line) return cosmo, data, command_line, True
return pred if __name__ == '__main__': """ test """ import numpy as np from data import Data from valid import Valid from xgb_wrapper import XGBWrapper from sklearn.metrics import accuracy_score # pd.set_option('display.max_columns', None) # pd.set_option('display.max_rows', None) # pd.set_option('display.max_colwidth', 200) data = Data() train_x, train_y, test_x = data.processing() valid = Valid() xgb_wrap = XGBWrapper() pred_train_xgb, pred_test_xgb = valid.hold_out(xgb_wrap, train_x, train_y, test_x) # train acc pred_binary_train_xgb = np.where(pred_train_xgb > 0.5, 1, 0) acc_train_xgb = round( accuracy_score(train_y, pred_binary_train_xgb) * 100, 2) print('##### acc_xgb #####') print(acc_train_xgb)
This module implements the core of the GOAT model. GOAT is an acronym for the paper titled Gossip and Attend: A Context-sensitive Graph Representation Learning """ from utils import parse_train_args, Trainer from data import Data, InputDataset, DataLoader from model import Goat if __name__ == '__main__': args = parse_train_args() data_ = Data(args=args) train_dataset = InputDataset(input_data=data_, num_neg=args.num_neg, partition="train") dev_dataset = InputDataset(input_data=data_, num_neg=args.num_neg, partition="dev") train_loader = DataLoader(dataset=train_dataset, batch_size=args.batch) dev_loader = DataLoader(dataset=train_dataset, batch_size=args.batch) goat = Goat(num_nodes=data_.num_nodes, dim=args.dim, dropout=args.dropout) trainer = Trainer(config=args) trainer.fit(model=goat, train_loader=train_loader, dev_loader=dev_loader)