def __init__(self, tlc=None, order=1, torder=0, directory=None, **kwargs): # initialize the Instrument, using the extra columns in the light curve dict = {} dict['rescaling'] = 1.0 # a constant out of transit flux level try: dict['C'] = np.median(tlc.flux) except: dict['C'] = 1.0 try: # include terms of (ev)^power for all the external variables available for power in np.arange(order)+1: for evkey in tlc.externalvariables.keys(): dict[evkey + '_tothe{0:1d}'.format(power)] = 0.0 # include (t)^power terms for time (CURRENTLY SET UP ONLY FOR SINGLE TRANSITS!) for power in np.arange(torder)+1: dict['t_tothe{0:1d}'.format(power)] = 0.0 # include other custom instrument parameters for k in kwargs.keys(): dict[k] = kwargs[k] except AttributeError: pass # include all the parameters explicitly defined here Parameters.__init__(self, directory=directory, **dict)
def __init__(self, **kwargs): # define defaults Parameters.__init__(self, J=0.0, \ k=0.1, \ rsovera=1.0/10.0, \ b=0.0, \ period=10.0, \ t0=2456000.0, \ dt=0.0, \ semiamplitude=0.0, \ esinw=0.0, \ ecosw=0.0) # overwrite defaults with input keywords Parameters.__init__(self, **kwargs) # set up the parameter constraints self.J.parinfo['limited'] = [True, False] self.J.parinfo['limits'] = [0, 100] self.b.parinfo['limited'] = [True, True] self.b.parinfo['limits'] = [0.0, 1.0] #self.q.parinfo['limited'] = [True, False] #self.q.parinfo['limits'] = [0, 100] self.period.parinfo['limited'] = [True, False] self.period.parinfo['limits'] = [0, 100] self.t0.parinfo['limited'] = [True, False] self.t0.parinfo['limits'] = [0, 1000000000]
def extractFullMIToThesaurus(): accents = Accents() parameters = Parameters() max_qty_terms = parameters.getMaxQtyTerms() seeds = Seeds() dic_seeds = seeds.getSeeds() mi_file = Statistic(stat_temp+'IMT_FullStatisticalCorpus.txt') try: thesaurus_file = codecs.open('../Data/Output/T3/T3_Jaccard.xml', 'w', 'utf-8') except IOError: print 'ERROR: System cannot open the file ../Data/Output/T3/T3_Jaccard.xml' sys.exit() thesaurus_file.write('<?xml version="1.0" encoding="ISO-8859-1"?>\n<thesaurus>\n\t<ontology id="privacy">\n') for seed in dic_seeds: qty_terms = 0 dic_related = mi_file.getOrderedNounMIForTerm(seed) if dic_related != False: thesaurus_file.write('\t\t<seed term_id="" term_name="'+accents.buildAccents(seed)+'" type="">\n') for mi_related in dic_related: if qty_terms < max_qty_terms: thesaurus_file.write('\t\t\t<term id="" display="ON" similarity="'+mi_related[0]+'">'+accents.buildAccents(mi_related[1])+'</term>\n') qty_terms += 1 thesaurus_file.write('\t\t</seed>\n') thesaurus_file.write('\t</ontology>\n</thesaurus>') thesaurus_file.close()
def __init__(self, **kwargs): Parameters.__init__(self, jitter=0.0, u1=0.2, u2=0.6, gd=0.32, albedo=0, mass=0.25) Parameters.__init__(self, **kwargs) self.u1.parinfo['limited'] = [True, True] self.u1.parinfo['limits'] = [0, 1] self.u2.parinfo['limited'] = [True, True] self.u2.parinfo['limits'] = [0, 1]
def initialize(self): problem = Problem() parameters = Parameters() problem_type = parameters.problem_type low_bound, high_bound = parameters.set_bounds(problem_type) self.position = np.random.uniform(low_bound, high_bound, parameters.dimension) self.velocity = np.random.uniform(parameters.min_vel, parameters.max_vel, parameters.dimension) self.cost = problem.cost_function(self.position) self.best_position = self.position[:] self.best_cost = self.cost
def main(argv): parameters = Parameters(argv) hostname = parameters.getHostname() port = parameters.getPort() dbname = parameters.getDBName() language_1, language_2 = parameters.getLanguage() collection = parameters.getCollection() filexml_1 = parameters.getInputFile_1() filexml_2 = parameters.getInputFile_2() type_corpus = parameters.getType() print 'Using parameters of configuration: ' print '- Host : ',hostname print '- Port : ',port print '- Coll : ',collection print '- DBase: ',dbname print '- XML1 : ',filexml_1 print '- XML2 : ',filexml_2 database = Mongo(hostname, dbname, collection) dic_content_1 = OrderedDict() parserxml_1 = XML(filexml_1, language_1) dic_content_1 = parserxml_1.getContent() size_1 = len(dic_content_1) del parserxml_1 dic_content_2 = OrderedDict() parserxml_2 = XML(filexml_2, language_2) dic_content_2 = parserxml_2.getContent() size_2 = len(dic_content_2) del parserxml_2 counter = 1 if size_1 == size_2: #As both files come from WebAligner, they must have the same number of documents for id_order in dic_content_1: id_file_1 = dic_content_1[id_order]['id_file'] language_1 = dic_content_1[id_order]['language'] content_1 = dic_content_1[id_order]['content'] id_file_2 = dic_content_2[id_order]['id_file'] language_2 = dic_content_2[id_order]['language'] content_2 = dic_content_2[id_order]['content'] if database.exists(language_1, id_file_1): if not database.exists(language_2, id_file_2): database.insertInExisting(language_1, id_file_1, language_2, id_file_2, content_2) else: if database.exists(language_2, id_file_2): database.insertInExisting(language_2, id_file_2, language_1, id_file_1, content_1) else: database.insertNewData(language_1, id_file_1, content_1, language_2, id_file_2, content_2, type_corpus, counter) counter += 1 else: #Files have different number of documents, so they are not aligned print '\nError: Files not aligned. Please align them with WebAligner.'
def main(argv): parameters = Parameters(argv) hostname = parameters.getHostname() port = parameters.getPort() dbname = parameters.getDBName() language_1, language_2 = parameters.getLanguage() collection = parameters.getCollection() fileinput_1 = parameters.getInputFile_1() fileinput_2 = parameters.getInputFile_2() type_corpus = parameters.getType() print 'Using parameters of configuration: ' print '- Host : ',hostname print '- Port : ',port print '- Coll : ',collection print '- DBase: ',dbname print '- File1: ',fileinput_1 print '- File2: ',fileinput_2 database = Mongo(hostname, dbname, collection) id_file_1 = (fileinput_1.split('/'))[-1] id_file_2 = (fileinput_2.split('/'))[-1] try: file_1 = codecs.open(fileinput_1, 'r', 'utf-8') except IOError: print 'ERROR: System cannot open the '+fileinput_1+' file' sys.exit(2) try: file_2 = codecs.open(fileinput_2, 'r', 'utf-8') except IOError: print 'ERROR: System cannot open the '+fileinput_2+' file' sys.exit(2) #Sentences indexed by the number of the line : number_line = _id (sentence) line_number = 1 lines_2 = file_2.readlines() for counter, content_1 in enumerate(file_1): content_2 = lines_2[counter] if not database.exists(language_1, id_file_1) and not database.exists(language_2, id_file_2): database.insertNewData(language_1, id_file_1, content_1, language_2, id_file_2, content_2, type_corpus, line_number) else: if database.existsSentence(language_1, id_file_1, line_number): if not database.existsSentence(language_2, id_file_2, line_number): database.insertInExistingSentence(language_1, id_file_1, language_2, id_file_2, content_2, line_number) else: if database.existsSentence(language_2, id_file_2, line_number): database.insertInExistingSentence(language_2, id_file_2, language_1, id_file_1, content_1, line_number) else: database.insertNewSentence(language_1, id_file_1, content_1, language_2, id_file_2, content_2, line_number) if (line_number % 1000 == 0): print 'Indexing line: ',line_number line_number += 1
def __init__(self): self.corpus_folder = '../Data/Corpus/Raw/' self.full_corpus = '../Data/Corpus/Statistical/Full/' self.noun_corpus = '../Data/Corpus/Statistical/Noun/' self.parameters = Parameters() self.firstLoadFile = True self.__buildStatisticalCorpus__() command = "cat "+self.full_corpus+"*.txt >> "+self.full_corpus+"../FullStatisticalCorpus.txt" os.system(command) command = "cat "+self.noun_corpus+"*.txt >> "+self.full_corpus+"../NounStatisticalCorpus.txt" os.system(command)
def generate_dataset(san, train_prep, train_ds, test_prep, test_ds, gen_path, train_id="train", test_id="test", max_epoch=0, addParamFn=None, phys=1, san_acts=1, san_phys=1): """ Generate a new dataset for each epoch :param san: the sanitizer model :param train_prep: the preprocessing core for the train set :param train_dl: the train dataloader :param test_prep: the preprocessing core for the test set :param test_dl: the test data loader :param gen_path: the path where to save the generated dataset :param train_id: the id of the train set :param test_id: the id of the test set :param epoch: the current epoch :param addParamFn: function to add parameters in name """ bs = 1024 train_dl = data.DataLoader(train_ds, batch_size=bs, shuffle=False, num_workers=1) test_dl = data.DataLoader(test_ds, batch_size=bs, shuffle=False, num_workers=1) name = lambda n, e: "{}/{}_{}.csv".format(gen_path, n, addParamFn(e)) def _generate_(san, dataloader, dset, prep, phys=phys, san_acts=san_acts, san_phys=san_phys): df = pd.DataFrame() select = lambda x, y, c: x if c else y cat = lambda x, ax: x if ax is None else torch.cat((ax, x), 0) a_xs, a_p, a_a, a_s, a_uid = None, None, None, None, None for i, sample in enumerate(dataloader): x = sample["sensor"].to(DEVICE) noise = NOISE(r=x.shape[0]).to(DEVICE) a = sample["act"].to(DEVICE) p = sample["phy"].to(DEVICE) * phys other_data = torch.cat((p, CTF(a) * san_acts, noise), 1) s = sample["sens"] xs, acs, ps = san(x, other_data) a = select(acs, a, san_acts == 1) p = select(ps, p, san_phys == 1) a_xs = cat(xs, a_xs) a_p = cat(p, a_p) a_a = cat(a, a_a) a_s = cat(s, a_s) a_uid = cat(sample["uid"], a_uid) df = dset.__inverse_transform_conv__(sensor_tensor=a_xs, phy=a_p, act_tensor=a_a, sens_tensor=a_s, user_id_tensor=a_uid, trials=dset.trials, cpu_device=CPU_DEVICE) p = prep.copy(True) p.df = df.reset_index(drop=True) p.inverse_transform() return p.df for epoch in tqdm.tqdm(range(1, max_epoch + 1)): # Load the sanitizer Model M.load_classifier_state(san, epoch, P.ModelsDir(), ext="S", otherParamFn=P.ParamFunction) # Check if there is a generated data in the correct format if not tryReading(name(train_id, epoch)): # Set does not exits df = _generate_(san, train_dl, train_ds, train_prep) df.to_csv(name(train_id, epoch), index=False) if not tryReading(name(test_id, epoch)): # Set does not exits df = _generate_(san, test_dl, test_ds, test_prep) df.to_csv(name(test_id, epoch), index=False)
default=508, help='give number of hidden units as an integer | (default = 508)') parser.add_argument('--epochs', type=int, default=1, help='give number of epochs as an integer | (default = 1)') parser.add_argument('--gpu', type=str, default='cuda', help='cuda or cpu | (default = cuda)') arg_in = parser.parse_args() parameters = Parameters({ "arch": arg_in.arch, "epochs": arg_in.epochs, "gpu": arg_in.gpu, "hidden_units": arg_in.hidden_units, "learningrate": arg_in.learningrate, "save_directory": arg_in.save_directory }) # Function that greet user greetUser() # Function that checks command line arguments using in_arg parameters.displayParameters('training') query = command("Please Enter to continue") print('Do you want to modify input values?') while True: print('‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡‡') print("## What you Want modify? ##") print("## 0 : modify a architecture ##") print("## 1 : modify a learningrate ##")
class StatisticalCorpus: def __init__(self): self.corpus_folder = '../Data/Corpus/Raw/' self.full_corpus = '../Data/Corpus/Statistical/Full/' self.noun_corpus = '../Data/Corpus/Statistical/Noun/' self.parameters = Parameters() self.firstLoadFile = True self.__buildStatisticalCorpus__() command = "cat "+self.full_corpus+"*.txt >> "+self.full_corpus+"../FullStatisticalCorpus.txt" os.system(command) command = "cat "+self.noun_corpus+"*.txt >> "+self.full_corpus+"../NounStatisticalCorpus.txt" os.system(command) def __buildStatisticalCorpus__(self): try: root, dirs, files = os.walk(self.corpus_folder).next()[:3] except: print 'ERROR: It was not possible to open the ../Data/Corpus/Raw/ folder' sys.exit() accents = Accents() for corpus_file in files: if re.match('.*xml$', corpus_file): corpus_filename = corpus_file.split('.')[0] xmlfile = ParseXml(root+''+corpus_file) dic_terms = xmlfile.getDicTerms() dic_nouns = xmlfile.getNouns() dic_verbs = xmlfile.getVerbs() id_sentence = 1 id_word = 1 id_t = 's'+str(id_sentence)+'_'+str(id_word) string_full = '' string_nouns = '' while dic_terms.has_key(id_t): while dic_terms.has_key(id_t): if not re.match('^(pu|num|conj|art|prp|spec)', dic_terms[id_t]['pos']) and (re.search('[$]', dic_terms[id_t]['lemma']) is None) and (len(dic_terms[id_t]['lemma']) >= self.parameters.getMinWordSize()): lemma = accents.buildCodes(dic_terms[id_t]['lemma']) if dic_nouns.has_key(id_t): string_nouns += lemma+'__N ' string_full += lemma+'__N ' elif dic_verbs.has_key(id_t): string_nouns += lemma+'__V ' string_full += lemma+'__V ' else: string_full += lemma+'__O ' string_nouns = string_nouns.replace('-', '_') string_full = string_full.replace('-', '_') id_word += 1 id_t = 's'+str(id_sentence)+'_'+str(id_word) id_word = 1 id_sentence += 1 id_t = 's'+str(id_sentence)+'_'+str(id_word) self.__writeCorpusFile__(corpus_filename, string_full, string_nouns) def __writeCorpusFile__(self, corpus_filename, string_full, string_nouns): try: write_full = codecs.open(self.full_corpus+''+corpus_filename+'.txt', 'w', 'utf-8') except IOError: print 'ERROR: System cannot open the '+self.full_corpus+''+corpus_filename+'.txt file' sys.exit() try: write_nouns = codecs.open(self.noun_corpus+''+corpus_filename+'.txt', 'w','utf-8') except IOError: print 'ERROR: System cannot open the '+self.noun_corpus+''+corpus_filename+'.txt file' sys.exit() write_full.write(string_full) write_nouns.write(string_nouns) write_full.close() write_nouns.close()
def __init__(self): Parameters.__init__(self) self.registerFloatParameter( "gravity", "X-component of gravitational acceleration")
def main(type_atc, argv): list_relations = ['AN', 'SV', 'VO'] date_start = datetime.datetime.now() date_start = date_start.strftime("%Y-%m-%d %H:%M:%S") parameters = Parameters(type_atc, argv) contexts = parameters.getContexts() svd_dimension = int(parameters.getSvdDimension()) input_folder = parameters.getInputFolder() language = parameters.getLanguage() min_word_size = parameters.getMinWordSize() max_qty_terms = int(parameters.getMaxQtyTerms()) output_folder = parameters.getOutputFolder() temp_folder = parameters.getTempFolder() record_log = parameters.getRecordLog() record_intermediate = parameters.getRecordIntermediate() seeds_file = parameters.getSeedsFile() stoplist_file = parameters.getStoplistFile() sim_measure = parameters.getSimilarityMeasure() del parameters logfile = LogFile(record_log, str(date_start), svd_dimension, input_folder, language, stoplist_file, min_word_size, max_qty_terms, None, output_folder, None, temp_folder, seeds_file, sim_measure) #if contexts: # logfile.writeLogfile('- Building syntactics relations from '+temp_folder) # contexts = Contexts(temp_folder) # del contexts #else: # logfile.writeLogfile('- Building syntactics relations from '+input_folder) # ling_corpus = StanfordSyntacticContexts(input_folder, temp_folder, stoplist_file, min_word_size, record_intermediate) # del ling_corpus matrix_relation = Matrix(temp_folder, svd_dimension, record_intermediate) del matrix_relation #similarities = Similarities(seeds_file, temp_folder, 'cosine') #dic_topn = similarities.getTopNOrderedDic(10) #del Similarities #logfile.writeLogfile('- Building thesaurus in '+output_folder+'T_'+type_atc+'_'+sim_measure+'.xml') #thesaurus = Thesaurus(output_folder+'T_'+type_atc+'_'+sim_measure+'.xml',max_qty_terms) #thesaurus.write(dic_topn) #del thesaurus date_end = datetime.datetime.now() date_end = date_end.strftime("%Y-%m-%d %H:%M:%S") logfile.writeLogfile('- Thesaurus sucessfully built!\nEnding process at: '+str(date_end)+'.\n') del logfile
class Slave: def __init__(self, name, index=1, host='localhost', port=8080, debug=False, param_file=None, folder_loc='', delimeter=':::', cameras=12, acceptable_filetype_set=['.JPG','CR2'], destination=None): self.name = name self.debug = debug self.index = index self.port = port self.host = host self.delimeter = delimeter self.client = Client(host=host, port=port, debug=debug, delimeter=self.delimeter) self.param_file = param_file if self.param_file: self.writeParamHeaders() self.writeParamFilenames() self.closeParam() #functions that revolve around other instances self.cameras=cameras self.acceptable_filetype_set=acceptable_filetype_set self.folder_loc = folder_loc self.folder = Folder(self.folder_loc) self.checkFolders() self.client.connect() def __del__(self): if self.client: self.client.close() if os.path.exists(self.folder_loc): shutil.rmtree(self.folder_loc) '''Param File Helper Functions''' def setParameters(self, name, destination): self.param_file = Parameters(name=self.name,destination=destination) self.writeParamHeaders() self.writeParamFilenames() self.closeParam() def writeParamHeaders(self): #if self.debug: # print "writeHeaders" self.param_file.write("Name" + str(self.delimeter) + str(self.name) + "\n") self.param_file.write("Port" + str(self.delimeter) + str(self.port) + "\n") self.param_file.write("Host" + str(self.delimeter) + str(self.host) + "\n") self.param_file.write("Parameter File Location" + str(self.delimeter) + str(abspath(join(self.param_file.getDestination(), self.name + self.param_file.getFileType()))) + "\n") def writeParamFilenames(self): #if self.debug: # print "writeParamFilenames" file_list = self.getFilenames(self.folder) for f in file_list: self.param_file.write("File" + str(self.delimeter) + str(f) + "\n") def closeParam(self): self.param_file.close() def setFolderLocation(self, folder_loc): self.folder_loc = folder_loc def getFilenames(self, folder, file_list=[]): for file in folder.files: if file.type.upper() in self.acceptable_filetype_set: file_list.append(file.path) for folder in folder.folders: file_list = self.getFilenames(folder=folder, file_list=file_list) return file_list def getFilenamesFolders(self, folder, folder_list=[]): for file in folder.files: if file.type.upper() in self.acceptable_filetype_set: if folder not in folder_list: folder_list.append(folder) for folder in folder.folders: file_list = self.getFilenamesFolders(folder=folder, folder_list=folder_list) return folder_list def checkData(self, folder, bad_data=[]): data_ok = True for data_type in self.acceptable_filetype_set: count_types = 0 for file in folder.files: if file.type.upper() == data_type: count_types += 1 if (count_types != self.cameras) and (count_types > 0): if [count_types, folder.path] not in bad_data: data_ok = False bad_data.append([count_types, folder.path]) for folder in folder.folders: self.checkData(folder=folder, bad_data=bad_data) return bad_data def moveTree(self, sourceRoot, destRoot): if not exists(destRoot): return False ok = True for path, dirs, files in walk(sourceRoot): relPath = relpath(path, sourceRoot) destPath = join(destRoot, relPath) if not exists(destPath): makedirs(destPath) for file in files: destFile = join(destPath, file) if isfile(destFile): print "Skipping existing file: " + join(relPath, file) remove(join(sourceRoot, file)) ok = False continue srcFile = join(path, file) #print "rename", srcFile, destFile rename(srcFile, destFile) for path, dirs, files in walk(sourceRoot, False): if len(files) == 0 and len(dirs) == 0: rmdir(path) return ok def checkFolders(self): folders = self.getFilenamesFolders(self.folder) for folder in folders: bad_data = self.checkData(folder) while bad_data != []: if (len(bad_data)%2 == 0): print "Data problem in two folders..." data_set_1 = bad_data.pop() data_set_2 = bad_data.pop() size_of_datas = data_set_1[0]+data_set_2[0] if size_of_datas == self.cameras: print "Data appears to be fixable..." self.moveTree(data_set_1[1], data_set_2[1]) print "Files moved, problem resolved between: " print data_set_1[1] print data_set_2[1] else: print "Corrupt folders are not aligned... Please manually inspect them." continue self.folder = Folder(self.folder_loc) def run(self): self.client.sendParamFile(self.param_file.param_path) self.client.run() def close(self): self.client.close()
class Runner(object): """ A runner is a utilitary tool that allows to create environment, and run simulations more easily. This specific class as for main purpose to evaluate the performance of a trained :class:`grid2op.Agent`, rather than to train it. Of course, it is possible to adapt it for a specific training mechanisms. Examples of such will be made available in the future. Attributes ---------- envClass: ``type`` The type of the environment used for the game. The class should be given, and **not** an instance (object) of this class. The default is the :class:`grid2op.Environment`. If modified, it should derived from this class. actionClass: ``type`` The type of action that can be performed by the agent / bot / controler. The class should be given, and **not** an instance of this class. This type should derived from :class:`grid2op.Action`. The default is :class:`grid2op.TopologyAction`. observationClass: ``type`` This type represents the class that will be used to build the :class:`grid2op.Observation` visible by the :class:`grid2op.Agent`. As :attr:`Runner.actionClass`, this should be a type, and **not** and instance (object) of this type. This type should derived from :class:`grid2op.Observation`. The default is :class:`grid2op.CompleteObservation`. rewardClass: ``type`` Representes the type used to build the rewards that are given to the :class:`Agent`. As :attr:`Runner.actionClass`, this should be a type, and **not** and instance (object) of this type. This type should derived from :class:`grid2op.Reward`. The default is :class:`grid2op.ConstantReward` that **should not** be used to train or evaluate an agent, but rather as debugging purpose. gridStateclass: ``type`` This types control the mechanisms to read chronics and assign data to the powergrid. Like every "\.*Class" attributes the type should be pass and not an intance (object) of this type. Its default is :class:`grid2op.GridStateFromFile` and it must be a subclass of :class:`grid2op.GridValue`. legalActClass: ``type`` This types control the mechanisms to assess if an :class:`grid2op.Action` is legal. Like every "\.*Class" attributes the type should be pass and not an intance (object) of this type. Its default is :class:`grid2op.AllwaysLegal` and it must be a subclass of :class:`grid2op.LegalAction`. backendClass: ``type`` This types control the backend, *eg.* the software that computes the powerflows. Like every "\.*Class" attributes the type should be pass and not an intance (object) of this type. Its default is :class:`grid2op.PandaPowerBackend` and it must be a subclass of :class:`grid2op.Backend`. agentClass: ``type`` This types control the type of Agent, *eg.* the bot / controler that will take :class:`grid2op.Action` and avoid cascading failures. Like every "\.*Class" attributes the type should be pass and not an intance (object) of this type. Its default is :class:`grid2op.DoNothingAgent` and it must be a subclass of :class:`grid2op.Agent`. logger: A object than can be used to log information, either in a text file, or by printing them to the command prompt. init_grid_path: ``str`` This attributes store the path where the powergrid data are located. If a relative path is given, it will be extended as an absolute path. names_chronics_to_backend: ``dict`` See description of :func:`grid2op.ChronicsHelper.initialize` for more information about this dictionnary parameters_path: ``str``, optional Where to look for the :class:`grid2op.Environment` :class:`grid2op.Parameters`. It defaults to ``None`` which corresponds to using default values. parameters: :class:`grid2op.Parameters` Type of _parameters used. This is an instance (object) of type :class:`grid2op.Parameters` initialized from :attr:`Runner.parameters_path` path_chron: ``str`` Path indicatng where to look for temporal data. chronics_handler: :class:`grid2op.ChronicsHandler` Initialized from :attr:`Runner.gridStateclass` and :attr:`Runner.path_chron` it represents the input data used to generate grid state by the :attr:`Runner.env` backend: :class:`grid2op.Backend` Used to compute the powerflow. This object has the type given by :attr:`Runner.backendClass` env: :class:`grid2op.Environment` Represents the environment which the agent / bot / control must control through action. It is initialized from the :attr:`Runner.envClass` agent: :class:`grid2op.Agent` Represents the agent / bot / controler that takes action performed on a environment (the powergrid) to maximize a certain reward. verbose: ``bool`` If ``True`` then detailed output of each steps are written. gridStateclass_kwargs: ``dict`` Additional keyword arguments used to build the :attr:`Runner.chronics_handler` thermal_limit_a: ``numpy.ndarray`` The thermal limit for the environment (if any). """ def __init__(self, # full path where grid state is located, eg "./data/test_Pandapower/case14.json" init_grid_path: str, path_chron, # path where chronics of injections are stored parameters_path=None, names_chronics_to_backend=None, actionClass=TopologyAction, observationClass=CompleteObservation, rewardClass=FlatReward, legalActClass=AllwaysLegal, envClass=Environment, gridStateclass=GridStateFromFile, # type of chronics to use. For example GridStateFromFile if forecasts are not used, or GridStateFromFileWithForecasts otherwise backendClass=PandaPowerBackend, agentClass=DoNothingAgent, # class used to build the agent agentInstance=None, verbose=False, gridStateclass_kwargs={}, thermal_limit_a=None ): """ Initialize the Runner. Parameters ---------- init_grid_path: ``str`` Madantory, used to initialize :attr:`Runner.init_grid_path`. path_chron: ``str`` Madantory where to look for chronics data, used to initialize :attr:`Runner.path_chron`. parameters_path: ``str`` or ``dict``, optional Used to initialize :attr:`Runner.parameters_path`. If it's a string, this will suppose parameters are located at this path, if it's a dictionary, this will use the parameters converted from this dictionary. names_chronics_to_backend: ``dict``, optional Used to initialize :attr:`Runner.names_chronics_to_backend`. actionClass: ``type``, optional Used to initialize :attr:`Runner.actionClass`. observationClass: ``type``, optional Used to initialize :attr:`Runner.observationClass`. rewardClass: ``type``, optional Used to initialize :attr:`Runner.rewardClass`. Default to :class:`grid2op.ConstantReward` that *should not** be used to train or evaluate an agent, but rather as debugging purpose. legalActClass: ``type``, optional Used to initialize :attr:`Runner.legalActClass`. envClass: ``type``, optional Used to initialize :attr:`Runner.envClass`. gridStateclass: ``type``, optional Used to initialize :attr:`Runner.gridStateclass`. backendClass: ``type``, optional Used to initialize :attr:`Runner.backendClass`. agentClass: ``type``, optional Used to initialize :attr:`Runner.agentClass`. agentInstance: :class:`grid2op.Agent.Agent` Used to initialize the agent. Note that either :attr:`agentClass` or :attr:`agentInstance` is used at the same time. If both ot them are ``None`` or both of them are "not ``None``" it throw an error. verbose: ``bool``, optional Used to initialize :attr:`Runner.verbose`. thermal_limit_a: ``numpy.ndarray`` The thermal limit for the environment (if any). """ if not isinstance(envClass, type): raise Grid2OpException( "Parameter \"envClass\" used to build the Runner should be a type (a class) and not an object " "(an instance of a class). It is currently \"{}\"".format( type(envClass))) if not issubclass(envClass, Environment): raise RuntimeError("Impossible to create a runner without an evnrionment derived from grid2op.Environement" " class. Please modify \"envClass\" parameter.") self.envClass = envClass if not isinstance(actionClass, type): raise Grid2OpException( "Parameter \"actionClass\" used to build the Runner should be a type (a class) and not an object " "(an instance of a class). It is currently \"{}\"".format( type(actionClass))) if not issubclass(actionClass, Action): raise RuntimeError("Impossible to create a runner without an action class derived from grid2op.Action. " "Please modify \"actionClass\" parameter.") self.actionClass = actionClass if not isinstance(observationClass, type): raise Grid2OpException( "Parameter \"observationClass\" used to build the Runner should be a type (a class) and not an object " "(an instance of a class). It is currently \"{}\"".format( type(observationClass))) if not issubclass(observationClass, Observation): raise RuntimeError("Impossible to create a runner without an observation class derived from " "grid2op.Observation. Please modify \"observationClass\" parameter.") self.observationClass = observationClass if not isinstance(rewardClass, type): raise Grid2OpException( "Parameter \"rewardClass\" used to build the Runner should be a type (a class) and not an object " "(an instance of a class). It is currently \"{}\"".format( type(rewardClass))) if not issubclass(rewardClass, Reward): raise RuntimeError("Impossible to create a runner without an observation class derived from " "grid2op.Reward. Please modify \"rewardClass\" parameter.") self.rewardClass = rewardClass if not isinstance(gridStateclass, type): raise Grid2OpException( "Parameter \"gridStateclass\" used to build the Runner should be a type (a class) and not an object " "(an instance of a class). It is currently \"{}\"".format( type(gridStateclass))) if not issubclass(gridStateclass, GridValue): raise RuntimeError("Impossible to create a runner without an chronics class derived from " "grid2op.GridValue. Please modify \"gridStateclass\" parameter.") self.gridStateclass = gridStateclass if not isinstance(legalActClass, type): raise Grid2OpException( "Parameter \"legalActClass\" used to build the Runner should be a type (a class) and not an object " "(an instance of a class). It is currently \"{}\"".format( type(legalActClass))) if not issubclass(legalActClass, LegalAction): raise RuntimeError("Impossible to create a runner without a class defining legal actions derived " "from grid2op.LegalAction. Please modify \"legalActClass\" parameter.") self.legalActClass = legalActClass if not isinstance(backendClass, type): raise Grid2OpException( "Parameter \"legalActClass\" used to build the Runner should be a type (a class) and not an object " "(an instance of a class). It is currently \"{}\"".format( type(backendClass))) if not issubclass(backendClass, Backend): raise RuntimeError("Impossible to create a runner without a backend class derived from grid2op.GridValue. " "Please modify \"backendClass\" parameter.") self.backendClass = backendClass if agentClass is not None: if agentInstance is not None: raise RuntimeError("Impossible to build the backend. Only one of AgentClass or agentInstance can be " "used (both are not None).") if not isinstance(agentClass, type): raise Grid2OpException( "Parameter \"agentClass\" used to build the Runner should be a type (a class) and not an object " "(an instance of a class). It is currently \"{}\"".format( type(agentClass))) if not issubclass(agentClass, Agent): raise RuntimeError("Impossible to create a runner without an agent class derived from grid2op.Agent. " "Please modify \"agentClass\" parameter.") self.agentClass = agentClass self._useclass = True self.agent = None elif agentInstance is not None: if not isinstance(agentInstance, Agent): raise RuntimeError("Impossible to create a runner without an agent class derived from grid2op.Agent. " "Please modify \"agentInstance\" parameter.") self.agentClass = None self._useclass = False self.agent = agentInstance else: raise RuntimeError("Impossible to build the backend. Either AgentClass or agentInstance must be provided " "and both are None.") self.logger = ConsoleLog( DoNothingLog.INFO if verbose else DoNothingLog.ERROR) # store _parameters self.init_grid_path = init_grid_path self.names_chronics_to_backend = names_chronics_to_backend # game _parameters if isinstance(parameters_path, str): self.parameters_path = parameters_path self.parameters = Parameters(parameters_path) elif isinstance(parameters_path, dict): self.parameters = Parameters() self.parameters.init_from_dict(parameters_path) elif parameters_path is None: self.parameters_path = parameters_path self.parameters = Parameters() else: raise RuntimeError("Impossible to build the parameters. The argument \"parameters_path\" should either" "be a string or a dictionary.") # chronics of grid state self.path_chron = path_chron self.gridStateclass_kwargs = gridStateclass_kwargs self.chronics_handler = ChronicsHandler(chronicsClass=self.gridStateclass, path=self.path_chron, **self.gridStateclass_kwargs) # the backend, used to compute powerflows self.backend = self.backendClass() # build the environment self.env = None self.verbose = verbose self.thermal_limit_a = thermal_limit_a def _new_env(self, chronics_handler, backend, parameters): res = self.envClass(init_grid_path=self.init_grid_path, chronics_handler=chronics_handler, backend=backend, parameters=parameters, names_chronics_to_backend=self.names_chronics_to_backend, actionClass=self.actionClass, observationClass=self.observationClass, rewardClass=self.rewardClass, legalActClass=self.legalActClass) if self.thermal_limit_a is not None: res.set_thermal_limit(self.thermal_limit_a) if self._useclass: agent = self.agentClass(res.helper_action_player) else: agent = self.agent return res, agent def init_env(self): """ Function used to initialized the environment and the agent. It is called by :func:`Runner.reset`. Returns ------- ``None`` """ self.env, self.agent = self._new_env( self.chronics_handler, self.backend, self.parameters) def reset(self): """ Used to reset an environment. This method is called at the beginning of each new episode. If the environment is not initialized, then it initializes it with :func:`Runner.make_env`. Returns ------- ``None`` """ if self.env is None: self.init_env() else: self.env.reset() def run_one_episode(self, indx=0, path_save=None): """ Function used to run one episode of the :attr:`Runner.agent` and see how it performs in the :attr:`Runner.env`. Parameters ---------- indx: ``int`` The number of episode previously run path_save: ``str``, optional Path where to save the data. See the description of :mod:`grid2op.Runner` for the structure of the saved file. Returns ------- cum_reward: ``float`` The cumulative reward obtained by the agent during this episode time_step: ``int`` The number of timesteps that have been played before the end of the episode (because of a "game over" or because there were no more data) """ self.reset() res = self._run_one_episode(self.env, self.agent, self.logger, indx, path_save) return res @staticmethod def _run_one_episode(env, agent, logger, indx, path_save=None): done = False time_step = int(0) dict_ = {} time_act = 0. cum_reward = 0. # reset the environment env.chronics_handler.tell_id(indx-1) # the "-1" above is because the environment will be reset. So it will increase id of 1. obs = env.reset() # compute the size and everything if it needs to be stored nb_timestep_max = env.chronics_handler.max_timestep() efficient_storing = nb_timestep_max > 0 nb_timestep_max = max(nb_timestep_max, 0) if path_save is None: # i don't store anything on drive, so i don't need to store anything on memory nb_timestep_max = 0 times = np.full(nb_timestep_max, fill_value=np.NaN, dtype=np.float) rewards = np.full(nb_timestep_max, fill_value=np.NaN, dtype=np.float) actions = np.full((nb_timestep_max, env.action_space.n), fill_value=np.NaN, dtype=np.float) env_actions = np.full( (nb_timestep_max, env.helper_action_env.n), fill_value=np.NaN, dtype=np.float) observations = np.full( (nb_timestep_max+1, env.observation_space.n), fill_value=np.NaN, dtype=np.float) disc_lines = np.full( (nb_timestep_max, env.backend.n_line), fill_value=np.NaN, dtype=np.bool) disc_lines_templ = np.full( (1, env.backend.n_line), fill_value=False, dtype=np.bool) if path_save is not None: # store observation at timestep 0 if efficient_storing: observations[time_step, :] = obs.to_vect() else: observations = np.concatenate((observations, obs.to_vect())) episode = EpisodeData(actions=actions, env_actions=env_actions, observations=observations, rewards=rewards, disc_lines=disc_lines, times=times, observation_space=env.observation_space, action_space=env.action_space, helper_action_env=env.helper_action_env, path_save=path_save, disc_lines_templ=disc_lines_templ, logger=logger, name=env.chronics_handler.get_name()) episode.set_parameters(env) beg_ = time.time() reward = env.reward_range[0] done = False while not done: beg__ = time.time() act = agent.act(obs, reward, done) end__ = time.time() time_act += end__ - beg__ obs, reward, done, info = env.step(act) # should load the first time stamp cum_reward += reward time_step += 1 episode.incr_store(efficient_storing, time_step, end__ - beg__, reward, env.env_modification, act, obs, info) end_ = time.time() episode.set_meta(env, time_step, cum_reward) li_text = ["Env: {:.2f}s", "\t - apply act {:.2f}s", "\t - run pf: {:.2f}s", "\t - env update + observation: {:.2f}s", "Agent: {:.2f}s", "Total time: {:.2f}s", "Cumulative reward: {:1f}"] msg_ = "\n".join(li_text) logger.info(msg_.format( env._time_apply_act + env._time_powerflow + env._time_extract_obs, env._time_apply_act, env._time_powerflow, env._time_extract_obs, time_act, end_ - beg_, cum_reward)) episode.set_episode_times(env, time_act, beg_, end_) episode.to_disk() name_chron = env.chronics_handler.get_name() return name_chron, cum_reward, int(time_step) def run_sequential(self, nb_episode, path_save=None): """ This method is called to see how well an agent performed on a sequence of episode. Parameters ---------- nb_episode: ``int`` Number of episode to play. path_save: ``str``, optional If not None, it specifies where to store the data. See the description of this module :mod:`Runner` for more information Returns ------- res: ``list`` List of tuple. Each tuple having 5 elements: - "id_chron" unique identifier of the episode - "name_chron" name of chronics - "cum_reward" the cumulative reward obtained by the :attr:`Runner.Agent` on this episode i - "nb_time_step": the number of time steps played in this episode. - "max_ts" : the maximum number of time steps of the chronics """ res = [(None, None, None, None, None) for _ in range(nb_episode)] for i in range(nb_episode): name_chron, cum_reward, nb_time_step = self.run_one_episode(path_save=path_save, indx=i) id_chron = self.chronics_handler.get_id() max_ts = self.chronics_handler.max_timestep() res[i] = (id_chron, name_chron, cum_reward, nb_time_step, max_ts) return res @staticmethod def _one_process_parrallel(runner, episode_this_process, process_id, path_save=None): chronics_handler = ChronicsHandler(chronicsClass=runner.gridStateclass, path=runner.path_chron, **runner.gridStateclass_kwargs) parameters = copy.deepcopy(runner.parameters) backend = runner.backendClass() nb_episode_this_process = len(episode_this_process) res = [(None, None, None) for _ in range(nb_episode_this_process)] for i, p_id in enumerate(episode_this_process): env, agent = runner._new_env(chronics_handler=chronics_handler, backend=backend, parameters=parameters) name_chron, cum_reward, nb_time_step = Runner._run_one_episode( env, agent, runner.logger, p_id, path_save) id_chron = chronics_handler.get_id() max_ts = chronics_handler.max_timestep() res[i] = (id_chron, name_chron, cum_reward, nb_time_step, max_ts) return res def run_parrallel(self, nb_episode, nb_process=1, path_save=None): """ This method will run in parrallel, independantly the nb_episode over nb_process. Note that it restarts completely the :attr:`Runner.backend` and :attr:`Runner.env` if the computation is actually performed with more than 1 cores (nb_process > 1) It uses the python multiprocess, and especially the :class:`multiprocess.Pool` to perform the computations. This implies that all runs are completely independant (they happen in different process) and that the memory consumption can be big. Tests may be recommended if the amount of RAM is low. It has the same return type as the :func:`Runner.run_sequential`. Parameters ---------- nb_episode: ``int`` Number of episode to simulate nb_process: ``int``, optional Number of process used to play the nb_episode. Default to 1. path_save: ``str``, optional If not None, it specifies where to store the data. See the description of this module :mod:`Runner` for more information Returns ------- res: ``list`` List of tuple. Each tuple having 3 elements: - "i" unique identifier of the episode (compared to :func:`Runner.run_sequential`, the elements of the returned list are not necessarily sorted by this value) - "cum_reward" the cumulative reward obtained by the :attr:`Runner.Agent` on this episode i - "nb_time_step": the number of time steps played in this episode. - "max_ts" : the maximum number of time steps of the chronics """ if nb_process <= 0: raise RuntimeError( "Runner: you need at least 1 process to run episodes") if nb_process == 1: warnings.warn( "Runner.run_parrallel: number of process set to 1. Failing back into sequential mod.") return [self.run_sequential(nb_episode, path_save=path_save)] else: if self.env is not None: self.env.close() self.env = None self.backend = self.backendClass() nb_process = int(nb_process) process_ids = [[] for i in range(nb_process)] for i in range(nb_episode): process_ids[i % nb_process].append(i) res = [] with Pool(nb_process) as p: tmp = p.starmap(Runner._one_process_parrallel, [(self, pn, i, path_save) for i, pn in enumerate(process_ids)]) for el in tmp: res += el return res def run(self, nb_episode, nb_process=1, path_save=None): """ Main method of the :class:`Runner` class. It will either call :func:`Runner.run_sequential` if "nb_process" is 1 or :func:`Runner.run_parrallel` if nb_process >= 2. Parameters ---------- nb_episode: ``int`` Number of episode to simulate nb_process: ``int``, optional Number of process used to play the nb_episode. Default to 1. path_save: ``str``, optional If not None, it specifies where to store the data. See the description of this module :mod:`Runner` for more information Returns ------- res: ``list`` List of tuple. Each tuple having 3 elements: - "i" unique identifier of the episode (compared to :func:`Runner.run_sequential`, the elements of the returned list are not necessarily sorted by this value) - "cum_reward" the cumulative reward obtained by the :attr:`Runner.Agent` on this episode i - "nb_time_step": the number of time steps played in this episode. """ if nb_episode < 0: raise RuntimeError( "Impossible to run a negative number of scenarios.") if nb_episode == 0: res = [] else: if nb_process <= 0: raise RuntimeError( "Impossible to run using less than 1 process.") if nb_process == 1: self.logger.info("Sequential runner used.") res = self.run_sequential(nb_episode, path_save=path_save) else: self.logger.info("Parrallel runner used.") res = self.run_parrallel( nb_episode, nb_process=nb_process, path_save=path_save) return res
print("Main_Both2") print('Data loading...') t1,t_init = time(),time() args.device = device args.date_time = datetime.datetime.now() print(args.date_time) if args.method.lower() in ['tenet']: if args.knn_graph == 'True': dataset = EmbedDataset(args) ##change according to method else: dataset = TenetDataset(args) ##change according to method else: dataset = Dataset(args) params = Parameters(args,dataset) print("""Load data done [%.1f s]. #user:%d, #list:%d, #item:%d, #train:%d, #valid:%d, #test:%d"""% (time() - t1, params.num_user, params.num_list, params.num_item,params.num_train_instances,params.num_valid_instances,params.num_test_instances)) args.args_str = params.get_args_to_string() t1 = time() print("args str: ",args.args_str) print("leng from list_items_list: ",len(utils.get_value_lists_as_list(params.list_items_dct))) print("leng from trainArrTriplets: ", len((params.trainArrTriplets[0]))) print("non-zero entries in train_matrix: ", params.train_matrix.nnz) # model-loss-optimizer defn ======================================================================= models = Models(params,device=device) model = models.get_model()
def setUp(self): """ The case file is a representation of the case14 as found in the ieee14 powergrid. :return: """ self.tolvect = 1e-2 self.tol_one = 1e-5 self.game_rules = GameRules() # pdb.set_trace() self.rewardClass = L2RPNReward self.reward_helper = self.rewardClass() self.obsClass = CompleteObservation self.parameters = Parameters() # powergrid self.backend = PandaPowerBackend() self.path_matpower = PATH_DATA_TEST_PP self.case_file = "test_case14.json" # chronics self.path_chron = os.path.join(PATH_CHRONICS, "chronics_with_forecast") self.chronics_handler = ChronicsHandler( chronicsClass=GridStateFromFileWithForecasts, path=self.path_chron) self.tolvect = 1e-2 self.tol_one = 1e-5 self.id_chron_to_back_load = np.array( [0, 1, 10, 2, 3, 4, 5, 6, 7, 8, 9]) # force the verbose backend self.backend.detailed_infos_for_cascading_failures = True self.names_chronics_to_backend = { "loads": { "2_C-10.61": 'load_1_0', "3_C151.15": 'load_2_1', "14_C63.6": 'load_13_2', "4_C-9.47": 'load_3_3', "5_C201.84": 'load_4_4', "6_C-6.27": 'load_5_5', "9_C130.49": 'load_8_6', "10_C228.66": 'load_9_7', "11_C-138.89": 'load_10_8', "12_C-27.88": 'load_11_9', "13_C-13.33": 'load_12_10' }, "lines": { '1_2_1': '0_1_0', '1_5_2': '0_4_1', '9_10_16': '8_9_2', '9_14_17': '8_13_3', '10_11_18': '9_10_4', '12_13_19': '11_12_5', '13_14_20': '12_13_6', '2_3_3': '1_2_7', '2_4_4': '1_3_8', '2_5_5': '1_4_9', '3_4_6': '2_3_10', '4_5_7': '3_4_11', '6_11_11': '5_10_12', '6_12_12': '5_11_13', '6_13_13': '5_12_14', '4_7_8': '3_6_15', '4_9_9': '3_8_16', '5_6_10': '4_5_17', '7_8_14': '6_7_18', '7_9_15': '6_8_19' }, "prods": { "1_G137.1": 'gen_0_4', "3_G36.31": "gen_2_1", "6_G63.29": "gen_5_2", "2_G-56.47": "gen_1_0", "8_G40.43": "gen_7_3" }, } # _parameters for the environment self.env_params = Parameters() self.env = Environment( init_grid_path=os.path.join(self.path_matpower, self.case_file), backend=self.backend, chronics_handler=self.chronics_handler, parameters=self.env_params, names_chronics_to_backend=self.names_chronics_to_backend, rewardClass=self.rewardClass) self.dict_ = { 'name_gen': ['gen_1_0', 'gen_2_1', 'gen_5_2', 'gen_7_3', 'gen_0_4'], 'name_load': [ 'load_1_0', 'load_2_1', 'load_13_2', 'load_3_3', 'load_4_4', 'load_5_5', 'load_8_6', 'load_9_7', 'load_10_8', 'load_11_9', 'load_12_10' ], 'name_line': [ '0_1_0', '0_4_1', '8_9_2', '8_13_3', '9_10_4', '11_12_5', '12_13_6', '1_2_7', '1_3_8', '1_4_9', '2_3_10', '3_4_11', '5_10_12', '5_11_13', '5_12_14', '3_6_15', '3_8_16', '4_5_17', '6_7_18', '6_8_19' ], 'name_sub': [ 'sub_0', 'sub_1', 'sub_10', 'sub_11', 'sub_12', 'sub_13', 'sub_2', 'sub_3', 'sub_4', 'sub_5', 'sub_6', 'sub_7', 'sub_8', 'sub_9' ], 'sub_info': [3, 6, 4, 6, 5, 6, 3, 2, 5, 3, 3, 3, 4, 3], 'load_to_subid': [1, 2, 13, 3, 4, 5, 8, 9, 10, 11, 12], 'gen_to_subid': [1, 2, 5, 7, 0], 'line_or_to_subid': [0, 0, 8, 8, 9, 11, 12, 1, 1, 1, 2, 3, 5, 5, 5, 3, 3, 4, 6, 6], 'line_ex_to_subid': [ 1, 4, 9, 13, 10, 12, 13, 2, 3, 4, 3, 4, 10, 11, 12, 6, 8, 5, 7, 8 ], 'load_to_sub_pos': [5, 3, 2, 5, 4, 5, 4, 2, 2, 2, 3], 'gen_to_sub_pos': [4, 2, 4, 1, 2], 'line_or_to_sub_pos': [0, 1, 0, 1, 1, 0, 1, 1, 2, 3, 1, 2, 0, 1, 2, 3, 4, 3, 1, 2], 'line_ex_to_sub_pos': [0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 2, 1, 1, 2, 0, 2, 3, 0, 3], 'load_pos_topo_vect': [8, 12, 55, 18, 23, 29, 39, 42, 45, 48, 52], 'gen_pos_topo_vect': [7, 11, 28, 34, 2], 'line_or_pos_topo_vect': [ 0, 1, 35, 36, 41, 46, 50, 4, 5, 6, 10, 15, 24, 25, 26, 16, 17, 22, 31, 32 ], 'line_ex_pos_topo_vect': [ 3, 19, 40, 53, 43, 49, 54, 9, 13, 20, 14, 21, 44, 47, 51, 30, 37, 27, 33, 38 ], 'gen_type': ['solar', 'nuclear', 'nuclear', 'nuclear', 'thermal'], 'gen_pmin': [0.0, 0.0, 0.0, 0.0, 0.0], 'gen_pmax': [50.0, 80.0, 120.0, 120.0, 190.0], 'gen_redispatchable': [False, True, True, True, True], 'gen_max_ramp_up': [50.0, 80.0, 120.0, 120.0, 190.0], 'gen_max_ramp_down': [50.0, 80.0, 120.0, 120.0, 190.0], 'gen_min_uptime': [0, 0, 0, 0, 0], 'gen_min_downtime': [0, 1, 0, 0, 0], 'gen_cost_per_MW': [0.0, 0.0, 0.0, 0.0, 10.0], 'gen_startup_cost': [0.0, 0.0, 0.0, 0.0, 0.0], 'gen_shutdown_cost': [0.0, 0.0, 0.0, 0.0, 0.0], 'subtype': 'Observation.CompleteObservation' } self.dtypes = np.array([ dtype('int64'), dtype('int64'), dtype('int64'), dtype('int64'), dtype('int64'), dtype('int64'), dtype('float64'), dtype('float64'), dtype('float64'), dtype('float64'), dtype('float64'), dtype('float64'), dtype('float64'), dtype('float64'), dtype('float64'), dtype('float64'), dtype('float64'), dtype('float64'), dtype('float64'), dtype('float64'), dtype('float64'), dtype('bool'), dtype('int64'), dtype('int64'), dtype('int64'), dtype('int64'), dtype('int64'), dtype('int64'), dtype('int64'), dtype('int64'), dtype('float64'), dtype('float64') ], dtype=object) self.shapes = np.array([ 1, 1, 1, 1, 1, 1, 5, 5, 5, 11, 11, 11, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 56, 20, 20, 14, 20, 20, 20, 5, 5 ]) self.size_obs = 454
) parser.add_argument( '--category_names', type=str, default='cat_to_name.json', help='file for flower name dictionary | (default = cat_to_name.json)') parser.add_argument('--gpu', type=str, default='cuda', help='cuda or cpu | (default = cuda)') arg_in = parser.parse_args() parameters = Parameters({ "imagefilepath": arg_in.imagefilepath, "checkpoint": arg_in.checkpoint, "top_k": arg_in.top_k, "category_names": arg_in.category_names, "gpu": arg_in.gpu }) # Function that greet user greetUser() # Function that checks command line arguments using in_arg parameters.displayParameters('predict') # Ask for image filepath to make predictions if not parameters.in_arg['imagefilepath']: parameters.in_arg['imagefilepath'] = command( "Enter the image path, to predict its flower class \n e.g. flowers/test/101/image_07949.jpg \n Your input:" ) print('Do you want to modify input values?')
import sys import time from datetime import datetime import numpy as np import tensorflow as tf from sklearn.metrics import confusion_matrix from utils.Dataset_hdf5 import DatasetHDF5 from Parameters import Parameters # ===============get basic folder===================== BASE_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.append(BASE_DIR) sys.path.append(os.path.join(BASE_DIR, 'models')) para = Parameters(evaluation=True) if para.gpu: gpus = tf.config.experimental.list_physical_devices('GPU') for gpu in gpus: tf.config.experimental.set_memory_growth(gpu, True) else: os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # set parameters BN_INIT_DECAY = 0.5 BN_DECAY_DECAY_RATE = 0.5 BN_DECAY_DECAY_STEP = float(para.decay_step) BN_DECAY_CLIP = 0.99 MODEL = importlib.import_module(para.model) # import network module LOG_MODEL = para.logmodelDir EVAL = para.evallog
def main(argv): parameters = Parameters(argv) hostname = parameters.getHostname() port = parameters.getPort() dbname = parameters.getDBName() language_1, language_2 = parameters.getLanguage() collection = parameters.getCollection() input_folder = parameters.getInputFolder() type_corpus = parameters.getType() print 'Using parameters of configuration: ' print '- Host : ',hostname print '- Port : ',port print '- Coll : ',collection print '- DBase: ',dbname print '- Input: ',input_folder database = Mongo(hostname, dbname, collection) try: root, dirs, files = os.walk(input_folder).next()[:3] except IOError: print 'ERROR: It was not possible to open the '+input_folder+' folder' sys.exit(2) name_folder = (input_folder.split('/'))[-2] dic_files = {} for corpus_file in files: print 'Working on file: '+corpus_file if not re.match('~$', corpus_file): id_file = corpus_file[0:-7] language = corpus_file[-6:-4] if not dic_files.has_key(id_file): dic_files[id_file] = {'language_1': language} else: dic_files[id_file]['language_2'] = language counter = 1 for filename in dic_files: language_1 = dic_files[filename]['language_1'] language_2 = dic_files[filename]['language_2'] id_file_1 = name_folder+'_'+filename+'_'+language_1 id_file_2 = name_folder+'_'+filename+'_'+language_2 try: file_1 = codecs.open(input_folder+''+filename+'_'+language_1+'.snt', 'r', 'utf-8') except IOError: print 'ERROR: System cannot open the '+input_folder+''+filename+'_'+language_1+'.snt file' sys.exit(2) try: file_2 = codecs.open(input_folder+''+filename+'_'+language_2+'.snt', 'r', 'utf-8') except IOError: print 'ERROR: System cannot open the '+input_folder+''+filename+'_'+language_2+'.snt file' sys.exit(2) content_1 = '' for line in file_1: #if line.strip(): content_1 += line content_2 = '' for line in file_2: #if line.strip(): content_2 += line if database.exists(language_1, id_file_1): if not database.exists(language_2, id_file_2): database.insertInExisting(language_1, id_file_1, language_2, id_file_2, content_2) else: if database.exists(language_2, id_file_2): database.insertInExisting(language_2, id_file_2, language_1, id_file_1, content_1) else: database.insertNewData(language_1, id_file_1, content_1, language_2, id_file_2, content_2, type_corpus, counter) counter += 1
class Network: dirs = [] listNames = [] parameters = Parameters() @classmethod def start(cls): cls.dirs = os.listdir( os.path.abspath(os.getcwd()) + "/" + cls.parameters.folder) X, Y = cls.get_data() print(len(Y), len(X)) model = cls.creatModel(X) print(K.eval(model.optimizer.lr)) history = model.fit([X[:, 0], X[:, 1]], Y, batch_size=64, epochs=cls.parameters.num_epochs, verbose=2, validation_split=.25) print(K.eval(model.optimizer.lr)) cls.graph(history) return model @classmethod def test(cls, img, model): print("Тест...") cls.dirs = os.listdir( os.path.abspath(os.getcwd()) + "/" + cls.parameters.folder) matrImage = cls.readDataTest() masImg = np.zeros([1, img.shape[0], img.shape[1], 1]) masImg[0, :, :, 0] = img masImg = masImg / 255 list = [] # распознаём изображение i = 0 while i < cls.parameters.numMan * cls.parameters.sample: #img = matrImage[i] * 255 #cv2.imwrite("image/" + str(i) + ".jpg", img[0,:,:,:]) res = model.predict([masImg, matrImage[i]]) list.append(res) #print(res, str(i)) i += 1 minZn = min(list) if minZn < 0.4: p = list.index(minZn) #print(minZn) p += 1 print(cls.dirs[math.ceil(p / cls.parameters.sample) - 1]) else: print("Изображение не идентифицировано.") @classmethod def creatModel(cls, mas): img_a = Input(shape=mas.shape[2:]) img_b = Input(shape=mas.shape[2:]) base_network = cls.build_base_network(mas.shape[2:]) # Получаем вектора признаков feat_vecs_a = base_network(img_a) feat_vecs_b = base_network(img_b) distance = Lambda(cls.euclidean_distance)([feat_vecs_a, feat_vecs_b]) rms = RMSprop(learning_rate=cls.parameters.learning_rate) model = Model([img_a, img_b], distance) model.compile(loss=cls.contrastive_loss, optimizer=rms, metrics=[cls.accuracy]) #opt = tensorflow.keras.optimizers.Adam(lr=0.001) #model.compile(optimizer=opt, loss='contrastive_crossentropy', metrics=['accuracy']) return model @classmethod def get_data(cls): total_sample_size = cls.parameters.total_sample_size width = cls.parameters.width height = cls.parameters.height sample = cls.parameters.sample numMan = cls.parameters.numMan nameFolder = cls.parameters.folder img = cv2.imread(nameFolder + "\\" + cls.dirs[0] + "\\" + str(1) + '.' + cls.parameters.typeel) img = cv2.resize(img, (width, height), interpolation=cv2.INTER_AREA) image = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) dim1 = image.shape[0] dim2 = image.shape[1] count = 0 x_geuine_pair = np.zeros([total_sample_size, 2, dim1, dim2, 1]) y_genuine = np.zeros([total_sample_size, 1]) for i in range(numMan): print(cls.dirs[i]) #for j in range(int((numMan * (numMan - 1)) / 2)): for j in range(int(total_sample_size / numMan)): ind1 = 0 ind2 = 0 # read images from same directory (genuine pair) while ind1 == ind2: ind1 = np.random.randint(sample) ind2 = np.random.randint(sample) # read the two images img1 = cv2.imread(nameFolder + "\\" + cls.dirs[i] + "\\" + str(ind1 + 1) + "." + cls.parameters.typeel) img2 = cv2.imread(nameFolder + "\\" + cls.dirs[i] + "\\" + str(ind2 + 1) + "." + cls.parameters.typeel) img1 = cv2.resize(img1, (width, height), interpolation=cv2.INTER_AREA) img2 = cv2.resize(img2, (width, height), interpolation=cv2.INTER_AREA) # to gray img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY) img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY) # store the images to the initialized numpy array x_geuine_pair[count, 0, :, :, 0] = img1 x_geuine_pair[count, 1, :, :, 0] = img2 # as we are drawing images from the same directory we assign label as 1. (genuine pair) y_genuine[count] = 1 count += 1 count = 0 x_imposite_pair = np.zeros([total_sample_size, 2, dim1, dim2, 1]) y_imposite = np.zeros([total_sample_size, 1]) #for i in range(int((numMan * (numMan - 1)) / 2)): for i in range(int(total_sample_size / sample)): for j in range(sample): ind1 = 0 ind2 = 0 # read images from different directory (imposite pair) while ind1 == ind2: ind1 = np.random.randint(numMan) ind2 = np.random.randint(numMan) img1 = cv2.imread(nameFolder + "\\" + cls.dirs[ind1] + "\\" + str(j + 1) + "." + cls.parameters.typeel) img2 = cv2.imread(nameFolder + "\\" + cls.dirs[ind2] + "\\" + str(j + 1) + "." + cls.parameters.typeel) img1 = cv2.resize(img1, (width, height), interpolation=cv2.INTER_AREA) img2 = cv2.resize(img2, (width, height), interpolation=cv2.INTER_AREA) # to gray img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY) img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY) x_imposite_pair[count, 0, :, :, 0] = img1 x_imposite_pair[count, 1, :, :, 0] = img2 # as we are drawing images from the different directory we assign label as 0. (imposite pair) y_imposite[count] = 0 count += 1 # now, concatenate, genuine pairs and imposite pair to get the whole data X = np.concatenate([x_geuine_pair, x_imposite_pair], axis=0) / 255 Y = np.concatenate([y_genuine, y_imposite], axis=0) return X, Y @classmethod def build_base_network(cls, input_shape): """ filters - кол-во выходных фильтров kernel_size - ширина и высота ядра двумерной свертки pool_size - размер окна пулинга strides - шаг сдвига окна activation - функция активации """ seq = Sequential() nb_filter = [6, 12] kernel_size = 3 seq.add( Conv2D(filters=nb_filter[0], kernel_size=(kernel_size, kernel_size), padding='valid', input_shape=input_shape, activation='relu')) seq.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) seq.add(Dropout(0.25)) seq.add( Conv2D(filters=nb_filter[1], kernel_size=(kernel_size, kernel_size), padding='valid', activation='relu')) seq.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) seq.add(Dropout(0.25)) seq.add(Flatten()) seq.add(Dense(100, activation='softmax')) return seq @classmethod def continueTraining(cls, model): cls.dirs = os.listdir( os.path.abspath(os.getcwd()) + "/" + cls.parameters.folder) newX, newY = cls.get_data() history = model.fit([newX[:, 0], newX[:, 1]], newY, validation_split=.25, batch_size=64, epochs=cls.parameters.num_epochs, verbose=2) cls.graph(history) return model @classmethod def euclidean_distance(cls, vects): x, y = vects sum_square = K.sum(K.square(x - y), axis=1, keepdims=True) return K.sqrt(K.maximum(sum_square, K.epsilon())) @classmethod def contrastive_loss(cls, y_true, y_pred): margin = 1 square_pred = K.square(y_pred) margin_square = K.square(K.maximum(margin - y_pred, 0)) return K.mean(y_true * square_pred + (1 - y_true) * margin_square) @classmethod def compute_accuracy(cls, predictions, labels): pred = labels.ravel() < 0.5 return np.mean(pred == predictions) @classmethod def accuracy(cls, y_true, y_pred): return K.mean(K.equal(y_true, K.cast(y_pred < 0.5, y_true.dtype))) @classmethod def graph(cls, history): plt.subplot(2, 2, 1) plt.plot(history.history['accuracy']) plt.title('Train accuracy') plt.ylabel('Точность') plt.xlabel('Эпохи') plt.legend(['Обучающая'], loc='upper left') plt.subplot(2, 2, 2) plt.plot(history.history['loss']) plt.title('Train loss') plt.ylabel('Потеря') plt.xlabel('Эпохи') plt.legend(['Обучающая'], loc='upper left') plt.subplot(2, 2, 3) plt.plot(history.history['val_accuracy']) plt.title('Val accuracy') plt.ylabel('Точность') plt.xlabel('Эпохи') plt.legend(['Обучающая'], loc='upper left') plt.subplot(2, 2, 4) plt.plot(history.history['val_loss']) plt.title('Val loss') plt.ylabel('Потеря') plt.xlabel('Эпохи') plt.legend(['Обучающая'], loc='upper left') plt.show() @classmethod def modelSave(cls, model): #model.save_weights('Models/model_w.h5') model.save('Models/model_w.h5') @classmethod def modelLoad(cls, fl): mas = np.zeros([ cls.parameters.total_sample_size * 2, 2, cls.parameters.height, cls.parameters.width, 1 ]) new_model = cls.creatModel(mas) new_model.load_weights(fl) return new_model @classmethod def readDataTest(cls): count = 0 sample = cls.parameters.sample numMan = cls.parameters.numMan width = cls.parameters.width height = cls.parameters.height matrImage = np.zeros([sample * numMan, 1, height, width, 1]) for i in range(0, numMan): for j in range(0, sample): img = cv2.imread(cls.parameters.folder + "\\" + cls.dirs[i] + "\\" + str(j + 1) + '.' + cls.parameters.typeel) img = cv2.resize(img, (width, height), interpolation=cv2.INTER_AREA) img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) matrImage[count, 0, :, :, 0] = img count += 1 matrImage = matrImage / 255 return matrImage
import jieba from Parameters import Parameters import numpy as np from random import randint from random import shuffle import gensim from gensim.models import KeyedVectors from sklearn import feature_extraction from sklearn.feature_extraction.text import TfidfTransformer from sklearn.feature_extraction.text import CountVectorizer from gensim.models import word2vec pm = Parameters() # model = KeyedVectors.load_word2vec_format('model/sgns.weibo.word') model = gensim.models.Word2Vec.load('model/Word60.model') '''获取停用词表''' def makeStopWord(): with open(pm.stop_word, 'r', encoding='utf-8') as f: lines = f.read() words = jieba.lcut(lines, cut_all=False) stopWord = [] for word in words: stopWord.append(word) return stopWord '''所有词语转化为向量'''
def parameters(): from Parameters import Parameters return Parameters()
def __init__(self, # full path where grid state is located, eg "./data/test_Pandapower/case14.json" init_grid_path: str, path_chron, # path where chronics of injections are stored parameters_path=None, names_chronics_to_backend=None, actionClass=TopologyAction, observationClass=CompleteObservation, rewardClass=FlatReward, legalActClass=AllwaysLegal, envClass=Environment, gridStateclass=GridStateFromFile, # type of chronics to use. For example GridStateFromFile if forecasts are not used, or GridStateFromFileWithForecasts otherwise backendClass=PandaPowerBackend, agentClass=DoNothingAgent, # class used to build the agent agentInstance=None, verbose=False, gridStateclass_kwargs={}, thermal_limit_a=None ): """ Initialize the Runner. Parameters ---------- init_grid_path: ``str`` Madantory, used to initialize :attr:`Runner.init_grid_path`. path_chron: ``str`` Madantory where to look for chronics data, used to initialize :attr:`Runner.path_chron`. parameters_path: ``str`` or ``dict``, optional Used to initialize :attr:`Runner.parameters_path`. If it's a string, this will suppose parameters are located at this path, if it's a dictionary, this will use the parameters converted from this dictionary. names_chronics_to_backend: ``dict``, optional Used to initialize :attr:`Runner.names_chronics_to_backend`. actionClass: ``type``, optional Used to initialize :attr:`Runner.actionClass`. observationClass: ``type``, optional Used to initialize :attr:`Runner.observationClass`. rewardClass: ``type``, optional Used to initialize :attr:`Runner.rewardClass`. Default to :class:`grid2op.ConstantReward` that *should not** be used to train or evaluate an agent, but rather as debugging purpose. legalActClass: ``type``, optional Used to initialize :attr:`Runner.legalActClass`. envClass: ``type``, optional Used to initialize :attr:`Runner.envClass`. gridStateclass: ``type``, optional Used to initialize :attr:`Runner.gridStateclass`. backendClass: ``type``, optional Used to initialize :attr:`Runner.backendClass`. agentClass: ``type``, optional Used to initialize :attr:`Runner.agentClass`. agentInstance: :class:`grid2op.Agent.Agent` Used to initialize the agent. Note that either :attr:`agentClass` or :attr:`agentInstance` is used at the same time. If both ot them are ``None`` or both of them are "not ``None``" it throw an error. verbose: ``bool``, optional Used to initialize :attr:`Runner.verbose`. thermal_limit_a: ``numpy.ndarray`` The thermal limit for the environment (if any). """ if not isinstance(envClass, type): raise Grid2OpException( "Parameter \"envClass\" used to build the Runner should be a type (a class) and not an object " "(an instance of a class). It is currently \"{}\"".format( type(envClass))) if not issubclass(envClass, Environment): raise RuntimeError("Impossible to create a runner without an evnrionment derived from grid2op.Environement" " class. Please modify \"envClass\" parameter.") self.envClass = envClass if not isinstance(actionClass, type): raise Grid2OpException( "Parameter \"actionClass\" used to build the Runner should be a type (a class) and not an object " "(an instance of a class). It is currently \"{}\"".format( type(actionClass))) if not issubclass(actionClass, Action): raise RuntimeError("Impossible to create a runner without an action class derived from grid2op.Action. " "Please modify \"actionClass\" parameter.") self.actionClass = actionClass if not isinstance(observationClass, type): raise Grid2OpException( "Parameter \"observationClass\" used to build the Runner should be a type (a class) and not an object " "(an instance of a class). It is currently \"{}\"".format( type(observationClass))) if not issubclass(observationClass, Observation): raise RuntimeError("Impossible to create a runner without an observation class derived from " "grid2op.Observation. Please modify \"observationClass\" parameter.") self.observationClass = observationClass if not isinstance(rewardClass, type): raise Grid2OpException( "Parameter \"rewardClass\" used to build the Runner should be a type (a class) and not an object " "(an instance of a class). It is currently \"{}\"".format( type(rewardClass))) if not issubclass(rewardClass, Reward): raise RuntimeError("Impossible to create a runner without an observation class derived from " "grid2op.Reward. Please modify \"rewardClass\" parameter.") self.rewardClass = rewardClass if not isinstance(gridStateclass, type): raise Grid2OpException( "Parameter \"gridStateclass\" used to build the Runner should be a type (a class) and not an object " "(an instance of a class). It is currently \"{}\"".format( type(gridStateclass))) if not issubclass(gridStateclass, GridValue): raise RuntimeError("Impossible to create a runner without an chronics class derived from " "grid2op.GridValue. Please modify \"gridStateclass\" parameter.") self.gridStateclass = gridStateclass if not isinstance(legalActClass, type): raise Grid2OpException( "Parameter \"legalActClass\" used to build the Runner should be a type (a class) and not an object " "(an instance of a class). It is currently \"{}\"".format( type(legalActClass))) if not issubclass(legalActClass, LegalAction): raise RuntimeError("Impossible to create a runner without a class defining legal actions derived " "from grid2op.LegalAction. Please modify \"legalActClass\" parameter.") self.legalActClass = legalActClass if not isinstance(backendClass, type): raise Grid2OpException( "Parameter \"legalActClass\" used to build the Runner should be a type (a class) and not an object " "(an instance of a class). It is currently \"{}\"".format( type(backendClass))) if not issubclass(backendClass, Backend): raise RuntimeError("Impossible to create a runner without a backend class derived from grid2op.GridValue. " "Please modify \"backendClass\" parameter.") self.backendClass = backendClass if agentClass is not None: if agentInstance is not None: raise RuntimeError("Impossible to build the backend. Only one of AgentClass or agentInstance can be " "used (both are not None).") if not isinstance(agentClass, type): raise Grid2OpException( "Parameter \"agentClass\" used to build the Runner should be a type (a class) and not an object " "(an instance of a class). It is currently \"{}\"".format( type(agentClass))) if not issubclass(agentClass, Agent): raise RuntimeError("Impossible to create a runner without an agent class derived from grid2op.Agent. " "Please modify \"agentClass\" parameter.") self.agentClass = agentClass self._useclass = True self.agent = None elif agentInstance is not None: if not isinstance(agentInstance, Agent): raise RuntimeError("Impossible to create a runner without an agent class derived from grid2op.Agent. " "Please modify \"agentInstance\" parameter.") self.agentClass = None self._useclass = False self.agent = agentInstance else: raise RuntimeError("Impossible to build the backend. Either AgentClass or agentInstance must be provided " "and both are None.") self.logger = ConsoleLog( DoNothingLog.INFO if verbose else DoNothingLog.ERROR) # store _parameters self.init_grid_path = init_grid_path self.names_chronics_to_backend = names_chronics_to_backend # game _parameters if isinstance(parameters_path, str): self.parameters_path = parameters_path self.parameters = Parameters(parameters_path) elif isinstance(parameters_path, dict): self.parameters = Parameters() self.parameters.init_from_dict(parameters_path) elif parameters_path is None: self.parameters_path = parameters_path self.parameters = Parameters() else: raise RuntimeError("Impossible to build the parameters. The argument \"parameters_path\" should either" "be a string or a dictionary.") # chronics of grid state self.path_chron = path_chron self.gridStateclass_kwargs = gridStateclass_kwargs self.chronics_handler = ChronicsHandler(chronicsClass=self.gridStateclass, path=self.path_chron, **self.gridStateclass_kwargs) # the backend, used to compute powerflows self.backend = self.backendClass() # build the environment self.env = None self.verbose = verbose self.thermal_limit_a = thermal_limit_a
import sys import os import re import codecs from collections import defaultdict from StatisticalCorpus import StatisticalCorpus from Parameters import Parameters from Seeds import Seeds from Accents import Accents temp_folder = '../Temp/' stat_corpus = '../Data/Corpus/Statistical/' stat_temp = temp_folder+'Statistical/' output_folder = '../Data/Output/' parameters = Parameters() max_qty_terms = parameters.getMaxQtyTerms() seeds = Seeds() list_seeds = seeds.getSeeds() accents = Accents() def mainscript(): StatisticalCorpus() executeMutualInformation('Full') executeMutualInformation('Noun') getThesaurusFromSeeds('Full') getThesaurusFromSeeds('Noun') def executeMutualInformation(typefile): command = 'count.pl --ngram 2 --window '+str(parameters.getWindowSize())+' '+stat_temp+'W'+str(parameters.getWindowSize())+'_'+typefile+'StatisticalCorpus.txt '+stat_corpus+''+typefile+'StatisticalCorpus.txt' os.system(command)
def main(argv): parameters = Parameters(argv) hostname = parameters.getHostname() port = parameters.getPort() dbname = parameters.getDBName() language_1, language_2 = parameters.getLanguage() collection = parameters.getCollection() input_folder = parameters.getInputFolder() type_corpus = parameters.getType() print 'Using parameters of configuration: ' print '- Host : ',hostname print '- Port : ',port print '- Coll : ',collection print '- DBase: ',dbname print '- Input: ',input_folder database = Mongo(hostname, dbname, collection) try: root, dirs, files = os.walk(input_folder+''+language_1+'/').next()[:3] except IOError: print 'ERROR: It was not possible to open the '+input_folder+'en/ folder' sys.exit(2) for corpus_file in files: #if (corpus_file ~ "/~/$"): if not '.txt~' in corpus_file: print 'Working on file: '+corpus_file id_file_1 = language_1+'_'+corpus_file[0:-4] id_file_2 = language_2+'_'+corpus_file[0:-4] try: file_1 = codecs.open(input_folder+''+language_1+'/'+corpus_file, 'r', 'utf-8') except IOError: print 'ERROR: System cannot open the '+root+''+corpus_file+' file' sys.exit(2) try: file_2 = codecs.open(input_folder+''+language_2+'/'+corpus_file, 'r', 'utf-8') except IOError: print 'ERROR: System cannot open the '+root+'../'+language_2+'/'+corpus_file+' file' sys.exit(2) #Sentences indexed by the number of the line : number_line = _id (sentence) line_number = 1 lines_2 = file_2.readlines() content_1 = '' content_2 = '' for counter, line in enumerate(file_1): if re.match('(^<)', line): if content_1 != '' and content_2 != '': if not database.exists(language_1, id_file_1) and not database.exists(language_2, id_file_2): database.insertNewData(language_1, id_file_1, content_1, language_2, id_file_2, content_2, type_corpus, line_number) else: if database.existsSentence(language_1, id_file_1, line_number): if not database.existsSentence(language_2, id_file_2, line_number): database.insertInExistingSentence(language_1, id_file_1, language_2, id_file_2, content_2, line_number) else: if database.existsSentence(language_2, id_file_2, line_number): database.insertInExistingSentence(language_2, id_file_2, language_1, id_file_1, content_1, line_number) else: database.insertNewSentence(language_1, id_file_1, content_1, language_2, id_file_2, content_2, line_number) line_number += 1 content_1 = '' content_2 = '' if (line_number % 100 == 0): print 'Indexing line: ',line_number else: content_1 += line content_2 += lines_2[counter] file_1.close() file_2.close()
test_images_src = './test_images' test_images_des = 'test_images_results' test_vedios_src = './test_vedios' test_vedios_des = './output_vedios' # Calibration logger_obj = Logger(str(results_directory)) calibration_obj = Calibration(calibration_src_images, calibration_dst_images, (9, 6), logger_obj) ## Get calibration data camera_matrix, distortion_coefficent = calibration_obj.get_calibration_parameters( ) ## Get Parameters pipeline_params = Parameters(logger_obj, camera_matrix, distortion_coefficent) # Get Pipeline object Runner = Pipeline(pipeline_params) def main(): # Images Runner.process_test_images(test_images_src) # Project vedio Runner.process_test_vedio(test_vedios_src, 'project_video', test_vedios_des) if __name__ == "__main__":
def main(type_atc, argv): date_start = datetime.datetime.now() date_start = date_start.strftime("%Y-%m-%d %H:%M:%S") parameters = Parameters(type_atc, argv) contexts = parameters.getContexts() input_folder = parameters.getInputFolder() language = parameters.getLanguage() min_word_size = parameters.getMinWordSize() max_qty_terms = int(parameters.getMaxQtyTerms()) output_folder = parameters.getOutputFolder() temp_folder = parameters.getTempFolder() record_log = parameters.getRecordLog() record_intermediate = parameters.getRecordIntermediate() seeds_file = parameters.getSeedsFile() stoplist_file = parameters.getStoplistFile() sim_measure = parameters.getSimilarityMeasure() del parameters logfile = LogFile(record_log, str(date_start), None, input_folder, language, stoplist_file, min_word_size, max_qty_terms, None, output_folder, None, temp_folder, seeds_file, sim_measure) if contexts: logfile.writeLogfile('- Building syntactics relations from '+temp_folder) contexts = Contexts(temp_folder) del contexts else: logfile.writeLogfile('- Building syntactics relations from '+input_folder) ling_corpus = StanfordSyntacticContexts(input_folder, temp_folder, stoplist_file, min_word_size, record_intermediate) del ling_corpus logfile.writeLogfile('- Merging terms to '+temp_folder+'Relations2ndOrder.txt') command = 'cat '+temp_folder+'AN_Relations.txt '+temp_folder+'SV_Relations.txt '+temp_folder+'VO_Relations.txt '+' > '+temp_folder+'Relations2ndOrder.txt' os.system(command) logfile.writeLogfile('- Calculating similarity using '+sim_measure) measures = Measures(temp_folder+'Relations2ndOrder.txt', seeds_file) dic_topn = measures.getTopNToAllSeeds(sim_measure, max_qty_terms) del measures logfile.writeLogfile('- Building thesaurus in '+output_folder+'T_'+type_atc+'_'+sim_measure+'.xml') thesaurus = Thesaurus(output_folder+'T_'+type_atc+'_'+sim_measure+'.xml',max_qty_terms) thesaurus.write(dic_topn) del thesaurus date_end = datetime.datetime.now() date_end = date_end.strftime("%Y-%m-%d %H:%M:%S") logfile.writeLogfile('- Thesaurus sucessfully built!\nEnding process at: '+str(date_end)+'.\n') del logfile
def main(type_atc, argv): list_relations = ['AN', 'SV', 'VO'] date_start = datetime.datetime.now() date_start = date_start.strftime("%Y-%m-%d %H:%M:%S") parameters = Parameters(type_atc, argv) contexts = parameters.getContexts() svd_dimension = int(parameters.getSvdDimension()) input_folder = parameters.getInputFolder() language = parameters.getLanguage() min_word_size = parameters.getMinWordSize() max_qty_terms = int(parameters.getMaxQtyTerms()) output_folder = parameters.getOutputFolder() temp_folder = parameters.getTempFolder() record_log = parameters.getRecordLog() record_intermediate = parameters.getRecordIntermediate() seeds_file = parameters.getSeedsFile() stoplist_file = parameters.getStoplistFile() sim_measure = parameters.getSimilarityMeasure() del parameters logfile = LogFile(record_log, str(date_start), svd_dimension, input_folder, language, stoplist_file, min_word_size, max_qty_terms, None, output_folder, None, temp_folder, seeds_file, sim_measure) #if contexts: # logfile.writeLogfile('- Building syntactics relations from '+temp_folder) # contexts = Contexts(temp_folder) # del contexts #else: # logfile.writeLogfile('- Building syntactics relations from '+input_folder) # ling_corpus = StanfordSyntacticContexts(input_folder, temp_folder, stoplist_file, min_word_size, record_intermediate) # del ling_corpus matrix_relation = Matrix(temp_folder, svd_dimension, record_intermediate) del matrix_relation #similarities = Similarities(seeds_file, temp_folder, 'cosine') #dic_topn = similarities.getTopNOrderedDic(10) #del Similarities #logfile.writeLogfile('- Building thesaurus in '+output_folder+'T_'+type_atc+'_'+sim_measure+'.xml') #thesaurus = Thesaurus(output_folder+'T_'+type_atc+'_'+sim_measure+'.xml',max_qty_terms) #thesaurus.write(dic_topn) #del thesaurus date_end = datetime.datetime.now() date_end = date_end.strftime("%Y-%m-%d %H:%M:%S") logfile.writeLogfile( '- Thesaurus sucessfully built!\nEnding process at: ' + str(date_end) + '.\n') del logfile
from java.awt import Panel, Dimension from java.lang import StringBuilder script_path = os.path.dirname(os.path.realpath(__file__)) if "Fiji.app" in script_path: ss = script_path.split("Fiji.app") final_folder = os.path.basename(script_path) script_path = os.path.join(ss[0], "Fiji.app", "plugins", "Scripts", "Plugins", final_folder) sys.path.insert(0, os.path.join(script_path, 'modules')) sys.path.insert(0, os.path.join(script_path, 'classes')) from Parameters import Parameters readme_fpath = os.path.join(script_path, "README.txt") params = Parameters() title = "Membrane Blebbing version " + Parameters._version_string try: f = open(readme_fpath, "rb") text = f.readlines() except: raise IOError("Error reading README.txt") finally: f.close() sb = StringBuilder() for line in text: sb.append(line)
def __init__(self): Parameters.__init__(self) self.registerFloatParameter("rho", "Density value") self.registerFloatParameter("p", "Pressure value") self.registerFloatParameter("T", "Temperature value")
if __name__ == '__main__': args = parse_args() print(args) filepath = args.path + args.dataset result_file = args.res_file cdcf_flag = True # Data processing t1 = time() print('Data loading...') dataset = Dataset(filepath, cdcf_flag) user_input, item_input, train_rating, train_domain = dataset.trainArrQuadruplets valid_user_input, valid_item_input, valid_rating, valid_domain = dataset.validArrQuadruplets test_user_input, test_item_input, test_rating, test_domain = dataset.testArrQuadruplets params = Parameters(args, dataset) #cdcf user_cdcf_input = params.num_users * train_domain + user_input valid_user_cdcf_input = params.num_users * valid_domain + valid_user_input test_user_cdcf_input = params.num_users * test_domain + test_user_input params.set_input(user_input, item_input, train_rating, train_domain, user_cdcf_input, valid_user_input, valid_item_input, valid_rating, valid_domain, valid_user_cdcf_input, test_user_input, test_item_input, test_rating, test_domain, test_user_cdcf_input) print( """Load data done [%.1f s]. #user:%d, #item:%d, #src_item:%d,#tar_item:%d, #train:%d, #test:%d, #valid:%d""" %
def __init__(self): Parameters.__init__(self)
def init(self, rootfolder, cli=False, conditions=None): """ Initialize dependencies :param string rootfolder: full path to root folder :type rootfolder: str :param cli: Run in command line (True) :type cli: bool :param conditions: dictionary providing experiment conditions (to be used instead of conditions.json) :type conditions: dict """ # Get experiment self.experimentsFolder = "{}/experiments/".format(rootfolder) self.expname = self.__get_experiment(self.experimentsFolder, cli=cli) # Get logger if not isdir('{}/logs'.format(rootfolder)): mkdir('{}/logs'.format(rootfolder)) self.logger = self.get_logger(rootfolder, self.expname) # Import experiment self.__import_experiment(self.experimentsFolder, self.expname) self.exp_version = RunTrial.version if hasattr(RunTrial, 'version') else '1.0.0' # Print welcome message print("\n##############################\n") self.logger.info("# Welcome to {} (version {})".format( self.appname, v.__version__)) self.logger.info("# {}".format(v.__copyright__)) self.logger.info("# Date: {}".format( time.strftime("%d-%m-%y %H:%M:%S"))) self.logger.info("# Experiment: {} (version {})".format( self.expname, self.exp_version)) print("\n##############################\n") # Get and set configuration self.config = Config(rootfolder=rootfolder, expname=self.expname, cli=cli) self.config.setup() self.settings = self.config.settings self.folders = self.config.folders self.files = self.config.files # Get experiment parameters self.parameters = Parameters(self.files['parameters']) # Create user self.user = User(data_folder=self.folders['data'], expname='{}_v{}'.format( self.expname, self.exp_version.replace('.', '-')), **self.__filter_args(User, self.settings['setup'])) self.user.setup(cli=cli) self.files['design'] = self.user.designfile # Make factorial design self.design = Design(conditionfile=self.files['conditions'], userfile=self.files['design'], folder=self.folders['expFolder'], conditions=conditions, **self.__filter_args(Design, self.settings['setup'])) self.design.make() # Devices self.devices = Devices(exp_folder=self.folders['expFolder'], base_name=self.user.dftName, cli=cli) # Screen self.screen = Screen(expfolder=self.folders['expFolder'], expname=self.expname, **self.__filter_args(Screen, self.settings['display']))
def sanitization_generation_metrics(feature_order=None, alpha_=P.Alpha, lambda_=P.Lambda, san_loss=P.SanLoss, pred_loss=P.PredLoss, disc_loss=P.DiscLoss, max_epoch=P.Epoch, k_pred=P.KPred, k_disc=P.KDisc, scale=P.Scale): # Return models and datasets # Take the first 70% timestep as training. train_prep = D.Preprocessing(P.TrainPath, prep_excluded=P.PreprocessingExcluded, scale=P.Scale, prep_included=P.PreprocessingIncluded) train_prep.set_features_ordering(feature_order) train_prep.fit_transform() test_prep = D.Preprocessing(P.TestPath, prep_excluded=P.PreprocessingExcluded, scale=P.Scale, prep_included=P.PreprocessingIncluded) test_prep.set_features_ordering(feature_order) test_prep.fit_transform() train_ds = D.MotionSenseDataset(train_prep, window_overlap=P.Window_overlap) test_ds = D.MotionSenseDataset(test_prep, window_overlap=P.Window_overlap) # Shape of unique values uniq_act = np.unique(train_ds.activities) uniq_sens = np.unique(train_ds.sensitive) uniq_uid = np.unique(train_ds.users_id) phys_cols = train_ds.phy_data.shape[1] try: act_cols = train_ds.activities.shape[1] except IndexError: act_cols = 1 # Discriminator target disc_target_values = uniq_sens pred_target_values = uniq_act # Load dataset # Create dataloader # build data loaders batch_size = P.BatchSize s_train_dl = data.DataLoader(train_ds, batch_size=batch_size, shuffle=True, num_workers=4) d_train_dl = data.DataLoader(train_ds.copy(True), batch_size=batch_size, shuffle=True, num_workers=4) d_dl_iter = iter(d_train_dl) p_train_dl = data.DataLoader(train_ds.copy(True), batch_size=batch_size, shuffle=True, num_workers=4) p_dl_iter = iter(p_train_dl) # Create models: sanitizer = M.SanitizerConv(input_channels=train_ds.input_channels, seq_len=train_ds.seq_len, kernel_sizes=[5, 5], strides=[1, 1], conv_paddings=[0, 0], phyNodes=phys_cols, noiseNodes=P.NoiseNodes, actNodes=act_cols) # Adding physio data can prevent the sensor information to be dependent of such attribute because the disc model # Can not predict the sensitive value even though the height weight and other are given. Or we know that if an # attribute is strongly correlated, then the model will find such correlation. Example: Create a model to predict # something and in train set, give the target as data input the predict the same target. The model will learn to dis- # regard other columns # Predictor model output should be of same shape as necessary for NLLLoss. (model output is a matrix while target is # a vector). def get_models(input_channels=train_ds.input_channels, seq_len=train_ds.seq_len, pred_out_size=pred_target_values.shape[0], disc_out_size=disc_target_values.shape[0], phys_cols=phys_cols, act_cols=act_cols): predictor = M.PredictorConv(input_channels=input_channels, seq_len=seq_len, output_size=pred_out_size, physNodes=phys_cols) predictor.to(DEVICE) pred_optim = M.get_optimizer(predictor, ) discriminator = M.DiscriminatorConv(input_channels=input_channels, seq_len=seq_len, output_size=disc_out_size, physNodes=phys_cols + act_cols) discriminator.to(DEVICE) disc_optim = M.get_optimizer(discriminator, ) return predictor, pred_optim, discriminator, disc_optim def reset_weights(m): try: m.reset_parameters() except AttributeError as e: pass # print(e) # print("Layer not affected") predictor, pred_optim, discriminator, disc_optim = get_models() # Send models on GPU or CPU sanitizer.to(DEVICE) # Check the latest Epoch to start sanitization start_epoch = M.get_latest_states(P.ModelsDir(), sanitizer, discriminator, predictor, otherParamFn=P.ParamFunction) # Initialise losses san_loss = Cl.SanitizerBerLoss(alpha_=alpha_, lambda_=lambda_, recOn=P.RecOn, optim_type=P.OptimType, device=DEVICE) # pred_loss = Cl.AccuracyLoss(device=DEVICE) pred_loss = Cl.BalancedErrorRateLoss(targetBer=0, device=DEVICE) disc_loss = Cl.BalancedErrorRateLoss(targetBer=0, device=DEVICE) # Optimizers san_optim = M.get_optimizer(sanitizer, ) losses_frame_path = "{}/{}.csv".format(P.ModelsDir(), P.ParamFunction("losses")) san_losses = [[], [], []] disc_losses = [] pred_losses = [] if (start_epoch > 1) and tryReading(losses_frame_path): losses_frame = pd.read_csv(losses_frame_path) disc_losses = losses_frame["disc"].values.tolist() pred_losses = losses_frame["pred"].values.tolist() san_losses = losses_frame.drop(["pred", "disc"], axis=1).T.values.tolist() # Function to differentiate and integrate the activities. (Ignore for the predictor, integrate for the sanitizer) act_fn_disc = lambda ps, act: torch.cat( (ps, act * P.DecorrelateActAndSens), 1) act_fn_pred = lambda ps, act: ps # Init figure fig = "asdfoijbnad" plt.figure(fig, figsize=(14, 14)) # Sanitize print("Starting Sanitizing ......>") for epoch in tqdm.tqdm(range(start_epoch, max_epoch + 1)): print("Current Epoch: {}".format(epoch)) if P.TrainingResetModelsStates: predictor.apply(reset_weights) discriminator.apply(reset_weights) # del predictor # del discriminator # del disc_optim # del pred_optim # predictor, pred_optim, discriminator, disc_optim = get_models() for sample in s_train_dl: # Train the sanitizer l = train_sanitizer( sample, sanitizer, discriminator, predictor, san_loss, san_optim, act_fn=act_fn_disc, act_select=P.ActivitySelection, phys_select=P.PhysiologSelection, phys=P.PhysInput, san_acts=P.SanitizeActivities, ) san_losses[0].append( l[0].mean().to(CPU_DEVICE).data.numpy().reshape(-1)[0]) san_losses[1].append( l[1].to(CPU_DEVICE).data.numpy().reshape(-1)[0]) san_losses[2].append( l[2].to(CPU_DEVICE).data.numpy().reshape(-1)[0]) # Train the predictor l, p_dl_iter = train_predictor( pred_losses, k_pred, sanitizer, predictor, p_train_dl, p_dl_iter, pred_loss, pred_optim, act_fn=act_fn_pred, act_select=P.ActivitySelection, phys_select=P.PhysiologSelection, target_key="act", sens_key="sens", phys=P.PhysInput, san_acts=P.SanitizeActivities, ) pred_losses.append(l.to(CPU_DEVICE).data.numpy().reshape(-1)[0]) # Train the discriminator l, d_dl_iter = train_predictor( disc_losses, k_pred, sanitizer, discriminator, d_train_dl, d_dl_iter, disc_loss, disc_optim, act_fn=act_fn_disc, act_select=P.ActivitySelection, phys_select=P.PhysiologSelection, target_key="sens", sens_key="sens", phys=P.PhysInput, san_acts=P.SanitizeActivities, ) disc_losses.append(l.to(CPU_DEVICE).data.numpy().reshape(-1)[0]) print("***") # Save losses, and models states. # Saving models States. M.save_classifier_states(sanitizer, epoch, P.ModelsDir(), otherParamFn=P.ParamFunction, ext="S") M.save_classifier_states(discriminator, epoch, P.ModelsDir(), otherParamFn=P.ParamFunction, ext="D") M.save_classifier_states(predictor, epoch, P.ModelsDir(), otherParamFn=P.ParamFunction, ext="P") # Saving and plotting losses losses_frame = pd.DataFrame.from_dict({ "san_rec": san_losses[0], "san_act": san_losses[1], "san_sens": san_losses[2], "disc": disc_losses, "pred": pred_losses, }) losses_frame.to_csv(losses_frame_path, index=False) losses_frame["san_sens"] = san_loss.disc_loss.get_true_value( losses_frame["san_sens"].values) if epoch % P.PlotRate == 0: plt.subplot(5, 1, 1) sns.lineplot(x="index", y="san_rec", data=losses_frame.reset_index()) plt.subplot(5, 1, 2) sns.lineplot(x="index", y="san_act", data=losses_frame.reset_index()) plt.subplot(5, 1, 3) sns.lineplot(x="index", y="san_sens", data=losses_frame.reset_index()) plt.subplot(5, 1, 4) sns.lineplot(x="index", y="disc", data=losses_frame.reset_index()) plt.subplot(5, 1, 5) sns.lineplot(x="index", y="pred", data=losses_frame.reset_index()) plt.savefig("{}/{}.png".format(P.FiguresDir(), P.ParamFunction("losses"))) plt.clf() # Check datasets and generate # def generate_dataset(san, train_prep, train_ds, train_dl, test_prep, test_ds, test_dl, gen_path, train_id="train", # test_id="test", max_epoch=0, addParamFn=None, phys=1, san_acts=1, san_phys=1): print("Generating Sanitized Datasets") generate_dataset(sanitizer, train_prep, train_ds, test_prep, test_ds, P.GenDataDir(), max_epoch=P.Epoch, addParamFn=P.ParamFunction, phys=P.PhysInput, san_acts=P.SanitizeActivities, san_phys=P.SanitizePhysio) # Check if everything has been correctly generated #print("Computing Metrics") # If device == cpu_device, then we are not supposed to use gpu as there might not be anyone """metrics_computation(input_channels=train_ds.input_channels, seq_len=train_ds.seq_len,
def setUp(self): # powergrid self.backend = PandaPowerBackend() self.path_matpower = PATH_DATA_TEST_PP self.case_file = "test_case14.json" # chronics self.path_chron = os.path.join(PATH_CHRONICS, "chronics") self.chronics_handler = ChronicsHandler( chronicsClass=GridStateFromFile, path=self.path_chron) self.tolvect = 1e-2 self.tol_one = 1e-5 self.id_chron_to_back_load = np.array( [0, 1, 10, 2, 3, 4, 5, 6, 7, 8, 9]) # force the verbose backend self.backend.detailed_infos_for_cascading_failures = True self.names_chronics_to_backend = { "loads": { "2_C-10.61": 'load_1_0', "3_C151.15": 'load_2_1', "14_C63.6": 'load_13_2', "4_C-9.47": 'load_3_3', "5_C201.84": 'load_4_4', "6_C-6.27": 'load_5_5', "9_C130.49": 'load_8_6', "10_C228.66": 'load_9_7', "11_C-138.89": 'load_10_8', "12_C-27.88": 'load_11_9', "13_C-13.33": 'load_12_10' }, "lines": { '1_2_1': '0_1_0', '1_5_2': '0_4_1', '9_10_16': '8_9_2', '9_14_17': '8_13_3', '10_11_18': '9_10_4', '12_13_19': '11_12_5', '13_14_20': '12_13_6', '2_3_3': '1_2_7', '2_4_4': '1_3_8', '2_5_5': '1_4_9', '3_4_6': '2_3_10', '4_5_7': '3_4_11', '6_11_11': '5_10_12', '6_12_12': '5_11_13', '6_13_13': '5_12_14', '4_7_8': '3_6_15', '4_9_9': '3_8_16', '5_6_10': '4_5_17', '7_8_14': '6_7_18', '7_9_15': '6_8_19' }, "prods": { "1_G137.1": 'gen_0_4', "3_G36.31": "gen_2_1", "6_G63.29": "gen_5_2", "2_G-56.47": "gen_1_0", "8_G40.43": "gen_7_3" }, } # _parameters for the environment self.env_params = Parameters() self.env = Environment( init_grid_path=os.path.join(self.path_matpower, self.case_file), backend=self.backend, chronics_handler=self.chronics_handler, parameters=self.env_params, names_chronics_to_backend=self.names_chronics_to_backend)
def metrics_computation(input_channels, seq_len, features_order, kernel_size=[5, 5], strides=[1, 1], conv_paddings=[0, 0], gpu=True, gpu_device="cuda:0", gen_path=P.GenDataDir(), alpha_=P.Alpha, lambda_=P.Lambda, train_id="train", test_id="test", epoch=P.Epoch, addParamFn=P.ParamFunction, seed=42, verbose=False): """ """ # Add restarting mechanism. metrics = Me.Metrics(input_channels=input_channels, seq_len=seq_len, kernel_sizes=kernel_size, strides=strides, conv_paddings=conv_paddings, gpu=gpu, distance_metric=P.DistanceMetric, gpu_device=gpu_device, prep_included=P.PreprocessingIncluded, prep_excluded=P.PreprocessingExcluded, scale=P.Scale, features_ordering=features_order, seed=seed, verbose=verbose, data_fmt_class="MotionSenseDataset", window_overlap=P.Window_overlap) results = R.Results(resultDir=P.ResultsDir()) sr = R.StopRestart(resultDir=P.ResultsDir()) # Read data and compute metrics, # Baseline metric_epochs = 200 metric_batch = 256 def shaping(path): # Set the data as the same for all, the generated ones and the original ones such that we have the same # computation graph. data = D.MotionSenseDataset(path, window_overlap=P.Window_overlap) return data.__inverse_transform_conv__(sensor_tensor=data.sensor, phy=data.phy_data, act_tensor=data.activities, sens_tensor=data.sensitive, user_id_tensor=data.users_id, trials=data.trials, cpu_device=CPU_DEVICE) o_test = shaping(P.TestPath) if not sr.computed(epoch=0, alpha_=np.NaN, lambda_=np.NaN, Attribute="act") or \ not sr.computed(epoch=0, alpha_=np.NaN, lambda_=np.NaN, Attribute="gender"): o_train = shaping(P.TrainPath) sp = metrics.sensitive_attribute( train_set=o_train, test_set=o_test, use_accuracy=False, act_name="act", drop=[], sens_name="gender", ms_act_name="act", ms_sens_name="sens", sklearn_data_process=None, use_phys=True, physNodes=3, phys_names=["height", "weight", "age"], ms_phys_name="phy", epoch=metric_epochs, batch_size=metric_batch, loss_fn=None) tp = metrics.task(train_set=o_train, test_set=o_test, act_name="act", drop=[], sens_name="gender", ms_act_name="act", ms_sens_name="sens", sklearn_data_process=None, use_phys=True, physNodes=3, phys_names=["height", "weight", "age"], ms_phys_name="phy", epoch=metric_epochs, batch_size=metric_batch, loss_fn=None) di = metrics.distance(o_test, o_test) # Add and save results results.add_result(distance=di, s_acc=sp[0], s_ber=sp[1], t_acc=tp[0], t_ber=tp[1], sens_name="gender", act_name="act", epoch=0, alpha_=np.NaN, lambda_=np.NaN) # Sanitization name = lambda n, e: "{}/{}_{}.csv".format(gen_path, n, addParamFn(e)) for epoch in tqdm.tqdm(range(1, epoch + 1)): if not sr.computed(epoch=epoch, alpha_=alpha_, lambda_=lambda_, Attribute="act") or \ not sr.computed(epoch=epoch, alpha_=alpha_, lambda_=lambda_, Attribute="gender"): train = shaping(name(train_id, epoch)) test = shaping(name(test_id, epoch)) sp = metrics.sensitive_attribute( train_set=train, test_set=test, use_accuracy=False, act_name="act", drop=[], sens_name="gender", ms_act_name="act", ms_sens_name="sens", sklearn_data_process=None, use_phys=True, physNodes=3, phys_names=["height", "weight", "age"], ms_phys_name="phy", epoch=metric_epochs, batch_size=metric_batch, loss_fn=None, learning_rate=5e-4, weight_decay=0) tp = metrics.task(train_set=train, test_set=test, act_name="act", drop=[], sens_name="gender", ms_act_name="act", ms_sens_name="sens", sklearn_data_process=None, use_phys=True, physNodes=3, phys_names=["height", "weight", "age"], ms_phys_name="phy", epoch=metric_epochs, batch_size=metric_batch, loss_fn=None, learning_rate=5e-4, weight_decay=0) di = metrics.distance(o_test_set=o_test, test_set=test) # Add and save results results.add_result(distance=di, s_acc=sp[0], s_ber=sp[1], t_acc=tp[0], t_ber=tp[1], sens_name="gender", act_name="act", epoch=epoch, alpha_=alpha_, lambda_=lambda_)
from datetime import datetime import numpy as np import tensorflow as tf from sklearn.metrics import classification_report from sklearn.metrics import confusion_matrix from utils.Dataset_hdf5 import DatasetHDF5 from utils.Dataset_hdf5_cv import DatasetHDF5_Kfold from Parameters import Parameters # ===============get basic folder===================== BASE_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.append(BASE_DIR) sys.path.append(os.path.join(BASE_DIR, 'models')) para = Parameters() # log MODEL = importlib.import_module(para.model) # import network module LOG_DIR = para.logDir LOG_MODEL = para.logmodelDir LOG_FOUT = open(os.path.join(LOG_DIR, f'{para.expName}.txt'), 'w') LOG_FOUT.write(str(para.__dict__) + '\n') # set parameters if para.gpu: gpus = tf.config.experimental.list_physical_devices('GPU') for gpu in gpus: tf.config.experimental.set_memory_growth(gpu, True) else: os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
def setUp(self): """ The case file is a representation of the case14 as found in the ieee14 powergrid. :return: """ # from ADNBackend import ADNBackend # self.backend = ADNBackend() # self.path_matpower = "/home/donnotben/Documents/RL4Grid/RL4Grid/data" # self.case_file = "ieee14_ADN.xml" # self.backend.load_grid(self.path_matpower, self.case_file) self.tolvect = 1e-2 self.tol_one = 1e-5 self.game_rules = GameRules() # pdb.set_trace() self.rewardClass = L2RPNReward self.reward_helper = self.rewardClass() self.obsClass = CompleteObservation self.parameters = Parameters() # powergrid self.backend = PandaPowerBackend() self.path_matpower = PATH_DATA_TEST_PP self.case_file = "test_case14.json" # chronics self.path_chron = os.path.join(PATH_CHRONICS, "chronics_with_forecast") self.chronics_handler = ChronicsHandler( chronicsClass=GridStateFromFileWithForecasts, path=self.path_chron) self.tolvect = 1e-2 self.tol_one = 1e-5 self.id_chron_to_back_load = np.array( [0, 1, 10, 2, 3, 4, 5, 6, 7, 8, 9]) # force the verbose backend self.backend.detailed_infos_for_cascading_failures = True self.names_chronics_to_backend = { "loads": { "2_C-10.61": 'load_1_0', "3_C151.15": 'load_2_1', "14_C63.6": 'load_13_2', "4_C-9.47": 'load_3_3', "5_C201.84": 'load_4_4', "6_C-6.27": 'load_5_5', "9_C130.49": 'load_8_6', "10_C228.66": 'load_9_7', "11_C-138.89": 'load_10_8', "12_C-27.88": 'load_11_9', "13_C-13.33": 'load_12_10' }, "lines": { '1_2_1': '0_1_0', '1_5_2': '0_4_1', '9_10_16': '8_9_2', '9_14_17': '8_13_3', '10_11_18': '9_10_4', '12_13_19': '11_12_5', '13_14_20': '12_13_6', '2_3_3': '1_2_7', '2_4_4': '1_3_8', '2_5_5': '1_4_9', '3_4_6': '2_3_10', '4_5_7': '3_4_11', '6_11_11': '5_10_12', '6_12_12': '5_11_13', '6_13_13': '5_12_14', '4_7_8': '3_6_15', '4_9_9': '3_8_16', '5_6_10': '4_5_17', '7_8_14': '6_7_18', '7_9_15': '6_8_19' }, "prods": { "1_G137.1": 'gen_0_4', "3_G36.31": "gen_2_1", "6_G63.29": "gen_5_2", "2_G-56.47": "gen_1_0", "8_G40.43": "gen_7_3" }, } # _parameters for the environment self.env_params = Parameters() self.env = Environment( init_grid_path=os.path.join(self.path_matpower, self.case_file), backend=self.backend, chronics_handler=self.chronics_handler, parameters=self.env_params, names_chronics_to_backend=self.names_chronics_to_backend, rewardClass=self.rewardClass)
from Serializer import Serializer, serialize_data from Vocabs import Vocabs, make_train_data from Seq2Seq import Seq2Seq from Parameters import Parameters from MakeSeq import make_seq if __name__ == u'__main__': # args parser = argparse.ArgumentParser() parser.add_argument('--gpu', '-g', default=None, type=int) args = parser.parse_args() # calculation parameters parameters = Parameters() # get data vocabs = Vocabs() serializer = Serializer(vocabs) train_data = make_train_data(vocabs, parameters.time_steps, parameters.data_num) print('--- first 10 data ---') for data in train_data[0: 10]: print(''.join(data)) train_data, result_data = serialize_data(serializer, train_data, parameters.time_steps)
print('wait for it...') seed = 0 args = { 'L': sp.L, 'T': sp.T, 'K': sp.K, 'lambda_': sp.lambda_, 'epsilon': sp.epsilon, 'c': sp.c, 'eta': sp.eta, 'seed': seed, } p = Parameters(**args) args = { 'N': sc.N, 'm': sc.m, 'n': sc.n, 'A': sc.A, 'B': sc.B, 'C': sc.C, 'D': sc.D, 'P': sc.P, 'F': sc.F, 'X': zeros_like(sc.X), } m = MJLS(**args)
def setParameters(self, name, destination): self.param_file = Parameters(name=self.name,destination=destination) self.writeParamHeaders() self.writeParamFilenames() self.closeParam()
def main(): # ensure consistent preference settings Prefs.blackBackground = False # get locations for previous and new outputs params = Parameters() output_folder_old, output_folder = mbio.rerun_location_chooser( params.input_image_path) params.loadParametersFromJson( os.path.join(output_folder_old, 'parameters used.json')) params.setOutputPath(os.path.join(output_folder, "looped analysis")) os.makedirs(params.output_path) # prompt user for path to text file describing the looped analysis loop_file_path = mbio.input_file_location_chooser( params.output_path, filt='*.txt', message="Please choose a text file containing the looping data...") # get original image file (for overlays etc.) if not os.path.isfile(params.input_image_path): mbui.warning_dialog([ "The original data can't be found at the location specified in saved parameters. ", "(Possibly something as simple as a change in network drive mapping has occurred)", "Please specify the location of the original image file..." ]) params.setInputImagePath( mbio.input_file_location_chooser(params.output_path)) import_opts, params = mbui.choose_series(params.input_image_path, params) imps = bf.openImagePlus(import_opts) imp = imps[0] if imp.getNSlices() > 1: mbui.warning_dialog([ "More than one Z plane detected.", "I will do a maximum projection before proceeding", "Continue?" ]) imp = ZProjector.run(imp, "max all") params = mbio.get_metadata(params) params.setCurvatureLengthUm( round(params.curvature_length_um / params.pixel_physical_size) * params.pixel_physical_size) params.persistParameters() IJ.run(imp, "Set Scale...", "distance=0 known=0 pixel=1 unit=pixel") imp.show() if imp.getNChannels() > 1: imp.setPosition(params.membrane_channel_number, 1, 1) mbui.autoset_zoom(imp) IJ.run("Enhance Contrast", "saturated=0.35") # prompt user to select ROI original_imp = Duplicator().run(imp) _, crop_params = mbui.crop_to_ROI(imp, params) imp.show() if crop_params is not None: params.perform_spatial_crop = True mbui.autoset_zoom(imp) imp.updateAndDraw() review_crop = mb.check_cropping(output_folder_old, params) keep_crop = not review_crop if review_crop: keep_crop = mbui.crop_review() if not keep_crop: imp.changes = False imp.close() imp = original_imp else: original_imp.close() else: original_imp.close() # prompt user to do time cropping imp, start_end_tuple = mbui.time_crop(imp, params) params.setTimeCropStartEnd(start_end_tuple) # import edges membrane_edges = mbio.load_qcd_edges2( os.path.join(output_folder_old, "user_defined_edges.zip")) mbio.save_qcd_edges2(membrane_edges, params.output_path) # generate list of Parameters objects to loop over params_list, loop_param_name, loop_param_vals = generate_params_list( loop_file_path, params) dummy_output_path = params.output_path for params, loop_param_val in zip(params_list, loop_param_vals): params.setOutputPath( os.path.join(dummy_output_path, (loop_param_name + " = " + str(loop_param_val)))) params.close_on_completion = True params.setCurvatureLengthUm( round(params.curvature_length_um / params.pixel_physical_size) * params.pixel_physical_size) os.makedirs(params.output_path) calculated_objects = CalculatedObjects() calculated_objects.membrane_edges = membrane_edges if params.time_crop_start_end[0] is not None: calculated_objects.timelist = [ idx * params.frame_interval for idx in range(params.time_crop_start_end[0], params.time_crop_start_end[1] + 1) ] else: calculated_objects.timelist = [ idx * params.frame_interval for idx in range(imp.getNFrames()) ] split_channels = mbfs.split_image_plus(imp, params) membrane_channel_imp = split_channels[0] actin_channel_imp = split_channels[1] segmentation_channel_imp = None if params.photobleaching_correction: if os.path.isfile( os.path.join(output_folder_old, 'binary_membrane_stack.tif')): segmentation_binary_path = os.path.join( output_folder_old, 'binary_membrane_stack.tif') segmentation_channel_imp = IJ.openImage( segmentation_binary_path) else: segmentation_channel_imp = split_channels[2] segmentation_channel_imp = mb.make_and_clean_binary( segmentation_channel_imp, params.threshold_method) split_channels[2] = segmentation_channel_imp calculated_objects = mbfs.calculate_outputs(params, calculated_objects, split_channels) # output colormapped images and kymographs fig_imp_list = mbfs.generate_and_save_figures( imp, calculated_objects, params, membrane_channel_imp, segmentation_channel_imp) mbfs.save_csvs(calculated_objects, params) params.saveParametersToJson( os.path.join(params.output_path, "parameters used.json")) imp.changes = False IJ.setTool("zoom") if params.close_on_completion: for fig_imp in fig_imp_list: fig_imp.close() if params.close_on_completion: imp.close() return
vetor_normal=[0] aux_vetor_normal=0 aux_flexion_left_knee=0 simetria_comprimento_passo=[0] t=[0] data_json = {} data_json['dados']=[0] movimento=0 ##Time up:0, Em círculos:1, Em linha reta:2, Elevação excessiva:3, Assimétrica:4 e Circundação do pé:5 altura_pe_esquerdo=0 altura_pe_direito=0 altura_calcanhar=0.135 ponto_tornozelo_direito=[] ponto_tornozelo_esquerdo=[] k=0 slide= 0 #input("Digite um valor para o silde: 0, 2 ou 4 ") slide_result=Parameters.slide_gait_cycle(slide) array_coordenadas=[] ##Array de coordenadas do esqueleto matrix_coordenadas=[] aux_movimento=[movimento] CAPTURA='RGB' angulo_nathan=[0] #Aqui começa a análise dos vídeos para cada frame for it_frames in range(video_loader.n_frames()): video_loader.load_next() tempo_anterior=time.time() aux_tempo=time.time() #atualiza o instante de tempo entre os frames
w = ((w_init - w_end) * (9999 - ite)/9999) + w_end return w def update_position(particle, low_bound, high_bound): for i, value in enumerate(particle.position): particle.position[i] = value + particle.velocity[i] if particle.position[i] > high_bound: particle.position[i] = high_bound - abs(particle.position[i] - high_bound) particle.velocity[i] *= -1.0 elif particle.position[i] < low_bound: particle.position[i] = low_bound + abs(particle.position[i] - low_bound) particle.velocity[i] *= -1.0 if __name__ == "__main__": ## Configurations ## parameters = Parameters() dimensions = parameters.dimension population = parameters.population problem_type = parameters.problem_type # Search space bounds low_bound, high_bound = parameters.set_bounds(problem_type) # Algorithm configurations iterations = parameters.iterations C1 = parameters.C1 C2 = parameters.C2 w = parameters.w min_vel = parameters.min_vel max_v = parameters.max_v topology = parameters.topology pso_type = parameters.pso_type ## End Configurations ##
# Project: https://github.com/HugoCMU/SolarTree # Description: Definitions for all parameters, runs/calls all other functions import serial import numpy as np from Sensor import Sensor from PinMaster import PinMaster from Parameters import Parameters import math import sys # Serial communication with Arduino ser = serial.Serial('/dev/ttyACM0', 9600) # Create Parameters object params = Parameters() # Add Navigation Parameters (name, value, description) params.addParam('DATA_SAMPLE_SIZE', 3, 'Sample size for data (increase to stabilize at cost of speed)') params.addParam('MAX_ITER', 2, 'Maximum number of Sense-Plan-Act Cycles') params.addParam('MOVE_PER_TURN', 1, 'How far you want the robot to move each step (increments of 10cm)') params.addParam('TIMEOUT', 0.1, 'How many seconds until sensor loop times out and returns a bunch of zeros') params.addParam('FORWARD_VECTOR', [0, 1, 0], 'Describes the forward (Theta = 0) direction in the robot frame') # Add SLAM Parameters (name, value, description) params.addParam('RAND_DIST_MU', 0, 'Center of distribution (cm)') params.addParam('RAND_DIST_SIGMA', 1, 'Standard deviation (cm)') params.addParam('RAND_ANG_MU', 0, 'Degrees') params.addParam('RAND_ANG_SIGMA', 10, 'Degrees') params.addParam('RAND_NUM', 10, 'Number of random samples')
def __init__(self): Parameters.__init__(self, 167, 3, 128, 61, 20, 18)
import serial import numpy as np from Sensor import Sensor from PinMaster import PinMaster from Parameters import Parameters # ---------------------------------------------------- # INITIALIZE GLOBAL VARIABLES # ---------------------------------------------------- # Serial communication with Arduino ser = serial.Serial('/dev/ttyACM0', 9600) # Create Parameters object params = Parameters() # Add Navigation Parameters (name, value, description) params.addParam('DATA_SAMPLE_SIZE', 3, 'Sample size for data (increase to stabilize at cost of speed)') params.addParam('MAX_ITER', 10, 'Maximum number of Sense-Plan-Act Cycles') params.addParam('MOVEMENT_WEIGHT', 5.0, 'Weight factor for movement vector vs move units to be traveled') params.addParam('TIMEOUT', 0.1, 'How many seconds until sensor loop times out and returns a bunch of zeros') params.addParam('FORWARD_VECTOR', [0, 1, 0], 'Describes the forward (Theta = 0) direction in the robot frame') params.addParam('BACKWARD_VECTOR', [0, -1, 0], 'Describes the backwards (Theta = 0) direction in the robot frame') params.addParam('WAIT_TIME', 2, 'How many seconds in between exploration steps') # Add Tuning Parameters params.addParam('DISTANCE_WEIGHT', [0.1, 0.1, 0.2], 'Weighting of [X, Y, Theta] each when determining distance metric') # Add Motor Parameters
args = parse_args() print(args) args_str = get_args_to_string(args) args.args_str = args_str print args_str print('Data loading...') t1, t_init = time(), time() #pdb.set_trace() if args.method.lower() in ['sorecgatitem']: dataset = SocialItem_Dataset(args) elif args.method.lower() in ['sorecgatuser']: dataset = SocialUser_Dataset(args) else: dataset = Dataset(args) params = Parameters(args, dataset) print( """Load data done [%.1f s]. #user:%d, #item:%d, #dom:%d, #train:%d, #test:%d, #valid:%d""" % (time() - t1, params.num_users, params.num_items, params.num_doms, params.num_train_instances, params.num_test_instances, params.num_valid_instances)) print('Method: %s' % (params.method)) if params.method in ['sorecgatitem', 'sorecgatuser']: model = Models(params) model.define_model() model.define_loss('all') print("Model definition completed: in %.2fs" % (time() - t1)) train_step = get_optimizer(params.learn_rate, params.optimizer).minimize(model.loss) init = tf.global_variables_initializer()
def main(type_atc, argv): date_start = datetime.datetime.now() date_start = date_start.strftime("%Y-%m-%d %H:%M:%S") parameters = Parameters(type_atc, argv) contexts = parameters.getContexts() input_folder = parameters.getInputFolder() language = parameters.getLanguage() min_word_size = int(parameters.getMinWordSize()) max_qty_terms = int(parameters.getMaxQtyTerms()) mi_precision = parameters.getMIPrecision() output_folder = parameters.getOutputFolder() window_size = parameters.getWindowSize() temp_folder = parameters.getTempFolder() record_log = parameters.getRecordLog() record_intermediate = parameters.getRecordIntermediate() seeds_file = parameters.getSeedsFile() sim_measure = parameters.getSimilarityMeasure() del parameters logfile = LogFile(record_log, str(date_start), None, input_folder, language, None, min_word_size, max_qty_terms, mi_precision, output_folder, window_size, temp_folder, seeds_file, sim_measure) stat_corpus = StatisticalCorpus(input_folder, temp_folder, min_word_size, window_size) if not contexts: logfile.writeLogfile('- Building statistical corpus at '+temp_folder) if language == 'pt': stat_corpus.buildCorpus_pt() param_nsp = '--token ../misc/tokens_nsp.pl' elif language == 'en': stat_corpus.buildCorpus_en() param_nsp = '' """ Uses count.pl from NGram Statistical Package (NSP) to get Bigrams in a window """ logfile.writeLogfile('- Getting bigrams to W'+window_size+'_Statistical_corpus.txt') command = 'count.pl --ngram 2 '+param_nsp+' --window '+window_size+' '+temp_folder+'W'+window_size+'_Statistical_corpus.txt '+temp_folder+'Statistical_corpus.txt' os.system(command) logfile.writeLogfile('- Using '+sim_measure+' as similarity measure') if sim_measure == 'mutual_information': mi = MutualInformation(temp_folder, 'W'+window_size+'_Statistical_corpus.txt', seeds_file, mi_precision) dic_terms = mi.getDicMI() del mi else: stat_corpus.buildSTRelations('W'+window_size+'_Statistical_corpus.txt', seeds_file) measures = Measures(temp_folder+'W'+window_size+'_Relations.txt', seeds_file) dic_terms = measures.getTopNToAllSeeds(sim_measure, max_qty_terms) del measures else: measures = Measures(temp_folder+'W'+window_size+'_Relations.txt', seeds_file) dic_terms = measures.getTopNToAllSeeds(sim_measure, max_qty_terms) del measures del stat_corpus logfile.writeLogfile('- Building thesaurus in '+output_folder+'T'+window_size+'_'+type_atc+'_'+sim_measure+'.xml') thesaurus = Thesaurus(output_folder+'T'+window_size+'_'+type_atc+'_'+sim_measure+'.xml',max_qty_terms) thesaurus.write(dic_terms) del thesaurus date_end = datetime.datetime.now() date_end = date_end.strftime("%Y-%m-%d %H:%M:%S") logfile.writeLogfile('- Thesaurus sucessfully built!\nEnding process at: '+str(date_end)+'.\n') del logfile