def main(): curr_full_path = FileUtils.getCurrentDirFullPath() config_fn = 'portal_activity_job_config.yaml' cI = ConfigUtils(curr_full_path + "/configs/", config_fn) configItems = cI.getConfigs() configItems[ 'config_dir'] = curr_full_path + "/" + configItems['config_dir'] configItems['curr_full_path'] = curr_full_path db_ini = configItems['config_dir'] + configItems['database_config'] conn_alq, meta_alq = PostgresStuff.connect_alq(db_ini) conn = PostgresStuff.connect(db_ini) db_tbl = configItems['activity_table'] first_run = getFirstRun(conn) if first_run == 0: print("****First RUN! No new created datasets in the past " + configItems['activity']['create']['time_interval'] + "*****") exit(0) insert_created = updateCreatedDatasets( conn, configItems['activity']['create']['time_interval']) #print insert_created created_datasets = MonitorPortal.generateActivityReport( conn_alq, configItems, 'create') if (not (created_datasets)): print("**** No new created datasets in the past " + configItems['activity']['create']['time_interval'] + "*****") exit(0) datasetid_notified = MonitorPortal.generateEmail(conn_alq, configItems, 'create', created_datasets) updted_notified_cnt = MonitorPortal.updateNotifiedDatasetIds( conn, configItems, 'create', datasetid_notified) print("******Notfied that " + str(updted_notified_cnt) + " datasets were created****") print("******Updated " + str(updted_notified_cnt) + " rows in the created_dataset table****")
def train(self): for e in xrange(self.starting_epoch, self.max_epochs): # training phase self.__reset_context() self.training_set.seek(0) previous_word = 0 for current_word in self.training_set: current_word = int(current_word) self.__feedforward(previous_word) self.__backpropagate(previous_word, current_word) self.neu_context = np.copy(self.neu_hidden) previous_word = current_word # reset context at the end of each sentence if(self.reset_context_each_sentence and current_word==0): self.__reset_context() # validation phase self.__reset_context() self.validation_set.seek(0) previous_word = 0 logp = 0 word_counter = 0 for current_word in self.validation_set: current_word = int(current_word) self.__feedforward(previous_word) # word==-1 are not in vocabulary if(current_word!=-1): word_counter += 1 logp += np.log2(self.neu_output[current_word]) self.neu_context = np.copy(self.neu_hidden) previous_word = current_word # reset context at the end of each sentence if(self.reset_context_each_sentence and current_word==0): self.__reset_context() # print progress print("*******************") print("{}".format(time.strftime("%Y-%m-%d %H:%M:%S"))) print("Epoch {}".format(e)) print("Learning rate {}".format(self.learning_rate)) print("Validation log probability {}".format(logp)) print("Validation words counter {}".format(word_counter)) if(word_counter>0): print("Validation PPL {}".format(np.power(2.0, -logp / word_counter))) # check improvement if(logp < self.logp_previous): self.__restore_weights() else: self.__save_weights() if(logp*self.min_improvement < self.logp_previous): if (not self.learning_rate_divide): self.learning_rate_divide = True else: break if (self.learning_rate_divide): self.learning_rate /= 2 self.logp_previous = logp # log last epoch for recovery cu.log_current_epoch(e+1, self.learning_rate, self.logp_previous, self.learning_rate_divide, self.recovery_config_file)
def get_filter_list(): global saved_filters if len(saved_filters) == 0: ConfigUtils.read_config( saved_filters, './config/savedfilters.txt' ) filters = saved_filters.keys() filters.sort() return filters
def TrainModel(model, session, mnist, cfg): data_tr = mnist.train data_val = mnist.validation loss_history = [] logs = [] acc_val_best_so_far = 0 current_step = session.run(model.global_step) lg = ConfigUtils.LogGenerator() for iteration in range(current_step, cfg.max_batch+1): batch = data_tr.next_batch(cfg.batch_size) learn_rate = cfg.lr * cfg.lr_mapper.GetFactor(iteration) feed_dict = {model.pl_images:_SubtractByMean(batch[0]), model.pl_labels:batch[1], model.pl_dropout_rate:0.5, model.pl_phase:1, model.pl_lr:learn_rate} loss, _ = session.run([model.loss, model.op_train], feed_dict=feed_dict) loss_history.append(loss) if cfg.is_save and iteration % cfg.save_every==0: model.saver.save(session, cfg.save_dir, iteration) if iteration % cfg.print_every != 0: continue feed_dict[model.pl_dropout_rate] = 0 feed_dict[model.pl_phase] = 0 acc_tr, loss_softmax_tr, loss_re_tr = session.run([model.accuracy, model.loss_softmax, model.loss_reg], feed_dict=feed_dict) acc_val, loss_softmax_val, loss_re_val = EvaluateModel(model, session, data_val, cfg) acc_val_best_so_far = acc_val if acc_val > acc_val_best_so_far else acc_val_best_so_far log = lg.GetContent() log.iteration = iteration log.acc_tr = acc_tr log.loss_softmax_tr = loss_softmax_tr log.loss_re_tr = loss_re_tr log.loss_tr = loss_softmax_tr + loss_re_tr log.acc_val = acc_val log.loss_softmax_val = loss_softmax_val log.loss_re_val = loss_re_val log.loss_val = loss_softmax_val + loss_re_val logs.append(log) if cfg.verbose: print('iteration: {0}\t acc_tr: {1:4.3f}\t acc_te: {2:4.3f}\t loss_tr: {3:4.4f}\t loss_val: {4:4.4f}\t lr: {5:4.2f}'.format(iteration, acc_tr, acc_val, log.loss_tr, log.loss_val, np.log(learn_rate))) acc_val, loss_softmax_val, loss_re_val = EvaluateModel(model, session, data_val, cfg) rg = ConfigUtils.ReportGenerator() report = rg.GetContent() report.acc = acc_val report.best_acc = acc_val_best_so_far report.loss_softmax =loss_softmax_val report.loss_re = loss_re_val report.loss = loss_softmax_val + loss_re_val report.traing_loss_hist = loss_history report.distribution = model.cfg.distribution report.reg = model.cfg.reg report.lr = cfg.lr return report, logs
def delete_filter(filter_name): global saved_filters if len(saved_filters) == 0: ConfigUtils.read_config( saved_filters, './config/savedfilters.txt' ) name = filter_name.title() try: del saved_filters[name] ConfigUtils.write_config( saved_filters, './config/savedfilters.txt' ) except KeyError: pass
def main(): fieldConfigFile, config_inputdir, jobType = parse_opts() cI = ConfigUtils(config_inputdir, fieldConfigFile) configItems = cI.getConfigs() configItems['dataset_name'] = jobType lg = pyLogger(configItems) logger = lg.setConfig() logger.info("****************JOB START******************") sc = SocrataClient(config_inputdir, configItems, logger) client = sc.connectToSocrata() clientItems = sc.connectToSocrataConfigItems() scrud = SocrataCRUD(client, clientItems, configItems, logger) sQobj = SocrataQueries(clientItems, configItems, logger) mmdd_fbf = configItems['dd']['master_dd']['fbf'] field_profiles_fbf = configItems['dd']['field_profiles']['fbf'] base_url = configItems['baseUrl'] field_type_fbf = configItems['dd']['field_type']['fbf'] load_mm_dd = ProfileFields.getBaseDatasetJson(sQobj, configItems, mmdd_fbf) #print load_mm_dd #load_mm_dd = True current_field_profiles = ProfileFields.getCurrentFieldProfiles( sQobj, base_url, field_profiles_fbf) if load_mm_dd: master_dfList = ProfileFields.get_dataset_as_dfList( configItems['pickle_data_dir'], configItems['mm_dd_json_fn'], base_url) ProfileFields.removeDeletedFields(scrud, field_profiles_fbf, master_dfList, current_field_profiles) current_field_profiles = ProfileFields.getCurrentFieldProfiles( sQobj, base_url, field_profiles_fbf) dataset_info = ProfileFields.buildInsertFieldProfiles( sQobj, scrud, configItems, master_dfList, current_field_profiles) print dataset_info dsse = JobStatusEmailerComposer(configItems, logger, jobType) if dataset_info['DatasetRecordsCnt'] > 1: dsse.sendJobStatusEmail([dataset_info]) else: dataset_info = { 'Socrata Dataset Name': configItems['dataset_name'], 'SrcRecordsCnt': 0, 'DatasetRecordsCnt': 0, 'fourXFour': "Nothing to Insert" } dataset_info['isLoaded'] = 'success' dsse.sendJobStatusEmail([dataset_info])
def save_filter(filter_name, filter_str): global saved_filters if len(saved_filters) == 0: ConfigUtils.read_config( saved_filters, './config/savedfilters.txt' ) name = filter_name.title() name = name.replace(' ', '_') saved_filters[name] = filter_str ConfigUtils.write_config( saved_filters, './config/savedfilters.txt' ) return
def get_saved_filter( filter_name ): global saved_filters if len(saved_filters) == 0: ConfigUtils.read_config( saved_filters, './config/savedfilters.txt' ) filter_list = [] try: name = filter_name.title() filter_str = saved_filters[name] filter_list = filter_str.split('+') except: pass return filter_list
def Tune(): ''' for small net re_max = -4 re_min = -14 lr_max = -6 lr_min = -10 ''' re_max = -4 re_min = -14 lr_max = -6 lr_min = -10 distribution = np.array(['uniform', 'normal']) iteration = 1 max_batch = 1000 verbose = True #list_lr = np.exp(np.linspace(start=-9, stop=-13, num=iteration)) list_lr = np.exp(np.random.uniform(low=lr_min, high=lr_max, size=iteration)) list_re = np.exp(np.random.uniform(low=re_min, high=re_max, size=iteration)) mask_distribution = np.random.randint(low=0, high=len(distribution), size=iteration) list_dstribution = distribution[mask_distribution] _Notice() list_report = [] list_logs = [] mcg = ConfigUtils.ModelCfgGenerator() ccg = ConfigUtils.TrainCfgGenerator() for i in range(iteration): print('number of hyperparam setting tried: {}/{}'.format(i+1, iteration)) cfg_model = mcg.GetContent(distribution=list_dstribution[i], reg=list_re[i]) cfg_train = ccg.GetContent(lr=list_lr[i], max_batch=max_batch, verbose=verbose) mnist = input_data.read_data_sets(cfg_train.data_dir , one_hot=False, seed=231) report, logs = Run(cfg_model,cfg_train, mnist) log_df = Logs2Df(logs) list_report.append(report) list_logs.append(log_df) df = Reports2Df(list_report) return df, list_logs
def get_saved_filter_json( global_config, filter_name, store_data_to_file=False ): global saved_filters competition = global_config['this_competition'] + global_config['this_season'] if len(saved_filters) == 0: ConfigUtils.read_config( saved_filters, './config/savedfilters.txt' ) result = [] result.append('{ "filters": [\n') if filter_name != None: try: filter_name = filter_name.title() filter_str = saved_filters[filter_name] except: pass result.append(' { "name": "%s", "filter_str": "%s" }\n' % (filter_name,filter_str)) else: for name, filter_str in saved_filters.iteritems(): result.append(' { "name": "%s", "filter_str": "%s" }' % (name,filter_str)) result.append(',\n') if len(saved_filters) > 0: result = result[:-1] result.append('] }\n') json_str = ''.join(result) if store_data_to_file: try: if filter_name == None: file_name = 'attrfilters.json' else: file_name = 'attrfilter_%s.json' % filter_name FileSync.put( global_config, '%s/EventData/%s' % (competition,file_name), 'text', json_str) except: raise return json_str
def runWebTask(configItems): cItemsWebtask = ConfigUtils(configItems['inputConfigDir'], configItems['webtask_config_fn'] ) configItemsWebTasks = cItemsWebtask.getConfigs() url = configItemsWebTasks['webtask_url'] attempts = 0 while attempts < 3: try: response = urllib2.urlopen(url, timeout = 15) content = response.read() if(content): f = open( configItems['log_dir'] + "algolia-sync-results.json", 'w' ) f.write( content ) f.close() return True break except urllib2.URLError as e: attempts += 1 print type(e) return False return False
def main(): curr_full_path = FileUtils.getCurrentDirFullPath() config_fn = 'portal_activity_job_config.yaml' cI = ConfigUtils(curr_full_path + "/configs/", config_fn) configItems = cI.getConfigs() configItems[ 'config_dir'] = curr_full_path + "/" + configItems['config_dir'] configItems['curr_full_path'] = curr_full_path db_ini = configItems['config_dir'] + configItems['database_config'] conn_alq, meta_alq = PostgresStuff.connect_alq(db_ini) conn = PostgresStuff.connect(db_ini) db_tbl = configItems['activity_table'] rotate_items = rotateActivityData(conn) activity = "rotate_portal_activity" subject_line = configItems['activity'][activity]['email_msg'][ 'subject_line'] msg_body = configItems['email_msg_template']['header'] + configItems[ 'activity'][activity]['email_msg']['msg'] msg_body = msg_body + configItems['email_msg_template']['footer'] em = Emailer(configItems) em.sendEmails(subject_line, msg_body.encode('utf-8').strip())
def main(): curr_full_path = FileUtils.getCurrentDirFullPath() config_fn = 'portal_activity_job_config.yaml' cI = ConfigUtils(curr_full_path + "/configs/", config_fn) configItems = cI.getConfigs() configItems[ 'config_dir'] = curr_full_path + "/" + configItems['config_dir'] configItems['curr_full_path'] = curr_full_path db_ini = configItems['config_dir'] + configItems['database_config'] conn_alq, meta_alq = PostgresStuff.connect_alq(db_ini) conn = PostgresStuff.connect(db_ini) db_tbl = configItems['activity_table'] digest_items = digestStaleDelayedDatasets(conn_alq, configItems, 'stale_delayed_digest') if (not (digest_items)): print("**** No digest items " + configItems['activity']['update']['time_interval'] + "*****") exit(0) datasetid_notified = MonitorPortal.generateEmail(conn_alq, configItems, 'stale_delayed_digest', digest_items)
def main(): curr_full_path = FileUtils.getCurrentDirFullPath() config_fn = 'portal_activity_job_config.yaml' cI = ConfigUtils(curr_full_path+ "/configs/" , config_fn) configItems = cI.getConfigs() configItems['config_dir'] = curr_full_path+ "/" + configItems['config_dir'] configItems['curr_full_path'] = curr_full_path db_ini = configItems['config_dir'] + configItems['database_config'] conn_alq, meta_alq =PostgresStuff.connect_alq(db_ini) conn = PostgresStuff.connect(db_ini) db_tbl = configItems['activity_table'] insert_late_updated = updateStaleDelayedDatasets(conn, configItems['activity']['update']['time_interval']) print (insert_late_updated) stale_late_datasets = MonitorPortal.generateActivityReport(conn_alq, configItems, 'update') print ( stale_late_datasets) if (not (stale_late_datasets)): print ("**** No changes for stale or deleyed datasets " + configItems['activity']['update']['time_interval'] + "*****") exit (0) datasetid_notified = MonitorPortal.generateEmail(conn_alq, configItems, 'update', stale_late_datasets) updted_notified_cnt = MonitorPortal.updateNotifiedDatasetIds(conn, configItems, 'update', datasetid_notified) print ("******Notfied that " +str(updted_notified_cnt) + " datasets are late or stale****" ) print ("******Updated" + str(updted_notified_cnt) + " rows in the late_updated_dataset table****" )
def emit(self, record): msg = self.format(record) _filePath = datetime.datetime.now().strftime(self.filefmt) filename = self.formatter.filename suffix = '/' + filename + '_' newpath = _filePath.replace('/', suffix) _dir = os.path.dirname(newpath) try: if os.path.exists(_dir) is False: os.makedirs(_dir) except Exception: print "can not make dirs" print "filepath is " + newpath pass try: global newfilepath if hasattr(self, 'newfile'): newpath = self.newfile with open(newpath, 'a') as _fobj: filesize = self.get_FileSize(newpath) logFileSize = ConfigUtils.getWebPorperties('LogFileSize') if filesize < logFileSize: _fobj.write(str(msg)) _fobj.write("\n") _fobj.flush() _fobj.close() else: newFilFmt = os.path.join("logs", "%Y%m%d%H%M%S.log") filePath = datetime.datetime.now().strftime(newFilFmt) suffix_new = '/' + filename + '_' self.newfile = filePath.replace('/', suffix_new) with open(self.newfile, 'a') as _fobj: _fobj.write(str(msg)) _fobj.write("\n") _fobj.flush() _fobj.close() # self.queryFileInDir(str(msg),filename) except Exception, e: print "can not write to file" print "filepath is " + newpath print e.message
def __init__(self): self.window = Tk() self.window.title("Dimens Convert Tool") windowW = 420 if (platform.system() == "Windows"): windowW = 420 else: windowW = 490 self.window.minsize(windowW, 355) self.center_wind(self.window, windowW, 355) frame = Frame(self.window) frame.pack() config_path = os.path.split( os.path.realpath(__file__))[0] + "/" + "configs.ini" self.rootDir = os.path.split(os.path.realpath(__file__))[0] self.config_path = config_path conf = ConfigUtils.Config(config_path) self.conf = conf fontSize = 35 menubar = Menu(self.window) helpmenu = Menu(menubar, tearoff=0) helpmenu.add_command(label="使用方法", command=self.help) helpmenu.add_separator() helpmenu.add_command(label="关于", command=self.about) menubar.add_cascade(label="帮助", menu=helpmenu) self.window.config(menu=menubar) index = 0 designDevice = DesignDevice.Design(conf, index, frame, fontSize, self.rootDir) index = designDevice.layout() self.designDevice = designDevice targetDevice = TargetDevice.Design(conf, index, frame, fontSize) index = targetDevice.layout() self.targetDevice = targetDevice btApply = Button(frame, text="生成", command=self.processButtonGeneral) btApply.grid(row=index + 1, column=1, columnspan=2) btCancel = Button(frame, text="取消", command=self.processButtonCancel) btCancel.grid(row=index + 1, column=2, columnspan=2) # 监测事件直到window被关闭 self.window.mainloop()
def _Test(): import ConfigUtils cfg = ConfigUtils.ModelCfg() model = BigModel(cfg) session = tf.InteractiveSession() session.run(model.op_init)
) parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="Be verbose.") parser.add_option( "-c", "--config", action="store", dest="configFile", default="config.json", help="Config file (json)" ) options, args = parser.parse_args() except Exception, e: print e print "For help use --help" sys.exit(2) ble = BleAutomator(options.interface, options.verbose) addresses = ConfigUtils.readAddresses(options.configFile) if not addresses: sys.exit(1) address_ind = 0 if len(addresses) == 1: cycle = 0 ble.connect(addresses[0]) else: cycle = 1 # Endless loop: while True: # Connect to peer device. if cycle == 1: ble.connect(addresses[address_ind])
default=None, help='Target address. (Can be found by running "hcitool lescan")') options, args = parser.parse_args() except Exception, e: print e print "For help use --help" sys.exit(2) ble = BleAutomator(options.interface, options.verbose) if (options.address): addresses = [options.address] else: addresses = ConfigUtils.readAddresses(options.configFile) if (not addresses): sys.exit(1) address_ind = 0 switch = 1 if (len(addresses) == 1): if not ble.connect(addresses[0]): exit(1) # Subscribe for notifications handle = ble.getHandle(CHAR_POWER_SAMPLES) + 2 if (not ble.writeCharacteristicHandle( handle, Conversion.uint16_to_uint8_array(1))): exit(1) while (True): getSamples(addresses[0])
def main(): # instacia das variaveis globais global counter global shutdown global fanPort global minFanUpTime global refreshRate global maxTemp global minTemp global channel_id global write_key global tskrefresh global lastUpdate global channel global isRelay global onValue global offValue global useSocCmd # Carrega as configura??es (fanPort,minFanUpTime,refreshRate,maxTemp,minTemp,channel_id,write_key,tskrefresh,isRelay,useSocCmd) = configs.loadConfig() if(isRelay): onValue = 0 offValue = 1 else: onValue = 1 offValue = 0 if(channel_id != -1): channel = thingspeak.Channel(id=channel_id,write_key=write_key) parser = optparse.OptionParser() parser.add_option("-v", "--version", action="store_true", dest="version", help="Show the software version.", default = False) parser.add_option("-c", "--config", action="store_true", dest="config", help="Generates a default config file. WARNING: this can overwrite existing settings file.", default = False) parser.add_option("--reloadConf", action="store_true", dest="reload", help="Reloads the config file to apply changes.", default = False) group = optparse.OptionGroup(parser, "Controll Options") group.add_option("-f", "--force", type="string", nargs = 1, dest="force", help="force the fan to be always ON/OFF", default = "null") group.add_option("-r", "--restore", action="store_true", dest="restore", help="restore the fan to auto mode", default = False) parser.add_option_group(group) group = optparse.OptionGroup(parser, "Status Options") group.add_option("-a", "--appear", action="store_true", dest="appear", help="force run even if a process is already running", default=False) group.add_option("-s", "--status", action="store_true", dest="fanstatus", help="show if the fan is ON/OFF", default=False) group.add_option("-t", "--temp", action="store_true", dest="temp", help="shows the current temperature", default=False) parser.add_option_group(group) group = optparse.OptionGroup(parser, "Installation Options") group.add_option("--install", action="store_true", dest="install", help="makes 'fan' command available to bash command line", default=False) group.add_option("--uninstall", action="store_true", dest="uninstall", help="uninstall 'fan' command from bash command line", default=False) group.add_option("--autoinit", type="string", nargs = 1, dest="autoinit", help="sets the auto init true/false, if true the process will start at boot up", default = "null") parser.add_option_group(group) group = optparse.OptionGroup(parser, "Dangerous Options","Use this options with caution.") group.add_option("--clear", action="store_true", dest="clear", help="free shared memory, this may cause some stability issues, try using '--restore' after using this option.", default = False) parser.add_option_group(group) (options, args) = parser.parse_args() # para o processo caso o computador comece a desligar signal.signal(signal.SIGTERM, stop) if(options.reload): try: configReload = sysv_ipc.SharedMemory(19021999) #procura a memoria compartilhada utils.write_to_memory(configReload,"reload") print("The configuration was reloaded.") except: print("Fail to reload the configuration.") sys.exit() elif(options.config): try: configs.createConfig() print('Config file created.') except: print('Fail to create the config file.') sys.exit() elif(options.version): print("Fan version: %s" % (version)) sys.exit() elif(options.clear): try: setGPIO() fanForce = sysv_ipc.SharedMemory(22061995) sysv_ipc.remove_shared_memory(fanForce.id) print("Memory Cleared.") except: print("There was no memory to clear.") GPIO.cleanup(fanPort) print("GPIO Cleared.") sys.exit() elif(options.install): try: installFan() print("'fan' was installed to the command line.") except: print("Fail to install 'fan' to the command line.") sys.exit() elif(options.uninstall): try: os.remove("/usr/local/bin/fan") print("'fan' was removed from command line.") except: print("Fail to remove 'fan' from command line.") sys.exit() elif(options.autoinit == "true"): try: installAutoInit() print("Autoinit set to True, restart your system apply changes.") except: print("Fail to set Autoinit to True.") sys.exit() elif(options.autoinit == "false"): try: uninstallAutoInit() print("Autoinit set to False") except: print("Fail to set Autoinit to False.") sys.exit() elif(options.autoinit != "null"): print("Invalid parameter.") print("") print("Usage: --autoinit [true/false]") sys.exit() # se receber o comando -a forca abrir uma instancia nova elif (options.appear): setGPIO() # desliga a fan GPIO.output(fanPort,offValue) elif(options.force == "on"): try: fanForce = sysv_ipc.SharedMemory(22061995) #procura a memoria compartilhada utils.write_to_memory(fanForce,"on") setGPIO() GPIO.output(fanPort,onValue) print("The Fan was forced to be on.") except: print("Fail forcing fan to be on.") sys.exit() elif(options.force == "off"): try: fanForce = sysv_ipc.SharedMemory(22061995) #procura a memoria compartilhada utils.write_to_memory(fanForce,"off") setGPIO() GPIO.output(fanPort,offValue) print("The Fan was forced to be off. WARNING: The fan will not auto turn on anymore.") except: print("Fail forcing fan to be off.") sys.exit() elif(options.force != "null"): print("Invalid parameter.") print("") print("Usage: --force [on/off]") sys.exit() elif(options.restore): try: fanForce = sysv_ipc.SharedMemory(22061995) #procura a memoria compartilhada utils.write_to_memory(fanForce,"default") print("The Fan was restored to auto-mode.") except: print("The Fan was restored to auto-mode.") sys.exit() elif(options.fanstatus): setGPIO() if(GPIO.input(fanPort) == offValue): print("The fan is inactive.") else: print("The fan is active.") sys.exit() elif(options.temp): print ("Temperature: %0.2f 'C" % (getTemp())) sys.exit() else: parser.error("fan requires an argument.") quit() try: fanForce = sysv_ipc.SharedMemory(22061995,sysv_ipc.IPC_CREX) #cria memoria compartilhada configReload = sysv_ipc.SharedMemory(19021999,sysv_ipc.IPC_CREX) #cria memoria compartilhada except: fanForce = sysv_ipc.SharedMemory(22061995) #cria memoria compartilhada configReload = sysv_ipc.SharedMemory(19021999) #cria memoria compartilhada utils.write_to_memory(fanForce,"default") utils.write_to_memory(configReload,"default") try: while (shutdown == False): # descobre se a fan esta ligada status = GPIO.input(fanPort) # se receber um comando de force via console para de rodar if(utils.read_from_memory(fanForce) == "default"): # se a temperatura for maior ou igual a maxTemp graus liga a fan if (getTemp() >= maxTemp): if(status == offValue): # liga a fan GPIO.output(fanPort,onValue) # envia o status para o thingspeak k=0 # aguarda o tempo minimo de execucao da fan while(k < minFanUpTime and shutdown == False): # se receber um comando de controle de fan sai da espera if(utils.read_from_memory(fanForce) == "default"): # envia o status para o thingspeak updateThingspeak() if(utils.read_from_memory(configReload) == "reload"): reloadConfigs() utils.write_to_memory(configReload,"default") time.sleep(1) k+=1 else: break counter=0 else: updateThingspeak() if(utils.read_from_memory(configReload) == "reload"): reloadConfigs() utils.write_to_memory(configReload,"default") time.sleep(refreshRate)#apos o tempo minimo ele aguarda o tempo de refresh hate definido counter=0 # se estiver no limite da transicao aguarda 2x o refresh rate para desligar a fan, assim evitando liga e desliga de fan elif (counter < 2): counter += 1 time.sleep(refreshRate) elif (minTemp < getTemp()): counter = 0 updateThingspeak() if(utils.read_from_memory(configReload) == "reload"): reloadConfigs() utils.write_to_memory(configReload,"default") else: if(status == onValue): GPIO.output(fanPort,offValue) updateThingspeak() if(utils.read_from_memory(configReload) == "reload"): reloadConfigs() utils.write_to_memory(configReload,"default") time.sleep(refreshRate) else: updateThingspeak() if(utils.read_from_memory(configReload) == "reload"): reloadConfigs() utils.write_to_memory(configReload,"default") time.sleep(refreshRate) except KeyboardInterrupt: pass finally: GPIO.cleanup(fanPort) sysv_ipc.remove_shared_memory(fanForce.id) sysv_ipc.remove_shared_memory(configReload.id)
import ConfigUtils as cu except: printException('Failed to load support scripts.',sys.exc_info()) #Ref0002 End ######################################################################################################### ######################################################################################################### # Gets configuration data from the repository ######################################################################################################### ######################################################################################################### True=1 False=0 #Ref0002 Begin try: cu.printMsg('Begin run.',False) except: print 'ERROR, cannot continue. Failed to load support scripts. Run wsadmin -f ConfigDump.py from the same directory it exists in.' sys.exit(1) excludeTypeList = [] #Ref0044 regexList=[] #Ref0038 if len(sys.argv) > 0: if " ".join(sys.argv).find("=") == -1: #Ref0038 cu.setRepositoryConfigFileName(sys.argv[0]) #Ref0002 End if len(sys.argv) > 1: #Ref0037 cu.setCellSuffix(sys.argv[1]) #Ref0037 #Ref0038 Begin else: for arg in sys.argv:
if __name__ == "__main__": # Decode the input augument options, args = decodeCommandLine() print "Input filelist : %s" % options.inList print "Histogram list : %s" % options.histoList print "Output directory : %s" % options.outDir if options.selectionList: print "Selection criteria: %s" % options.selctionList else: print "No selection criterion is applied." # Read in the input filelist sList = {} nSources, sList = ConfigUtils.getFileList( options.inList ) # Read in the histogram list Histos = {} tree, Histos = ConfigUtils.getHistoNames( options.histoList ) # Create the output directory createDir() # Copy the input file list to the output directory os.system('cp %s %s' % ( options.inList, options.outDir ) ) # Read in the selection criteria Selection = {} if options.selectionList: Selection = ConfigUtils.getSelection( options.selctionList )
def __init__(self, vocabulary_filename, training_set_size, training_set_filename, validation_set_filename, test_set_filename, config_file): # load configuration file options = cu.init_configuration(config_file) self.recovery_config_file = options['recovery_config_file'] self.recovery_path = options['recovery_path'] # load dataset files self.training_set = open(training_set_filename, 'r') self.validation_set = open(validation_set_filename, 'r') self.test_set = open(test_set_filename, 'r') # load vocabulary v = open(vocabulary_filename, 'r') vocabulary = json.load(v) v.close() ss_occurrences = vocabulary.pop("</s>") vocabulary = OrderedDict({"</s>": ss_occurrences}.items() + OrderedDict(sorted(vocabulary.items(), key=lambda x: (-x[1], x[0]))).items()) self.vocabulary_size = len(vocabulary) # smoothing factor self.occurrences_smoothing = options['occurrences_smoothing'] if(self.occurrences_smoothing): occurrences = vocabulary.values() log_training_set_size = np.log(training_set_size) self.smoothing_factors = [] for i in xrange(self.vocabulary_size): self.smoothing_factors.insert(i, log_training_set_size - np.log(occurrences[i])) # normalize between 0.2 and 1.2 old_min = min(self.smoothing_factors) old_max = max(self.smoothing_factors) new_min = 1.0 new_max = 2.0 norm_range = (new_max-new_min)/(old_max-old_min) for i in xrange(self.vocabulary_size): factor = new_min + (self.smoothing_factors[i] - old_min) * norm_range self.smoothing_factors[i] = factor # net parameters self.learning_rate = options['learning_rate'] self.starting_epoch = options['starting_epoch'] self.max_epochs = options['max_epochs'] self.min_improvement = options['min_validation_logp_improvement'] self.logp_previous = options['logp_previous'] self.learning_rate_divide = options['learning_rate_divide'] self.reset_context_each_sentence = options['reset_context_each_sentence'] # init neurons self.hidden_layer_size = options['hidden_layer_size'] self.neu_input_index = -1 self.neu_context = [0.1] * self.hidden_layer_size self.neu_hidden = np.zeros(self.hidden_layer_size) self.neu_output = np.zeros(self.vocabulary_size) # init synapses try: self.__restore_weights() except IOError: # weights between input and hidden layer self.syn_input = get_random_matrix( self.vocabulary_size + self.hidden_layer_size, self.hidden_layer_size, options['weight_min_value'], options['weight_max_value']) # weights between hidden and output layer self.syn_hidden = get_random_matrix( self.hidden_layer_size, self.vocabulary_size, options['weight_min_value'], options['weight_max_value'])
import ConfigUtils logs_file_name_prefix = "/home/tomek/logs/logs_" + ConfigUtils.get_datetime_prefix() + "_" logs_file_append_pid = True projects_db_path = '/home/tomek/workspace/project_db.json' # autoscan autoscan_server_runnable_files = ['app_server.sh', 'standalone.sh'] autoscan_folders_to_skip = ['target', 'tests', 'standalone', 'glassfish', 'ACA'] actions = { 'clean-build': ('cb', 'mvn clean install'), 'clean-build2': ('cb2', 'mvn2 clean install'), 'clean-build3': ('cb3', 'mvn3 clean install'), 'mvn': ('m', 'mvn {0} install', ['']), 'reset-hard': ('rh', 'git reset --hard && git checkout {0} && git fetch && git reset --hard origin/{0}', ['master']), 'apply-new-code': ('anc', 'git add -A && git stash && git fetch && git reset --hard origin/{0} && git stash apply', ['master']), 'run': ('r', '{0}', []), 'deploy-core-full': ('dcf', '{0} undeploy && {0} stop && {0} clean && {0} deploy', []), } continue_on_fail = False
def reloadConfigs(): global fanPort global minFanUpTime global refreshRate global maxTemp global minTemp global channel_id global write_key global tskrefresh global channel global isRelay global onValue global offValue global useSocCmd lastFanPort = fanPort (fanPort,minFanUpTime,refreshRate,maxTemp,minTemp,channel_id,write_key,tskrefresh,isRelay,useSocCmd) = configs.loadConfig() if(channel_id != -1): channel = thingspeak.Channel(id=channel_id,write_key=write_key) if(isRelay): onValue = 0 offValue = 1 else: onValue = 1 offValue = 0 GPIO.cleanup(lastFanPort) setGPIO()
def main(): fieldConfigFile, config_inputdir, jobType, hourly = parse_opts() cI = ConfigUtils(config_inputdir, fieldConfigFile) configItems = cI.getConfigs() configItems['dataset_name'] = jobType lg = pyLogger(configItems) logger = lg.setConfig() logger.info("****************JOB START******************") sc = SocrataClient(config_inputdir, configItems, logger) client = sc.connectToSocrata() clientItems = sc.connectToSocrataConfigItems() scrud = SocrataCRUD(client, clientItems, configItems, logger) sQobj = SocrataQueries(clientItems, configItems, logger) mmdd_fbf = configItems['dd']['master_dd']['fbf'] ds_profiles_fbf = configItems['dd']['ds_profiles']['fbf'] base_url = configItems['baseUrl'] field_type_fbf = configItems['dd']['field_type']['fbf'] asset_inventory_fbf = configItems['dd']['asset_inventory']['fbf'] ds_profiles = ProfileDatasets.getCurrentDatasetProfiles( sQobj, base_url, ds_profiles_fbf) update_counter = 0 updated_datasets = [] #if int(hourly) == 1: # print "****hourly update****" for datasetid, last_updt in ds_profiles.iteritems(): mm_profiles_to_updt = ProfileDatasets.getViewsLastUpdatedAt( datasetid, last_updt, clientItems) #print datasetid #print mm_profiles_to_updt if ('cols' in mm_profiles_to_updt.keys()): dataset_info_mm = { 'Socrata Dataset Name': configItems['dataset_name'], 'SrcRecordsCnt': 0, 'DatasetRecordsCnt': 0, 'fourXFour': mmdd_fbf, 'row_id': 'columnid' } dataset_info_mm['DatasetRecordsCnt'] = 0 dataset_info_mm['SrcRecordsCnt'] = len(mm_profiles_to_updt['cols']) #print mm_profiles_to_updt.keys() print "***************" print "Updating " + mm_profiles_to_updt['dataset_name'] print "*************" print "**************" dataset_info_mm = scrud.postDataToSocrata( dataset_info_mm, mm_profiles_to_updt['cols']) update_counter = update_counter + 1 updated_datasets.append(mm_profiles_to_updt['dataset_name']) datasets = ProfileDatasets.getBaseDatasets(sQobj, base_url, mmdd_fbf) asset_inventory_dict = ProfileDatasets.getAssetInventoryInfo( sQobj, base_url, asset_inventory_fbf) #delete datasets from the profile that no longer exist ProfileDatasets.removeDeletedDatasets(scrud, ds_profiles_fbf, asset_inventory_dict, ds_profiles, datasets) ds_profiles = ProfileDatasets.getCurrentDatasetProfiles( sQobj, base_url, ds_profiles_fbf) field_types = ProfileDatasets.getFieldTypes(sQobj, base_url, field_type_fbf) dataset_info = ProfileDatasets.buildInsertDatasetProfiles( sQobj, scrud, configItems, datasets, ds_profiles, field_types, asset_inventory_dict) ## triggers a webtask to update the agolia index updated_algolia = False if (WebTasks.runWebTask(configItems)): updated_algolia = True print "******Ran Updated *** agolia script****" + str(updated_algolia) print dataset_info dsse = JobStatusEmailerComposer(configItems, logger, jobType) if dataset_info['DatasetRecordsCnt'] > 1 and updated_algolia: print "update complete" dsse.sendJobStatusEmail([dataset_info]) else: dataset_info = { 'Socrata Dataset Name': configItems['dataset_name'], 'SrcRecordsCnt': 0, 'DatasetRecordsCnt': 0, 'fourXFour': "Nothing to Insert" } dataset_info['isLoaded'] = 'success' dsse.sendJobStatusEmail([dataset_info])
# command line options handling parser = OptionParser() parser.add_option( "-l","--processloop",dest="processloop", default='0', help='Process Team Files') parser.add_option( "-a","--aliases",dest="comp_alias_file",default='ScoutingAppEventAliases.txt', help="Competition Alias Configuration File") # Parse the command line arguments (options,args) = parser.parse_args() global_config = {} ConfigUtils.read_config(global_config, './config/ScoutingAppConfig.txt') logger = Logger.get_logger('./config', 'logging.conf', 'scouting.fileproc') global_config['logger'] = logger # load the competition alias file if one is specified if options.comp_alias_file != '': comp_alias_file = './config/' + options.comp_alias_file logger.debug('Loading Competition Alias file: %s' % comp_alias_file) CompAlias.read_comp_alias_config(comp_alias_file) session = DbSession.open_db_session((global_config['db_name'] + global_config['this_season']), DataModel) counter = 0 done = False
True = 1 ######################################################################################################### ######################################################################################################### # Execution logic begins here ######################################################################################################### ######################################################################################################### if len(sys.argv) < 3: #Ref0036 print 'Syntax:' print ' python ' + sys.argv[0] + ' ReportFileName FileList1 FileList3 ...' print ' ReportFileName is the name of output report (fully qualified or relative)' print ' FileList parameters are lists of files generated by CollectFileData.sh' #Ref0036 # print ' LibertyTarX is a tar file created by ConfigDumpLiberty.sh' sys.exit(0) cu.printMsg('Begin run.', False) fileIdx = 1 if sys.argv[0].find('.py') > 0: fileIdx = 2 ReportFileName = sys.argv[1] cu.printMsg('Report file name: %s' % ReportFileName, False) #Get information from input files FileNames = sys.argv[fileIdx:] for idx in range(0, len(FileNames) - 1): FileNames[idx] = FileNames[idx].strip() ServerDataList = cu.createServerFileDataList(FileNames) #Ref0030 #Run reports
["category=", "origin=", "config="]) except getopt.GetoptError: print('Usage: ' + progname + ' -c <category> -o <origin> -f <configfile>') sys.exit(2) for opt, arg in opts: if opt == '-h': print(progname + ' -c <category -o <origin> -f <configfile>') sys.exit() elif opt in ("-c", "--category"): category = arg elif opt in ("-o", "--origin"): origin = arg elif opt in ("-f", "--config"): configfile = arg ConfigUtils.init(configfile) localSyncFolderConfig = SyncFolderConfig('local', category, 'out') cloudSyncFolderConfig = SyncFolderConfig('cloud', category, origin) logger = LoggerFactory.createLogger( "upload", os.path.join(localSyncFolderConfig.subfolder, "upload.log")) if __name__ == "__main__": path = localSyncFolderConfig.folder event_handler = EventHandler() observer = Observer() observer.schedule(event_handler, path, recursive=True) observer.start() UploadFolder.upload(localSyncFolderConfig, cloudSyncFolderConfig) try: