def create(cls, serviceID=None): self = cls() try: createSql = "INSERT INTO {}.{} "\ " (CHANNEL_ID , MEDIA_TYPES, STEP, STATUS, DATA_STATUS, TASK_FQ_CONTENT, " \ " CREATE_TIME, UPDATE_TIME, CREATE_USER, UPDATE_USER, DELETE_FLAG )" \ " SELECT DISTINCT {}, 1, 5, 0 , 0, null, '{}', null, 'system', null, 0 " \ " FROM {}.{} " \ " WHERE EXISTS ( " \ " SELECT * FROM {}.{} " \ " WHERE CHANNEL_ID = {} AND STEP = 4 AND STATUS = 2)" \ " AND NOT EXISTS ( " \ " SELECT * FROM {}.{} " \ " WHERE CHANNEL_ID = {} AND STEP = 5 )".format( self.__db, self.__table, serviceID, utils.now_str(), self.__db, self.__table, self.__db, self.__table, serviceID, self.__db, self.__table, serviceID) mysqlSvrClient(host=self.__host, user=self.__user, passwd=self.__passwd, port=3306, db=self.__db).execute_data(createSql) except Exception, e: logging.error(e)
def train(): # ======= paramter ============== use_local_net = False epoch = 20 num_epoches = 40 batch_size = 32 # =======net ================ model_path = 'model_without_sigmoid\\' + \ 'model' + '_' + str(epoch - 1) + model_ext if use_local_net: model = load_model(model_path) logger.info('use local model from %d epoch', epoch) else: model = net() logger.info('model inited') test_cells, test_masks = get_test_data() # ===========train ============= while epoch < num_epoches: logger.info('%s epoch %d started', now_str(), epoch) generator = batch_data_generator(batch_size=batch_size, use_shuffle=True) batch_idx = 0 for cell_mask in generator: cell_list = np.array([item[0] for item in cell_mask]) mask_list = np.array([item[1] for item in cell_mask]) model.train_on_batch(cell_list, mask_list) batch_idx += 1 if batch_idx % 20: logger.info('%s epoch %d batch_index %d', now_str(), epoch, batch_idx) batch_idx = 0 # ===========test================== predict_masks = model.predict(test_cells, batch_size=64) error = test_error(test_masks, predict_masks) logger.info('predict error %f in epoch %d', error, epoch) model.save('model_without_sigmoid\\' + 'model' + '_' + str(epoch) + model_ext) logger.info('%s epoch %d model saved', now_str(), epoch) epoch += 1
def train(): # =====parameters =========== col, tokenizer= db_init() use_loaded_model =True batch_size = 64 batch_idx = 0 n_epoch = 15 # ==========currnt========== epoch = 1 # =======net =============== net = Net() if use_loaded_model: try: net = load_model(model_dir+model_name+'_'+ str(epoch)+'_'+model_ext) logger.info('init net from local, current epoch is %d',epoch) except Exception: pass # =========train =============== while epoch < n_epoch: logger.info('%s %d / %d epoch start',now_str(),epoch,n_epoch) while True: if batch_idx % 100 == 0: logger.info('%s %d epoch %d batch start',now_str(),epoch,batch_idx) data,labels = get_train_batch_data_labels( csv_path=train_data_path,col=col,tokenizer=tokenizer, batch_index=batch_idx,batch_size=batch_size) if len(data) > 1: sample_weight = calc_sample_weight(labels) net.train_on_batch(data,labels,sample_weight=sample_weight) else: logger.debug('error!,data in batch_index %d len(data) <= 1',batch_idx) break batch_idx +=1 # ===save===== logger.info('save model...') batch_idx = 0 net.save(model_dir+model_name+'_'+ str(epoch)+'_'+model_ext) epoch+=1
print(f'Starting ({now_str()})') iter_count = len(data_set) // BATCH_SIZE while epoch < first_epoch + EPOCH_COUNT: iter_metrics = Metrics() lr = scheduler.get_lr()[0] for i, (inputs, labels) in enumerate(data_loader): inputs = inputs.to(device) labels = labels.to(device) optimizer.zero_grad() outputs = model(inputs).to(device) loss = criterion(outputs, labels) coef = Coef.calc(outputs, labels) iter_metrics.append_loss(loss.item()) iter_metrics.append_coef(coef) pp('epoch[{ep}]:{i}/{I} iou:{c.pjac:.4f} acc:{c.pdice:.4f} loss:{loss:.4f} lr:{lr:.4f} ({t})'.format( ep=epoch, i=i+1, I=iter_count, lr=lr, t=now_str(), loss=loss.item(), c=coef)) loss.backward() optimizer.step() pp('epoch[{ep}]:Done. iou:{c.pjac:.4f} acc:{c.pdice:.4f} gsi:{c.gsensi:.4f} gsp:{c.gspec:.4f} tsi:{c.tsensi:.4f} tsp:{c.tspec:.4f} loss:{loss:.4f} lr:{lr:.4f} ({t})'.format( ep=epoch, t=now_str(), lr=lr, loss=iter_metrics.avg('losses'), c=iter_metrics.avg_coef() )) gc.collect() print() weight_path = os.path.join(DEST_DIR, f'{Model.__name__.lower()}_{epoch}.pt') weights = model.module.cpu().state_dict() if USE_MULTI_GPU else model.cpu().state_dict() metrics.append_coef(iter_metrics.avg_coef()) metrics.append_loss(iter_metrics.avg_loss()) store.set_states(weights, optimizer.state_dict(), metrics.state_dict()) store.save(weight_path) print(f'save weights to {weight_path}') model = model.to(device)
def log(msg): if DEBUG: utils.eprint('[%s] %s' % (utils.now_str(), msg))
def log(msg): if DEBUG: utils.eprint('[%s] %s' % (utils.now_str(), msg)) if __name__ == '__main__': options = Options() opts = options.parse(sys.argv[1:]) DEBUG = opts.verbose g_file = opts.graphml_file multiples = opts.multiplier exid_file = opts.exclude_ids_file STARTING_TIME = utils.now_str() log('Starting at %s\n' % STARTING_TIME) log('reading graph %s' % g_file) in_g = nx.read_graphml(g_file) t_file = opts.tweets_file log('reading tweets %s' % t_file) in_f = (gzip.open(t_file, 'rt', encoding='utf-8') if t_file[-1] in 'zZ' else open(t_file, 'r', encoding='utf-8')) post_counts = {} retweeted = {} # ot_id : [rting_acct_ids] line_count = 0 for l in in_f: line_count = utils.log_row_count(line_count, DEBUG) t = json.loads(l)
import csv import os.path import shutil import sys import utils from os import path # Sorts the rows in a CSV output of raw_to_csv.py into timestamp order. # Useful when you've created the CSV from an input file not in timestamp order # (e.g., the RU-IRA tweets) if __name__ == '__main__': print('Starting: %s' % utils.now_str()) in_f = sys.argv[1] backup = '%s.bkp' % in_f i = 1 backup_base = backup while path.exists(backup): backup = '%s%d' % (backup_base, i) i += 1 print('Backing up to %s' % backup) shutil.copyfile(in_f, backup) print('Reading in %s' % backup) rows = []
def index(): return render_template('index.html', title='flask test', message=now_str())
DEBUG = False def log(msg): if DEBUG: utils.eprint('[%s] %s' % (utils.now_str(), msg)) if __name__ == '__main__': options = Options() opts = options.parse(sys.argv[1:]) DEBUG = opts.verbose STARTING_TIME = utils.now_str() log('Starting at %s' % STARTING_TIME) # read in HCCs file and determine HCCs and their members g = nx.read_graphml(opts.hccs_file) lcn_g = nx.read_graphml(opts.lcn_file) log('graphs loaded.') bot_data = {} if opts.bots_file: try: bot_data = utils.load_botornot_results(opts.bots_file) except FileNotFoundError as e: log('bot file missing: %s' % opts.bots_file) log('bot data loaded.')