def create(cls, serviceID=None):
self = cls()
try:
createSql = "INSERT INTO {}.{} "\
" (CHANNEL_ID , MEDIA_TYPES, STEP, STATUS, DATA_STATUS, TASK_FQ_CONTENT, " \
" CREATE_TIME, UPDATE_TIME, CREATE_USER, UPDATE_USER, DELETE_FLAG )" \
" SELECT DISTINCT {}, 1, 5, 0 , 0, null, '{}', null, 'system', null, 0 " \
" FROM {}.{} " \
" WHERE EXISTS ( " \
" SELECT * FROM {}.{} " \
" WHERE CHANNEL_ID = {} AND STEP = 4 AND STATUS = 2)" \
" AND NOT EXISTS ( " \
" SELECT * FROM {}.{} " \
" WHERE CHANNEL_ID = {} AND STEP = 5 )".format( self.__db, self.__table,
serviceID,
utils.now_str(),
self.__db, self.__table,
self.__db, self.__table,
serviceID,
self.__db, self.__table,
serviceID)
mysqlSvrClient(host=self.__host,
user=self.__user,
passwd=self.__passwd,
port=3306,
db=self.__db).execute_data(createSql)
except Exception, e:
logging.error(e)
def train():
# ======= paramter ==============
use_local_net = False
epoch = 20
num_epoches = 40
batch_size = 32
# =======net ================
model_path = 'model_without_sigmoid\\' + \
'model' + '_' + str(epoch - 1) + model_ext
if use_local_net:
model = load_model(model_path)
logger.info('use local model from %d epoch', epoch)
else:
model = net()
logger.info('model inited')
test_cells, test_masks = get_test_data()
# ===========train =============
while epoch < num_epoches:
logger.info('%s epoch %d started', now_str(), epoch)
generator = batch_data_generator(batch_size=batch_size,
use_shuffle=True)
batch_idx = 0
for cell_mask in generator:
cell_list = np.array([item[0] for item in cell_mask])
mask_list = np.array([item[1] for item in cell_mask])
model.train_on_batch(cell_list, mask_list)
batch_idx += 1
if batch_idx % 20:
logger.info('%s epoch %d batch_index %d', now_str(), epoch,
batch_idx)
batch_idx = 0
# ===========test==================
predict_masks = model.predict(test_cells, batch_size=64)
error = test_error(test_masks, predict_masks)
logger.info('predict error %f in epoch %d', error, epoch)
model.save('model_without_sigmoid\\' + 'model' + '_' + str(epoch) +
model_ext)
logger.info('%s epoch %d model saved', now_str(), epoch)
epoch += 1
def train():
# =====parameters ===========
col, tokenizer= db_init()
use_loaded_model =True
batch_size = 64
batch_idx = 0
n_epoch = 15
# ==========currnt==========
epoch = 1
# =======net ===============
net = Net()
if use_loaded_model:
try:
net = load_model(model_dir+model_name+'_'+ str(epoch)+'_'+model_ext)
logger.info('init net from local, current epoch is %d',epoch)
except Exception:
pass
# =========train ===============
while epoch < n_epoch:
logger.info('%s %d / %d epoch start',now_str(),epoch,n_epoch)
while True:
if batch_idx % 100 == 0:
logger.info('%s %d epoch %d batch start',now_str(),epoch,batch_idx)
data,labels = get_train_batch_data_labels(
csv_path=train_data_path,col=col,tokenizer=tokenizer,
batch_index=batch_idx,batch_size=batch_size)
if len(data) > 1:
sample_weight = calc_sample_weight(labels)
net.train_on_batch(data,labels,sample_weight=sample_weight)
else:
logger.debug('error!,data in batch_index %d len(data) <= 1',batch_idx)
break
batch_idx +=1
# ===save=====
logger.info('save model...')
batch_idx = 0
net.save(model_dir+model_name+'_'+ str(epoch)+'_'+model_ext)
epoch+=1
print(f'Starting ({now_str()})')
iter_count = len(data_set) // BATCH_SIZE
while epoch < first_epoch + EPOCH_COUNT:
iter_metrics = Metrics()
lr = scheduler.get_lr()[0]
for i, (inputs, labels) in enumerate(data_loader):
inputs = inputs.to(device)
labels = labels.to(device)
optimizer.zero_grad()
outputs = model(inputs).to(device)
loss = criterion(outputs, labels)
coef = Coef.calc(outputs, labels)
iter_metrics.append_loss(loss.item())
iter_metrics.append_coef(coef)
pp('epoch[{ep}]:{i}/{I} iou:{c.pjac:.4f} acc:{c.pdice:.4f} loss:{loss:.4f} lr:{lr:.4f} ({t})'.format(
ep=epoch, i=i+1, I=iter_count, lr=lr, t=now_str(), loss=loss.item(), c=coef))
loss.backward()
optimizer.step()
pp('epoch[{ep}]:Done. iou:{c.pjac:.4f} acc:{c.pdice:.4f} gsi:{c.gsensi:.4f} gsp:{c.gspec:.4f} tsi:{c.tsensi:.4f} tsp:{c.tspec:.4f} loss:{loss:.4f} lr:{lr:.4f} ({t})'.format(
ep=epoch, t=now_str(), lr=lr, loss=iter_metrics.avg('losses'), c=iter_metrics.avg_coef()
))
gc.collect()
print()
weight_path = os.path.join(DEST_DIR, f'{Model.__name__.lower()}_{epoch}.pt')
weights = model.module.cpu().state_dict() if USE_MULTI_GPU else model.cpu().state_dict()
metrics.append_coef(iter_metrics.avg_coef())
metrics.append_loss(iter_metrics.avg_loss())
store.set_states(weights, optimizer.state_dict(), metrics.state_dict())
store.save(weight_path)
print(f'save weights to {weight_path}')
model = model.to(device)
def log(msg):
if DEBUG: utils.eprint('[%s] %s' % (utils.now_str(), msg))
def log(msg):
if DEBUG: utils.eprint('[%s] %s' % (utils.now_str(), msg))
if __name__ == '__main__':
options = Options()
opts = options.parse(sys.argv[1:])
DEBUG = opts.verbose
g_file = opts.graphml_file
multiples = opts.multiplier
exid_file = opts.exclude_ids_file
STARTING_TIME = utils.now_str()
log('Starting at %s\n' % STARTING_TIME)
log('reading graph %s' % g_file)
in_g = nx.read_graphml(g_file)
t_file = opts.tweets_file
log('reading tweets %s' % t_file)
in_f = (gzip.open(t_file, 'rt', encoding='utf-8')
if t_file[-1] in 'zZ' else open(t_file, 'r', encoding='utf-8'))
post_counts = {}
retweeted = {} # ot_id : [rting_acct_ids]
line_count = 0
for l in in_f:
line_count = utils.log_row_count(line_count, DEBUG)
t = json.loads(l)
import csv
import os.path
import shutil
import sys
import utils
from os import path
# Sorts the rows in a CSV output of raw_to_csv.py into timestamp order.
# Useful when you've created the CSV from an input file not in timestamp order
# (e.g., the RU-IRA tweets)
if __name__ == '__main__':
print('Starting: %s' % utils.now_str())
in_f = sys.argv[1]
backup = '%s.bkp' % in_f
i = 1
backup_base = backup
while path.exists(backup):
backup = '%s%d' % (backup_base, i)
i += 1
print('Backing up to %s' % backup)
shutil.copyfile(in_f, backup)
print('Reading in %s' % backup)
rows = []
def index():
return render_template('index.html', title='flask test', message=now_str())
DEBUG = False
def log(msg):
if DEBUG: utils.eprint('[%s] %s' % (utils.now_str(), msg))
if __name__ == '__main__':
options = Options()
opts = options.parse(sys.argv[1:])
DEBUG = opts.verbose
STARTING_TIME = utils.now_str()
log('Starting at %s' % STARTING_TIME)
# read in HCCs file and determine HCCs and their members
g = nx.read_graphml(opts.hccs_file)
lcn_g = nx.read_graphml(opts.lcn_file)
log('graphs loaded.')
bot_data = {}
if opts.bots_file:
try:
bot_data = utils.load_botornot_results(opts.bots_file)
except FileNotFoundError as e:
log('bot file missing: %s' % opts.bots_file)
log('bot data loaded.')
