def get(self, **kwargs):
bus_stop = db.get(self.request.values.get('busstop_key'))
directions = self.request.values.get('directions', '')
if len(directions.split(',')) == 2:
directions = None
else:
directions = directions
bus_times = bus_stop.get_next_bus_times(settings.NEXT_BUS_TIME_MINUTES,
direction=directions)
bus_direction = BusDirection.all()
bus_direction.filter('direction =', directions)
# README
# BadValueError: Filtering on lists is not supported
# We can't use "bus_direction.filter('bus_line =', bus_stop.lines)"
# So, in the while we will use "bus_stop.lines[0]" because a BusStop only
# has one line for the moment
bus_direction.filter('bus_line =', bus_stop.lines[0])
bus_direction = bus_direction.fetch(1)
bus_direction = bus_direction[0]
info_content = '<b>%s</b><br /> %s<br /><em style="font-size: 10px">%s / Hacia: %s (%s)</em>' % \
(bus_stop.name, bus_stop.address, utils.get_weekday_display(), bus_direction.to_direction, bus_direction.direction)
for bus_time in bus_times:
left_time = relativedelta(bus_time.time_1970(), utils.now_time()).minutes
bus_already_gone = False
if left_time < 0:
if bus_time.time_1970().hour == 0:
# Time over 00hs
next_day = bus_time.time_1970() + datetime.timedelta(days=1)
left_time = relativedelta(next_day, utils.now_time()).minutes
else:
# This bus has already gone by
bus_already_gone = True
time = bus_time.time.strftime('%H:%M')
time_content = '<br /><b>%s min:</b><span> %s hs</span>' % (left_time, time)
if bus_already_gone:
info_content += '<span style="color: red">' + time_content + '</span>'
else:
info_content += time_content
if bus_time.comments:
info_content += '<em> ('
for comment in bus_time.comments:
info_content += '%s, ' % comment
info_content = info_content[:-2]
info_content += ')</em>'
return render_json_response({
'info_content': info_content
})
def delete_group(chat_id, username):
print("DATABASE: Delete Group Started")
row = find_row_by_id(item_id=chat_id)[0]
print("DATABASE: Got row number")
group_info = groupchats.row_values(row)
print("DATABASE: Got Row info")
try:
calls = spreadsheet.worksheet(str(chat_id))
print("DATABASE: Got calls worksheet")
# DELETE CALENDAR EVENTS
events_info = calls.get_all_values()
events_info.pop(0)
print("DATABASE: Event Info: ", events_info)
for event in events_info:
print("DATABASE: Event", event)
gcalendar.delete_event(event[0])
# ARCHIVE TRELLO CALL CARDS
# try:
# call_card_id = get_group_card(chat_id)
# trelloc.delete_group(call_card_id)
# print("DATABASE: Deleted Trello card")
# except:
# print("DATABASE: No Trello Card found")
# DELETE LINKED CALL SHEET
spreadsheet.del_worksheet(calls)
except:
print("DATABASE: No Calls sheet found")
# DELETE CARD IN TRELLO
try:
card_id = get_group_card(chat_id)
trelloc.delete_group(card_id)
print("DATABASE: Deleted Trello card")
except:
print("DATABASE: No Trello Card found")
# DELETE CHILDREN LINKS IN DATABASE
children = find_row_by_id(item_id=chat_id, col=11)
print("DATABASE: Children: ", children, " Type: ", type(children))
if children[0] != -1:
for child in children:
groupchats.update_cell(child, 11, '')
print("DATABASE: Deleted children")
# DROP UNNECESSAY INFO
unwanted = [0, 1, 9, 12, 13]
for ele in sorted(unwanted, reverse=True):
del group_info[ele]
group_info.extend((utils.now_time().strftime("%Y/%m/%d %H:%M"), username))
print("DATABASE: Dropped info")
# SAVE INFO IN DELETE SHEET
deleted.append_row(group_info)
# REMOVE ROW FROM GROUPS DATABASE
groupchats.delete_row(find_row_by_id(item_id=chat_id)[0])
def task_uuid():
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('8.8.8.8', 80))
ip = s.getsockname()[0]
# ip str = ip.replace('.','-')
task_id = now_time() + '-' + ip + '-'
return task_id
def download_img():
success = 0
path = os.path.join(os.getcwd(), 'imgs')
if not os.path.exists(path):
os.mkdir(path)
for img_name in imgs:
img_dict = imgs[img_name]
try:
img_data = requests.get(img_dict).content
with open(os.path.join(path, img_name + '.jpg'), 'wb') as file:
file.write(img_data)
file.close()
success += 1
except Exception as e:
print('[' + utils.now_time() + ']网络较差，页面访问失败，下载图片失败')
print('[' + utils.now_time() + ']成功下载' + str(success) + '张图片,保存在imgs目录下！')
def into_log():
fp = open('result.txt', 'a')
s = '[StarFollow]\n'
fp.write(s)
s = '于' + utils.now_time() + '收集' + str(img_count) + '张图片(user_id=' \
+ user_id + ',next_since_id=' + str(since_id) + ')\n'
fp.write(s)
for pic in imgs:
fp.write(imgs[pic] + '\n')
fp.close()
def get_next_bus_times(self, next_minutes, direction=None):
weekday = utils.get_weekday_display()
now = utils.now_time() - datetime.timedelta(minutes=settings.PREVIOUS_BUS_TIME_MINUTES)
next_minutes = now + datetime.timedelta(minutes=next_minutes)
query = db.Query(BusTime)
query.filter('bus_stop =', self.key())
query.filter('time <=', next_minutes)
query.filter('time >=', now)
query.filter('days =', weekday)
if direction:
query.filter('direction =', direction)
query.order('time')
count = query.count()
results = query.fetch(count)
if next_minutes.day != now.day:
# next_minutes is over 00hs, so we have to subs a day
# TODO: I couldn't find the way to use the OR Logical operator
next_minutes = next_minutes - datetime.timedelta(days=1)
# results = BusTime.gql('WHERE bus_stop = :1 AND (time <= :2 OR time) >= :3 AND days = :4 ORDER BY time',
# self.key(), next_minutes, now, weekday)
query = db.Query(BusTime)
query.filter('bus_stop =', self.key())
query.filter('time <=', next_minutes)
# query.filter('time >=', now)
query.filter('days =', weekday)
if direction:
query.filter('direction =', direction)
query.order('time')
count = query.count()
results.extend(query.fetch(count))
# new_results.extend(results)
# results = new_results
# Try again if there are any time.
# if count == 0:
# # FIXME: this could be dangerous (infinite recursion)
# return self.get_next_bus_times(settings.NEXT_BUS_TIME_MINUTES + 30, direction)
return results
def get_online_player(self, uid, room_id):
try:
headers = {'host': 'live.ximalaya.com'}
url = 'http://183.6.210.144/lamia/v10/live/room?id={}&roomId={}&timeToPreventCaching={}'
response = self.session.get(url.format(uid, room_id, now_time()),
headers=headers)
info = response.json().get('data')
item = dict()
# 获取粉丝团信息
item['fans_name'] = info.get('fansClubVo').get('clubName')
item['fans_count'] = info.get('fansClubVo').get('count')
# 直播间信息
item['name'] = info.get('recordInfo').get('name')
start_time = int(
info.get('recordInfo').get('actualStartAt') / 1000)
item['start_time'] = time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(start_time)) if start_time else ''
end_time = int(info.get('recordInfo').get('actualStopAt') / 1000)
item['end_time'] = time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(end_time)) if end_time else ''
# 分组ID
item['category_id'] = info.get('recordInfo').get('categoryId')
# 在线人数
item['online_count'] = info.get('recordInfo').get('onlineCount')
# 参与人数
item['play_count'] = info.get('recordInfo').get('playCount')
# 用户头像
item['avatar'] = info.get('userInfo').get('largeAvatar')
return item
except JSONDecodeError as e:
print(e)
except requests.exceptions.ReadTimeout as e:
print(e)
return None
def start():
loop_num = 1
global headers, json
headers = {
'Host':
utils.host,
'Referer':
'https://m.weibo.cn/api/container/getIndex?uid=' + user_id +
'&containerid=107603' + user_id,
'User-Agent':
utils.user_agent
}
if input_cho == 2:
headers[
'Referer'] = 'https://m.weibo.cn/api/container/getIndex?containerid=' + user_id + '_-_feed'
try:
while img_count <= img_min:
fail_count = 1
print('[' + utils.now_time() + ']开始第' + str(loop_num) + '轮收集...')
loop_num += 1
if get_single_page(since_id):
get_page_img()
if img_count > img_min:
print('[' + utils.now_time() + ']共收集到' + str(img_count) +
'张照片')
break
else:
print('[' + utils.now_time() + ']共收集到' + str(img_count) +
'张照片，系统休息1分钟')
time.sleep(60)
else:
if fail_count <= 3:
print('[' + utils.now_time() + ']网络较差，页面访问失败，第' +
str(fail_count) + '次重试')
get_single_page(since_id)
fail_count += 1
else:
print('[' + utils.now_time() + ']页面访问失败，退出程序')
except Exception as e:
print(e)
print('[' + utils.now_time() + ']系统发生错误，退出程序')
sys.exit()
def start_work(times, params):
time.sleep(5)
# return 'work - 1'
return ([{params: 'start'}, {times: now_time()}])
def end_work(times, params):
time.sleep(15)
return ([{params: 'end'}, {times: now_time()}])
def async_work(times, params):
time.sleep(10)
return ([{params: 'async'}, {times: now_time()}])
embedding = torch.from_numpy(embedding).float()
model = DCTModel(embed_dim=embed_dim,
hidden_dim=hidden_dim,
out_dim=output_dim,
vocab_size=len(embedding),
pretrain_weight=None) #embedding)
# setting optimizer
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
#%%
training_loss = []
validate_loss = []
losses = AverageMeter()
training_date = now_time()
for epoch in range(num_epochs):
# start training
losses.reset()
model.train()
for idx, (input, target) in enumerate(train_dataloader):
optimizer.zero_grad()
output = model(input)
# apply L1 regularisation and L2 loss
l1_penalty = 0
for param in model.parameters():
l1_penalty += torch.norm(param, p=1)
loss = F.mse_loss(output.squeeze(),
target.float()) + l1_penalty_coef * l1_penalty
def generate(data):
# Turn on evaluation mode which disables dropout.
model.eval()
idss_predict = []
context_predict = []
rating_predict = []
with torch.no_grad():
while True:
user, item, rating, seq, feature = data.next_batch()
user = user.to(device) # (batch_size,)
item = item.to(device)
bos = seq[:, 0].unsqueeze(0).to(device) # (1, batch_size)
feature = feature.t().to(device) # (1, batch_size)
if args.use_feature:
text = torch.cat([feature, bos],
0) # (src_len - 1, batch_size)
else:
text = bos # (src_len - 1, batch_size)
start_idx = text.size(0)
for idx in range(args.words):
# produce a word at each step
if idx == 0:
log_word_prob, log_context_dis, rating_p, _ = model(
user, item, text, False
) # (batch_size, ntoken) vs. (batch_size, ntoken) vs. (batch_size,)
rating_predict.extend(rating_p.tolist())
context = predict(log_context_dis,
topk=args.words) # (batch_size, words)
context_predict.extend(context.tolist())
else:
log_word_prob, _, _, _ = model(
user, item, text, False, False,
False) # (batch_size, ntoken)
word_prob = log_word_prob.exp() # (batch_size, ntoken)
word_idx = torch.argmax(
word_prob, dim=1
) # (batch_size,), pick the one with the largest probability
text = torch.cat([text, word_idx.unsqueeze(0)],
0) # (len++, batch_size)
ids = text[start_idx:].t().tolist() # (batch_size, seq_len)
idss_predict.extend(ids)
if data.step == data.total_step:
break
# rating
predicted_rating = [
(r, p) for (r, p) in zip(data.rating.tolist(), rating_predict)
]
RMSE = root_mean_square_error(predicted_rating, corpus.max_rating,
corpus.min_rating)
print(now_time() + 'RMSE {:7.4f}'.format(RMSE))
MAE = mean_absolute_error(predicted_rating, corpus.max_rating,
corpus.min_rating)
print(now_time() + 'MAE {:7.4f}'.format(MAE))
# text
tokens_test = [
ids2tokens(ids[1:], word2idx, idx2word) for ids in data.seq.tolist()
]
tokens_predict = [
ids2tokens(ids, word2idx, idx2word) for ids in idss_predict
]
BLEU1 = bleu_score(tokens_test, tokens_predict, n_gram=1, smooth=False)
print(now_time() + 'BLEU-1 {:7.4f}'.format(BLEU1))
BLEU4 = bleu_score(tokens_test, tokens_predict, n_gram=4, smooth=False)
print(now_time() + 'BLEU-4 {:7.4f}'.format(BLEU4))
USR, USN = unique_sentence_percent(tokens_predict)
print(now_time() + 'USR {:7.4f} | USN {:7}'.format(USR, USN))
feature_batch = feature_detect(tokens_predict, feature_set)
DIV = feature_diversity(feature_batch) # time-consuming
print(now_time() + 'DIV {:7.4f}'.format(DIV))
FCR = feature_coverage_ratio(feature_batch, feature_set)
print(now_time() + 'FCR {:7.4f}'.format(FCR))
feature_test = [idx2word[i]
for i in data.feature.squeeze(1).tolist()] # ids to words
FMR = feature_matching_ratio(feature_batch, feature_test)
print(now_time() + 'FMR {:7.4f}'.format(FMR))
text_test = [' '.join(tokens) for tokens in tokens_test]
text_predict = [' '.join(tokens) for tokens in tokens_predict]
tokens_context = [
' '.join([idx2word[i] for i in ids]) for ids in context_predict
]
ROUGE = rouge_score(text_test, text_predict) # a dictionary
for (k, v) in ROUGE.items():
print(now_time() + '{} {:7.4f}'.format(k, v))
text_out = ''
for (real, ctx, fake) in zip(text_test, tokens_context, text_predict):
text_out += '{}\n{}\n{}\n\n'.format(real, ctx, fake)
return text_out
def archive_group(chat_id, username):
print("DATABASE: Archive Group Started")
group_info = find_row_by_id(chat_id)[0]
group_info.append(str(utils.now_time()))
# ADD GROUP INFO TO ARCHIVE
archive.append_row(group_info)
def url(self):
if self._url and self._url.find('?ts') == -1:
return '{}?ts={}'.format(self._url, now_time())
return None
def start_task(params):
return start_work(now_time(), params)
def end_task(params):
return end_work(now_time(), params)
def async_task(params):
return async_work(now_time(), params)
def main():
tasks_name = random.choice(['start', 'async', 'end'])
params = random.choice([1, 2, 3, 4, 5, 6, 7, 8, 9])
print('bg_name-params----------------------------%s--%s--%s' % (tasks_name, params, now_time()))
return start(tasks_name=tasks_name, params=params)
def train(data):
# Turn on training mode which enables dropout.
model.train()
context_loss = 0.
text_loss = 0.
rating_loss = 0.
total_sample = 0
while True:
user, item, rating, seq, feature = data.next_batch(
) # (batch_size, seq_len), data.step += 1
batch_size = user.size(0)
user = user.to(device) # (batch_size,)
item = item.to(device)
rating = rating.to(device)
seq = seq.t().to(device) # (tgt_len + 1, batch_size)
feature = feature.t().to(device) # (1, batch_size)
if args.use_feature:
text = torch.cat([feature, seq[:-1]],
0) # (src_len + tgt_len - 2, batch_size)
else:
text = seq[:-1] # (src_len + tgt_len - 2, batch_size)
# Starting each batch, we detach the hidden state from how it was previously produced.
# If we didn't, the model would try backpropagating all the way to start of the dataset.
optimizer.zero_grad()
log_word_prob, log_context_dis, rating_p, _ = model(
user, item, text
) # (tgt_len, batch_size, ntoken) vs. (batch_size, ntoken) vs. (batch_size,)
context_dis = log_context_dis.unsqueeze(0).repeat(
(tgt_len - 1, 1,
1)) # (batch_size, ntoken) -> (tgt_len - 1, batch_size, ntoken)
c_loss = text_criterion(context_dis.view(-1, ntokens),
seq[1:-1].reshape((-1, )))
r_loss = rating_criterion(rating_p, rating)
t_loss = text_criterion(log_word_prob.view(-1, ntokens),
seq[1:].reshape((-1, )))
loss = args.text_reg * t_loss + args.context_reg * c_loss + args.rating_reg * r_loss
loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem.
torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip)
optimizer.step()
context_loss += batch_size * c_loss.item()
text_loss += batch_size * t_loss.item()
rating_loss += batch_size * r_loss.item()
total_sample += batch_size
if data.step % args.log_interval == 0 or data.step == data.total_step:
cur_c_loss = context_loss / total_sample
cur_t_loss = text_loss / total_sample
cur_r_loss = rating_loss / total_sample
print(
now_time() +
'context ppl {:4.4f} | text ppl {:4.4f} | rating loss {:4.4f} | {:5d}/{:5d} batches'
.format(math.exp(cur_c_loss), math.exp(cur_t_loss), cur_r_loss,
data.step, data.total_step))
context_loss = 0.
text_loss = 0.
rating_loss = 0.
total_sample = 0
if data.step == data.total_step:
break
if args.data_path is None:
parser.error('--data_path should be provided for loading data')
if args.index_dir is None:
parser.error('--index_dir should be provided for loading data splits')
print('-' * 40 + 'ARGUMENTS' + '-' * 40)
for arg in vars(args):
print('{:40} {}'.format(arg, getattr(args, arg)))
print('-' * 40 + 'ARGUMENTS' + '-' * 40)
# Set the random seed manually for reproducibility.
torch.manual_seed(args.seed)
if torch.cuda.is_available():
if not args.cuda:
print(
now_time() +
'WARNING: You have a CUDA device, so you should probably run with --cuda'
)
device = torch.device('cuda' if args.cuda else 'cpu')
if not os.path.exists(args.checkpoint):
os.makedirs(args.checkpoint)
model_path = os.path.join(args.checkpoint, 'model.pt')
prediction_path = os.path.join(args.checkpoint, args.outf)
###############################################################################
# Load data
###############################################################################
print(now_time() + 'Loading data')
corpus = DataLoader(args.data_path, args.index_dir, args.vocab_size)
img_data = requests.get(img_dict).content
with open(os.path.join(path, img_name + '.jpg'), 'wb') as file:
file.write(img_data)
file.close()
success += 1
except Exception as e:
print('[' + utils.now_time() + ']网络较差，页面访问失败，下载图片失败')
print('[' + utils.now_time() + ']成功下载' + str(success) + '张图片,保存在imgs目录下！')
if __name__ == '__main__':
print(
'----------------------------------欢迎使用 StarFollow 2.0版-------------------------------------'
)
print('[' + utils.now_time() + ']请选择是收集个人主页还是超话？(1.个人主页 2.超话)')
input_cho = utils.is_number(input())
while input_cho == -1:
if input_cho == 1 or input_cho == 2:
break
print('[' + utils.now_time() + ']输入1或2！请重新输入：')
input_cho = utils.is_number(input())
if input_cho == 1:
print('[' + utils.now_time() + ']请输入你想要收集的个人主页地址：')
print(
'[' + utils.now_time() +
']例如https://m.weibo.cn/u/xxxxxxxxxx... 或https://weibo.com/u/xxxxxxxxxx...'
)
else:
print('[' + utils.now_time() + ']请输入你想要收集的超话地址：')
