def get(self, **kwargs): bus_stop = db.get(self.request.values.get('busstop_key')) directions = self.request.values.get('directions', '') if len(directions.split(',')) == 2: directions = None else: directions = directions bus_times = bus_stop.get_next_bus_times(settings.NEXT_BUS_TIME_MINUTES, direction=directions) bus_direction = BusDirection.all() bus_direction.filter('direction =', directions) # README # BadValueError: Filtering on lists is not supported # We can't use "bus_direction.filter('bus_line =', bus_stop.lines)" # So, in the while we will use "bus_stop.lines[0]" because a BusStop only # has one line for the moment bus_direction.filter('bus_line =', bus_stop.lines[0]) bus_direction = bus_direction.fetch(1) bus_direction = bus_direction[0] info_content = '<b>%s</b><br /> %s<br /><em style="font-size: 10px">%s / Hacia: %s (%s)</em>' % \ (bus_stop.name, bus_stop.address, utils.get_weekday_display(), bus_direction.to_direction, bus_direction.direction) for bus_time in bus_times: left_time = relativedelta(bus_time.time_1970(), utils.now_time()).minutes bus_already_gone = False if left_time < 0: if bus_time.time_1970().hour == 0: # Time over 00hs next_day = bus_time.time_1970() + datetime.timedelta(days=1) left_time = relativedelta(next_day, utils.now_time()).minutes else: # This bus has already gone by bus_already_gone = True time = bus_time.time.strftime('%H:%M') time_content = '<br /><b>%s min:</b><span> %s hs</span>' % (left_time, time) if bus_already_gone: info_content += '<span style="color: red">' + time_content + '</span>' else: info_content += time_content if bus_time.comments: info_content += '<em> (' for comment in bus_time.comments: info_content += '%s, ' % comment info_content = info_content[:-2] info_content += ')</em>' return render_json_response({ 'info_content': info_content })
def delete_group(chat_id, username): print("DATABASE: Delete Group Started") row = find_row_by_id(item_id=chat_id)[0] print("DATABASE: Got row number") group_info = groupchats.row_values(row) print("DATABASE: Got Row info") try: calls = spreadsheet.worksheet(str(chat_id)) print("DATABASE: Got calls worksheet") # DELETE CALENDAR EVENTS events_info = calls.get_all_values() events_info.pop(0) print("DATABASE: Event Info: ", events_info) for event in events_info: print("DATABASE: Event", event) gcalendar.delete_event(event[0]) # ARCHIVE TRELLO CALL CARDS # try: # call_card_id = get_group_card(chat_id) # trelloc.delete_group(call_card_id) # print("DATABASE: Deleted Trello card") # except: # print("DATABASE: No Trello Card found") # DELETE LINKED CALL SHEET spreadsheet.del_worksheet(calls) except: print("DATABASE: No Calls sheet found") # DELETE CARD IN TRELLO try: card_id = get_group_card(chat_id) trelloc.delete_group(card_id) print("DATABASE: Deleted Trello card") except: print("DATABASE: No Trello Card found") # DELETE CHILDREN LINKS IN DATABASE children = find_row_by_id(item_id=chat_id, col=11) print("DATABASE: Children: ", children, " Type: ", type(children)) if children[0] != -1: for child in children: groupchats.update_cell(child, 11, '') print("DATABASE: Deleted children") # DROP UNNECESSAY INFO unwanted = [0, 1, 9, 12, 13] for ele in sorted(unwanted, reverse=True): del group_info[ele] group_info.extend((utils.now_time().strftime("%Y/%m/%d %H:%M"), username)) print("DATABASE: Dropped info") # SAVE INFO IN DELETE SHEET deleted.append_row(group_info) # REMOVE ROW FROM GROUPS DATABASE groupchats.delete_row(find_row_by_id(item_id=chat_id)[0])
def task_uuid(): s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.connect(('8.8.8.8', 80)) ip = s.getsockname()[0] # ip str = ip.replace('.','-') task_id = now_time() + '-' + ip + '-' return task_id
def download_img(): success = 0 path = os.path.join(os.getcwd(), 'imgs') if not os.path.exists(path): os.mkdir(path) for img_name in imgs: img_dict = imgs[img_name] try: img_data = requests.get(img_dict).content with open(os.path.join(path, img_name + '.jpg'), 'wb') as file: file.write(img_data) file.close() success += 1 except Exception as e: print('[' + utils.now_time() + ']网络较差,页面访问失败,下载图片失败') print('[' + utils.now_time() + ']成功下载' + str(success) + '张图片,保存在imgs目录下!')
def into_log(): fp = open('result.txt', 'a') s = '[StarFollow]\n' fp.write(s) s = '于' + utils.now_time() + '收集' + str(img_count) + '张图片(user_id=' \ + user_id + ',next_since_id=' + str(since_id) + ')\n' fp.write(s) for pic in imgs: fp.write(imgs[pic] + '\n') fp.close()
def get_next_bus_times(self, next_minutes, direction=None): weekday = utils.get_weekday_display() now = utils.now_time() - datetime.timedelta(minutes=settings.PREVIOUS_BUS_TIME_MINUTES) next_minutes = now + datetime.timedelta(minutes=next_minutes) query = db.Query(BusTime) query.filter('bus_stop =', self.key()) query.filter('time <=', next_minutes) query.filter('time >=', now) query.filter('days =', weekday) if direction: query.filter('direction =', direction) query.order('time') count = query.count() results = query.fetch(count) if next_minutes.day != now.day: # next_minutes is over 00hs, so we have to subs a day # TODO: I couldn't find the way to use the OR Logical operator next_minutes = next_minutes - datetime.timedelta(days=1) # results = BusTime.gql('WHERE bus_stop = :1 AND (time <= :2 OR time) >= :3 AND days = :4 ORDER BY time', # self.key(), next_minutes, now, weekday) query = db.Query(BusTime) query.filter('bus_stop =', self.key()) query.filter('time <=', next_minutes) # query.filter('time >=', now) query.filter('days =', weekday) if direction: query.filter('direction =', direction) query.order('time') count = query.count() results.extend(query.fetch(count)) # new_results.extend(results) # results = new_results # Try again if there are any time. # if count == 0: # # FIXME: this could be dangerous (infinite recursion) # return self.get_next_bus_times(settings.NEXT_BUS_TIME_MINUTES + 30, direction) return results
def get_online_player(self, uid, room_id): try: headers = {'host': 'live.ximalaya.com'} url = 'http://183.6.210.144/lamia/v10/live/room?id={}&roomId={}&timeToPreventCaching={}' response = self.session.get(url.format(uid, room_id, now_time()), headers=headers) info = response.json().get('data') item = dict() # 获取粉丝团信息 item['fans_name'] = info.get('fansClubVo').get('clubName') item['fans_count'] = info.get('fansClubVo').get('count') # 直播间信息 item['name'] = info.get('recordInfo').get('name') start_time = int( info.get('recordInfo').get('actualStartAt') / 1000) item['start_time'] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(start_time)) if start_time else '' end_time = int(info.get('recordInfo').get('actualStopAt') / 1000) item['end_time'] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(end_time)) if end_time else '' # 分组ID item['category_id'] = info.get('recordInfo').get('categoryId') # 在线人数 item['online_count'] = info.get('recordInfo').get('onlineCount') # 参与人数 item['play_count'] = info.get('recordInfo').get('playCount') # 用户头像 item['avatar'] = info.get('userInfo').get('largeAvatar') return item except JSONDecodeError as e: print(e) except requests.exceptions.ReadTimeout as e: print(e) return None
def start(): loop_num = 1 global headers, json headers = { 'Host': utils.host, 'Referer': 'https://m.weibo.cn/api/container/getIndex?uid=' + user_id + '&containerid=107603' + user_id, 'User-Agent': utils.user_agent } if input_cho == 2: headers[ 'Referer'] = 'https://m.weibo.cn/api/container/getIndex?containerid=' + user_id + '_-_feed' try: while img_count <= img_min: fail_count = 1 print('[' + utils.now_time() + ']开始第' + str(loop_num) + '轮收集...') loop_num += 1 if get_single_page(since_id): get_page_img() if img_count > img_min: print('[' + utils.now_time() + ']共收集到' + str(img_count) + '张照片') break else: print('[' + utils.now_time() + ']共收集到' + str(img_count) + '张照片,系统休息1分钟') time.sleep(60) else: if fail_count <= 3: print('[' + utils.now_time() + ']网络较差,页面访问失败,第' + str(fail_count) + '次重试') get_single_page(since_id) fail_count += 1 else: print('[' + utils.now_time() + ']页面访问失败,退出程序') except Exception as e: print(e) print('[' + utils.now_time() + ']系统发生错误,退出程序') sys.exit()
def start_work(times, params): time.sleep(5) # return 'work - 1' return ([{params: 'start'}, {times: now_time()}])
def end_work(times, params): time.sleep(15) return ([{params: 'end'}, {times: now_time()}])
def async_work(times, params): time.sleep(10) return ([{params: 'async'}, {times: now_time()}])
embedding = torch.from_numpy(embedding).float() model = DCTModel(embed_dim=embed_dim, hidden_dim=hidden_dim, out_dim=output_dim, vocab_size=len(embedding), pretrain_weight=None) #embedding) # setting optimizer optimizer = optim.Adam(model.parameters(), lr=learning_rate) #%% training_loss = [] validate_loss = [] losses = AverageMeter() training_date = now_time() for epoch in range(num_epochs): # start training losses.reset() model.train() for idx, (input, target) in enumerate(train_dataloader): optimizer.zero_grad() output = model(input) # apply L1 regularisation and L2 loss l1_penalty = 0 for param in model.parameters(): l1_penalty += torch.norm(param, p=1) loss = F.mse_loss(output.squeeze(), target.float()) + l1_penalty_coef * l1_penalty
def generate(data): # Turn on evaluation mode which disables dropout. model.eval() idss_predict = [] context_predict = [] rating_predict = [] with torch.no_grad(): while True: user, item, rating, seq, feature = data.next_batch() user = user.to(device) # (batch_size,) item = item.to(device) bos = seq[:, 0].unsqueeze(0).to(device) # (1, batch_size) feature = feature.t().to(device) # (1, batch_size) if args.use_feature: text = torch.cat([feature, bos], 0) # (src_len - 1, batch_size) else: text = bos # (src_len - 1, batch_size) start_idx = text.size(0) for idx in range(args.words): # produce a word at each step if idx == 0: log_word_prob, log_context_dis, rating_p, _ = model( user, item, text, False ) # (batch_size, ntoken) vs. (batch_size, ntoken) vs. (batch_size,) rating_predict.extend(rating_p.tolist()) context = predict(log_context_dis, topk=args.words) # (batch_size, words) context_predict.extend(context.tolist()) else: log_word_prob, _, _, _ = model( user, item, text, False, False, False) # (batch_size, ntoken) word_prob = log_word_prob.exp() # (batch_size, ntoken) word_idx = torch.argmax( word_prob, dim=1 ) # (batch_size,), pick the one with the largest probability text = torch.cat([text, word_idx.unsqueeze(0)], 0) # (len++, batch_size) ids = text[start_idx:].t().tolist() # (batch_size, seq_len) idss_predict.extend(ids) if data.step == data.total_step: break # rating predicted_rating = [ (r, p) for (r, p) in zip(data.rating.tolist(), rating_predict) ] RMSE = root_mean_square_error(predicted_rating, corpus.max_rating, corpus.min_rating) print(now_time() + 'RMSE {:7.4f}'.format(RMSE)) MAE = mean_absolute_error(predicted_rating, corpus.max_rating, corpus.min_rating) print(now_time() + 'MAE {:7.4f}'.format(MAE)) # text tokens_test = [ ids2tokens(ids[1:], word2idx, idx2word) for ids in data.seq.tolist() ] tokens_predict = [ ids2tokens(ids, word2idx, idx2word) for ids in idss_predict ] BLEU1 = bleu_score(tokens_test, tokens_predict, n_gram=1, smooth=False) print(now_time() + 'BLEU-1 {:7.4f}'.format(BLEU1)) BLEU4 = bleu_score(tokens_test, tokens_predict, n_gram=4, smooth=False) print(now_time() + 'BLEU-4 {:7.4f}'.format(BLEU4)) USR, USN = unique_sentence_percent(tokens_predict) print(now_time() + 'USR {:7.4f} | USN {:7}'.format(USR, USN)) feature_batch = feature_detect(tokens_predict, feature_set) DIV = feature_diversity(feature_batch) # time-consuming print(now_time() + 'DIV {:7.4f}'.format(DIV)) FCR = feature_coverage_ratio(feature_batch, feature_set) print(now_time() + 'FCR {:7.4f}'.format(FCR)) feature_test = [idx2word[i] for i in data.feature.squeeze(1).tolist()] # ids to words FMR = feature_matching_ratio(feature_batch, feature_test) print(now_time() + 'FMR {:7.4f}'.format(FMR)) text_test = [' '.join(tokens) for tokens in tokens_test] text_predict = [' '.join(tokens) for tokens in tokens_predict] tokens_context = [ ' '.join([idx2word[i] for i in ids]) for ids in context_predict ] ROUGE = rouge_score(text_test, text_predict) # a dictionary for (k, v) in ROUGE.items(): print(now_time() + '{} {:7.4f}'.format(k, v)) text_out = '' for (real, ctx, fake) in zip(text_test, tokens_context, text_predict): text_out += '{}\n{}\n{}\n\n'.format(real, ctx, fake) return text_out
def archive_group(chat_id, username): print("DATABASE: Archive Group Started") group_info = find_row_by_id(chat_id)[0] group_info.append(str(utils.now_time())) # ADD GROUP INFO TO ARCHIVE archive.append_row(group_info)
def url(self): if self._url and self._url.find('?ts') == -1: return '{}?ts={}'.format(self._url, now_time()) return None
def start_task(params): return start_work(now_time(), params)
def end_task(params): return end_work(now_time(), params)
def async_task(params): return async_work(now_time(), params)
def main(): tasks_name = random.choice(['start', 'async', 'end']) params = random.choice([1, 2, 3, 4, 5, 6, 7, 8, 9]) print('bg_name-params----------------------------%s--%s--%s' % (tasks_name, params, now_time())) return start(tasks_name=tasks_name, params=params)
def train(data): # Turn on training mode which enables dropout. model.train() context_loss = 0. text_loss = 0. rating_loss = 0. total_sample = 0 while True: user, item, rating, seq, feature = data.next_batch( ) # (batch_size, seq_len), data.step += 1 batch_size = user.size(0) user = user.to(device) # (batch_size,) item = item.to(device) rating = rating.to(device) seq = seq.t().to(device) # (tgt_len + 1, batch_size) feature = feature.t().to(device) # (1, batch_size) if args.use_feature: text = torch.cat([feature, seq[:-1]], 0) # (src_len + tgt_len - 2, batch_size) else: text = seq[:-1] # (src_len + tgt_len - 2, batch_size) # Starting each batch, we detach the hidden state from how it was previously produced. # If we didn't, the model would try backpropagating all the way to start of the dataset. optimizer.zero_grad() log_word_prob, log_context_dis, rating_p, _ = model( user, item, text ) # (tgt_len, batch_size, ntoken) vs. (batch_size, ntoken) vs. (batch_size,) context_dis = log_context_dis.unsqueeze(0).repeat( (tgt_len - 1, 1, 1)) # (batch_size, ntoken) -> (tgt_len - 1, batch_size, ntoken) c_loss = text_criterion(context_dis.view(-1, ntokens), seq[1:-1].reshape((-1, ))) r_loss = rating_criterion(rating_p, rating) t_loss = text_criterion(log_word_prob.view(-1, ntokens), seq[1:].reshape((-1, ))) loss = args.text_reg * t_loss + args.context_reg * c_loss + args.rating_reg * r_loss loss.backward() # `clip_grad_norm` helps prevent the exploding gradient problem. torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip) optimizer.step() context_loss += batch_size * c_loss.item() text_loss += batch_size * t_loss.item() rating_loss += batch_size * r_loss.item() total_sample += batch_size if data.step % args.log_interval == 0 or data.step == data.total_step: cur_c_loss = context_loss / total_sample cur_t_loss = text_loss / total_sample cur_r_loss = rating_loss / total_sample print( now_time() + 'context ppl {:4.4f} | text ppl {:4.4f} | rating loss {:4.4f} | {:5d}/{:5d} batches' .format(math.exp(cur_c_loss), math.exp(cur_t_loss), cur_r_loss, data.step, data.total_step)) context_loss = 0. text_loss = 0. rating_loss = 0. total_sample = 0 if data.step == data.total_step: break
if args.data_path is None: parser.error('--data_path should be provided for loading data') if args.index_dir is None: parser.error('--index_dir should be provided for loading data splits') print('-' * 40 + 'ARGUMENTS' + '-' * 40) for arg in vars(args): print('{:40} {}'.format(arg, getattr(args, arg))) print('-' * 40 + 'ARGUMENTS' + '-' * 40) # Set the random seed manually for reproducibility. torch.manual_seed(args.seed) if torch.cuda.is_available(): if not args.cuda: print( now_time() + 'WARNING: You have a CUDA device, so you should probably run with --cuda' ) device = torch.device('cuda' if args.cuda else 'cpu') if not os.path.exists(args.checkpoint): os.makedirs(args.checkpoint) model_path = os.path.join(args.checkpoint, 'model.pt') prediction_path = os.path.join(args.checkpoint, args.outf) ############################################################################### # Load data ############################################################################### print(now_time() + 'Loading data') corpus = DataLoader(args.data_path, args.index_dir, args.vocab_size)
img_data = requests.get(img_dict).content with open(os.path.join(path, img_name + '.jpg'), 'wb') as file: file.write(img_data) file.close() success += 1 except Exception as e: print('[' + utils.now_time() + ']网络较差,页面访问失败,下载图片失败') print('[' + utils.now_time() + ']成功下载' + str(success) + '张图片,保存在imgs目录下!') if __name__ == '__main__': print( '----------------------------------欢迎使用 StarFollow 2.0版-------------------------------------' ) print('[' + utils.now_time() + ']请选择是收集个人主页还是超话?(1.个人主页 2.超话)') input_cho = utils.is_number(input()) while input_cho == -1: if input_cho == 1 or input_cho == 2: break print('[' + utils.now_time() + ']输入1或2!请重新输入:') input_cho = utils.is_number(input()) if input_cho == 1: print('[' + utils.now_time() + ']请输入你想要收集的个人主页地址:') print( '[' + utils.now_time() + ']例如https://m.weibo.cn/u/xxxxxxxxxx... 或https://weibo.com/u/xxxxxxxxxx...' ) else: print('[' + utils.now_time() + ']请输入你想要收集的超话地址:')