def del_repo():
if request.method == 'DELETE':
repo = request.args.get('repo', None)
if repo:
desc = REDIS.hget('repos', repo)
desc = json.loads(desc)
desc.update(is_show='0')
REDIS.hset('repos', repo, json.dumps(desc))
return jsonify({"status": True})
def index():
repos = [(key, json.loads(value))
for key, value in REDIS.hgetall('repos').items()]
# repos.sort(key=lambda x:x.star)
# print repos
# repos.sort(key=lambda x: x['star'], reverse=True)
# print repos
repos.sort(key=lambda x: x[1]['star'], reverse=True)
return render_template('index.html',
repos=repos,
keywords=REDIS.get('keywords').decode('utf-8'))
def search_github(keyword):
# 爬取 20 页最新的列表
for i in range(1, 20):
res = requests.get(SEARCH_API % (keyword, i))
repo_list = res.json()['items']
for repo in repo_list:
repo_name = repo['html_url']
desc = {
'repo_desc': repo['description'],
'star': repo['stargazers_count'],
'is_show': REPO_SHOW
}
REDIS.hset('repos', repo_name, json.dumps(desc))
print(repo_name)
time.sleep(10)
def add_todo():
if request.method == 'POST':
if request.form['id'] != '':
todo = {}
todo_id = request.form['id']
todo['title'] = request.form['title']
todo['descript'] = request.form['descript']
todo['create'] = datetime.datetime.today().strftime('%Y/%m/%d %H:%M:%S')
todo['due'] = request.form['due']
todo['status'] = request.form['status']
REDIS.hset('todolist',todo_id,json.dumps(todo))
return redirect(url_for('index'))
def search_github(keyword):
# get latest 20 page list
for i in range(1, 21):
res = requests.get(SEARCH_API % (keyword, i))
repo_list = res.json()['items']
for repo in repo_list:
repo_name = repo['html_url']
desc = {
'repo_desc': repo['description'],
'star': repo['stargazers_count'],
'name': repo['name'],
'language': repo['language'],
'owner': repo['owner']['login'],
'is_show': REPO_SHOW
}
if REDIS.hsetnx('repos', repo_name, json.dumps(desc)):
print(repo_name)
time.sleep(10)
def index():
repos = [(key, json.loads(value)) for key, value in REDIS.hgetall('repos').items()]
return render_template('index.html', repos=repos, keywords=REDIS.get('keywords').decode('utf-8'))
for repo in repo_list:
repo_name = repo['html_url']
desc = {
'repo_desc': repo['description'],
'star': repo['stargazers_count'],
'is_show': REPO_SHOW
}
REDIS.hset('repos', repo_name, json.dumps(desc))
print(repo_name)
time.sleep(10)
if __name__ == '__main__':
#keywords = ['爬虫', 'spider', 'crawl','资料','电子书','题目','pdf','PDF','leetcode','签到','挂机','脚本','实用','spring','php','python']
#REDIS.set('keywords', ' '.join(keywords))
keywords = REDIS.get('keywords')
if (keywords is None or len(keywords) == 0):
keywords = [
'爬虫', 'spider', 'crawl', '资料', '电子书', '题目', 'pdf', 'PDF',
'leetcode', '签到', '挂机', '脚本', '实用', 'spring', 'php', 'python',
'QQ', 'bot'
]
REDIS.set('keywords', ' '.join(keywords))
else:
keywords = keywords.decode('utf-8').split(' ')
print(keywords)
while (True):
for keyword in keywords:
search_github(keyword)
#print("here")
keywords = REDIS.get('keywords')
def del_todo():
if request.method == 'DELETE':
todo_id = request.args.get('todo_id', None)
if todo_id:
REDIS.hdel('todolist',todo_id)
return jsonify({"status": True})
def index():
todolist = [(id.decode('utf8'), json.loads(value))
for id, value in REDIS.hgetall('todolist').items()]
return render_template('index.html', todolist=todolist)
git = repo.git
git.checkout("master")
repo.remotes[0].pull()
return repo
def search_github(keyword):
# 爬取 20 页最新的列表
for i in range(1, 21):
res = requests.get(SEARCH_API % (keyword, i))
repo_list = res.json()['items']
for repo in repo_list:
repo_name = repo['html_url']
desc = {
'repo_desc': repo['description'],
'star': repo['stargazers_count'],
'is_show': REPO_SHOW
}
if REDIS.hsetnx('repos', repo_name, json.dumps(desc)):
download_dataset(repo_name)
print repo_name
time.sleep(10)
if __name__ == '__main__':
keywords = ['爬虫', 'spider', 'crawl']
REDIS.set('keywords', ' '.join(keywords))
for keyword in keywords:
search_github(keyword)