def crawl_top_users():
users = find_all('users')
users = [user for user in users if len(user) == 1]
for i, user in enumerate(users):
print(f'crawling {i+1}/{len(users)}...')
repos = get_user_starred_repo(user['name'])
print(f"finish {user['name']} total: {len(repos)} update to db")
update_one('users', {'name': user['name']}, {'repos': repos})
def saveDictation():
'''Endpoint that saves a sound file for later acoustic adaptation.
Body:
cookie: Cookie of current user.
Files:
url: Sound file.
'''
cookie = request.form['cookie']
# Get current user
res = database.find_one('connections', {'_id': cookie})
email_name = res['email_name']
text = request.form['text']
url = request.files['url']
out = os.path.join('./data', email_name)
if not os.path.exists(out):
os.makedirs(out)
wav_path = os.path.join(out, 'wav')
if not os.path.exists(wav_path):
os.makedirs(wav_path)
# Keep the number of dictations in the database.
res = database.find_one('savedDictations', {'_id': email_name})
if res is None:
counter = 0
database.insert_one('savedDictations', {
'_id': email_name,
'num': counter
})
else:
counter = res['num'] + 1
database.update_one('savedDictations', {'_id': email_name},
{"$set": {
'num': counter
}})
# Keep a file that contains all the ids.
with open(os.path.join(out, 'ids'), 'a') as f:
f.write(str(counter) + '\n')
# Keep a file that contains all the transcriptions.
with open(os.path.join(out, 'transcriptions'), 'a') as f:
f.write('<s> ' + text.strip('\n') + ' </s> ' + ' (' + str(counter) +
')' + '\n')
# Save current dictation in filesystem.
url.save(os.path.join(wav_path, str(counter) + '.wav'))
return {'message': 'OK'}
def insert_repo_detail(repo_name, mode='token'):
repo_detail = get_repo_detail(repo_name, mode)
print(f"finish {repo_name} update to db")
update_one('top1000_repos_detail', {'name': repo_detail['name']},
repo_detail)
def insert_user_starred_repo(user, mode='token'):
repos = get_user_starred_repo(user['name'], mode)
print(f"finish {user['name']} total: {len(repos)} update to db")
update_one('users', {'name': user['name']}, {'repos': repos})
def update_row(self, table, values, designation):
database.update_one(self.conn, table, values, designation)