def setUp(self):
self.app = app
self.client = self.app.test_client
self.app.testing = True
self.Sir3n = User('*****@*****.**', 'Kali2017', 'Dhulkifli',
'Hussein')
self.Johndoe = User('*****@*****.**', 'Kali2017', 'John', 'Doe')
def save_to_firebase(self):
validation = self.validate()
if validation:
print("validation success")
email = self.lineEdit_Login.text()
password = self.lineEdit_Senha.text()
name = self.lineEdit_nome.text()
cpf = self.lineEdit_CPF.text()
#Remove dots and dashes from CPF
from re import sub as re_sub
cpf = re_sub('[.-]', '', cpf)
user = User(email, name, cpf, LevelOfAccess.COMMON_USER)
auth.create_user_with_email_and_password(email, password)
db.child('users').push(user.to_dict())
msg = QtWidgets.QMessageBox()
msg.setIcon(QtWidgets.QMessageBox.NoIcon)
msg.setText("Sucesso")
msg.setInformativeText("Cadastrado com sucesso!")
msg.setWindowTitle("Sucesso")
msg.exec_()
global loggedUser
if self.mainWindow:
if loggedUser != None and loggedUser.level == LevelOfAccess.ADMIN:
self.mainWindow.stackedWidget.setCurrentIndex(4)
else:
self.mainWindow.stackedWidget.setCurrentIndex(0)
else:
print("validation error")
def Create(request):
"""
@params: logintoken, name, username, password
"""
requestJSON = request.get_json(silent=True)
try:
assertJSON(requestJSON, ['logintoken', 'name', 'username', 'password'])
User.InputRuleSet.validateFromPropertyNames(
['name', 'username', 'password'], requestJSON)
with SessionHandler.app_and_db_session_scope(
requestJSON['logintoken'],
SessionHandler.PermissionLevel.ADMIN) as session:
# check if same username already exists
if User.loadFromDbByUsername(
session.db_session,
username=requestJSON['username']) is not None:
raise HttpException(
HttpErrorType.GenericError, 'User create',
'User with that username already exists in the database.')
addedUser = User.addNew(session=session.db_session,
name=requestJSON['name'],
username=requestJSON['username'],
password=encryptPassword(
requestJSON['password']))
return SessionHandler.OK(addedUser.toJSONObject())
except HttpException as exc:
return exc.GetResponse()
def seed_data():
Lucy_Wilson_key = Teacher(name="Lucy Wilson").put()
Chuck_Rosenthal_key = Teacher(name="Chuck Rosenthal").put()
Paul_A_Harris_key = Teacher(name="Paul A. Harris").put()
Michael_Berg_key = Teacher(name="Michael Berg").put()
Alissa_Crans_key = Teacher(name="Alissa Crans").put()
Blake_Mellor_key = Teacher(name="Blake Mellor").put()
David_Moffet_key = Teacher(name="David Moffet").put()
Eric_Strauss_key = Teacher(name="Eric Strauss").put()
John_Dorsey_key = Teacher(name="John Dorsey").put()
Jennifer_Abe_key = Teacher(name="Jennifer Abe").put()
Nadia_Y_Kim_key = Teacher(name="Nadia Y. Kim").put()
Alicia_Partnoy_key = Teacher(name="Alicia Partnoy").put()
English_key = Course(name = "English", teachers = [Lucy_Wilson_key, Chuck_Rosenthal_key, Paul_A_Harris_key]).put()
Math_key = Course(name = "Math", teachers = [Michael_Berg_key, Alissa_Crans_key, Blake_Mellor_key]).put()
Science_key = Course(name = "Science", teachers = [David_Moffet_key, Eric_Strauss_key, John_Dorsey_key]).put()
Social_Studies_key = Course(name = "SocialStudies", teachers = [Jennifer_Abe_key, Nadia_Y_Kim_key, Alicia_Partnoy_key]).put()
User_One_key = User(name="User One Name", username="******", email="*****@*****.**").put()
User_Two_key = User(name="User Two Name", username="******", email="*****@*****.**").put()
User_Three_key = User(name="User Three Name", username="******", email="*****@*****.**").put()
Enrollment(user=User_One_key, course=Math_key).put()
Enrollment(user=User_One_key, course=Science_key).put()
Enrollment(user=User_Two_key, course=Social_Studies_key).put()
Enrollment(user=User_Two_key, course=Science_key).put()
Enrollment(user=User_Three_key, course=English_key).put()
Enrollment(user=User_Three_key, course=Science_key).put()
def users():
if request.method == 'GET':
if request.args:
return User.scan(username=request.args['username'])
return User.scan()
if request.method == 'POST':
return User.new(request.json['username'], request.json['password'])
def summaryFavorite(self):
self.user = User(self.body['context']['System']['user']['userId'])
msg = []
for code in self.user.data['symbol']:
print(code)
msg += self.stockSummary(code, no_news=True)[1]
msg += ['자세한 뉴스 내용을 알고싶으시면 종목이름 뉴스 알려줘 라고 말씀해주세요.']
return 'SpeechList', msg, True, None
def signup():
success, error = False, ''
scan = User.scan(username=request.json['username'])
if scan['Count'] == 0:
User.new(request.json['username'], request.json['password'])
else:
error = 'Username taken'
return {'success': success, 'error': error}
def currentFavorite(self):
self.user = User(self.body['context']['System']['user']['userId'])
if len(self.user.data['symbol']) == 0:
return 'SimpleSpeech', '현재 관심종목이 없어요.', False, None
cfs = ', '.join(
[code_to_name[symbol] for symbol in self.user.data['symbol']])
print(self.user.data)
return 'SimpleSpeech', '현재 관심종목은 {}에요'.format(cfs), False, None
def user(user_id):
if request.method == 'GET':
return User.get(user_id)
if request.method == 'PUT':
user = User.get(user_id)
user.username = request.json['username']
user.password = request.json['password']
return user.update()
if request.method == 'DELETE':
return User.delete(user_id)
def setUp(self):
app.secret_key = '\xfd{H\xe5<\x95\xf9\xe3\x96.5\xd1\x01O<!\xd5\xa2\xa0\x9fR"\xa1\xa8'
self.recipe = Recipe()
self.user = User()
self.upVote = UpVote()
self.movie_data = {'id':'5XXXXX', 'title':'Movie one', 'details':'this movie is very funny', 'link':' https://www.youtube.com/embed/VIDEOID', 'created_by':'Moses','votes':0, 'status':False}
self.user_data = {'id':'5XXXXX','name':'Moses','email':'*****@*****.**','username':'******','password':'******', 'confirm':'12345'}
self.user_data2 = {'id':'5XXXXX','name':'Moses','email':'*****@*****.**','username':'******','password':'******', 'confirm':'12345'}
self.user_data3 = {'id':'5XXXXX','name':'Moses','email':'*****@*****.**','username':'******','password':'******', 'confirm':'12345'}
self.upvote_data = {'id':'1XXXXX', 'movie_id':'5XXXXX','voted_by':'Mosebadus'}
def register(message):
chat_id = message.chat.id
text = message.text
if not (len(text) < 15):
msg = bot.send_message(chat_id, 'incorrect login')
bot.register_next_step_handler(msg, register)
return
user = User(chat_id, text, 0, 0)
user.get_id(chat_id, text)
msg = bot.send_message(chat_id, 'Вперед!')
main_menu(msg)
def setup_user():
import yaml
from data import signage_db, User
with open("./web.yml", 'r') as ymlfile:
cfg = yaml.load(ymlfile)
username, password = cfg['upload_user'].split(':')
user = User(username=username)
user.hash_password(password)
signage_db.session.add(user)
signage_db.session.commit()
print("added access user")
def register():
if request.method == 'POST':
e_mail = request.form['email']
name = request.form['username']
passwrd = generate_password_hash(request.form['password'])
if bool(re.search(r'@', e_mail)) is False:
flash("Email must have an @ symbol")
return render_template('register.html')
if len(request.form['password']) > 12 or len(
request.form['password']) < 6:
flash("Password length must be between 12 and 6 chars")
return render_template('register.html')
User.create(username=name, password=passwrd, email=e_mail)
flash("New User added successfully!")
return render_template('register.html')
def register(username, password):
session = db_session.create_session()
user = session.query(User).filter(User.username == username).first()
if user:
raise LocalApi.DuplicateError(
f'User with this username already exists: "{username}"',
duptype='username')
if len(password) < 5:
raise LocalApi.PasswordError('This password is too simple')
user = User(username=username)
user.set_password(password)
session.add(user)
session.commit()
def removeFavorite(self):
self.user = User(self.body['context']['System']['user']['userId'])
try:
symbol = self.body['request']['intent']['slots']['symbol']['value']
symbol_code = name_to_code[symbol]
except (KeyError, TypeError) as e:
symbol = None if 'symbol' not in locals() else symbol
return self.no_symbol(symbol,
sessionAttributes={'name': 'addFavorite'})
self.user.data = self.user.data.loc[
self.user.data['symbol'] != symbol_code, :].dropna()
self.user.save_data()
return 'SimpleSpeech', symbol + '를 관심종목에서 제거하였어요. 계속 제거를 원하시면 종목 이름을 말씀해 주세요.', False, {
'name': 'removeFavorite'
}
def register_page():
form = RegisterForm()
if form.validate_on_submit():
session = create_session()
user = User().fill(email=form.email.data,
name=form.name.data,
surname=form.surname.data,
patronymic=form.patronymic.data,
city=form.city.data,
birthday=form.birthday.data,)
user.set_password(form.password.data)
session.add(user)
session.commit()
return redirect("/login")
return render_template("register.html", form=form)
def Login(request, logger):
"""
@params: username, password, [userAgent]
"""
requestJSON = request.get_json(silent=True)
try:
assertJSON(requestJSON, ['username', 'password'])
with SessionHandler.app_and_db_session_scope(
'', SessionHandler.PermissionLevel.NONE) as session:
# here verify password and login are valid, create a session and return a token
loadedUser = User.loadFromDbByUsername(
session.db_session, username=requestJSON['username'])
if not loadedUser:
encryptPassword(requestJSON['password'])
else:
if (verifyPassword(requestJSON['password'],
loadedUser.password)):
newSession = AppSession.addNewOrReturnLastIfValid(
session.db_session, loadedUser.id, 1,
requestJSON['userAgent']
if 'userAgent' in requestJSON else '')
return SessionHandler.OK({'logintoken': newSession.token})
logger.error(
'Failed logon attempt for username: {}, from remote address: {}'
.format(requestJSON['username'], str(request.remote_addr)))
raise HttpException(HttpErrorType.GenericError, 'login',
'Invalid username or password.')
except HttpException as exc:
return exc.GetResponse()
def get(self):
batchid = self.request.get('batchid')
batch = db.get(batchid)
uobjects = []
for user in batch.users:
if User.all().filter("nickname=", user).count() > 0:
continue
#TODO query FF for the users prescriptions
n = random.randint(1,10)
friends = [ ("user%02d" % (random.randint(0,19))) for i in range(n)]
u = User( nickname = user, friends = friends)
uobjects += [ u ]
db.put(uobjects)
batch.done = True
key = db.Key( str(batch.parent_key() ))
batch.delete()
#nbatches = Batch.all().ancestor( key ).filter("done=", True).count()
nbatches = Batch.all(keys_only=True)
nbatches = nbatches.ancestor( key )
nbatches = nbatches.count()
logging.info("nbatches=%d" % nbatches)
if nbatches == 0:
job = db.get( key )
job.done = True
job.put()
t = Task(method="GET", url="/tasks/FinishJob?jobid=%s" % job.key())
t.add()
def post(self):
user_id = self.request.get('user_id')
password = self.request.get('password')
# 验证是否重复添加
users = User.all().filter('owner =', current_user)
count = users.filter('user_id =', user_id).count()
if count:
json.dump({ 'error': True, 'reason': u'用户已存在' },
self.response.out)
return
# 验证用户名密码是否正确
headers = {
'Authorization': 'Basic ' +
base64.b64encode('%s:%s' %
(user_id.encode('utf8'), password.encode('utf8'))),
}
try:
resp = urlfetch.fetch(
'http://api.fanfou.com/account/verify_credentials.json',
headers=headers, deadline=10)
except Exception, e:
json.dump({ 'error': True, 'reason': u'错误 ' + repr(e) },
self.response.out)
def get(self):
google_login_template = jinja_env.get_template(
"/templates/google_login.html")
new_user_template = jinja_env.get_template("/templates/new_user.html")
user = users.get_current_user()
if user:
print("ACCOUNT EXISTS:")
print(user.email())
print(user.nickname())
existing_user = User.query().filter(
User.email == user.email()).get()
nickname = user.nickname()
if existing_user:
self.redirect('/addcourses')
if not existing_user:
fields = {
"nickname": nickname,
"logout_url": logout_url,
}
self.response.write(new_user_template.render(fields))
# else:
# self.redirect('/layout.html')
else:
self.response.write(
google_login_template.render({"login_url": login_url}))
def recur():
snm = "TEST" + str(randint(10000, 99999999))
go = True if User.query.filter_by(name=snm
).first() is None else False
if not go:
return recur()
db.session.add(User(snm, snm, role))
def post(self):
args = UsersResource.reg_pars.parse_args()
session = create_session()
abort_if_email_exist(session, args['email'])
user = User().fill(
email=args['email'],
name=args['name'],
surname=args['surname'],
patronymic=args['patronymic'],
city=args['city'],
birthday=args['birthday'],
)
user.set_password(args['password'])
session.add(user)
session.commit()
return jsonify({"success": "ok"})
def createUser(login_session):
newUser = User(name=login_session['username'], email=login_session[
'email'], picture=login_session['picture'])
session.add(newUser)
session.commit()
user = session.query(User).filter_by(email=login_session['email']).one()
return user.id
def get(self):
template = jinja_environment.get_template('schedule.html')
user = users.get_current_user()
date_format = "%m/%d/%Y"
#gets variables from the settings html
city = self.request.get('city').lower()
state = self.request.get('state').lower()
radius = self.request.get('radius')
dateStart = datetime.strptime(self.request.get('dateStart'), date_format)
dateEnd = datetime.strptime(self.request.get('dateEnd'), date_format)
#Finds the number of days
dateNum = (dateEnd - dateStart)
#checks to see if user has inputed all relevant fields
if city and state and radius and dateStart and dateEnd:
#Fetch calls
attractions = self.fetch_attractions(city, state, radius)
resturantsBreakfast = self.fetch_resturants_breakfast(city, state, radius)
resturantsGeneral = self.fetch_resturants_general(city, state, radius)
# if User.query().filter(User.user == users.get_current_user().email()):
# userCal.delete()
bizData = User(state = state, city = city, user= user.email(), attractions=attractions, resturantsBreakfast=resturantsBreakfast, resturantsGeneral= resturantsGeneral, dateNum=dateNum.days, id=user.email())
#bizData.put()
variables = {
'ad1p1': attractions,
'rd1b1': resturantsBreakfast[0:1],
'rd2b2': resturantsBreakfast[1:2],
'rd2b3': resturantsBreakfast[2:3],
'rd1g1': resturantsGeneral[0:1],
'rd1g2': resturantsGeneral[1:2],
'rd2g3': resturantsGeneral[2:3],
'rd2g4': resturantsGeneral[3:4],
'rd3g5': resturantsGeneral[4:5],
'rd3g6': resturantsGeneral[5:6],
'search_city': city,
'search_state': state
}
self.response.write(template.render(variables))
bizData.put() #Renders the schedule Html
else:
self.response.write("Please specify a city, state, radius, start date, and end date")
def add_user():
name = request.args.get('name')
email = request.args.get('email')
u = User(username=name, email=email)
db.session.add(u)
db.session.commit()
userstr = str(u)
return '%s - %s' % (u.username, u.email)
def __dumpFriends(self, uid, access_token, page):
conn = httplib.HTTPSConnection("api.weibo.com")
conn.request("GET", "/2/friendships/friends/bilateral.json?page=%d&count=%d&uid=%d&access_token=%s" %(page, self.pageSize, uid, access_token))
resp = conn.getresponse()
if resp.status != 200:
raise ApiException(resp)
strJson = resp.read()
return User.decodeList(strJson)
def addme(self, queue_name, first_name, last_name, username, user_id,
date):
queue_id = Queue.find_by_name(self.conn, queue_name)
if queue_id:
User(first_name, last_name, username, user_id).insert(self.conn)
UserQueue(user_id, queue_id, date).insert(self.conn)
else:
BOT.send_message(self.chat_id,
'There is no queue named "{}"'.format(queue_name))
def registration():
form = RegisterForm()
if form.validate_on_submit():
session = create_session()
user = User(
name=form.name.data,
surname=form.surname.data,
email=form.email.data,
age=form.age.data,
nickname=form.nickname.data,
)
user.set_password(form.password.data)
session.add(user)
session.commit()
return redirect('/login')
return render_template('registration.html',
title='Registration',
form=form)
def webhook_handler():
if flask.request.method == "POST":
# Retrieve the message in JSON and then transform it to Telegram object
update = telegram.Update.de_json(flask.request.get_json(force=True),
bot)
if update.message:
# Regular message
text = update.message.text
user_id = update.message.from_user.id
chat_id = update.message.chat_id
username = update.message.from_user.username
message_id = None
elif update.callback_query:
# Callback query
text = update.callback_query.data
user_id = update.callback_query.from_user.id
chat_id = update.callback_query.message.chat_id
username = update.callback_query.from_user.username
message_id = update.callback_query.message.message_id
else:
logging.error("Received unknown update!")
return constants.RESPONSE_OK
# User must have username
if not username:
bot.sendMessage(chat_id, constants.ERROR_NO_USERNAME)
return constants.RESPONSE_OK
# Retrieve/Create user
user = User.get_by_id(user_id)
if not user:
# New user
logging.info("User %s not found! Creating new user...", user_id)
user = User(id=user_id, chat_id=chat_id, username=username)
user.put()
else:
# Existing user
user.last_activity_date = datetime.now()
if username != user.username:
logging.debug("User %s has changed username from %s to %s",
user_id, user.username, username)
user.username = username
user.put()
commands.handle_input(user, text, message_id)
return constants.RESPONSE_OK
def task_matches():
logging.info("Performing match maintainance...")
matches = list(Match.query().fetch())
for match in matches:
# Delete all matches whose last move is older than the specified timeout
if match.last_move_date:
expired = (datetime.now() -
match.last_move_date) > timedelta(seconds=match.timeout)
else:
expired = False
if expired:
white = User.get_by_id(match.white_id)
black = User.get_by_id(match.black_id)
if match.is_user_turn(white):
looser = white
winner = black
else:
winner = white
looser = black
white.send_message(
constants.ERROR_TIMEOUT.format(
looser.username,
humanfriendly.format_timespan(match.timeout)))
black.send_message(
constants.ERROR_TIMEOUT.format(
looser.username,
humanfriendly.format_timespan(match.timeout)))
winner.end_game(GameResult.WIN)
looser.end_game(GameResult.LOSE)
# Delete match
logging.info("Deleting match %s. Last activity: %s",
match.key.id(), str(match.last_move_date))
match.key.delete()
def test_login_user_with_wrong_password(self):
self.Sir3n = User('*****@*****.**', 'Kali2017', 'Dhulkifli',
'Hussein')
login = self.app.test_client().post('/login',
data={
"email": '*****@*****.**',
"password": '******'
},
follow_redirects=True)
assert b'Password does not match. Please try again' in login.data
def loadlist(self, data):
self.active_users = []
json_dict = json.loads(data)
for k in json_dict["data"]:
user = User(k['username'], k['ip'])
self.active_users.append(user)
self.controller.client.active_users = self.active_users
gui_dict = {"name": "active_users", "data": self.active_users}
self.queue.put(gui_dict)
print(gui_dict)
def getUserByName(self, name, access_token):
conn = httplib.HTTPSConnection("api.weibo.com")
conn.request("GET", "https://api.weibo.com/2/users/show.json?screen_name=%s&access_token=%s" %(name, access_token))
resp = conn.getresponse()
if resp.status == 400:
return None
if resp.status != 200:
raise ApiException(resp)
strJson = resp.read()
return User.decodeFromJson(strJson)
def get(self):
jobid = self.request.get('jobid')
job = db.get(jobid)
job.ready = True
users = User.all().filter("nickname IN", job.users)
job.scores = [ 0. ] * len(job.users)
for user in users:
name = user.nickname
idx = job.users.index(name)
#TODO: calculate scores another way
n = len(filter(lambda x: job.users.count(name)>0, user.friends))
job.scores[idx] = float(n)
#TODO: don't delete if caching
user.delete()
job.put()
def post(self):
status = self.request.get('status')
user_id = self.request.get('user_id')
user = User.all().filter('owner =', current_user)
user = user.filter('user_id =', user_id).fetch(1)[0]
if status:
plan_time = datetime.strptime(self.request.get('plan_time'),
'%Y-%m-%d %H:%M') - utc_fix
interval = int(self.request.get('interval'))
if interval and plan_time <= datetime.now():
plan_time = next_time(plan_time, interval)
Plan(user=user,
status=status,
plan_time=plan_time,
interval=interval
).put()
json.dump(get_user_plans(user), self.response.out)
def post(self):
user_id = self.request.get('user_id')
users = User.all().filter('owner =', current_user)
the_user = users.filter('user_id =', user_id)
if not the_user.count():
return
the_user = the_user.fetch(1)[0]
plans = Plan.all().filter('user ='******'users': convert_users_list(users),
'plans': get_user_plans(users[0]) if users else [],
}
json.dump(data, self.response.out)
def get(self):
template = jinja_environment.get_template('calendar.html')
logout = {'logout':users.create_logout_url('/')}
#self.response.out.write(template.render(logout))
#user = User.query().filter().keys
#print user
#pulls list out of Datastore
userCal = User.query().filter(User.user == users.get_current_user().email())
if userCal.get() == None:
self.redirect('/intro')
else:
attractions= userCal.get().attractions
resturantsBreakfast= userCal.get().resturantsBreakfast
resturantsGeneral= userCal.get().resturantsGeneral
userDay= userCal.get().dateNum
city= userCal.get().city
state= userCal.get().state
variables = {
'ad1p1': attractions[0:3],
'ad1p2': attractions[3:6],
'ad2p1': attractions[6:9],
'ad2p2': attractions[9:12],
'ad3p1': attractions[12:15],
'ad3p2': attractions[15:18],
'rd1b1': resturantsBreakfast[0:1],
'rd2b2': resturantsBreakfast[1:2],
'rd2b3': resturantsBreakfast[2:3],
'rd1g1': resturantsGeneral[0:1],
'rd1g2': resturantsGeneral[1:2],
'rd2g3': resturantsGeneral[2:3],
'rd2g4': resturantsGeneral[3:4],
'rd3g5': resturantsGeneral[4:5],
'rd3g6': resturantsGeneral[5:6],
'search_city': city,
'search_state': state
}
self.response.write(template.render(variables))
def runFlterEmpty(uids):
global access_token
print 'step1: get empty uids'
for uid in uids:
try:
print 'step2: dump friends, uid=%d' % uid
friends = client.dumpFriends(uid, access_token)
except ApiException, e:
print e
if e.status == 403:
print "reload config"
const.loadConfig()
access_token = const.accessToken
time.sleep(60)
continue
friendIds = User.extractIds(friends)
if len(friendIds)==0:
print 'empty friends, uid=%d' % uid
print friends
friendsStore.delEmpty(uid)
continue
print 'step2: del empty uid'
friendsStore.delEmpty(uid)
print 'step3: save friend ids'
friendsStore.saveFriends(uid, friendIds)
print 'step4: save every friend in user store'
userStore.saveUsers(friends)
print 'step5: get enqueue list'
existList = friendsStore.existPipe(friendIds)
enqueueList = []
for i in range(len(friendIds)):
if existList[i] == False:
enqueueList.append(friendIds[i])
print 'step6: enqueue'
queue.enqueuePipe(enqueueList)
print 'friends.keyCount=%d' % friendsStore.keyCount()
print '\n\n--------------------------------\n\n'
def get_user(self):
user_cookie = self.read_secure_cookie()
if user_cookie:
user_cookie = user_cookie.split('|')
return User.get_by_id(int(user_cookie[0]))
return None
friends = client.dumpFriends(uid, access_token)
except ApiException, e:
print e
if e.status == 403:
print "reload config"
const.loadConfig()
access_token = const.accessToken
queue.putFront(uid)
time.sleep(60)
continue
except:
print 'exception'
queue.putFront(uid)
time.sleep(60)
continue
friendIds = User.extractIds(friends)
if len(friendIds)==0:
print 'empty friends, uid=%d' % uid
continue
print 'step3: save friend ids'
friendsStore.saveFriends(uid, friendIds)
print 'step4: save every friend in user store'
userStore.saveUsers(friends)
print 'step5: get enqueue list'
existList = friendsStore.existPipe(friendIds)
enqueueList = []
for i in range(len(friendIds)):
if existList[i] == False:
enqueueList.append(friendIds[i])
print 'step6: enqueue'
queue.enqueuePipe(enqueueList)
def crawler_comment_thread(requester, queue, index):
fout = file('./tmp/comment.part.%d' % index, 'w')
ferr = file('./tmp/comment.err.part.%d' % index, 'w')
failure_count = 1
while not queue.empty():
try:
url, count = queue.get(True, 30)
if count == 10:
log('10:' + url + '\n', ferr)
continue
html = requester.get(url)
log('%d:%d:%s' % (queue.qsize(), index, url), fout)
page = H.document_fromstring(html)
shop_id = shop_id_pattern.search(url).group()
comment_list_node = page.xpath(COMMENT_LIST_XPATH)
for comment_block in comment_list_node:
comment = Comment()
user = User()
comment.comment_id = comment_block.xpath(COMMENT_ID_XPATH)[0]
comment.shop_id = shop_id
user.user_id = comment_block.xpath(USER_ID_XPATH)[0]
comment.user_id = user.user_id
username_node = comment_block.xpath(USER_NAME_XPATH)
if len(username_node) < 1:
log('no username:%s' % url, ferr)
continue
user.username = comment_block.xpath(USER_NAME_XPATH)[0].text_content()
score_node = comment_block.xpath(SCORE_XPATH)
comment.star = int(number_pattern.search(comment_block.xpath(SCORE_XPATH)[0].split(' ')[1]).group()) / 10 if len(score_node) > 0 else 0
average_node = comment_block.xpath(AVERAGE_PER_XPATH)
comment.average = number_pattern.search(average_node[0].text_content()).group() if len(average_node) > 0 else 0
comment.date = fill_date(str(comment_block.xpath(DATE_XPATH)[0].text_content().encode('utf-8')).split('\xc2\xa0')[0], DATE_FORMAT)
content_extra_node = comment_block.xpath(CONTENT_EXTRA_XPATH)
if len(content_extra_node) > 0:
comment.content = content_extra_node[0].text_content().strip()
else:
comment.content = comment_block.xpath(CONTENT_XPATH)[0].text_content().strip()
other_score_node = comment_block.xpath(OTHER_SCORE_XPATH)
comment.taste_score = 0
comment.envir_score = 0
comment.service_score = 0
for each_node in other_score_node:
if TASTE_TAG in each_node.text_content():
comment.taste_score = each_node.text_content()[2]
elif ENVIR_TAG in each_node.text_content():
comment.envir_score = each_node.text_content()[2]
elif SERVICE_TAG in each_node.text_content():
comment.service_score = each_node.text_content()[2]
#has_other_score = len(other_score_node) > 0
#comment.taste_score = number_pattern.search(other_score_node[0].text_content()).group() if has_other_score else 0
#comment.envir_score = number_pattern.search(other_score_node[1].text_content()).group() if has_other_score else 0
#comment.service_score = number_pattern.search(other_score_node[2].text_content()).group() if has_other_score else 0
user.save()
comment.save()
next_page_node = page.xpath(COMMENT_NEXT_XPATH)
if len(next_page_node) > 0:
pageno = next_page_node[0]
comment_url_prefix = pageno_pattern.sub('', url)
next_url = comment_url_prefix + pageno
queue.put((next_url, 1))
failure_count = 1
except Empty,e:
log('%d:Empty' % index, fout)
break
except urllib2.HTTPError,e:
if e.code != 404:
# 403 forbbidden
queue.put((url, count))
sleep(10 * failure_count)
failure_count += 1
if failure_count == 10:
log('%d:403:%s' % (index, url), fout)
else:
log('%d:404:error:%s' % (index, url), fout)
}
try:
resp = urlfetch.fetch(
'http://api.fanfou.com/account/verify_credentials.json',
headers=headers, deadline=10)
except Exception, e:
json.dump({ 'error': True, 'reason': u'错误 ' + repr(e) },
self.response.out)
else:
if resp.status_code == 200:
user_data = json.loads(resp.content)
user_id = user_data['id']
name = user_data['name']
# 添加到数据库
user = User(user_id=user_id,
password=base64.b64encode(password),
name=name,
owner=current_user)
user.put()
# 输出到浏览器
json.dump({
'error': False,
'user_id': user_id,
'users': convert_users_list(get_users_list()),
'plans': get_user_plans(user),
}, self.response.out)
elif resp.status_code == 401:
json.dump({ 'error': True, 'reason': u'用户名或密码错误' },
self.response.out)
else:
json.dump({
'error': True,
def get(self, user_id):
user_id = urllib.unquote(user_id).decode('utf8')
user = User.all().filter('owner =', current_user)
user = user.filter('user_id =', user_id).fetch(1)[0]
json.dump(get_user_plans(user), self.response.out)
def get_users_list():
return [user for user in User.all().filter('owner =', current_user)]
def format_links(html):
'''
This monster of a function takes in the html from a post and returns a dict
containing html, text, summary.
Uses opengraph to try to get titles for all untitled links, and tries to hyperlink everything.
'''
edit_html = html
html = html.replace('"', '"')
soup = BeautifulSoup(re.sub(r'&(?!amp;)', r'&', html))
reformat_str = ''.join(random.sample(string.ascii_uppercase, 10)) + '__'
reformat_dict = {}
videos = []
image = None
# Set aside all <img> tags, because we need to treat them special and will add them in later.
for tag_index, img_tag in enumerate(soup.find_all('img')):
key = reformat_str + 'img' + str(tag_index)
img_tag.replace_with(key)
# handle the shitty case where a user inputs a non-http link
if img_tag.has_attr('src') and not img_tag['src'].startswith('http'):
new_src = 'http://' + img_tag['src']
img_tag['src'] = new_src
if not image:
image = img_tag['src']
reformat_dict[key] = img_tag
# Set aside all <a> tags, because we need to treat them special and will add them in later.
for tag_index, a_tag in enumerate(soup.find_all('a')):
key = reformat_str + 'a' + str(tag_index)
a_tag.replace_with(key)
# handle the shitty case where a user inputs a non-http link
if a_tag.has_attr('href'):
new_href = a_tag['href'].strip()
if not new_href.startswith('http'):
new_href = 'http://' + a_tag['href']
a_tag['href'] = new_href
embed_link = get_embed_link(new_href)
if embed_link:
videos.append(embed_link)
a_tag['target'] = '_blank'
try:
if a_tag.string and a_tag['href'] and a_tag.string in a_tag['href']:
og_title = get_opengraph(a_tag['href'], params=['title']).get('title')
a_tag.string = og_title.strip()
except:
pass
reformat_dict[key] = a_tag
mentions = []
# Find all mentions and format them
mention_regex = re.compile(r'(@\S+(?:\s\S+)?)')
for mention_index, mention_str in enumerate(soup(text=mention_regex)):
key = reformat_str + 'm' + str(mention_index)
mention_split_list = mention_regex.split(mention_str)
parent_tag = Tag(name='span')
for piece in mention_split_list:
if type(piece) in [unicode, str]:
s = mention_regex.search(piece)
if s:
first_letter = re.search(r"@\S+", piece).group()[1]
names = [u.name for u in User.objects(name__istartswith=first_letter)]
for i in range(len(piece) - 1):
query_name = re.compile(piece[1:len(piece) - i], flags=re.IGNORECASE)
matches = len([name for name in names if query_name.match(name)])
if matches == 1:
a_tag = Tag(name='a')
target_user = User.objects(name=query_name).get()
a_tag['href'] = '/profile/%s' % str(target_user.id)
a_tag['target'] = '_blank'
a_tag['mention'] = 'Yes'
a_tag.string = '@' + query_name.pattern
parent_tag.append(a_tag)
parent_tag.append(NavigableString(piece[len(piece) - i:]))
mentions.append(str(target_user.id))
break
else:
# for/else structure
# catch an @ that didn't match any users
parent_tag.append(NavigableString(piece))
else:
parent_tag.append(NavigableString(piece))
reformat_dict[key] = parent_tag
mention_str.replace_with(key)
opengraph_index = 0
opengraph_objects = []
# Find all plaintext links and format them.
for p in soup.find_all('p'):
p_text = unicode(p.text)
if link_regex.search(p_text):
new_p = Tag(name='p')
opengraph_only = False
p_opengraph_objects = []
link_split_list = link_regex.split(p_text)
for piece in link_split_list:
if type(piece) in [unicode, str]:
s = link_regex.search(piece)
if s:
link_text = s.group().strip()
if not link_text.startswith('http'):
link_text = 'http://' + link_text
opengraph = get_opengraph(link_text)
a_tag = Tag(name='a')
a_tag.string = opengraph.get('title', link_text) or link_text
a_tag['href'] = link_text
a_tag['target'] = '_blank'
if not image and opengraph['image']:
image = opengraph['image']
embed_link = get_embed_link(link_text)
if embed_link:
videos.append(embed_link)
else:
num_items = 0
for item in link_split_list:
if item and not re.match(r'^<.+>$', item):
num_items += 1
if num_items == 1:
opengraph_objects.append(opengraph)
p_opengraph_objects.append(opengraph)
opengraph_only = True
new_p.append(a_tag)
else:
new_p.append(NavigableString(piece))
if opengraph_only:
new_p = Tag(name='p')
for obj in p_opengraph_objects:
div = Tag(name='div db-opengraph')
div['site'] = 'comment.opengraph[%d]' % opengraph_index
opengraph_index += 1
new_p.append(div)
p.replace_with(new_p)
# Bring back all set-aside <a> and <img> tags
for key in reformat_dict:
soup(text=key)[0].replace_with(reformat_dict[key])
# Extract html from soup
html = unicode(soup)
html = clean_html(html)
# Anonymized html
for mention in soup.find_all('a', attrs={'mention': 'Yes'}):
mention.replace_with(NavigableString('@User'))
anonymized_html = unicode(soup)
anonymized_html = clean_html(anonymized_html)
# Generate text
text = MLStripper.strip_html(html)
anonymized_text = MLStripper.strip_html(anonymized_html)
# Generate summary
first_paragraph = re.compile('<p>.+?(<br/>|</p>)').search(html)
if first_paragraph:
summary = MLStripper.strip_html(first_paragraph.group())
if not summary and opengraph_objects:
summary = opengraph_objects[0]['title']
if not summary and text:
summary = text
if not summary:
summary = ""
# Generate anonymized summary
first_paragraph = re.compile('<p>.+?(<br/>|</p>)').search(anonymized_html)
if first_paragraph:
anonymized_summary = MLStripper.strip_html(first_paragraph.group())
if not anonymized_summary and opengraph_objects:
anonymized_summary = opengraph_objects[0]['title']
if not anonymized_summary and text:
anonymized_summary = text
if not anonymized_summary:
anonymized_summary = ""
# In summary, we should all the pesky double-spaces and truncate if necessary
summary = summary.replace(' ', ' ')
if len(summary) > 100:
summary = summary[:97] + '...'
anonymized_summary = anonymized_summary.replace(' ', ' ')
if len(anonymized_summary) > 100:
anonymized_summary = anonymized_summary[:97] + '...'
return {'html': html, 'edit_html': edit_html, 'summary': summary, 'text': text,
'anonymized_html': anonymized_html, 'anonymized_summary': anonymized_summary,
'mentions': mentions, 'videos': videos, 'opengraph': opengraph_objects, 'image': image}
