def get(self):
to_be_visited = ProfileList.query(
ProfileList.user_key == self.user.key,
ProfileList.visited == False).order(-ProfileList.date_added)
to_be_visited = gql_json_parser(to_be_visited)
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps(to_be_visited))
def add_unique_profiles(user, profile_list):
# logging.info('entered add_unique_profiles')
# logging.info(profile_list)
# logging.info(user.key)
# logging.info(user)
# logging.info(user.key.id())
stripped_profiles = get_stripped_profiles(profile_list)
# logging.info('got stripped_profiles')
new_profiles_to_add = []
for profile in stripped_profiles:
if not ProfileList.get_by_id(str(user.key.id()) + '|' + profile):
new_profile = ProfileList(
parent = user.key,
id = str(user.key.id()) + '|' + profile,
user_key = user.key,
profile = profile,
visited = False)
new_profiles_to_add.append(new_profile)
ndb.put_multi(new_profiles_to_add)
def get(self):
logging.info(ProfileList)
to_be_visited = ProfileList.query(
ProfileList.visited == False,
ancestor = self.user.key).order(-ProfileList.date_added)
params = {
'profiles':to_be_visited
}
self.render_template('template_files/getlist.html', params)
def post(self):
num = int(self.request.get('num'))
profiles_visited = 0
auth_id = self.request.get('auth_id')
user = User.get_by_auth_id(auth_id)
logging.info(user.key)
okcupid_username = user.okcupid_username
okcupid_pw = decode_pw(user.okcupid_pw)
logging.info('okcupid username: %s' % okcupid_username)
logging.info('okcupid password: %s' % okcupid_pw)
# Create browser
br = browser.create_logged_in_browser(okcupid_username, okcupid_pw)
# Get the to be visited list
to_be_visited = ProfileList.query(
ProfileList.visited == False,
ancestor = user.key).order(ProfileList.date_added).fetch(num)
# If it doesn't exist, get the seed profile list
if not to_be_visited:
browser.get_first_profiles(user, br)
profiles_visited_counter = 0
while profiles_visited_counter < num:
to_be_visited = ProfileList.query(
ProfileList.visited == False,
ancestor = user.key).order(ProfileList.date_added).fetch(num-profiles_visited_counter)
for item in to_be_visited:
browser.visit_page(user, br, item.profile)
item.visited = True
item.put()
secs = random.randint(0, 2500) / 1000.0
sleep(secs)
profiles_visited_counter = user.profiles_visited_counter
user.is_crawling = False
user.profiles_visited_today += user.profiles_visited_counter
user.profiles_visited_counter = 0
user.put()