def __crawl_house_info(self, house):
attrs = {
'name': self.__get_location(house),
'price': self.__get_price(house),
'subway_distance': self.__get_subway_distance(house),
'area': self.__get_area(house),
'url': self.__get_url(house)
}
subway_name = self.__get_subway(house)
sw = session.query(Subway).filter(Subway.name == subway_name).first()
attrs['subway'] = sw
attrs.update(self.__get_detail(attrs.get('url')))
h = session.query(House).filter(
House.number == attrs.get('number')).first()
if h is None:
h = House(**attrs)
Logger.info('获取 {} 价格 {}'.format(h.name, h.price))
self._add.append(h)
else:
if attrs.get('subway').name != h.subway.name:
tmp_subway = attrs['subway']
del attrs['subway']
session.query(House).filter(House.name == h.name).update(attrs)
if 'subway_2' in locals():
h.subway = tmp_subway
if h in session.dirty:
Logger.info('{} 数据更新'.format(h.name))
self._changes.append(h)
else:
Logger.info('{} 没有变化'.format(h.name))
session.add(h)
session.commit()
def create_house(pigeon_id, name, cover_url, category):
house = House(cover_url=cover_url,
name=name,
category=category,
view=1,
num_of_subed=0)
house.key = ndb.Key(House, name, parent=ndb.Key(Pigeon, pigeon_id))
house.put()
def main():
my_house = House(
address=[
AddressData(
address_data=[
AddressParts(
house_number=1023,
street_name="Main Street",
city_name="Anytown"
)
]
)
]
)
# also called automatically when `.save()` is called
my_house.full_clean()
out = pickle.loads(pickle.dumps(my_house))
# out.full_clean()
out.validate()
# clean also automatically wraps dicts in appropriate classes so this is
# also acceptable:
my_house = House(address=[{'address_data': [{
'house_number': 1023,
'street_name': "Main Street",
'city_name': "Anytown"
}]}])
my_house.full_clean()
def add_number_of_schools():
try:
request_content = PostRequestInput(request.data)
house = House(request_content.get_house_location())
request_content.add_to_house_details(
key="Schools", value=house.get_amenity_frequency("school"))
except json.decoder.JSONDecodeError:
return fail(400, "Could convert body to JSON")
except TypeError:
return fail(400, "Something is wrong with the input's data stractures")
except KeyError:
return fail(404, "Input should have lat and lon")
except OverpassServiceError:
return fail(503, "Please try again later")
return jsonify(request_content.build_response())
def create_house(**kwargs):
session = Session()
house = House()
for k, v in kwargs.iteritems():
setattr(house, k, v)
session.add(house)
session.commit()
def approve_pull_request(house_name, user_id, date):
pigeon_key = ndb.Key(Pigeon, user_id)
house_list = House.query(House.name == house_name).fetch()
house_key = house_list[0].key
pr_list = PullRequest.query(PullRequest.pigeon_key == pigeon_key,
PullRequest.house_key == house_key,
PullRequest.date_str == date).fetch()
if pr_list:
pr = pr_list[0]
if pr.mode == 'add':
add_card(pr.house_key.get().name, pr.new_key, pr.new_value)
pr.key.delete()
return
elif pr.mode == 'remove':
remove_card(pr.house_key.get().name, pr.card_key.get().card_key)
pr.key.delete()
return
elif pr.mode == 'key':
edit_card_key(pr.house_key.get().name,
pr.card_key.get().card_key, pr.new_key)
pr.key.delete()
return
elif pr.mode == 'content':
edit_card_content(pr.house_key.get().name,
pr.card_key.get().card_key, pr.new_value)
pr.key.delete()
return
else:
return
def art_add():
form = ArtForm()
if form.validate_on_submit():
data = form.data
print(data['category'],)
if data['category'] == 1:
category = '租房'
if data['category'] == 2:
category = '二手房'
if data['category'] == 3:
category = '商业地产'
# 保存数据
art = House(
name= data['name'],
category= category,
number= data['number'],
company= data['company'],
address= data['address'],
shop_address= data['shop_address'],
created_time=datetime.datetime.now()
)
field.session.add(art)
field.session.commit()
flash(u"添加成功", 'ok')
return render_template('art_add.html', title=u'添加', form=form)
def add_post(house_name, user_id, content):
pigeon_key = ndb.Key(Pigeon, user_id)
house_list = House.query(House.name == house_name).fetch()
house_key = house_list[0].key
post_list = Post.query(Post.pigeon_key == pigeon_key,
Post.house_key == house_key).fetch()
if post_list:
length = len(post_list)
post = Post(pigeon_key=pigeon_key,
house_key=house_key,
content=content,
number=length + 1,
date_str='')
post.put()
date2str = str(post.date)
str_list = date2str.split('.')
post.date_str = str_list[0]
post.put()
return
else:
post = Post(pigeon_key=pigeon_key,
house_key=house_key,
content=content,
number=1,
date_str='')
post.put()
date2str = str(post.date)
str_list = date2str.split('.')
post.date_str = str_list[0]
post.put()
return
def remove_all_sub(house_name):
house_list = House.query(House.name == house_name).fetch()
house_key = house_list[0].key
sub_list = Subscription.query(Subscription.house_key == house_key).fetch()
for sub in sub_list:
sub.key.delete()
return
def add_card(house_name, key, value):
house_list = House.query(House.name == house_name).fetch()
if house_list:
card = Card(card_key=key, value=value)
house = house_list[0]
card.key = ndb.Key(Card, key, parent=house.key)
card.put()
return
def create_houses():
with open("../anapioficeandfire/houses.json") as house_file:
houses = json.load(house_file)
for d in houses:
id = d["url"].split("/").pop()
name = d["name"]
region = d["region"]
coatOfArms = d["coatOfArms"]
words = d["words"]
founded = d["founded"]
diedOut = d["diedOut"]
h = House.query.get(id)
if not h:
h = House(id=id, name=name, region=region, coatOfArms=coatOfArms,
words=words, founded=founded, diedOut=diedOut)
else:
h.name = name
h.region = region
h.coatOfArms = coatOfArms
h.words = words
h.founded = founded
h.diedOut = diedOut
create_rel_currentlord(h, d["currentLord"])
create_rel_heir(h, d["heir"])
create_rel_overlord(h, d["overlord"])
create_rel_founder(h, d["founder"])
db.session.add(h)
print(h)
def card_exists(house_name, card_key_str):
house_list = House.query(House.name == house_name).fetch()
house_key = house_list[0].key
card_key = ndb.Key(Card, card_key_str, parent=house_key)
card = card_key.get()
if card:
return True
else:
return False
def edit_card_content(house_name, card_key_str, new_card_content):
house_list = House.query(House.name == house_name).fetch()
house_key = house_list[0].key
card_key = ndb.Key(Card, card_key_str, parent=house_key)
card = card_key.get()
card.value = new_card_content
card.put()
return
def get_category(category):
house_list = House.query(House.category == category).fetch()
return map(
lambda s: {
"house_name": s.name,
"cover_url": s.cover_url,
"num_of_subed": s.num_of_subed,
"view": s.view
}, house_list)
def sortinghat():
user = json.loads(request.form.get('payload'))['user']['id']
house = json.loads(request.form.get('payload'))['actions'][0]['value']
house = House.get(name=house)
person = Person.get(slack_id=user)
person.house = house
person.save()
return "Well, if you're sure...better be... {0} !!".format(
house.name.title())
def is_subscribed(house_name, pigeon_id):
pigeon_key = ndb.Key(Pigeon, pigeon_id)
house_list = House.query(House.name == house_name).fetch()
house_key = house_list[0].key
sub_list = Subscription.query(Subscription.pigeon_key == pigeon_key,
Subscription.house_key == house_key).fetch()
if sub_list:
return True
else:
return False
def edit_card_key(house_name, card_key_str, new_card_key):
house_list = House.query(House.name == house_name).fetch()
house_key = house_list[0].key
card_key = ndb.Key(Card, card_key_str, parent=house_key)
card = card_key.get()
value = card.value
remove_card(house_name, card_key_str)
add_card(house_name, new_card_key, value)
return
def get_all_house():
house_list = House.query().fetch()
return map(
lambda s: {
"house_name": s.name,
"cover_url": s.cover_url,
"category": s.category,
"view": s.view,
"num_of_subed": s.num_of_subed
}, house_list)
def get_self_house(pigeon_id):
house_list = House.query(ancestor=ndb.Key(Pigeon, pigeon_id)).fetch()
return map(
lambda s: {
"house_name": s.name,
"cover_url": s.cover_url,
"category": s.category,
"view": s.view,
"num_of_subed": s.num_of_subed
}, house_list)
def get_trending_view():
house_list = House.query().fetch()
sorted_list = sorted(house_list, key=lambda h: h.view, reverse=True)
return map(
lambda s: {
"house_name": s.name,
"cover_url": s.cover_url,
"category": s.category,
"view": s.view
}, sorted_list)
def get_single_house_2(house_name):
house_list = House.query(House.name == house_name).fetch()
if house_list:
house = house_list[0]
house.view = house.view + 1
house.put()
card_list = Card.query(ancestor=house.key).fetch()
return map(lambda s: {str(s.card_key): str(s.value)}, card_list)
else:
return
def get_num_per_day(user_id, house_name):
pigeon_key = ndb.Key(Pigeon, user_id)
house_list = House.query(House.name == house_name).fetch()
house_key = house_list[0].key
sub_list = Subscription.query(Subscription.pigeon_key == pigeon_key,
Subscription.house_key == house_key).fetch()
if sub_list:
return sub_list[0].num_per_day
else:
return
def process_item(self, item, spider):
# 写入小区房价
if isinstance(item, FangtianxiaRentItem):
db_house = self.session.query(House).filter(
House.house_name == item['name']).filter(
House.city_name == item['city_name']).first()
if db_house:
if db_house.property_corp == u'暂无数据':
db_house.property_corp = item['property_corp']
if db_house.developers == u'暂无数据':
db_house.developers = item['developers']
if db_house.building_date == u'暂无数据':
db_house.building_date = item['building_date']
if db_house.building_date == u'暂无数据':
db_house.building_date = item['building_date']
# 需要添加判断, 以防同一个价格数据插入多次
p = Price(price=item['price'],
origin=item['origin'],
months=item['months'],
crawl_time=item['crawl_date'],
uid=db_house.id)
db_house.price.append(p)
else:
house = House(house_name=item['name'],
city_name=item['city_name'],
url=item['url'],
latitude=item['latitude'],
longitude=item['longitude'],
address=item['location'],
building_date=item['building_date'],
building_type=item['building_type'],
property_corp=item['property_corp'],
developers=item['developers'],
district=item['district'])
price = Price(price=item['price'],
origin=item['origin'],
months=item['months'],
crawl_time=item['crawl_date'])
house.price.append(price)
self.session.add(house)
try:
self.session.commit()
except Exception, e:
print e
self.session.rollback()
return item
def newhome():
if request.method == 'POST':
#May not need to format types of input
LandlordFName = request.form['landlordFName'].encode('ascii', 'ignore')
LandlordLName = request.form['landlordLName'].encode('ascii', 'ignore')
LandlordEmail = request.form['landlordEmail']
Address1 = request.form['address1'].encode('ascii', 'ignore')
Address2 = request.form['address2'].encode('ascii', 'ignore')
City = request.form['city'].encode('ascii', 'ignore')
State = request.form['state'].encode('ascii', 'ignore')
Zipcode = request.form['zip'] # parseFloat deprecated
Rooms = int(request.form['bedrooms'])
ParkingSpots = int(request.form['parking'])
MonthlyRent = int(request.form['rent'])
UtilitiesIncluded = True if request.form[
'utilities'] == 'true' else False
Laundry = True if request.form['laundry'] == 'true' else False
Pets = True if request.form['pets'] == 'true' else False
Latitude = request.form['latitude']
Longitude = request.form['longitude']
DistFromCC = request.form['disttocc']
#Finding corresponding landlord based on email
someLandlord = Landlord.query.filter_by(Email=LandlordEmail).first()
#If no landlord exists by that email
if someLandlord == None:
return jsonify([{
'status':
400,
'message':
'Landlord does not match any on file, please check the email.'
}])
house = House(someLandlord.Id, Address1, Address2, City, State,
Zipcode, Rooms, ParkingSpots, MonthlyRent,
UtilitiesIncluded, Laundry, Pets, Latitude, Longitude,
DistFromCC)
db.session.add(house)
#Handling SQLalchemy errors when a house cannot be inputted/already has the address
#Will need to readjust once unique key is handled
try:
db.session.commit()
except exc.IntegrityError:
db.session.rollback()
return jsonify([{
'status':
400,
'message':
'This house has already been listed as active'
}])
return jsonify([{'status': 200}])
else:
if 'username' in session:
return render_template('newhome.html')
else:
return redirect(url_for('index'))
def get_single_house_info(house_name):
house_list = House.query(House.name == house_name).fetch()
if house_list:
s = house_list[0]
return {
"house_name": s.name,
"cover_url": s.cover_url,
"category": s.category,
"view": s.view,
"num_of_subed": s.num_of_subed
}
def get_learn_factor(user_id, house_name, card_key_str):
pigeon_key = ndb.Key(Pigeon, user_id)
house_list = House.query(House.name == house_name).fetch()
house_key = house_list[0].key
card_key = ndb.Key(Card, card_key_str, parent=house_key)
progress_list = Progress.query(Progress.pigeon_key == pigeon_key,
Progress.card_key == card_key).fetch()
if progress_list:
return progress_list[0].learn_factor
else:
_initailize_progress(pigeon_key, card_key)
return 1
def set_schedule(user_id, house_name, num_per_day):
pigeon_key = ndb.Key(Pigeon, user_id)
house_list = House.query(House.name == house_name).fetch()
house_key = house_list[0].key
sub_list = Subscription.query(Subscription.pigeon_key == pigeon_key,
Subscription.house_key == house_key).fetch()
if sub_list:
sub_list[0].num_per_day = num_per_day
sub_list[0].put()
return
else:
return
def get_trending_subscription():
house_list = House.query().fetch()
sorted_list = sorted(house_list,
key=lambda h: h.num_of_subed,
reverse=True)
return map(
lambda s: {
"house_name": s.name,
"cover_url": s.cover_url,
"category": s.category,
"num_of_subed": s.num_of_subed
}, sorted_list)
def hello_world():
query = House.select(
House.name,
pw.fn.COUNT(Point.id).alias('ct')).group_by(House.name).join(
Point, pw.JOIN.LEFT_OUTER).where(House.name != "unknown").order_by(
House.id).dicts()
vals = [x.get('ct') for x in query]
vals.append(1)
factor = (50 / float(max(vals)))
for y in query:
y['height'] = y['ct'] * factor - 50.00
return render_template('index.jade', points=query, houses=houses)
def newhome():
#TOODshould there be a sessions check here?? Probably...
if request.method == 'POST':
#May not need to format types of input
LandlordFName = request.form['landlordFName'].encode('ascii', 'ignore')
LandlordLName = request.form['landlordLName'].encode('ascii', 'ignore')
LandlordEmail = request.form['landlordEmail']
Address1 = request.form['address1'].encode('ascii', 'ignore')
Address2 = request.form['address2'].encode('ascii', 'ignore')
City = request.form['city'].encode('ascii', 'ignore')
State = request.form['state'].encode('ascii', 'ignore')
Zipcode = request.form['zip'] # parseFloat deprecated
Rooms = int(request.form['bedrooms'])
ParkingSpots = int(request.form['parking'])
MonthlyRent = int(request.form['rent'])
UtilitiesIncluded = True if request.form['utilities'] == 'true' else False
Laundry = True if request.form['laundry'] == 'true' else False
Pets = True if request.form['pets'] == 'true' else False
Latitude = request.form['latitude']
Longitude = request.form['longitude']
DistFromCC = request.form['disttocc']
DayAvailable = int(request.form['availableday'])
MonthAvailable = int(request.form['availablemonth'])
YearAvailable = int(request.form['availableyear'])
DateAvailable = datetime(YearAvailable, MonthAvailable, DayAvailable, 0, 0)
LeaseTerm = int(request.form['leaseterm'])
#Finding corresponding landlord based on email
someLandlord = Landlord.query.filter_by(Email=LandlordEmail).first()
#If no landlord exists by that email
if someLandlord == None:
return jsonify([{'status':400, 'message':'Landlord does not match any on file, please check the email.'}])
house = House(someLandlord.Id, Address1, Address2, City, State, Zipcode, \
Rooms, ParkingSpots, MonthlyRent, UtilitiesIncluded, Laundry, \
Pets, Latitude, Longitude, DistFromCC, DateAvailable, LeaseTerm,\
datetime.now(), datetime.now(), True)
db.session.add(house)
#Handling SQLalchemy errors when a house cannot be inputted/already has the address
#Will need to read just once unique key is handled
try:
db.session.commit()
mc.delete("Houses") # flush cache, it's now stale
mc.delete("AllIds") # flush cache, it's now stale
except exc.IntegrityError:
db.session.rollback()
return jsonify([{'status':400, 'message':'This house has already been listed as active'}])
return jsonify([{'status':201, "houseID":house.Id}])
else:
if 'username' in session:
usertype = {"type": session['usertype']}
return render_template('newhome.html', usertype=usertype)
else:
return redirect(url_for('auth_page.index'))
def resolve_issue(house_name, user_id, date):
pigeon_key = ndb.Key(Pigeon, user_id)
house_list = House.query(House.name == house_name).fetch()
house_key = house_list[0].key
issue_list = Issue.query(Issue.pigeon_key == pigeon_key,
Issue.house_key == house_key,
Issue.date_str == date).fetch()
if issue_list:
issue = issue_list[0]
issue.key.delete()
return
else:
return
def get(self):
# CLEAR All
query = Department.query()
entries =query.fetch()
entry_keys =[entry.key for entry in entries]
ndb.delete_multi(entry_keys)
query = House.query()
entries =query.fetch()
entry_keys =[entry.key for entry in entries]
ndb.delete_multi(entry_keys)
query = Class.query()
entries =query.fetch()
entry_keys =[entry.key for entry in entries]
ndb.delete_multi(entry_keys)
query = Student.query()
entries =query.fetch()
entry_keys =[entry.key for entry in entries]
ndb.delete_multi(entry_keys)
user = users.get_current_user()
if user and users.is_current_user_admin():
# Create houses
houses = ['Gryffindor', 'Hufflepuff', 'Ravenclaw', 'Slytherin']
for house in houses:
House(house_name=house).put()
# Create departments
deps = [
{'code': 'CS', 'name': 'Computer Science'},
{'code': 'BA', 'name': 'Business Administration'},
{'code': 'PHY', 'name': 'Physics'},
{'code': 'MAT', 'name': 'Mathematics'},
{'code': 'DRA', 'name': 'Drama'},
{'code': 'ECE', 'name': 'Electrical Engineering'},
]
for dep in deps:
Department(id = dep['code'], dept_name = dep['name']).put()
# Create 40 classes for department, update depts
classes_count = 40
for dept in deps:
department_key = ndb.Key('Department', dept['code'])
department_obj = department_key.get()
for i in range(classes_count):
class_full_name = "%s - Class %d" % (dept['name'] ,i)
class_key = Class(id='%s-%d'%(dept['code'], i),
class_full_name = class_full_name,
department = department_key).put()
department_obj.classes.append(class_key)
department_obj.put()
# Create students
students_count = 1000
random.seed()
all_classes = Class.query().fetch()
all_houses = House.query().fetch()
for i in range(students_count):
house = random.choice(all_houses)
house_key = house.key
student_name = 'Student %d'% i
student_year = random.randint(1,4)
current_classes = random.sample(all_classes, 5)
current_classes_keys = [el.key for el in current_classes]
all_classes_keys = [el.key for el in all_classes]
leftovers = set(all_classes_keys) - set(current_classes_keys)
completed_classes_keys = random.sample(leftovers, 6*(student_year-1))
student_key = Student(
student_name = student_name,
student_year = student_year,
current_classes = current_classes_keys,
completed_classes = completed_classes_keys,
house = house_key).put()
house.students.append(student_key)
for el in current_classes:
el.students_enrolled.append(student_key)
# save new status of houses and current classes
for house in all_houses:
house.put()
for current_class in all_classes:
current_class.put()
elif user and not users.is_current_user_admin():
self.redirect(users.create_logout_url('/create_entries'))
else:
self.redirect(users.create_login_url('/create_entries'))
def get(self):
# CLEAR All
query = Department.query()
entries = query.fetch()
entry_keys = [entry.key for entry in entries]
ndb.delete_multi(entry_keys)
query = House.query()
entries = query.fetch()
entry_keys = [entry.key for entry in entries]
ndb.delete_multi(entry_keys)
query = Class.query()
entries = query.fetch()
entry_keys = [entry.key for entry in entries]
ndb.delete_multi(entry_keys)
query = Student.query()
entries = query.fetch()
entry_keys = [entry.key for entry in entries]
ndb.delete_multi(entry_keys)
user = users.get_current_user()
if user and users.is_current_user_admin():
# Create houses
houses = ["Gryffindor", "Hufflepuff", "Ravenclaw", "Slytherin"]
for house in houses:
House(house_name=house).put()
# Create departments
deps = [
{"code": "CS", "name": "Computer Science"},
{"code": "BA", "name": "Business Administration"},
{"code": "PHY", "name": "Physics"},
{"code": "MAT", "name": "Mathematics"},
{"code": "DRA", "name": "Drama"},
{"code": "ECE", "name": "Electrical Engineering"},
]
for dep in deps:
Department(id=dep["code"], dept_name=dep["name"]).put()
# Create 40 classes for department, update depts
classes_count = 40
for dept in deps:
department_key = ndb.Key("Department", dept["code"])
department_obj = department_key.get()
for i in range(classes_count):
class_full_name = "%s - Class %d" % (dept["name"], i)
department = department_key
class_key = Class(id="%s-%d" % (dept["code"], i), class_full_name=class_full_name).put()
department_obj.classes.append(class_key)
department_obj.put()
# Create students
students_count = 1000
random.seed()
all_classes = Class.query().fetch()
all_houses = House.query().fetch()
for i in range(students_count):
house = random.choice(all_houses)
house_key = house.key
student_name = "Student %d" % i
student_year = random.randint(1, 4)
current_classes = random.sample(all_classes, 5)
current_classes_keys = [el.key for el in current_classes]
all_classes_keys = [el.key for el in all_classes]
leftovers = set(all_classes_keys) - set(current_classes_keys)
completed_classes_keys = random.sample(leftovers, 6 * (student_year - 1))
student_key = Student(
student_name=student_name,
student_year=student_year,
current_classes=current_classes_keys,
completed_classes=completed_classes_keys,
house=house_key,
).put()
house.students.append(student_key)
for el in current_classes:
el.students_enrolled.append(student_key)
# save new status of houses and current classes
for house in all_houses:
house.put()
for current_class in all_classes:
current_class.put()
elif user and not users.is_current_user_admin():
self.redirect(users.create_logout_url("/create_entries"))
else:
self.redirect(users.create_login_url("/create_entries"))