def delete_category(id):
if request.method == 'OPTIONS':
return setHTTPResponse(status=200)
id = convert_to_integer(id)
if id == ErrorMessages.NOT_NUMBER:
return setHTTPResponse(status=406)
c = db.query(Category).get(id)
_fire_hydrants = db.query(FireHydrant).filter_by(category_id=id)
if _fire_hydrants:
for fh in _fire_hydrants:
try:
db.delete(fh)
db.commit()
except:
db.rollback()
traceback.print_exc()
return setHTTPResponse(500, body={"msg": "CHILD_FAILED"})
if not c:
return setHTTPResponse(status=404)
try:
db.delete(c)
db.commit()
return setHTTPResponse(status=200)
except:
db.rollback()
traceback.print_exc()
return setHTTPResponse(status=500)
def delete_category(id):
if request.method == 'OPTIONS':
return setHTTPResponse(status=200)
id = convert_to_integer(id)
if id == ErrorMessages.NOT_NUMBER:
return setHTTPResponse(status=406)
c = db.query(Category).get(id)
_fire_hydrants = db.query(FireHydrant).filter_by(category_id=id)
if _fire_hydrants:
for fh in _fire_hydrants:
try:
db.delete(fh)
db.commit()
except:
db.rollback()
traceback.print_exc()
return setHTTPResponse(500, body={"msg": "CHILD_FAILED"})
if not c:
return setHTTPResponse(status=404)
try:
db.delete(c)
db.commit()
return setHTTPResponse(status=200)
except:
db.rollback()
traceback.print_exc()
return setHTTPResponse(status=500)
def update_category(id):
if request.method == 'OPTIONS':
return setHTTPResponse(status=200)
id = convert_to_integer(id)
if id == ErrorMessages.NOT_NUMBER:
return setHTTPResponse(status=406,
body=json.dumps({'id': 'NOT_NUMBER'}))
if not "name" in request.forms:
return setHTTPResponse(status=400)
name = request.forms.get("name")
_name = db.query(Category).filter_by(name=name).first()
if _name:
return setHTTPResponse(status=409,
body=json.dumps({'name': 'DUBLICATE'}))
c = db.query(Category).get(id)
c.name = name
if not c.validate():
try:
db.commit()
response.content_type = "application/json"
return c.to_json()
except Exception as err:
db.rollback()
traceback.print_exc()
return setHTTPResponse(status=500)
return setHTTPResponse(status=406, body=json.dumps(c.validate()))
def update_fire_hydrant(id):
if request.method == "OPTIONS":
return setHTTPResponse(status=200)
id = convert_to_integer(id)
if id == ErrorMessages.NOT_NUMBER:
return setHTTPResponse(status=406,
body=json.dumps({'id': 'NOT_NUMBER'}))
req_flag = False
for req in required_values:
if not req in request.forms:
req_flag = True
if req_flag:
return setHTTPResponse(status=400,
body=json.dumps({'msg': 'REQUIRED_VALUES'}))
cat_id = request.forms.get('category_id')
lat = request.forms.get('latitude')
long = request.forms.get('longitude')
description = ""
try:
description = request.forms.get('description')
except:
pass
trunk_line = ""
try:
trunk_line = request.forms.get('trunk_line_diameter')
except:
pass
fh = db.query(FireHydrant).get(id)
if not fh:
return setHTTPResponse(status=404)
#Store previous values
_cat_id = fh.category_id
_lat = fh.latitude
_long = fh.longitude
_desc = fh.description
_trunk_line = fh.trunk_line_diameter
#Update with new values
fh.category_id = cat_id
fh.description = description
fh.latitude = lat
fh.longitude = long
fh.trunk_line_diameter = trunk_line
errs = fh.validate()
if errs:
#Restore previuous values
fh.category_id = _cat_id
fh.latitude = _lat
fh.longitude = _long
fh.description = _desc
fh.trunk_line_diameter = _trunk_line
return setHTTPResponse(status=406, body=errs)
else:
try:
db.commit()
response.content_type = "application/json"
return fh.to_json()
except:
db.rollback()
traceback.print_exc()
return setHTTPResponse(status=500)
def get_categories():
data = []
_categories = db.query(Category).all()
for category in _categories:
data.append(category.get_data())
response.content_type = "application/json"
json_val = json.dumps(data)
return json_val
def get_categories():
data = []
_categories = db.query(Category).all()
for category in _categories:
data.append(category.get_data())
response.content_type = "application/json"
json_val = json.dumps(data)
return json_val
def update_fire_hydrant(id):
if request.method == "OPTIONS":
return setHTTPResponse(status=200)
id = convert_to_integer(id)
if id == ErrorMessages.NOT_NUMBER:
return setHTTPResponse(status=406, body=json.dumps({'id': 'NOT_NUMBER'}))
req_flag = False
for req in required_values:
if not req in request.forms:
req_flag = True
if req_flag:
return setHTTPResponse(status=400, body=json.dumps({'msg': 'REQUIRED_VALUES'}))
cat_id = request.forms.get('category_id')
lat = request.forms.get('latitude')
long = request.forms.get('longitude')
description = ""
try:
description = request.forms.get('description')
except:
pass
trunk_line = ""
try:
trunk_line = request.forms.get('trunk_line_diameter')
except:
pass
fh = db.query(FireHydrant).get(id)
if not fh:
return setHTTPResponse(status=404)
#Store previous values
_cat_id= fh.category_id
_lat = fh.latitude
_long = fh.longitude
_desc = fh.description
_trunk_line = fh.trunk_line_diameter
#Update with new values
fh.category_id = cat_id
fh.description = description
fh.latitude = lat
fh.longitude = long
fh.trunk_line_diameter = trunk_line
errs = fh.validate()
if errs:
#Restore previuous values
fh.category_id = _cat_id
fh.latitude = _lat
fh.longitude = _long
fh.description = _desc
fh.trunk_line_diameter = _trunk_line
return setHTTPResponse(status=406, body=errs)
else:
try:
db.commit()
response.content_type = "application/json"
return fh.to_json()
except:
db.rollback()
traceback.print_exc()
return setHTTPResponse(status=500)
def get_fire_hydrants():
if request.method == "OPTIONS":
return setHTTPResponse(status=200)
db.flush()
_fire_hydrants = db.query(FireHydrant).all()
data = []
for fh in _fire_hydrants:
data.append(fh.get_data())
return json.dumps(data)
def get_fire_hydrants():
if request.method == "OPTIONS":
return setHTTPResponse(status=200)
db.flush()
_fire_hydrants = db.query(FireHydrant).all()
data = []
for fh in _fire_hydrants:
data.append(fh.get_data())
return json.dumps(data)
def get_category(id):
id = convert_to_integer(id)
if id == ErrorMessages.NOT_NUMBER:
return setHTTPResponse(status=406, body={})
c = db.query(Category).get(id)
if not c:
return setHTTPResponse(status=404, body={})
response.content_type = "application/json"
return c.to_json()
def get_category(id):
id = convert_to_integer(id)
if id == ErrorMessages.NOT_NUMBER:
return setHTTPResponse(status=406, body={})
c = db.query(Category).get(id)
if not c:
return setHTTPResponse(status=404, body={})
response.content_type = "application/json"
return c.to_json()
def get_fire_hydrant(id):
if request.method == "OPTIONS":
return setHTTPResponse(status=200)
id = convert_to_integer(id)
if id == ErrorMessages.NOT_NUMBER:
return setHTTPResponse(status=406, body=json.dumps({'id': 'NOT_NUMBER'}))
fh = db.query(FireHydrant).get(id)
if not fh:
return setHTTPResponse(status=404, body=json.dumps({'msg': 'NOT_FOUND'}))
return fh.to_json()
def get_data(self):
cat = db.query(Category).get(self.category_id)
d = {
"id": self.id,
"category": cat.get_data(),
"latitude": self.latitude,
"longitude": self.longitude,
"description": self.description,
"trunk_line_diameter": self.trunk_line_diameter
}
return d
def get_data(self):
cat = db.query(Category).get(self.category_id)
d = {
"id": self.id,
"category": cat.get_data(),
"latitude": self.latitude,
"longitude": self.longitude,
"description": self.description,
"trunk_line_diameter": self.trunk_line_diameter
}
return d
def validate_category_id(self):
is_empty = validator.validate_empty(self.category_id)
if is_empty == validator.ErrorMessages.EMPTY_VAL:
self.errors['category_id'] = _empty_val
return
is_not_valid = validator.validate_integer(self.category_id)
if is_not_valid == validator.ErrorMessages.NOT_NUMBER:
self.errors['category_id'] = _not_integer
return
cat = db.query(Category).get(self.category_id)
if not cat:
self.errors['category_id'] = _not_existing
def validate_category_id(self):
is_empty = validator.validate_empty(self.category_id)
if is_empty == validator.ErrorMessages.EMPTY_VAL:
self.errors['category_id'] = _empty_val
return
is_not_valid = validator.validate_integer(self.category_id)
if is_not_valid == validator.ErrorMessages.NOT_NUMBER:
self.errors['category_id'] = _not_integer
return
cat = db.query(Category).get(self.category_id)
if not cat:
self.errors['category_id'] = _not_existing
def get_fire_hydrant(id):
if request.method == "OPTIONS":
return setHTTPResponse(status=200)
id = convert_to_integer(id)
if id == ErrorMessages.NOT_NUMBER:
return setHTTPResponse(status=406,
body=json.dumps({'id': 'NOT_NUMBER'}))
fh = db.query(FireHydrant).get(id)
if not fh:
return setHTTPResponse(status=404,
body=json.dumps({'msg': 'NOT_FOUND'}))
return fh.to_json()
def update_category(id):
if request.method == 'OPTIONS':
return setHTTPResponse(status=200)
id = convert_to_integer(id)
if id == ErrorMessages.NOT_NUMBER:
return setHTTPResponse(status=406, body=json.dumps({'id': 'NOT_NUMBER'}))
if not "name" in request.forms:
return setHTTPResponse(status=400)
name = request.forms.get("name")
_name = db.query(Category).filter_by(name=name).first()
if _name:
return setHTTPResponse(status=409, body=json.dumps({'name': 'DUBLICATE'}))
c = db.query(Category).get(id)
c.name = name
if not c.validate():
try:
db.commit()
response.content_type = "application/json"
return c.to_json()
except Exception as err:
db.rollback()
traceback.print_exc()
return setHTTPResponse(status=500)
return setHTTPResponse(status=406, body=json.dumps(c.validate()))
def get_fire_hydrant_csv():
global csv_file
fire_hydrants = db.query(FireHydrant).all()
rows = []
#first line
rows.append(["Id", "Lat", "Lon", "Name", "Note", "Text"])
for fire_hydrant in fire_hydrants:
rows.append([
fire_hydrant.id, fire_hydrant.latitude, fire_hydrant.longitude,
fire_hydrant.get_category_name().encode('utf-8'),
fire_hydrant.description, fire_hydrant.trunk_line_diameter
])
print type(fire_hydrant.get_category_name())
csvw = CSVWriter()
_file, _path = csvw.write_csv(rows)
csv_file = _path
return static_file(_file, root=os.path.join(os.sep, "tmp"))
def delete_fire_hydrant(id):
if request.method == "OPTIONS":
return setHTTPResponse(status=200)
id = convert_to_integer(id)
if id == ErrorMessages.NOT_NUMBER:
return setHTTPResponse(status=406)
fh = db.query(FireHydrant).get(id)
if not fh:
return setHTTPResponse(status=404)
try:
db.delete(fh)
db.commit()
return setHTTPResponse(status=200)
except Exception:
db.rollback()
traceback.print_exc()
return setHTTPResponse(status=500)
def delete_fire_hydrant(id):
if request.method == "OPTIONS":
return setHTTPResponse(status=200)
id = convert_to_integer(id)
if id == ErrorMessages.NOT_NUMBER:
return setHTTPResponse(status=406)
fh = db.query(FireHydrant).get(id)
if not fh:
return setHTTPResponse(status=404)
try:
db.delete(fh)
db.commit()
return setHTTPResponse(status=200)
except Exception:
db.rollback()
traceback.print_exc()
return setHTTPResponse(status=500)
def get_fire_hydrant_csv():
global csv_file
fire_hydrants = db.query(FireHydrant).all()
rows = []
#first line
rows.append(["Id", "Lat", "Lon", "Name", "Note", "Text"])
for fire_hydrant in fire_hydrants:
rows.append([
fire_hydrant.id,
fire_hydrant.latitude,
fire_hydrant.longitude,
fire_hydrant.get_category_name().encode('utf-8'),
fire_hydrant.description,
fire_hydrant.trunk_line_diameter
])
print type(fire_hydrant.get_category_name())
csvw = CSVWriter()
_file, _path = csvw.write_csv(rows)
csv_file = _path
return static_file(_file, root=os.path.join(os.sep, "tmp"))
def new_category():
if "name" in request.forms:
name = request.forms.getunicode('name')
print type(name)
_name = db.query(Category).filter_by(name=name).first()
if _name:
return setHTTPResponse(status=409, body=json.dumps({"name": "DUBLICATE"}))
c = Category(name)
if not c.validate():
try:
db.add(c)
db.commit()
response.content_type = "application/json"
return c.to_json()
except Exception as err:
db.rollback()
traceback.print_exc()
return setHTTPResponse(status=500)
return setHTTPResponse(status=406, body=json.dumps(c.validate()))
return setHTTPResponse(status=400)
def new_category():
if "name" in request.forms:
name = request.forms.getunicode('name')
print type(name)
_name = db.query(Category).filter_by(name=name).first()
if _name:
return setHTTPResponse(status=409,
body=json.dumps({"name": "DUBLICATE"}))
c = Category(name)
if not c.validate():
try:
db.add(c)
db.commit()
response.content_type = "application/json"
return c.to_json()
except Exception as err:
db.rollback()
traceback.print_exc()
return setHTTPResponse(status=500)
return setHTTPResponse(status=406, body=json.dumps(c.validate()))
return setHTTPResponse(status=400)
def spider(url, db, article):
browser.get(url)
html = browser.page_source
if len(html) > 90000:
box = re.compile(box_pattern, re.I).findall(html)
if len(box) > 0:
info = json.loads(box[0])
data = info['props']['initialState']['note']['data']
tmp_list = {}
status = 'DONE'
tmp_list['UID'] = str(article['UID'])
tmp_list['AID'] = str(article['ID'])
tmp_list['DataJson'] = db.self_escape_string(json.dumps(data)).strip()
tmp_list['LastUpdatedAt'] = datetime.datetime.fromtimestamp(data['last_updated_at']).strftime(
"%Y-%m-%d %H:%M:%S")
tmp_list['Title'] = db.self_escape_string(data['public_title']).strip()
tmp_list['CommentsCount'] = str(data['comments_count'])
tmp_list['Content'] = db.self_escape_string(data['free_content']).strip()
tmp_list['LikesCount'] = str(data['likes_count'])
tmp_list['PaidType'] = data['paid_type']
tmp_list['Wordage'] = str(data['wordage'])
tmp_list['FeaturedCommentsCount'] = str(data['featured_comments_count'])
tmp_list['DownvotesCount'] = str(data['downvotes_count'])
tmp_list['TotalRewardsCount'] = str(data['total_rewards_count'])
tmp_list['FirstSharedAt'] = datetime.datetime.fromtimestamp(data['first_shared_at']).strftime(
"%Y-%m-%d %H:%M:%S")
tmp_list['ViewsCount'] = str(data['views_count'])
tmp_list['NotebookId'] = str(data['notebook_id'])
tmp_list['AddTime'] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
tmp_list['UpdateTime'] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
column_str = ','.join(tmp_list.keys())
value_str = "','".join(tmp_list.values())
value_str = "'" + value_str + "'"
sql = "insert into jianshu_article_info (%s) values (%s)" % (column_str, value_str)
try:
db.db_reconnect()
db.query(sql)
except BaseException:
status = 'FAILED'
print('------------- error ------------')
print(sql.encode('utf-8'))
print('------------- error ------------')
else:
db.db_reconnect()
db.query(
'update jianshu_user set SucArticles = IFNULL(SucArticles, 0) + 1 where ID = ' + str(article['UID']))
db.db_reconnect()
db.query(
"update jianshu_article_list set CommentsNum = '" + str(tmp_list['CommentsCount']) + "', Wordage = '" +
str(tmp_list['Wordage']) + "', TotalRewardsCount = '" + str(tmp_list[
'TotalRewardsCount']) + "', DownvotesCount = '" + str(tmp_list[
'DownvotesCount']) + "', FeaturedCommentsCount = '" + str(tmp_list[
'FeaturedCommentsCount']) + "', PaidType = '" + str(tmp_list['PaidType']) + "', LikeNum = '" +
str(tmp_list['LikesCount']) + "', NotebookId = '" + str(tmp_list['NotebookId']) + "', ViewsCount = '" +
str(tmp_list['ViewsCount']) + "', LastUpdatedAt = '" + str(tmp_list[
'LastUpdatedAt']) + "', FirstSharedAt = '" + str(tmp_list['FirstSharedAt']) + "' where ID = " +
str(article['ID']))
sql = "update jianshu_article_list set Status = '%s' where ArticleID = '%s'" % (
status, article['ArticleID'])
db.db_reconnect()
db.query(sql)
sleep(1)
def get_category_name(self):
cat = db.query(Category).get(self.category_id)
return cat.name
def get_category_name(self):
cat = db.query(Category).get(self.category_id)
return cat.name