def post(self):
self.response.headers['Content-Type'] = 'application/json'
data = json.loads(self.request.body)
Mail = Object.extend('Mail')
newMail = Mail()
newMail.set('subject',data['subject'])
newMail.set('to',data['to'])
newMail.set('html',data['html'])
newMail.save()
type = data['type']
Timer = Object.extend('Timer')
timer = Timer()
timer.set('mailId',newMail.id)
timer.set('status','unsent')
if type == 'byInterval':
now = datetime.datetime.now()
timeUnit = data['timeUnit']
intervalCount = data['intervalCount']
seconds = 0;
if timeUnit == 'day':
seconds = intervalCount * 86400
elif timeUnit == 'hour':
seconds = intervalCount * 3600
elif timeUnit == 'minute':
seconds = intervalCount * 60
timeAfter = now + datetime.timedelta(seconds = seconds)
timestamp = int(time.mktime(timeAfter.timetuple()))
elif type == 'byTime':
timeStr = data['timeStr']
timeArray = time.strptime(timeStr, "%Y-%m-%d %H:%M")
timestamp = int(time.mktime(timeArray))
timer.set('timestamp',timestamp)
timer.save()
self.response.out.write(json.dumps({'code':0}))
def getYesterdayKDJ(rlist):
cyStockData = Object.extend('CYStockData')
query = Query(cyStockData)
query.equal_to('kdjback', 1)
query.greater_than('kdj',0)
query.equal_to('volume',1)
cyresults = query.find()
for x in cyresults:
rlist.append(x.get("stcode"))
szStockData = Object.extend('SZStockData')
query = Query(szStockData)
query.equal_to('kdjback', 1)
query.greater_than('kdj',0)
query.equal_to('volume',1)
szresults = query.find()
for x in szresults:
rlist.append(x.get("stcode"))
shStockData = Object.extend('SHStockData')
query = Query(shStockData)
query.equal_to('kdjback', 1)
query.greater_than('kdj',0)
query.equal_to('volume',1)
shresults = query.find()
for x in shresults:
rlist.append(x.get("stcode"))
def test_find_unsaved_children():
album = Album()
unsaved_children = []
unsaved_files = []
Object._find_unsaved_children(album, unsaved_children, unsaved_files)
assert unsaved_children == [album]
assert unsaved_files == []
def test_batch():
foo = Object.create("Foo")
bar = Object.create("Bar")
bar.set("baz", "baz")
foo.set("bar", bar)
bar.save()
foo.save()
def create_demo_application(table_name='DemoApplication'):
all_real_app = get_all_real_applications()
DbTable = Object.extend(table_name)
for index, app in enumerate(all_real_app):
db_table = DbTable()
relation = app.relation('tracker')
# query = Query(DbTable)
query = relation.query()
user_list = query.find()
relation = db_table.relation('tracker')
for user in user_list:
ADbTable = Object.extend('Tracker')
query = Query(ADbTable)
query.equal_to('objectId',user.id)
result_list = query.find()
if result_list:
tracker = result_list[0]
relation.add(tracker)
else:
print 'tracker not exists and objectId is: %s' %(str(user.id))
db_table.set('app_name',app.get('app_name'))
db_table.set('origin_name',app.get('name'))
db_table.save()
def test_batch(): # type: () -> None
foo = Object.create('Foo')
bar = Object.create('Bar')
bar.set('baz', 'baz')
foo.set('bar', bar)
bar.save()
foo.save()
def consume(self):
queue = SQS()
rs = queue.read()
print 'Start consuming:'
for record in rs:
mailId = record.get_body()
print mailId
Timer = Object.extend('Timer')
timerQuery = Query(Timer)
timerQuery.equal_to('mailId', mailId)
firstTimer = timerQuery.first()
if firstTimer.get('status') != 'sent':
Mail = Object.extend('Mail')
query = Query(Mail)
mailObj = query.get(mailId)
sender = mailer()
mailToSent = {
'to':mailObj.get('to'),
'html':mailObj.get('html'),
'subject':mailObj.get('subject')
}
sender.send(mailToSent)
firstTimer.set('status','sent')
firstTimer.save()
queue.remove(record)
return self;
def test_find_unsaved_children():
album = Album()
unsaved_children = []
unsaved_files = []
Object._find_unsaved_children(album, unsaved_children, unsaved_files)
assert unsaved_children == [album]
assert unsaved_files == []
def test_find_unsaved_children_2(): # type: ignore
album = Album()
band = Band()
album.set('band', band)
unsaved_children = []
unsaved_files = []
Object._find_unsaved_children(album, unsaved_children, unsaved_files)
assert unsaved_children == [band, album]
def test_find_unsaved_children_2(): # type: ignore
album = Album()
band = Band()
album.set('band', band)
unsaved_children = []
unsaved_files = []
Object._find_unsaved_children(album, unsaved_children, unsaved_files)
assert unsaved_children == [band, album]
def test_find_unsaved_children_2():
album = Album()
band = Band()
album.set("band", band)
unsaved_children = []
unsaved_files = []
Object._find_unsaved_children(album, unsaved_children, unsaved_files)
assert unsaved_children == [band, album]
def test_relation(): # type: () -> None
foo = Object.extend('Foo')()
foo.set('a', 1)
bar = Object.extend('Bar')()
bar.set('baz', 'baz')
bar.save()
relation = foo.relation('list')
relation.add(bar)
foo.save()
def test_pointer_query():
foo = Object.create('Foo')
bar = Object.create('Bar')
bar.save()
foo.set('bar', bar)
foo.save()
q = Query('Foo').equal_to('bar', bar)
assert len(q.find()) == 1
def test_relation():
foo = Object.extend('Foo')()
foo.set('a', 1)
bar = Object.extend('Bar')()
bar.set('baz', 'baz')
bar.save()
relation = foo.relation('list')
relation.add(bar)
foo.save()
def test_pointer_query():
foo = Object.create('Foo')
bar = Object.create('Bar')
bar.save()
foo.set('bar', bar)
foo.save()
q = Query('Foo').equal_to('bar', bar)
assert len(q.find()) == 1
def test_relation(): # type: () -> None
foo = Object.extend("Foo")()
foo.set("a", 1)
bar = Object.extend("Bar")()
bar.set("baz", "baz")
bar.save()
relation = foo.relation("list")
relation.add(bar)
foo.save()
def test_relation():
foo = Object.extend("Foo")()
foo.set("a", 1)
bar = Object.extend("Bar")()
bar.set("baz", "baz")
bar.save()
relation = foo.relation("list")
relation.add(bar)
foo.save()
def updata_backend_info(parse_dict):
print parse_dict
user_id = parse_dict['user_id']
# get user Object
query = Query(Object.extend('_User'))
query.equal_to('objectId', user_id)
user = query.find()[0] if query.count() else None
# get app Object
query = Query(Object.extend('BindingInstallation'))
query.equal_to('user', user)
result_list = query.find()
app_set = set()
for result in result_list:
app_set.add(result.attributes['application'].id)
app_id_list = list(app_set)
for app_id in app_id_list:
query = Query(Object.extend('Application'))
query.equal_to('objectId', app_id)
app = query.find()[0]
table_dash = Object.extend('DashboardSource')
query = Query(table_dash)
query.equal_to('app', app)
query.equal_to('user', user)
dst_table = query.find()
if not dst_table:
dst_table = table_dash()
else:
dst_table = dst_table[0]
dst_table.set('app', app)
for key, value in parse_dict.items():
if key is 'user_id':
dst_table.set('user', user)
elif key is 'home_office_status':
home_office_status = dst_table.get('home_office_status') or {}
for k, v in parse_dict['home_office_status'].items():
home_office_status[k] = v
dst_table.set('home_office_status', home_office_status)
elif key is 'event':
event = dst_table.get('event') or {}
for k, v in parse_dict['event'].items():
event[k] = v
dst_table.set('event', event)
elif key is 'location':
location = dst_table.get('location') or {}
for k, v in parse_dict['location'].items():
location[k] = v
dst_table.set('location', location)
else:
dst_table.set(key, value)
dst_table.save()
return True
def associate_event_and_activity(db_name='MergedUserContext'):
application_event_dict = get_all_event()
print 'already get the application_event_dict'
print str(len(application_event_dict.keys()))
DBTable = Object.extend(db_name)
# print 'event_list: %s' %str(application_event_dict.values())
for application,event_dict in application_event_dict.items():
# print event_dict
if event_dict:
print 'application_event_dict values first count: %s' %str(event_dict.keys())
EventActivity = Object.extend('FakeEventActivity')
for event_name,event_list in event_dict.items():
total_count = len(event_list)
print 'event_list total_count: %s with event_name is: %s' %(str(total_count),event_name)
print 'application id is: %s' %str(application.id)
event_activity = EventActivity()
relation = event_activity.relation('event')
activity_dict = {}
for index,event in enumerate(event_list):
relation.add(event)
query = Query(DBTable)
query.equal_to('tracker',event.get('tracker'))
query.less_than_or_equal_to('startTime',event.get('timestamp'))
query.greater_than_or_equal_to('endTime',event.get('timestamp'))
activity_list = query.find()
if len(activity_list) == 1 or len(activity_list) == 2 :
# for the convenience of adding the dimension of time to the analyzer
event.set('activity',activity_list[0])
event.save()
# activity = activity_list[0].get('eventType')[0]
activity = activity_list[0].get('eventType')[0]
if activity in activity_dict.keys():
activity_dict[activity]+=1
else:
activity_dict[activity] =1
else:
event.destroy()
print 'length of activity_list: %s' %(str(len(activity_list)))
print 'Seems to be an error,index: %s,user: %s; timestamp: %s \n' %(str(index),str(event.get('tracker').id ),str(event.get('timestamp')))
other_activity_total_count =total_count-sum(activity_dict.values())
if other_activity_total_count:
activity_dict['others'] = other_activity_total_count
# EventActivity = Object.extend('EventActivity')
# event_activity = EventActivity()
event_activity.set('application',application)
event_activity.set('event_name',event_list[0].get('event_name'))
event_activity.set('activity_dict',activity_dict)
event_activity.save()
def get_age_and_gender_data_dict(self, table_name="AppStaticInfo", filed_name="app"):
try:
Application = Object.extend("Application")
query = Query(Application)
query.equal_to("app_id", self.app_id)
result_list = query.find()
length = len(result_list)
if length == 0:
print "error: application not exists in table Applicaiton"
return 0
elif length != 1:
print "error: multi application exists in table Applicaiton"
return 0
else:
app = result_list[0]
DbTable = Object.extend(table_name)
query = Query(DbTable)
query.equal_to(filed_name, app)
result_list = query.find()
length = len(result_list)
if length == 0:
print "error: application not exists in table %s" % (str(table_name))
return 0
elif length != 1:
print "error: multi application exists in table %s" % (str(table_name))
return 0
else:
app_static_info = result_list[0]
age_and_gender_dict = app_static_info.get("age_and_gender")
return age_and_gender_dict
# WeightedStaticInfo = Object.extend('WeightedStaticInfo')
# query = Query(WeightedStaticInfo)
# query.exists('objectId')
# query.select('age','gender')
# staticInfoList = query.find()
# gender_type_list =['man','woman']
# age_type_list = ['16down','16to35','35to55','55up']
# dataDict ={gender_type:{age_type:0 for age_type in age_type_list} for gender_type in gender_type_list}
#
# for staticInfo in staticInfoList:
# gender = 'man' if staticInfo.get('gender') >0 else 'woman'
# age_info_dict= staticInfo.get('age')
# dataDict[gender][age_info_dict.keys()[0]] += 1
# # dataDict ={'man' if staticInfo.get('gender') >0 else 'woman':dataDict['man' if staticInfo.get('gender') >0 else 'woman'][staticInfo.get('age').keys()[0]] +=1 for staticInfo in staticInfoList}
# new_data_dict = {key:[0 for i in range(4)] for key in dataDict.keys()}
# for index ,age_type in enumerate(age_type_list):
# for gender_type in dataDict.keys():
# new_data_dict[gender_type][index] = dataDict[gender_type][age_type]
except LeanCloudError, e:
raise e
def process_item(self, item, spider):
Answers = Object.extend('Answers')
Users = Object.extend('Users')
for index ,ques in enumerate(item['floorNumList']):
answer = Answers()
user = Users()
queryAnswer = Query(Answers)
queryUser = Query(Users)
queryAnswer.equal_to('questionLink',item['questionLink'])
queryAnswer.equal_to('userId',item['userIdList'][index])
queryAnswer.equal_to('answerTime',item['answerTimeList'][index])
try:
if queryAnswer.find():
pass
else:
answer.set('questionLink',item['questionLink'])
answer.set('answerPageNum',item['answerPageNum'])
answer.set('floorNum',item['floorNumList'][index])
answer.set('answerPosition',item['answerPositionList'][index])
answer.set('userId',item['userIdList'][index])
answer.set('answerTime',item['answerTimeList'][index])
answer.set('answerIp',item['answerIpList'][index])
answer.set('answerContent',item['answerContentList'][index])
try:
answer.save()
except LeanCloudError,e:
print e
except LeanCloudError,e:
print e
queryUser.equal_to('userId',item['userIdList'][index])
try:
if queryUser.find():
pass
else:
user.set('userId',item['userIdList'][index])
user.set('userImgLink',item['userImgLinkList'][index])
user.set('userName',item['userNameList'][index])
user.set('userClass',item['userClass'][index])
user.set('userQuestionCount',item['userQuestionCountList'][index])
user.set('userScore',item['userScore'][index])
try:
user.save()
except LeanCloudError,e:
print e
except LeanCloudError,e:
print e
def test_save_and_destroy_all():
ObjToDelete = Object.extend('ObjToDelete')
objs = [ObjToDelete() for _ in range(3)]
Object.save_all(objs)
assert all(not x.is_new() for x in objs)
Object.destroy_all(objs)
for obj in objs:
try:
leancloud.Query(ObjToDelete).get(obj.id)
except leancloud.LeanCloudError as e:
assert e.code == 101
def test_save_and_destroy_all():
ObjToDelete = Object.extend('ObjToDelete')
objs = [ObjToDelete() for _ in range(3)]
Object.save_all(objs)
assert all(not x.is_new() for x in objs)
Object.destroy_all(objs)
for obj in objs:
try:
leancloud.Query(ObjToDelete).get(obj.id)
except leancloud.LeanCloudError as e:
assert e.code == 101
def parseDetailPage(url, word):
# url = 'http://www.hujiang.com/ciku/noresult/'
try:
print url
global category
global type_code
headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36'
}
req = requests.get(url, headers=headers)
req.encoding = 'utf-8'
content = req.text.replace('<br>', '\n')
content = content.replace('<br />', '\n')
content = content.replace('<br/>', '\n')
soup = BeautifulSoup(content, "html5lib")
article_content = soup.find('div', id="article_content")
article_title = soup.find('h1', id='article_title')
if article_title and article_content:
title = article_title.text.strip()
content = getContent(article_content.text.strip(), title)
if isExit(title):
print 'exit and return'
return
else:
print word
print title
print content
FilmDetail = Object.extend('HJWordStudyCList')
mFilmDetail = FilmDetail()
mFilmDetail.set('title', title)
mFilmDetail.set('word', word)
mFilmDetail.set('word_des', content)
mFilmDetail.set('category', category)
mFilmDetail.set('type', type_code)
mFilmDetail.set('source_url', url)
mFilmDetail.save()
print('save item')
else:
print 'has no result'
except:
print 'exception:' + word
print traceback.format_exc()
FilmDetail = Object.extend('HJWordStudyCList')
mFilmDetail = FilmDetail()
mFilmDetail.set('word', word)
mFilmDetail.set('category', category)
mFilmDetail.set('type', type_code)
mFilmDetail.set('source_url', url)
mFilmDetail.save()
def test_pointer_query(): # type: () -> None
foo = Object.create('Foo')
bar = Object.create('Bar')
bar.save()
foo.set('bar', bar)
foo.save()
q = Query('Foo').equal_to('bar', bar)
assert len(q.find()) == 1
inner_query = leancloud.Query('Post')
inner_query.exists("image")
query = leancloud.Query('Comment')
query.matches_query("post", inner_query)
assert query.dump() == {'where': {'post': {'$inQuery': {'className': 'Post', 'where': {'image': {'$exists': True}}}}}}
def index():
college_news = Query(Object.extend('College')).descending(key='news_time'). \
limit(10).select('news_title', 'news_time').find()
work_news = Query(Object.extend('Work')).descending(key='news_time'). \
limit(10).select('news_title', 'news_time').find()
notice_news = Query(Object.extend('College')).equal_to(key='news_type', value=10).\
descending(key='news_time').limit(10).select('news_title', 'news_time').find()
recruit_news = Query(Object.extend('Work')).equal_to(key='news_type', value=16).\
descending(key='news_time').limit(10).select('news_title', 'news_time').find()
return render_template('front/index.html', college_news=college_news, work_news=work_news,
notice_news=notice_news, recruit_news=recruit_news)
def test_pointer_query(): # type: () -> None
foo = Object.create('Foo')
bar = Object.create('Bar')
bar.save()
foo.set('bar', bar)
foo.save()
q = Query('Foo').equal_to('bar', bar)
assert len(q.find()) == 1
inner_query = leancloud.Query('Post')
inner_query.exists("image")
query = leancloud.Query('Comment')
query.matches_query("post", inner_query)
assert query.dump() == {'where': {'post': {'$inQuery': {'className': 'Post', 'where': {'image': {'$exists': True}}}}}}
def after_event_save(event):
print("after event save started")
institution = event.get('institution')
print "institution:" + institution
Conversation = Object.extend('_Conversation')
query1 = Conversation.query
query2 = Conversation.query
query1.equal_to('name', institution)
query2.equal_to('sys', True)
query = Query.and_(query1, query2)
query_list = query.find()
if len(query_list) == 0:
raise LeanEngineError('没有找到系统对话')
else:
conversation = query_list[0]
conversation_id = conversation.get('objectId')
print "conversationId:" + conversation_id
eventId = event.get('objectId')
headers = {'Content-Type': 'application/json', \
'X-LC-Id': APP_ID, \
'X-LC-Key': MASTER_KEY + ',master'}
data = {"from_peer": "sys", \
"message": "{\"_lctype\":-1,\"_lctext\":\"eventCreated\", \
\"_lcattrs\":{\"snType\":" + str(snEventCreated) + \
",\"eventId\": \"" + eventId + "\"}}", \
"conv_id": conversation_id}
requests.post(subscriber_url, data=json.dumps(data), headers=headers)
print("after event save ended")
def process_item(self, item, spider):
global itemId
print '-----------------process_item-----------------------'
if is_exit(item['title'],item['category']):
print 'already exit'
return item
else:
content = ''
contents = item['content']
for con in contents.splitlines():
content += con.strip()
content += '\n\n'
itemId += 1
Composition = Object.extend('Reading')
mComposition = Composition()
mComposition.set('item_id', itemId)
mComposition.set('title', item['title'])
mComposition.set('img_url', item['img_url'])
mComposition.set('content', content)
mComposition.set('type_name', item['type_name'])
mComposition.set('publish_time', item['publish_time'])
mComposition.set('type_id', item['type_id'])
mComposition.set('source_url', item['source_url'])
mComposition.set('source_name', item['source_name'])
mComposition.set('category', item['category'])
mComposition.set('type', item['type'])
mComposition.set('img_type', 'url')
mComposition.set('media_url', item['media_url'])
# mComposition.set('category_2', '')
mComposition.save()
print('save item')
return item
def test_encode(): # type: () -> None
Foo = Object.extend("Foo")
obj = Foo()
assert utils.encode(obj) == {
"className": "Foo",
"__type": "Pointer",
"objectId": None,
}
acl = ACL()
assert utils.encode(acl) == {}
acl.set_read_access("xxx", True)
assert utils.encode(acl) == {"xxx": {"read": True}}
point = GeoPoint()
assert utils.encode(point) == {
"__type": "GeoPoint",
"longitude": 0,
"latitude": 0,
}
assert utils.encode([obj, acl, point]) == [
{"className": "Foo", "__type": "Pointer", "objectId": None},
{"xxx": {"read": True}},
{"__type": "GeoPoint", "longitude": 0, "latitude": 0},
]
assert utils.encode({"a": obj, "b": acl}) == {
"a": {"className": "Foo", "__type": "Pointer", "objectId": None},
"b": {"xxx": {"read": True}},
}
def test_save_and_destroy_all(): # type: () -> None
ObjToDelete = Object.extend('ObjToDelete')
objs = [ObjToDelete() for _ in range(3)]
already_saved_obj = ObjToDelete()
already_saved_obj.save()
objs.append(already_saved_obj)
Object.save_all(objs)
assert all(not x.is_new() for x in objs)
Object.destroy_all(objs)
for obj in objs:
try:
leancloud.Query(ObjToDelete).get(obj.id)
except leancloud.LeanCloudError as e:
assert e.code == 101
def get_occupation_data_dict(table_name='WeightedStaticInfo',tracker_list=None):
try:
field_name = 'user'
DBTable = Object.extend(table_name)
gender_type_list =['man','woman']
age_type_list = ['16down','16to35','35to55','55up']
dataDict ={gender_type:{age_type:0 for age_type in age_type_list} for gender_type in gender_type_list}
# new_data_dict = {key:[0 for i in range(4)] for key in dataDict.keys()}
total_count = len(tracker_list)
for index, tracker in enumerate(tracker_list):
query = Query(DBTable)
query.equal_to(field_name,tracker)
query.select('age','gender')
result_list = query.find()
length = len(result_list)
if length!=1:
print 'error: the length of result_list is %s with index: %s with tracker_objectId: %s' %(str(length),str(index),tracker.id)
if length >=1:
result = result_list[0]
else:
continue
print 'index: %s gender: %s age: %s ' %(str(index),str(result.get('gender')),str(result.get('age')))
gender = 'man' if result.get('gender') >0 else 'woman'
age_info_dict= result.get('age')
dataDict[gender][age_info_dict.keys()[0]] += 1
# dataDict ={'man' if staticInfo.get('gender') >0 else 'woman':dataDict['man' if staticInfo.get('gender') >0 else 'woman'][staticInfo.get('age').keys()[0]] +=1 for staticInfo in staticInfoList}
# for index ,age_type in enumerate(age_type_list):
# for gender_type in dataDict.keys():
# new_data_dict[gender_type][index] = dataDict[gender_type][age_type]
known_count = sum(dataDict['man'].values())+sum(dataDict['woman'].values())
dataDict['unknown'] = total_count -known_count
except LeanCloudError, e:
raise e
def checkIfConfirmationCodeMatches(**params):
print "check if confirmation code matches starts"
if 'email' in params and 'code' in params:
try:
email = params['email']
print "email: " + email
code = params['code']
print "code: " + code
ConfirmationCode = Object.extend('ConfirmationCode')
twentyMinutesAgo = datetime.datetime.now() - datetime.timedelta(
minutes=20)
print(twentyMinutesAgo)
query1 = ConfirmationCode.query
query2 = ConfirmationCode.query
query3 = ConfirmationCode.query
query1.equal_to('email', email)
query2.equal_to('code', code)
query3.greater_than_or_equal_to('updatedAt', twentyMinutesAgo)
query12 = Query.and_(query1, query2)
query = Query.and_(query12, query3)
query_list = query.find()
print "check if confirmation code matches ends"
if len(query_list) == 0:
return False
else:
return True
except Exception as e:
print e
print "check if confirmation code matches ends"
raise LeanEngineError('系统错误：无法验证验证码')
else:
print "email and code cannot be empty"
print "check if confirmation code matches ends"
raise LeanEngineError('邮箱以及验证码都不能为空')
def nowplaying_movies(url, typeId, publish_time):
global item_id
contents = ''
img_url = ''
type_name = ''
# browser = webdriver.PhantomJS(executable_path='/root/phantomjs/bin/phantomjs')
browser = webdriver.PhantomJS(
executable_path=
'/Users/luli/Downloads/phantomjs-2.1.1-macosx/bin/phantomjs')
browser.get(url)
# print 'current_url:'+browser.current_url.encode('utf-8')
# time.sleep(3)
try:
title = browser.find_element_by_css_selector('h1.ph').text
if is_exit(title):
# print('already exit'.encode('utf-8'))
return
else:
soup = BeautifulSoup(browser.page_source, "html5lib")
wholeLayer = soup.find('div', id='wholeLayer').find_all(
'td', attrs={"align": "left"})
contents = ''
if len(wholeLayer) <= 0:
return
for td in wholeLayer:
if td.text.strip():
contents += td.text.strip() + '\n\n'
type_name = get_type_name(typeId)
print(title.encode('utf-8'))
print(contents.encode('utf-8'))
print('img_url:' + img_url.encode('utf-8'))
print(typeId.encode('utf-8'))
print(type_name.encode('utf-8'))
item_id += 1
Composition = Object.extend('Reading')
mComposition = Composition()
mComposition.set('item_id', item_id)
mComposition.set('title', title)
mComposition.set('img_url', img_url)
mComposition.set('img_type', 'url')
mComposition.set('content', contents)
mComposition.set('type_name', type_name)
mComposition.set('type_id', str(typeId))
mComposition.set('publish_time', publish_time)
mComposition.set('source_url', url)
mComposition.set('source_name', '酷悠双语网')
mComposition.set('category', 'shuangyu_reading')
mComposition.set('type', 'text')
# mComposition.save()
except:
print 'except return'.encode('utf-8')
# time.sleep(5)
# nowplaying_movies(url,typeId,publish_time)
return
browser.quit()
def saveBalanceLeanCloud(b):
# print('balance',b)
if b:
Balance = Object.extend('Balance')
balance = Balance()
balance.set('balance', b[0])
balance.save()
def get_sport_data_dict(self, app_table=APPLICATION_CLASS, field_name="app", kind=None):
try:
field = "sport"
app = self.get_the_app(kind=kind)
static_info_table = STATIC_INFO_TABLE
DbTable = Object.extend(static_info_table)
query = Query(DbTable)
query.equal_to(field_name, app)
query.exists(field)
result_list = query.find()
length = len(result_list)
if length == 0:
print "error: application not exists in table %s" % (str(static_info_table))
return {}
elif length > 1:
print "error: multi application exists in table %s" % (str(static_info_table))
# return 0
app_static_info = result_list[0]
data_dict = app_static_info.get(field)
# return age_and_gender_dict
except LeanCloudError, e:
raise e
def saveQiangquan(user, msg):
Balance = Object.extend('Qiangquan')
balance = Balance()
balance.set('msg', msg)
balance.set('user', user)
balance.save()
def saveZhongJiang(user, msg):
Balance = Object.extend('ZhongJiang')
balance = Balance()
balance.set('msg', msg)
balance.set('user', user)
balance.save()
def tupianxiaohua():
global sign
for page in range(1,2):
para = '&page=%s&maxResult=20&time=2015-07-10' % (page)
url = 'http://route.showapi.com/341-2'+ sign + para
# print page
# print url
req = requests.get(url)
# print req.text
result = json.loads(req.text)
if 0 == result['showapi_res_code']:
newslist = result['showapi_res_body']['contentlist']
for data in newslist:
if is_exit(data['title'].strip()):
# print 'exit and continue'
continue
else:
# print data['title'].strip()
# print data['img']
ratio = getImageRatio(data['img'])
if ratio == 5.5555:
continue
Reading = Object.extend('Joke')
mReading = Reading()
mReading.set('text', data['title'].strip())
mReading.set('img', data['img'])
mReading.set('ratio', ratio)
mReading.set('type', '1')
mReading.set('category', '101')
mReading.save()
def see_record_num(period_label_dic):
valid_num=0
valid_period=[]
####init
connect_db()
#####get all
UserSensor = Object.extend('UserSensor')
query = Query(UserSensor)
#
for label,periods in period_label_dic.items():
for period in periods:
stamp_range=generate_stamp(period)
#
query.equal_to("deviceId",device).not_equal_to("events",None).\
equal_to("sensorType",sensor).\
less_than("timestamp", stamp_range[1]).greater_than("timestamp",stamp_range[0])
#equal_to("motion",class_type).\
if query.count()>10:
valid='valid'
valid_num+=query.count()
valid_period.append([label,period])
else:valid='not valid'
print label,period,'count',query.count(),valid
print 'total valid record:',valid_num
def get_tracker_of_app(app_id):
query = Query(Object.extend('Application'))
query.equal_to('app_id', app_id)
app_list = query.find()
if not app_list:
return []
the_app = app_list[0]
query = Query(Object.extend('BindingInstallation'))
query.equal_to('application', the_app)
query.select('user')
installation_list = query.find()
user_set = set()
for installation in installation_list:
user_set.add(installation.attributes['user'].id)
return list(user_set)
def create_new_app(self, app_name):
try:
if not app_name:
return 0
user = self.user.become(self.session_token)
print "Got the user"
Application = Object.extend(APPLICATION_CLASS)
application = Application()
query = Query(Application)
query.equal_to("user", user)
query.equal_to("app_name", app_name)
if query.find():
print "Application name exists!"
return 0
else:
print "Application name not exists! "
application.set("app_name", app_name)
application.set("user", user)
application.save()
app_id = application.id
app_key = (signer.sign(app_id).split(app_id + "."))[1]
# app_key = app_id+"this is app_key"
application.set("app_id", app_id)
application.set("app_key", app_key)
application.save()
return 1
except LeanCloudError, e:
print e
return 0
def test_save_and_destroy_all(): # type: () -> None
ObjToDelete = Object.extend('ObjToDelete')
objs = [ObjToDelete() for _ in range(3)]
already_saved_obj = ObjToDelete()
already_saved_obj.save()
objs.append(already_saved_obj)
Object.save_all(objs)
assert all(not x.is_new() for x in objs)
Object.destroy_all(objs)
for obj in objs:
try:
leancloud.Query(ObjToDelete).get(obj.id)
except leancloud.LeanCloudError as e:
assert e.code == 101
def get_tracker_of_app(self, app_id=""):
try:
self.tracker_list = []
# user = self.user.become(self.session_token)
Application = Object.extend("Application")
query = Query(Application)
# query.equal_to('user', user)
# app_id = 'demo 55f7e36f60b2fe7115171b4b'
print "@@@@@@@@@@@@@@@@@@" + app_id
query.equal_to("app_id", app_id)
app_list = query.find()
if len(app_list) != 1:
return []
the_app = app_list[0]
# Tracker = Object.extend('BindingInstallation')
# query = Query(Tracker)
# query.equal_to('application', the_app)
# app_list = query.find()
relation = the_app.relation("tracker")
print relation,
return 1
except LeanCloudError, e:
print e
return 0
def query2():
location_list = []
finger_print = Object.extend('fingerprint')
my_query = finger_print.query
my_query.select('name', 'info')
query_list = my_query.find()
for test in query_list:
temp_dict = {}
name = test.get('name')
fp = test.get('info')
temp_dict['Tag'] = name
temp_dict['FingerPrint'] = eval(fp)
# print(temp_dict['FingerPrint'])
location_list.append(temp_dict)
# print(name, fp)
for location in location_list:
# if re.search('[7][0-9]{3}', location['Tag']):
for fp in location['FingerPrint']:
if fp['mac'] not in mac_list:
mac_list.append(fp['mac'])
for location in location_list:
for mac in mac_list:
flag = 1
for fp in location['FingerPrint']:
if mac == fp['mac']:
flag = 0
break
if flag:
temp_dict = {'mac': mac, 'rss': '-99'}
location['FingerPrint'].append(temp_dict)
return location_list
def xiaohuadaquan_gif():
global sign
for page in range(1,2):
para = '&page=%s&maxResult=20' % (page)
url = 'http://route.showapi.com/341-3'+ sign + para
# print page
# print url
req = requests.get(url)
# print req.text
result = json.loads(req.text)
if 0 == result['showapi_res_code']:
newslist = result['showapi_res_body']['contentlist']
for data in newslist:
if is_exit(data['title'].strip()):
# print 'exit and continue'
continue
else:
publish_time = datetime.strptime(str(data['ct']), "%Y-%m-%d %H:%M:%S.%f")
# print data['title'].strip()
# print publish_time
# print data['img']
ratio = getImageRatio(data['img'])
if ratio == 5.5555:
continue
Reading = Object.extend('Joke')
mReading = Reading()
mReading.set('text', data['title'].strip())
mReading.set('img', data['img'])
mReading.set('ratio', ratio)
# mReading.set('source_url', data['url'])
mReading.set('publish_time', publish_time)
mReading.set('type', '3')
mReading.set('category', '101')
mReading.save()
def get_tracker_data(table_name=None,tracker_list=None,field_name=None):
DBTable = Object.extend(table_name)
tracker_data_dict = {}
for index,tracker in enumerate(tracker_list):
#这样处理是因为可能一个user的记录超过了一次可以读取的数量（1K条）
query = Query(DBTable)
query.equal_to(field_name,tracker)
query.less_than('createdAt',currentTime)
total_count=query.count()
# print 'TotalCount %s' %str(total_count)
query_times=(total_count+query_limit-1)/query_limit
#如果想在这里按timestamp排序取出每个user的记录是不靠谱的，可能读取时还有插入，而且timestamp介于之间
for index in range(query_times):
# print 'querying index: %s' %str(index)
query = Query(DBTable)
query.equal_to(field_name,tracker)
query.less_than('createdAt',currentTime)
query.ascending('createdAt')
query.limit(query_limit)
query.skip(index*query_limit)
if tracker in userDataDict.keys():
tracker_data_dict.get(tracker).extend(query.find())
else :
tracker_data_dict[tracker]=query.find()
return tracker_data_dict
def laifudaotupianxiaohua():
global sign
url = 'http://route.showapi.com/107-33'+ sign
# print url
req = requests.get(url)
# print req.text
result = json.loads(req.text)
if 0 == result['showapi_res_code']:
newslist = result['showapi_res_body']['list']
for data in newslist:
if is_exit(data['title'].strip()):
# print 'exit and continue'
continue
else:
time.sleep(3)
# print data['title'].strip()
# print data['thumburl']
ratio = getImageRatio(data['thumburl'])
Reading = Object.extend('Joke')
mReading = Reading()
mReading.set('text', data['title'].strip())
mReading.set('img', data['thumburl'])
mReading.set('ratio', ratio)
mReading.set('type', '1')
mReading.set('category', '101')
if ratio != 5.5555:
mReading.save()
def process_item(self, item, spider):
global itemId
print '-----------------process_item-----------------------'
if is_exit(item['title'], item['category']):
print 'already exit'
return item
else:
itemId += 1
Composition = Object.extend('Reading')
mComposition = Composition()
mComposition.set('item_id', itemId)
mComposition.set('title', item['title'])
mComposition.set('img_url', item['img_url'])
mComposition.set('content', item['content'])
mComposition.set('type_name', item['type_name'])
mComposition.set('publish_time', item['publish_time'])
mComposition.set('type_id', item['type_id'])
mComposition.set('source_url', item['source_url'])
mComposition.set('source_name', item['source_name'])
mComposition.set('category', item['category'])
mComposition.set('type', item['type'])
mComposition.set('img_type', 'url')
mComposition.set('img_urls', item['img_urls'])
mComposition.set('media_url', item['media_url'])
mComposition.set('category_2', '')
mComposition.save()
print('save item')
return item
def get_daocloud_app():
print('get func begin run..')
base_url = 'https://openapi.daocloud.io/v1/apps'
DAOCLOUD_APITOKEN = os.environ.get('DAOCLOUD_APITOKEN')
token = "token " + DAOCLOUD_APITOKEN
requests.packages.urllib3.disable_warnings()
result = requests.get(base_url, headers={"Authorization": token})
data = json.loads(result.text)
DaoCloudApp = Object.extend('DaoCloudApp')
query = leancloud.Query('DaoCloudApp')
for daoapp in data['app']:
query.equal_to("appid", daoapp['id'])
try:
dao = query.first()
dao.set('state', daoapp['state'])
dao.set('last_operated_at', daoapp['last_operated_at'])
dao.save()
except leancloud.LeanCloudError as e:
if e.code == 101:
dao = DaoCloudApp()
dao.set('appid', daoapp['id'])
dao.set('name', daoapp['name'])
dao.set('state', daoapp['state'])
dao.set('last_operated_at', daoapp['last_operated_at'])
dao.save()
else:
raise e
print('get func OK')
def __init__(self):
leancloud.init('mctfj249nwy7c1ymu3cps56lof26s17hevwq4jjqeqoloaey', master_key='ao6h5oezem93tumlalxggg039qehcbl3x3u8ofo7crw7atok')
Users = Object.extend('Users')
query = Query(Users)
curTime = datetime.now()
query.exists('userId')
query.less_than('createdAt',curTime)
userCount = query.count()
print "userCounts: %s" %str(userCount)
queryLimit = 500
queryTimes = userCount/queryLimit + 1
self.urls = []
for index in range(queryTimes):
query = Query(Users)
query.exists('userId')
query.less_than('createdAt',curTime)
query.descending('createdAt')
query.limit(queryLimit)
query.skip(index*queryLimit)
query.select('userId')
userRet = query.find()
for user in userRet:
self.urls.append(self.baseUrl + user.get('userId') +".json")
pass
def gen_short_url(lurl):
'''Generates shortened URL from given long url, returns a string as the
shortened url's key.
'''
try:
if get_long(lurl) is not None: return lurl
except LeanCloudError as e:
if e.code == 101:
pass
else:
raise e
try:
return get_short(lurl).get('short')
except LeanCloudError as e:
if e.code == 101:
pass
else:
raise e
Shortened = Object.extend('Shortened')
shortened = Shortened()
# Hard coded size. Fix later ( or never )
size = 4
surl = gen_random_string(size=size)
shortened.set('long', lurl)
shortened.set('short', surl)
shortened.save()
return surl
def laifudaoxiaohua():
global sign
url = 'http://route.showapi.com/107-32'+ sign
# print url
req = requests.get(url)
# print req.text
result = json.loads(req.text)
if 0 == result['showapi_res_code']:
newslist = result['showapi_res_body']['list']
for data in newslist:
if is_exit(data['content'].strip()):
# print 'exit and continue'
continue
else:
# print data['content'].strip()
# print data['url']
Reading = Object.extend('Joke')
mReading = Reading()
mReading.set('text', data['content'].strip())
# mReading.set('img', data['picUrl'])
mReading.set('source_url', data['url'])
# mReading.set('publish_time', publish_time)
mReading.set('type', '5')
mReading.set('category', '101')
mReading.save()
def getPositionLeanCloud():
Position = Object.extend('Position')
position = Query(Position)
position.select('position')
query_list = position.find()
# print (str(query_list[0]))
return query_list
def get_event_to_location_data(self, event_name=None, app_table=APPLICATION_CLASS, kind=None):
# print app_table
try:
app = self.get_the_app(kind=kind)
db_name = "FakeEventActivity"
DbTable = Object.extend(db_name)
query = Query(DbTable)
# 这里只是测试知道是少于1K条的
query.equal_to("application", app)
query.exists("location_dict")
if event_name:
query.equal_to("event_name", event_name)
# query.equal_to('application_id',application_id)
query.descending("createdAt")
query.limit(1)
result_list = query.find()
if result_list:
# event_name = result_list[0].get('event_name')
# activity_statistics_dict = {result_list[0].get('event_name'):result_list[0].get('activity_dict')}
statistics_dict = result_list[0].get("location_dict")
else:
statistics_dict = []
except LeanCloudError, e:
raise e
def start_if_stop():
print('start func begin run..')
base_url = 'https://openapi.daocloud.io/v1/apps'
DAOCLOUD_APITOKEN = os.environ.get('DAOCLOUD_APITOKEN')
token = "token {token}".format(token=DAOCLOUD_APITOKEN)
requests.packages.urllib3.disable_warnings()
result = requests.get(base_url, headers={"Authorization": token})
data = json.loads(result.text)
DaoCloudApp = Object.extend('DaoCloudApp')
query = leancloud.Query('DaoCloudApp')
for daoapp in data['app']:
try:
if daoapp['state'] == 'stopped':
print(daoapp['name'] + ' stopped, start...')
action_id_text = requests.post(
'{0}/{1}/actions/start'.format(base_url, daoapp['id']),
headers={"Authorization": token}).text
action_id = json.loads(action_id_text)
action_result = 'in_process'
while action_result == 'in_process':
action_result_text = requests.get(
'{0}/{1}/actions/{2}'.format(base_url,
daoapp['id'], action_id['action_id']),
headers={"Authorization": token}).text
action_result = json.loads(action_result_text)['state']
print(action_result)
time.sleep(5)
print(daoapp['name'] + ' ' + action_result)
except leancloud.LeanCloudError as e:
raise e
print('start func OK')
def songci(message):
cmd, text = parse_cmd_text(message.text)
if text == None or len(text) == 0:
bot.sendMessage(chat_id=message.chat.id,
reply_to_message_id=message.message_id,
text='请使用 /songci <词名>')
return
bot.sendChatAction(chat_id=message.chat.id,
action=telegram.ChatAction.TYPING)
text = text.replace(' ', '·')
keyword = urllib2.quote(text)
response = urllib2.urlopen(songci_api + keyword)
data = json.loads(response.read())
Songci = Object.extend('Songci')
__songci = Songci()
__songci.set('keyword', keyword)
__songci.set('data', response.read())
__songci.save()
try:
a_songci = data['result']['list'][0]
except TypeError as e:
bot.sendMessage(chat_id=message.chat.id,
reply_to_message_id=message.message_id,
text='找不到对应的宋词')
return
__text = a_songci['title'] + '\n' + a_songci['author'] + '\n' + a_songci[
'content']
block_chars = '⓪①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮⑯⑰⑱⑲⑳❶❷❸❹❺❻❼❽❾❿⑴⑵⑶⑷⑸⑹⑺⑻⑼⑽⑾⑿⒀⒁⒂⒃⒄⒅⒆⒇'
temp = ''
for c in __text:
if not c in block_chars:
temp += c
__text = temp.replace(' ', ' ').replace('<br />', '\n')
bot.sendMessage(chat_id=message.chat.id, text=__text)
def saveErrorLog(user, msg):
Balance = Object.extend('ErrorLog')
balance = Balance()
balance.set('msg', msg)
balance.set('user', user)
balance.save()
def huabanfuli():
global sign
# types=['4001','4002','4003','4004','4007','4010','4011','4012','4013','4014']
types = ['4013', '4014']
for type in types:
para = '&page=1&num=20&type=%s' % (type)
url = 'http://route.showapi.com/852-2' + sign + para
print url
req = requests.get(url)
print req.text
result = json.loads(req.text)
if 0 == result['showapi_res_code']:
allPages = result['showapi_res_body']['pagebean']['allPages']
print allPages
if type == '4013':
allPages = 128
for page in range(allPages, 0, -1):
para1 = '&page=%s&num=20&type=%s' % (page, type)
url = 'http://route.showapi.com/852-2' + sign + para1
print url
print page
req = requests.get(url)
print req.text
sbody = json.loads(req.text)
if 0 == sbody['showapi_res_code']:
contentlist = sbody['showapi_res_body']['pagebean'][
'contentlist']
if len(contentlist) == 0:
print 'contentlist = 0'
continue
for item in contentlist:
title_h = item['title']
img_list = item['list']
for index, list_item in enumerate(img_list):
big_img = list_item['big']
title = title_h + str(index + 1)
if is_exit(title):
print 'title exit:' + title
continue
else:
if is_exit_img(big_img):
print 'big_img exit:' + big_img
continue
else:
ratio = getImageRatio(big_img)
if ratio == 5.5555:
print '5.5555 exit'
continue
else:
Reading = Object.extend('Joke')
mReading = Reading()
mReading.set('text', title)
mReading.set('img', big_img)
mReading.set('ratio', ratio)
mReading.set('type', '1')
mReading.set('category', '103')
mReading.save()
print('save item')
def nowplaying_movies(url, publish_time):
global item_id
contents = ''
img_url = ''
type_name = ''
headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36'
}
req = requests.get(url, headers=headers)
# req.encoding='utf-8'
soup = BeautifulSoup(req.text, "html5lib")
[script.extract() for script in soup.findAll('script')]
[style.extract() for style in soup.findAll('style')]
title = soup.find(
'div', class_='pull-right sider-content').find('h1').text.strip()
if is_exit(title, publish_time):
print('already exit')
return
your_string = soup.find('div', id='bodybox')
if your_string:
contents = getContent(your_string.get_text().strip())
type_name = ""
type_name_tag = soup.select('li.active > a')
if type_name_tag:
type_name = type_name_tag[0].text
typeId = get_type_id(type_name)
img_tag = soup.select("div#bodybox > div > img")
if img_tag:
img_url = "http://www.adreep.cn" + img_tag[0]['src']
print(title)
print(type_name)
print(typeId)
print(item_id)
print(contents)
print(img_url)
item_id += 1
Composition = Object.extend('Reading')
mComposition = Composition()
mComposition.set('item_id', item_id)
mComposition.set('title', title)
mComposition.set('img_url', img_url)
mComposition.set('img_type', 'url')
mComposition.set('content', contents)
mComposition.set('type_name', type_name)
mComposition.set('type_id', typeId)
mComposition.set('publish_time', publish_time)
mComposition.set('source_url', url)
mComposition.set('source_name', '水滴英语作文网')
mComposition.set('category', 'composition')
mComposition.set('type', 'text')
mComposition.save()
