def add_feed(request):
init_db()
nw = NewsWorker()
if request.POST['txtFeedTitle'] != "" and request.POST['txtFeedUrl'] != "" :
nw.add_feed(request.POST['cbFeedSource'],request.POST['txtFeedTitle'],request.POST['txtFeedUrl'],)
return source_manager(request)
pass
def read(self):
if self.is_crawl_time() == False:
return 0
urls = Feed.select().execute()
result = []
entries = []
news_count = 0
nw = NewsWorker()
for url in urls:
try:
try:
Setting.log.info("start crawing for "+url.url)
except:
pass
d = feedparser.parse(url.url)
rss_title = encode_to_utf8(d['feed']['title'])
for item in d.entries:
gmt_date,persian_date = self.compute_dates(item.published, item.published_parsed)
date,time = self.get_jalali_datetime(persian_date)
if nw.add_news(item.title, item.link, url.id, gmt_date) :
news_count += 1
except Exception as ex:
Setting.log.log_exception(ex,'rss reading main loop: ')
self.set_crawl_log(news_count)
return news_count
def home_page(header, request):
#raise()
Setting.log.log_request(request)
try:
forgot_key = None
#LogOut(header,request)
init_db()
user_name = ""
authenticated = False
session = request.session
if 'authenticated' in session and session['authenticated']:
authenticated = True
uw = UserWorker()
user_name = uw.get_user_full_name(session['username'])
if 'forgotPasswordKey' in request.params.keys():
forgot_key = request.params.getone('forgotPasswordKey')
nw = NewsWorker()
news = nw.get_news()
return {
'news': news,
'authenticated': authenticated,
'user_name': user_name,
'forgot_key': forgot_key
}
except Exception as ex:
Setting.log.log_exception(ex)
if Setting.debuging:
return Response(ex.__str__())
else:
return Response('sorry,problem')
def home_page(header,request):
#raise()
Setting.log.log_request(request)
try:
forgot_key = None
#LogOut(header,request)
init_db()
user_name = ""
authenticated = False
session = request.session
if 'authenticated' in session and session['authenticated']:
authenticated = True
uw = UserWorker()
user_name = uw.get_user_full_name(session['username'])
if 'forgotPasswordKey' in request.params.keys():
forgot_key = request.params.getone('forgotPasswordKey')
nw = NewsWorker()
news = nw.get_news()
return {'news':news,'authenticated':authenticated,'user_name':user_name,'forgot_key':forgot_key}
except Exception as ex:
Setting.log.log_exception(ex)
if Setting.debuging:
return Response(ex.__str__())
else:
return Response('sorry,problem')
def Read(self):
if self.IsCrawlTime() == False:
return 0
urls = Feed.select().execute()
result = []
entries = []
news_count = 0
nw = NewsWorker()
for url in urls:
try:
d = feedparser.parse(url.url)
rss_title = EncodeToUtf8(d['feed']['title'])
for item in d.entries:
gmt_date, persian_date = self.ComputeDates(
item.published, item.published_parsed)
date, time = self.GetJalaliDateTime(persian_date)
if nw.AddNews(item.title, item.link, url.id, gmt_date):
news_count += 1
except Exception as ex:
Setting.error_log.LogException(ex, 'rss reading main loop: ')
self.SetCrawlLog(news_count)
return news_count
def read(self):
if self.is_crawl_time() == False:
return 0
urls = Feed.select().execute()
result = []
entries = []
news_count = 0
nw = NewsWorker()
for url in urls:
try:
try:
Setting.log.info("start crawing for " + url.url)
except:
pass
d = feedparser.parse(url.url)
rss_title = encode_to_utf8(d['feed']['title'])
for item in d.entries:
gmt_date, persian_date = self.compute_dates(
item.published, item.published_parsed)
date, time = self.get_jalali_datetime(persian_date)
if nw.add_news(item.title, item.link, url.id, gmt_date):
news_count += 1
except Exception as ex:
Setting.log.log_exception(ex, 'rss reading main loop: ')
self.set_crawl_log(news_count)
return news_count
def add_source(request):
init_db()
nw = NewsWorker()
if request.POST['txtSourceTitle'] != "" and request.POST['txtSourceUrl'] != "" :
nw.add_source(request.POST['txtSourceTitle'],request.POST['txtSourceUrl'],request.POST['txtSourceIcon'])
return source_manager(request)
pass
def HomePage(header, request):
print("ff")
try:
#LogOut(header,request)
InitDB()
user_name = ""
authenticated = "false"
session = request.session
if 'authenticated' in session and session['authenticated']:
authenticated = "true"
uw = UserWorker()
user_name = uw.GetUserFullName(session['username'])
rr = RssReader()
rr.Read()
nw = NewsWorker()
news = nw.GetNews()
return {
'news': news,
'authenticated': authenticated,
'user_name': user_name
}
except Exception as ex:
Setting.error_log.LogException(ex)
if Setting.debuging:
return Response(ex.__str__())
else:
return Response('sorry,problem')
def home_page(header,request):
#raise()
Setting.log.log_request(request)
try:
query = ""
days = None
source=None
starred = None
alert_message = ""
if 'q' in request.params:
query = request.params.getone('q')
if 'date' in request.params:
days = request.params.getone('date')
if 'source' in request.params:
source = request.params.getone('source')
if 'starred' in request.params:
tmp = request.params.getone('starred')
if tmp == 'true':
starred = True
else:
starred = False
forgot_key = None
#LogOut(header,request)
init_db()
user_name = ""
authenticated = False
session = request.session
if 'authenticated' in session and session['authenticated']:
authenticated = True
uw = UserWorker()
user_name = uw.get_user_full_name(session['username'])
else:
if len(query) > 0 or days != None or source != None:
query = ""
days = None
source = None
alert_message = u'تنها کاربران عضو قادر به جستجو می باشند';
if 'forgotPasswordKey' in request.params.keys():
forgot_key = request.params.getone('forgotPasswordKey')
nw = NewsWorker()
news = nw.get_news(query,source,days,starred,session['user_id'] if 'user_id' in session else None)
sources = nw.get_all_sources()
return {'news':news,'authenticated':authenticated,'user_name':user_name,'forgot_key':forgot_key,
'search_text':query,'search_source':source,'search_date':days,'alert_message':alert_message,
'sources':sources,'starred':'true' if starred== True else 'false'}
except Exception as ex:
Setting.log.log_exception(ex)
if Setting.debuging:
return Response(ex.__str__())
else:
return Response('sorry,problem')
def toggle_star(request):
try:
init_db()
news_id = request.POST['news_id']
user_id = request.session['user_id']
nw = NewsWorker()
res = nw.add_user_news(user_id,news_id)
return {'success':True, 'result':res}
except Exception as ex:
return Response(ex.__str__())
def get_all_sources(request):
try:
init_db()
n = NewsWorker()
res = n.get_all_sources()
return {'rows':res}
except Exception as ex:
Setting.log.log_exception(ex)
if Setting.debuging:
return Response(ex.__str__())
else:
return Response('sorry,problem')
def extract_news(self,count=10):
n = NewsWorker()
items = n.get_unextracted_news(count)
for item in items:
try:
url = item.url
#print(url)
id = item.id
tag_name = item.tag_name
tag_class = item.tag_class
tag_id = item.tag_id
page = get_data(url)
text,html = self.extract(page.decode('utf-8'),tag_name,tag_class,tag_id)
n.add_content(id, mysql_str(html), mysql_str(text))
except Exception as ex:
Setting.log.log_exception(ex)
def home_page(header, request):
#raise()
Setting.log.log_request(request)
try:
query = ""
alert_message = ""
if 'q' in request.params:
query = request.params.getone('q')
forgot_key = None
#LogOut(header,request)
init_db()
user_name = ""
authenticated = False
session = request.session
if 'authenticated' in session and session['authenticated']:
authenticated = True
uw = UserWorker()
user_name = uw.get_user_full_name(session['username'])
else:
if len(query) > 0:
query = ""
alert_message = 'تنها کاربران عضو قادر به جستجو می باشند'
if 'forgotPasswordKey' in request.params.keys():
forgot_key = request.params.getone('forgotPasswordKey')
nw = NewsWorker()
news = nw.get_news(
query, session['user_id'] if 'user_id' in session else None)
return {
'news': news,
'authenticated': authenticated,
'user_name': user_name,
'forgot_key': forgot_key,
'search_text': query,
'alert_message': alert_message
}
except Exception as ex:
Setting.log.log_exception(ex)
if Setting.debuging:
return Response(ex.__str__())
else:
return Response('sorry,problem')
def extract_news(self, count=10):
n = NewsWorker()
items = n.get_unextracted_news(count)
for item in items:
try:
url = item.url
#print(url)
id = item.id
tag_name = item.tag_name
tag_class = item.tag_class
tag_id = item.tag_id
page = get_data(url)
text, html = self.extract(page.decode('utf-8'), tag_name,
tag_class, tag_id)
n.add_content(id, mysql_str(html), mysql_str(text))
except Exception as ex:
Setting.log.log_exception(ex)
def ExtractNews(self,count=10):
n = NewsWorker()
items = n.GetUnextractedNews(count)
for item in items:
try:
url = item.url
print(url)
id = item.id
tag_name = item.tag_name
tag_class = item.tag_class
tag_id = item.tag_id
page = GetData(url)
text,html = self.Extract(page.decode('utf-8'),tag_name,tag_class,tag_id)
n.AddContent(id, MysqlStr(html), MysqlStr(text))
except Exception as ex:
Setting.error_log.LogException(ex)
return
def source_manager(request):
init_db()
nw = NewsWorker()
feeds = []
sources = nw.get_all_sources()
current_source=-1
if 'cbFeedSource' in request.POST:
current_source = request.POST['cbFeedSource']
else:
current_source = sources[0]['id']
feeds = nw.get_source_feed(current_source)
for s in sources:
s['selected'] = 'selected' if s['id']==int(current_source) else None
print(s['id'])
return render_to_response('../frontend/templates/source_manager.pt',{'sources':sources,'feeds':feeds,'current_source':current_source},request=request)
def home_page(header, request):
# raise()
Setting.log.log_request(request)
try:
query = ""
alert_message = ""
if "q" in request.params:
query = request.params.getone("q")
forgot_key = None
# LogOut(header,request)
init_db()
user_name = ""
authenticated = False
session = request.session
if "authenticated" in session and session["authenticated"]:
authenticated = True
uw = UserWorker()
user_name = uw.get_user_full_name(session["username"])
else:
if len(query) > 0:
query = ""
alert_message = "تنها کاربران عضو قادر به جستجو می باشند"
if "forgotPasswordKey" in request.params.keys():
forgot_key = request.params.getone("forgotPasswordKey")
nw = NewsWorker()
news = nw.get_news(query)
return {
"news": news,
"authenticated": authenticated,
"user_name": user_name,
"forgot_key": forgot_key,
"search_text": query,
"alert_message": alert_message,
}
except Exception as ex:
Setting.log.log_exception(ex)
if Setting.debuging:
return Response(ex.__str__())
else:
return Response("sorry,problem")
def home_page(header,request):
#raise()
Setting.log.log_request(request)
try:
query = ""
alert_message = ""
if 'q' in request.params:
query = request.params.getone('q')
forgot_key = None
#LogOut(header,request)
init_db()
user_name = ""
authenticated = False
session = request.session
if 'authenticated' in session and session['authenticated']:
authenticated = True
uw = UserWorker()
user_name = uw.get_user_full_name(session['username'])
else:
if len(query) > 0 :
query = ""
alert_message = u'تنها کاربران عضو قادر به جستجو می باشند';
if 'forgotPasswordKey' in request.params.keys():
forgot_key = request.params.getone('forgotPasswordKey')
nw = NewsWorker()
news = nw.get_news(query,session['user_id'] if 'user_id' in session else None)
return {'news':news,'authenticated':authenticated,'user_name':user_name,'forgot_key':forgot_key,'search_text':query,'alert_message':alert_message}
except Exception as ex:
Setting.log.log_exception(ex)
if Setting.debuging:
return Response(ex.__str__())
else:
return Response('sorry,problem')
def read_all_bulletin(self):
query = (BulletinFilter.select(User.username,User.id,BulletinFilter.query_text,BulletinFilter.scan_period,
BulletinFilter.fk_source.alias('source'),BulletinFilter.starred,Source.title)
.join(Source,JOIN_LEFT_OUTER).switch(BulletinFilter).join(Bulletin).join(User).naive())
cnt = 0
if query.count() > 0 :
nw = NewsWorker()
for row in query.execute():
email = row.username
user_id = row.id
query_text = row.query_text
scan_period = row.scan_period
source = row.source
starred = row.starred
source_title = row.title
title = "خبرنامه:";
if source != None :
title += " {0} ".format(source_title)
else:
title += " کل اخبار ";
if scan_period != None:
title += " در {0} روز گذشته".format(scan_period)
if starred != None :
title += " در موارد ستاره دار "
if query_text != None:
title += " خبرهای شامل عبارت: "+query_text
html = '''<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<center>
<h2>یکسطر: درجریان باشید</h2>
<h2>''' +title+ '''</h2>
<div class="" style="direction:rtl;text-align:right;font-family:Tahoma;font-size:15px;width:650px; font: normal 12px/150% Tahoma; background: #fff; overflow: hidden; border: 1px solid #006699; -webkit-border-radius: 3px; -moz-border-radius: 3px; border-radius: 3px; " >
<table style="border-collapse: collapse; text-align: right; width: 100%;" ><tbody>''';
#print(user_id,email,source )
news = nw.get_news(query_text, source, scan_period, starred, user_id, 0, 100)
for index,item in enumerate( news):
css_style = '''line-height:20px;font-family:Tahoma;padding: 3px 10px; color: #00496B; border-left: 1px solid #E1EEF4;font-size: 12px;font-weight: normal;
'''
if index %2 == 0:
css_style += 'background: #E1EEF4; color: #00496B;'
html += """<tr>
<td style='border-left: none;width:24px;{0}'><img style='width:16px;height:16px' src='http://yeksatr.com/{1}'</td>
<td style='{0}' ><a style='text-decoration:none; color:black;cursor:pointer' href='{2}' target='_blank' >{3}</a></td>
<td style='width:80px;{0}' >{4}</td>
</tr>""".format(css_style,item.icon,item.link,item.title,item.persian_date)
html += "</tbody></table></div></center>"
sendmail(email, title, "", html)
cnt+=1
#file = open("c:\\temp\\1.html", mode='w', encoding="utf-8")
#file.write(html);
#file.flush();
#file.close();
#print (len(news))
print (str(cnt) + " bulletin sent ")
pass
def get_bulletin_dialog(request):
nw = NewsWorker()
sources = nw.get_all_sources()
return render_to_response('../frontend/templates/index.bulletin.pt',{'sources':sources},request=request)
def get_bulletin_filter_dialog(request):
init_db()
nw = NewsWorker()
sources = nw.get_all_sources()
return render_to_response('../frontend/templates/index.bulletin.filter.pt',
{'bulletin_id':request.POST['bulletin_id'],'sources':sources},request=request)
def add_bulletin(request):
init_db()
nw = NewsWorker()
nw.add_bulletin(request.session['user_id'], request.POST['query_text'], int(request.POST['scan_period']), int(request.POST['source']),
request.POST['starred'], int(request.POST['bulletin_id']))
return {'success':True}
def get_user_bulletin(request):
init_db()
nw = NewsWorker()
return {'success':True,'data':nw.get_user_bulletin(request.session['user_id'])}
def add_bulletin_filter(request):
init_db()
nw = NewsWorker()
nw.add_bulletin_filter(request.POST['bulletin_id'],int(request.POST['source_id']) if int(request.POST['source_id'])>=0 else None ,
request.POST['query_text'])
return {'success':True}
def get_bulletin_filter(request):
init_db()
nw = NewsWorker()
return {'success':True,'data':nw.get_bulletin_filter(request.POST['bulletin_id'],request.session['user_id'])}
def remove_bulletin(request):
init_db()
nw = NewsWorker()
res = nw.remove_bulletin(request.POST['bulletin_id'],request.session['user_id'])
return {'success':res}