def __init__(self):
self.sources = [
model.Page('mako', model.Image(os.path.join(IMAGE_DIR,
'mako.png'))),
model.Page('Haaretz הארץ',
model.Image(os.path.join(IMAGE_DIR, 'haaretz.png'))),
model.Page('חדשות 10',
model.Image(os.path.join(IMAGE_DIR, '10.png'))),
model.Page('וואלה! תרבות ובידור',
model.Image(os.path.join(IMAGE_DIR, 'walla.png'))),
model.Page('ynet', model.Image(os.path.join(IMAGE_DIR,
'ynet.jpg')))
]
def __init__(self):
self.sources = [
model.Page(
'The New York Times',
model.Image.from_file(
os.path.join(IMAGE_DIR, 'new_york_times.png'))),
model.Page(
'BBC News',
model.Image.from_file(os.path.join(IMAGE_DIR, 'bbc.png'))),
model.Page(
'CNN',
model.Image.from_file(os.path.join(IMAGE_DIR, 'cnn.png'))),
model.Page(
'Fox News',
model.Image.from_file(os.path.join(IMAGE_DIR, 'fox_news.png')))
]
def make_admin_report_panel(self, sortby='date', order='desc'):
sortby_type = sortby
sortby_dir = order
session = model.Session()
table = model.report
sql = table.select()
# Enforce limited moderation reign.
if self.user.account == staff.MODERATOR:
sql = sql.where(table.c.board.in_(self.user.reign))
# Determine order.
if sortby_type in ('board', 'postnum', 'date'):
try:
column = getattr(table.c, sortby_type)
except AttributeError:
raise WakaError('Sort-by column is absent from table.')
sort = column.desc
if sortby_dir == 'asc':
sort = column.asc
sql = sql.order_by(sort(), table.c.date.desc())
else:
sql = sql.order_by(table.c.date.desc())
# Paginate.
res = model.Page(sql, self.page, self.perpage)
# Hidden input fields.
inputs = [{
'name': 'task',
'value': 'reports'
}, {
'name': 'order',
'value': sortby_dir
}, {
'name': 'sortby',
'value': sortby_type
}]
rooturl = misc.make_script_url(task='reports',
sortby=sortby_type,
order=sortby_dir,
_amp=True)
Template.__init__(self,
'report_panel_template',
reports=res.rows,
sortby=sortby_type,
order=sortby_dir,
number_of_pages=res.total_pages,
rowcount=res.total_entries,
inputs=inputs,
rooturl=rooturl)
def session():
engine = create_engine('sqlite:///:memory:', echo=True)
Session = sessionmaker(bind=engine)
#Session = sessionmaker()
# Session.configure(bind=engine)
lorem_page = model.Page(content='Lorem Ipsum', heading='Ed Jones')
assert lorem_page.content == 'Lorem Ipsum'
assert lorem_page.heading == 'Ed Jones'
assert lorem_page.id == 1, 'lorem_page.id should be {} but is {} instead'.format(
1, lorem_page.id)
def generate_pages(self, articles, row_per_page=15):
page_group = self._chunk(articles, row_per_page)
total_page = len(page_group)
pages = [
model.Page(i, total_page, page)
for i, page in enumerate(page_group, start=1)
]
if not pages:
return None
self.generate_to('../gui/index.html',
'page.html',
page=pages[0],
**self.global_data)
for i in xrange(1, total_page):
self.generate_to('../gui/page/{0}.html'.format(i + 1),
'page.html',
page=pages[i],
**self.global_data)
def main(page_id, user_path, access_token, launch_time_date, since, until, number):
"""
Main program entry
:param page_id: The id of the page to crawl
:param user_path: The path where to store the data
:param access_token: The access token used to authenticate to the server
:param launch_time_date: The launch date and time
:param since: The date from where to begin the crawl
:param until: The date of the latest post
:param number: The number of latest posts
:return:
"""
# Create the log's file name using today's date
log_file_name = '{}-{}-{}-{}.{}'.format(page_id, launch_time_date, since, until, 'log')
# Configure the logger
logger = logger_config(log_file_name)
# To avoid any exception
try:
# Create an object Page
page = model.Page()
# Get page information
page.get_page_info(page_id, access_token)
# Move to the posts
page.get_posts(access_token, since=since, until=until, number=number, with_comments=True, with_replies=True,
with_reactions=True)
# Get stats about the page
page.get_stats()
# The directory where to save the files
write_directory = os.path.join(config.data_dir, user_path,
'{}_-_{}_-_{}_-_{}'.format(getattr(page, 'name').encode('utf-8'), page_id, since, until))
# Create the write directory if it doesn't exist
os_utils.create_directories(write_directory)
# Create the file name using today's date
filename = '{}.{}'.format(launch_time_date, 'json')
# Logging
logger.info('Writing all the data to file {}'.format(filename))
# Writing page info data to file
os_utils.export_json_data_to_file(page.to_json(), os.path.join(write_directory, filename))
print('All the information extracted from {} page !'.format(page_id))
except Exception:
print('An error occurred, please check the log file {}'.format(page_id))
# Exit with status code : -4
sys.exit(-4)
def make_admin_post_search_panel(self, search, text, caller='internal'):
board = self.board
session = model.Session()
table = board.table
board.check_access(self.user)
popup = caller != 'board'
if search.find('IP Address') != -1:
try:
sql = table.select()\
.where(table.c.ip == misc.dot_to_dec(text))
except ValueError:
raise WakaError('Please enter a valid IP.')
search_type = 'IP'
elif search.find('Text String') != -1:
sql = table.select().where(table.c.comment.like('%' + text + '%'))
search_type = 'text string'
elif search.find('Author') != -1:
sql = table.select().where(
or_(table.c.name.like('%' + text + '%'),
table.c.trip.like('%' + text + '%')))
search_type = 'author'
else:
sql = table.select().where(table.c.num == text)
search_type = 'ID'
if search_type != 'ID':
page = model.Page(sql, self.page, self.perpage)
rowcount = page.total_entries
total_pages = page.total_pages
posts = page.rows
if not posts:
raise WakaError("No posts found for %s %s" %
(search_type, text))
else:
rowcount = total_pages = 1
row = session.execute(sql).fetchone()
if not row:
raise WakaError("Post not found. (It may have just been"
" deleted.)")
posts = [row]
inputs = [{
'name': 'board',
'value': board.name
}, {
'name': 'task',
'value': 'searchposts'
}, {
'name': 'text',
'value': text
}, {
'name': 'caller',
'value': caller
}, {
'name': 'search',
'value': search
}]
rooturl = misc.make_script_url(task='searchposts',
board=board.name,
caller=caller,
search=search,
text=text,
_amp=True)
Template.__init__(self,
'post_search',
num=id,
posts=posts,
search=search,
text=text,
inputs=inputs,
number_of_pages=total_pages,
rooturl=rooturl,
rowcount=rowcount,
popup=popup)
def make_admin_activity_panel(self,
view='',
user_to_view=None,
action_to_view=None,
ip_to_view=None,
post_to_view=None,
sortby_name='date',
sortby_dir='desc'):
board = self.board
template_view = 'staff_activity_unfiltered'
action_name = action_content = ''
table = model.activity
account_table = model.account
dual_table_select = [
account_table.c.username, account_table.c.account,
account_table.c.disabled, table.c.action, table.c.info,
table.c.date, table.c.ip
]
sql = select(dual_table_select,
from_obj=[
table.join(
account_table,
table.c.username == model.account.c.username)
])
rooturl_args = dict(task='stafflog',
board=board.name,
view=view,
sortby=sortby_name,
order=sortby_dir,
_amp=True)
if view == 'user':
if not user_to_view:
raise WakaError('Please select a user to view.')
template_view = 'staff_activity_by_user'
sql = sql.where(table.c.username == user_to_view)
rooturl_args['usertoview'] = user_to_view
elif view == 'action':
if not action_to_view:
raise WakaError('Please select an action to view.')
template_view = 'staff_activity_by_actions'
(action_name, action_content) \
= staff_tasks.get_action_name(action_to_view, 1)
sql = sql.where(table.c.action == action_to_view)
rooturl_args['actiontoview'] = action_to_view
elif view == 'ip':
if not ip_to_view:
raise WakaError('Please specify an IP address to view.')
template_view = 'staff_activity_by_ip_address'
sql = sql.where(table.c.info.like('%' + ip_to_view + '%'))
rooturl_args['iptoview'] = ip_to_view
elif view == 'post':
if not post_to_view:
raise WakaError('Post key missing.')
template_view = 'staff_activity_by_post'
sql = sql.where(table.c.info.like('%' + post_to_view + '%'))
rooturl_args['posttoview'] = post_to_view
rooturl = misc.make_script_url(**rooturl_args)
# Acquire staff info.
session = model.Session()
staff_get = model.account.select()
staff = session.execute(staff_get).fetchall()
# Establish list of hidden inputs.
inputs = [{
'name': 'actiontoview',
'value': action_to_view
}, {
'name': 'task',
'value': 'stafflog'
}, {
'name': 'posttoview',
'value': post_to_view
}, {
'name': 'usertoview',
'value': user_to_view
}, {
'name': 'iptoview',
'value': ip_to_view
}, {
'name': 'order',
'value': sortby_dir
}, {
'name': 'sortby',
'value': sortby_name
}, {
'name': 'view',
'value': view
}]
if self.board:
inputs.append({'name': 'board', 'value': self.board.name})
# Apply sorting.
if sortby_name and hasattr(table.c, sortby_name):
order_col = getattr(table.c, sortby_name)
if sortby_dir.lower() == 'asc':
sort_spec = order_col.asc()
else:
sort_spec = order_col.desc()
sql = sql.order_by(sort_spec)
res = model.Page(sql, self.page, self.perpage)
Template.__init__(self,
template_view,
user_to_view=user_to_view,
entries=res.rows,
staff=staff,
rowcount=res.total_entries,
numberofpages=res.total_pages,
view=view,
order=sortby_dir,
action_name=action_name,
content_name=action_content,
sortby=sortby_name,
number_of_pages=res.total_pages,
rooturl=rooturl,
inputs=inputs)
# -*- coding: utf-8 -*-
"""
Created on Wed Aug 17 05:52:09 2016
@author: hclqaVirtualBox1
"""
from object_test import session
import random
import string
import model
test_page = model.Page()
N = 5
test_page.title = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(N))
test_page.content = u'Test content'
print(test_page.title)
session.add(test_page)
print("1 ----- TestPage ID")
print(test_page.id)
"""
At this point the test_page object is known to SQLAlchemy,
but not to the database. To send it to the database,
a flush operation can be forced:
"""
session.flush()
print("2 ----- TestPage ID")
print (test_page.id)
