def upgrade(migrate_engine):
"""By default start all installs out with a bookmark to bmark.us
It's cheesy, but helps with testing to have some base data and is good for
pubbing the links
"""
from datetime import datetime
import transaction
from bookie.models import initialize_sql
initialize_sql(migrate_engine)
from bookie.models import DBSession
from bookie.models import Bmark
bmark_us = Bmark('http://bmark.us',
desc="Bookie Website",
ext= "Bookie Documentation Home",
tags = "bookmarks")
bmark_us.stored = datetime.now()
bmark_us.updated = datetime.now()
DBSession.add(bmark_us)
DBSession.flush()
transaction.commit()
def upgrade(migrate_engine):
"""By default start all installs out with a bookmark to bmark.us
It's cheesy, but helps with testing to have some base data and is good for
pubbing the links
"""
from datetime import datetime
import transaction
from bookie.models import initialize_sql
initialize_sql(migrate_engine)
from bookie.models import DBSession
from bookie.models import Bmark
bmark_us = Bmark('http://bmark.us',
desc="Bookie Website",
ext="Bookie Documentation Home",
tags="bookmarks")
bmark_us.stored = datetime.now()
bmark_us.updated = datetime.now()
DBSession.add(bmark_us)
DBSession.flush()
transaction.commit()
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
# Update the settings with the current app root path
settings["app_root"] = abspath(dirname(dirname(__file__)))
initialize_sql(settings)
authn_policy = AuthTktAuthenticationPolicy(settings.get("auth.secret"), callback=UserMgr.auth_groupfinder)
authz_policy = ACLAuthorizationPolicy()
config = Configurator(
settings=settings,
root_factory="bookie.RootFactory",
authentication_policy=authn_policy,
authorization_policy=authz_policy,
)
config.set_request_factory(RequestWithUserAttribute)
config = build_routes(config)
config.add_static_view("static", "bookie:static")
config.add_renderer("jsonp", JSONP(param_name="callback"))
config.scan("bookie.views")
return config.make_wsgi_app()
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
# Update the settings with the current app root path
settings['app_root'] = abspath(dirname(dirname(__file__)))
initialize_sql(settings)
authn_policy = AuthTktAuthenticationPolicy(
settings.get('auth.secret'), callback=UserMgr.auth_groupfinder)
authz_policy = ACLAuthorizationPolicy()
config = Configurator(settings=settings,
root_factory='bookie.RootFactory',
authentication_policy=authn_policy,
authorization_policy=authz_policy)
config.set_request_factory(RequestWithUserAttribute)
config = build_routes(config)
config.add_static_view('static', 'bookie:static')
config.add_renderer('jsonp', JSONP(param_name='callback'))
config.scan('bookie.views')
return config.make_wsgi_app()
def importer_process_worker(iid):
"""Do the real import work
:param iid: import id we need to pull and work on
"""
logger = importer_process_worker.get_logger()
trans = transaction.begin()
initialize_sql(ini)
import_job = ImportQueueMgr.get(iid)
logger.info("IMPORT: RUNNING for {username}".format(**dict(import_job)))
try:
# process the file using the import script
import_file = open(import_job.file_path)
importer = Importer(
import_file,
import_job.username)
importer.process()
import_job.mark_done()
logger.info(
"IMPORT: COMPLETE for {username}".format(**dict(import_job)))
except Exception, exc:
# we need to log this and probably send an error email to the
# admin
logger = importer_process_worker.get_logger()
logger.error(str(exc))
import_job.mark_error()
def importer_depth():
"""Update the RRD data for the import queue depth."""
trans = transaction.begin()
initialize_sql(ini)
StatBookmarkMgr.count_importer_depth()
trans.commit()
celery.task.subtask(importer_depth_rrd).delay()
def importer_process_worker(iid):
"""Do the real import work
:param iid: import id we need to pull and work on
"""
logger = importer_process_worker.get_logger()
trans = transaction.begin()
initialize_sql(ini)
import_job = ImportQueueMgr.get(iid)
logger.info("IMPORT: RUNNING for {username}".format(**dict(import_job)))
try:
# process the file using the import script
import_file = open(import_job.file_path)
importer = Importer(import_file, import_job.username)
importer.process()
import_job.mark_done()
logger.info(
"IMPORT: COMPLETE for {username}".format(**dict(import_job)))
except Exception, exc:
# we need to log this and probably send an error email to the
# admin
logger = importer_process_worker.get_logger()
logger.error(str(exc))
import_job.mark_error()
def db_init_bookmark():
"""install the initial bookmark in a new install"""
require('hosts', provided_by=[sample])
require('ini', provided_by=[sample])
parse_ini(env["ini_file"])
from datetime import datetime
import transaction
from bookie.models import initialize_sql
from sqlalchemy import create_engine
engine = create_engine(env.ini.get('app:bookie', 'sqlalchemy.url'))
initialize_sql(engine)
from bookie.models import DBSession
from bookie.models import Bmark
bmark_us = Bmark(u'http://bmark.us',
u'admin',
desc=u"Bookie Website",
ext= u"Bookie Documentation Home",
tags = u"bookmarks")
bmark_us.stored = datetime.now()
bmark_us.updated = datetime.now()
DBSession.add(bmark_us)
DBSession.flush()
transaction.commit()
def importer_depth():
"""Update the RRD data for the import queue depth."""
trans = transaction.begin()
initialize_sql(ini)
StatBookmarkMgr.count_importer_depth()
trans.commit()
celery.task.subtask(importer_depth_rrd).delay()
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
# Update the settings with the current app root path
settings['app_root'] = abspath(dirname(dirname(__file__)))
initialize_sql(settings)
authn_policy = AuthTktAuthenticationPolicy(
settings.get('auth.secret'),
callback=UserMgr.auth_groupfinder)
authz_policy = ACLAuthorizationPolicy()
config = Configurator(settings=settings,
root_factory='bookie.RootFactory',
authentication_policy=authn_policy,
authorization_policy=authz_policy)
config.set_request_factory(RequestWithUserAttribute)
config = build_routes(config)
config.add_static_view('static', 'bookie:static')
config.scan('bookie.views')
return config.make_wsgi_app()
def new_user(username, email):
"""Add new user function, pass username, email
:param username: string of new user
:param email: string of new email
"""
require('hosts', provided_by=[sample])
require('ini', provided_by=[sample])
parse_ini(env["ini_file"])
import transaction
from bookie.models import initialize_sql
initialize_sql(dict(env.ini.items('app:main')))
from bookie.models import DBSession
from bookie.models.auth import get_random_word, User
sess = DBSession()
u = User()
u.username = unicode(username)
passwd = get_random_word(8)
u.password = passwd
u.email = unicode(email)
u.activated = True
u.is_admin = False
u.api_key = User.gen_api_key()
print dict(u)
print passwd
sess.add(u)
sess.flush()
transaction.commit()
def _init_sql(args):
"""Init the sql session for things to work out"""
ini = ConfigParser()
ini_path = path.join(path.dirname(path.dirname(path.dirname(__file__))),
args.ini)
ini.readfp(open(ini_path))
here = path.abspath(path.join(path.dirname(__file__), '../../'))
ini.set('app:main', 'here', here)
initialize_sql(dict(ini.items("app:main")))
def load_bookie_ini(ini_file):
"""Load the settings for the bookie.ini file."""
ini = ConfigParser()
ini_path = path.join(path.dirname(path.dirname(__file__)), ini_file)
ini.readfp(open(ini_path))
here = path.abspath(path.join(path.dirname(__file__), '../'))
ini.set('app:bookie', 'here', here)
initialize_sql(dict(ini.items("app:bookie")))
return ini
def load_bookie_ini(ini_file):
"""Load the settings for the bookie.ini file."""
ini = ConfigParser()
ini_path = path.join(path.dirname(path.dirname(__file__)), ini_file)
ini.readfp(open(ini_path))
here = path.abspath(path.join(path.dirname(__file__), '../'))
ini.set('app:bookie', 'here', here)
initialize_sql(dict(ini.items("app:bookie")))
return ini
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
engine = engine_from_config(settings, 'sqlalchemy.')
initialize_sql(engine)
config = Configurator(settings=settings)
config.add_static_view('static', 'bookie:static')
config = build_routes(config)
return config.make_wsgi_app()
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
engine = engine_from_config(settings, 'sqlalchemy.')
initialize_sql(engine)
unencrypt = UnencryptedCookieSessionFactoryConfig('itsaseekreet')
config = Configurator(settings=settings, session_factory=unencrypt)
config = build_routes(config)
config.add_static_view('static', 'bookie:static')
return config.make_wsgi_app()
def importer_process():
"""Check for new imports that need to be scheduled to run"""
initialize_sql(ini)
imports = ImportQueueMgr.get_ready(limit=1)
for i in imports:
# Log that we've scheduled it
logger = importer_process.get_logger()
logger.info("IMPORT: SCHEDULED for {username}.".format(**dict(i)))
# We need to mark that it's running to prevent it getting picked up
# again.
trans = transaction.begin()
i.mark_running()
trans.commit()
celery.task.subtask(importer_process_worker, args=(i.id, )).delay()
def importer_process():
"""Check for new imports that need to be scheduled to run"""
initialize_sql(ini)
imports = ImportQueueMgr.get_ready(limit=1)
for i in imports:
# Log that we've scheduled it
logger = importer_process.get_logger()
logger.info("IMPORT: SCHEDULED for {username}.".format(**dict(i)))
# We need to mark that it's running to prevent it getting picked up
# again.
trans = transaction.begin()
i.mark_running()
trans.commit()
celery.task.subtask(importer_process_worker, args=(i.id,)).delay()
def email_signup_user(email, msg, settings, message_data):
"""Do the real import work
:param iid: import id we need to pull and work on
"""
from bookie.lib.message import InvitationMsg
msg = InvitationMsg(email, msg, settings)
status = msg.send(message_data)
if status == 4:
from bookie.lib.applog import SignupLog
trans = transaction.begin()
initialize_sql(ini)
SignupLog(SignupLog.ERROR,
'Could not send smtp email to signup: ' + email)
trans.commit()
def email_signup_user(email, msg, settings, message_data):
"""Do the real import work
:param iid: import id we need to pull and work on
"""
from bookie.lib.message import InvitationMsg
msg = InvitationMsg(email, msg, settings)
status = msg.send(message_data)
if status == 4:
from bookie.lib.applog import SignupLog
trans = transaction.begin()
initialize_sql(ini)
SignupLog(SignupLog.ERROR,
'Could not send smtp email to signup: ' + email)
trans.commit()
def reset_password(username, password):
"""Reset a user's password"""
require('hosts', provided_by=[sample])
require('ini', provided_by=[sample])
parse_ini(env["ini_file"])
import transaction
from bookie.models import initialize_sql
initialize_sql(dict(env.ini.items('app:main')))
from bookie.models import DBSession
from bookie.models.auth import UserMgr
sess = DBSession()
u = UserMgr.get(username=username)
u.password = password
sess.flush()
transaction.commit()
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
engine = engine_from_config(settings, 'sqlalchemy.')
initialize_sql(engine)
authn_policy = AuthTktAuthenticationPolicy(
settings.get('auth.secret'),
callback=UserMgr.auth_groupfinder)
authz_policy = ACLAuthorizationPolicy()
config = Configurator(settings=settings,
root_factory='bookie.RootFactory',
authentication_policy=authn_policy,
authorization_policy=authz_policy)
config.set_request_factory(RequestWithUserAttribute)
config = build_routes(config)
config.add_static_view('static', 'bookie:static')
config.scan('bookie.views')
return config.make_wsgi_app()
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
initialize_sql(settings)
authn_policy = AuthTktAuthenticationPolicy(
settings.get('auth.secret'),
callback=UserMgr.auth_groupfinder)
authz_policy = ACLAuthorizationPolicy()
config = Configurator(settings=settings,
root_factory='bookie.RootFactory',
authentication_policy=authn_policy,
authorization_policy=authz_policy)
config.set_request_factory(RequestWithUserAttribute)
import bookie.bcelery.celeryapp
bookie.bcelery.celeryapp.load_config(settings)
config = build_routes(config)
config.add_static_view('static', 'bookie:static')
config.scan('bookie.views')
return config.make_wsgi_app()
return args
if __name__ == "__main__":
args = parse_args()
# we need to make sure you submitted an ini file to use:
# args.ini
ini = ConfigParser()
ini_path = path.join(path.dirname(path.dirname(path.dirname(__file__))), args.ini)
ini.readfp(open(ini_path))
db_url = ini.get('app:bookie', 'sqlalchemy.url')
engine = create_engine(db_url, echo=False)
initialize_sql(engine)
ct = 0
all = False
while(not all):
if args.new_only:
# we take off the offset because each time we run, we should have
# new ones to process. The query should return the 10 next
# non-imported urls
url_list = Hashed.query.outerjoin(Readable).\
filter(Readable.imported == None).\
limit(PER_TRANS).all()
elif args.retry_errors:
def count_total():
"""Count the total number of bookmarks in the system"""
trans = transaction.begin()
initialize_sql(ini)
StatBookmarkMgr.count_total_bookmarks()
trans.commit()
def count_total():
"""Count the total number of bookmarks in the system"""
trans = transaction.begin()
initialize_sql(ini)
StatBookmarkMgr.count_total_bookmarks()
trans.commit()
def count_unique():
"""Count the unique number of bookmarks/urls in the system"""
trans = transaction.begin()
initialize_sql(ini)
StatBookmarkMgr.count_unique_bookmarks()
trans.commit()
def _initTestingDB():
from sqlalchemy import create_engine
from bookie.models import initialize_sql
session = initialize_sql(create_engine('sqlite://'))
return session
print read.content
else:
print "Url is an image"
else:
# we need to make sure you submitted an ini file to use:
# args.ini
ini = ConfigParser()
ini_path = path.join(path.dirname(path.dirname(
path.dirname(__file__))),
args.ini)
ini.readfp(open(ini_path))
here = path.abspath(path.join(path.dirname(__file__), '../../'))
ini.set('app:main', 'here', here)
initialize_sql(dict(ini.items("app:main")))
ct = 0
all = False
while(not all):
# start the queue up we'll use to thread the url fetching
enclosure_queue = Queue()
if args.new_only:
# we take off the offset because each time we run, we should
# have new ones to process. The query should return the 10
# next non-imported urls
url_list = Bmark.query.outerjoin(Readable, Bmark.readable).\
filter(Readable.imported == None).\
limit(PER_TRANS).all()
def count_unique():
"""Count the unique number of bookmarks/urls in the system"""
trans = transaction.begin()
initialize_sql(ini)
StatBookmarkMgr.count_unique_bookmarks()
trans.commit()
"""
import transaction
from collections import defaultdict
from ConfigParser import ConfigParser
from os import path
from bookie.models import initialize_sql
if __name__ == "__main__":
ini = ConfigParser()
ini_path = path.join(path.dirname(path.dirname(path.dirname(__file__))),
'bookie.ini')
ini.readfp(open(ini_path))
initialize_sql(dict(ini.items("app:main")))
from bookie.models import DBSession
from bookie.models import Bmark
bookmarks = Bmark.query.all()
index = defaultdict(list)
to_delete = []
for b in bookmarks:
key = (b.username, b.hash_id)
index[key].append(b)
for k, v in index.iteritems():
if len(v) > 1:
first = v[0]
second = v[1]
import transaction
from whoosh.store import LockError
from whoosh.writing import IndexingError
from bookie.lib.importer import Importer
from bookie.lib.readable import ReadUrl
from bookie.models import initialize_sql
from bookie.models import Bmark
from bookie.models import Readable
from bookie.models.auth import UserMgr
from bookie.models.stats import StatBookmarkMgr
from bookie.models.queue import ImportQueueMgr
from .celery import load_ini
INI = load_ini()
initialize_sql(INI)
logger = get_task_logger(__name__)
@celery.task(ignore_result=True)
def hourly_stats():
"""Hourly we want to runa series of numbers to track
Currently we're monitoring:
- Total number of bookmarks in the system
- Unique number of urls in the system
- Total number of tags in the system
"""
import transaction
from datetime import datetime
from ConfigParser import ConfigParser
from os import path
from bookie.models import initialize_sql
if __name__ == "__main__":
ini = ConfigParser()
ini_path = path.join(path.dirname(path.dirname(path.dirname(__file__))),
'bookie.ini')
ini.readfp(open(ini_path))
initialize_sql(dict(ini.items("app:bookie")))
from bookie.models import DBSession
from bookie.models import Bmark
bmark_us = Bmark(u'http://bmark.us',
u'admin',
desc=u'Bookie Website',
ext= u'Bookie Documentation Home',
tags = u'bookmarks')
bmark_us.stored = datetime.now()
bmark_us.updated = datetime.now()
DBSession.add(bmark_us)
DBSession.flush()
transaction.commit()
