def new_authorized_apps_collection(db):
"""Move the authorized_apps information from the users collection
to an authorized_apps collection.
"""
app_cache = {}
auth = Authorizator(db)
scopes = ['read-passwords', 'write-passwords', 'read-userinfo']
for user in db.users.find():
authorized_apps = user['authorized_apps']
# create an authorized_apps document for every authorized app
for app_id in authorized_apps:
if app_id not in app_cache:
app_cache[app_id] = db.applications.find_one({'_id': app_id})
app = app_cache[app_id]
credentials = {
'client_id': app['client_id'],
'user': user,
'redirect_uri': app['callback_url'],
'response_type': 'code',
}
auth.store_user_authorization(scopes, credentials)
safe_print('Storing authorized app "%s" for user %s' % (
app['client_id'],
get_user_display_name(user),
))
# remove the authorized_apps attribute from all users
db.users.update({}, {'$unset': {'authorized_apps': ''}}, multi=True)
def users():
result = setup_simple_command(
"users",
"Report information about users and their passwords.",
)
if isinstance(result, int):
return result
else:
settings, closer, env, args = result
try:
for user in Session.query(User).order_by(User.creation):
info = _get_user_info(user)
providers = info['providers']
text = (
'%s (%s)\n'
'\tPasswords: %d\n'
'\tProviders:%s\n'
'\tVerified: %s\n'
'\tDate joined: %s\n'
'\tLast login: %s\n' % (
info['display_name'],
user.id,
info['passwords'],
' ' + providers if providers else '',
info['verified'],
info['date_joined'],
info['last_login'],
)
)
safe_print(text)
finally:
closer()
def applications():
result = setup_simple_command(
"applications",
"Report information about oauth2 client applications.",
)
if isinstance(result, int):
return result
else:
settings, closer, env, args = result
try:
db = settings['mongodb'].get_database()
for app in db.applications.find():
info = _get_app_info(db, app)
text = ('%s\n'
'\tOwner: %s\n'
'\tMain URL: %s\n'
'\tCallback URL: %s\n'
'\tUsers: %d\n' % (
info['name'], info['owner'],
info['main_url'], info['callback_url'],
info['users'],
))
safe_print(text)
finally:
closer()
def add_attribute(collection, obj, obj_repr, attribute, value):
if attribute not in obj:
safe_print('Adding attribute "%s" to %s' % (attribute, obj_repr))
collection.update(
{'_id': obj['_id']},
{'$set': {attribute: value}},
)
def users():
result = setup_simple_command(
"users",
"Report information about users and their passwords.",
)
if isinstance(result, int):
return result
else:
settings, closer, env, args = result
try:
db = settings['mongodb'].get_database()
for user in db.users.find().sort('date_joined'):
info = _get_user_info(db, user)
text = ('%s (%s)\n'
'\tPasswords: %d\n'
'\tProviders: %s\n'
'\tVerified: %s\n'
'\tDate joined: %s\n'
'\tLast login: %s\n' % (
info['display_name'], user['_id'],
info['passwords'], info['providers'], info['verified'],
info['date_joined'], info['last_login'],
))
safe_print(text)
finally:
closer()
def clean_access_codes():
result = setup_simple_command(
"clean_access_codes",
"Deletes expired access codes"
)
if isinstance(result, int):
return result
else:
settings, closer, env, args = result
try:
now = datetime.datetime.utcnow()
with transaction.manager:
counter = 0
for access_code in Session.query(AccessCode).filter(
AccessCode.expiration < now
):
Session.delete(access_code)
counter += 1
if counter > 0:
safe_print('%d access codes were cleaned' % counter)
finally:
closer()
def applications():
result = setup_simple_command(
"applications",
"Report information about oauth2 client applications.",
)
if isinstance(result, int):
return result
else:
settings, closer, env, args = result
try:
for app in Session.query(Application).all():
text = (
'%s\n'
'\tOwner: %s\n'
'\tMain URL: %s\n'
'\tCallback URL: %s\n'
'\tUsers: %d\n' % (
app.name,
get_user_display_name(app.user),
app.main_url,
app.callback_url,
len(app.authorized_applications),
)
)
safe_print(text)
finally:
closer()
def migrate():
usage = "migrate: %prog config_uri migration_name"
description = "Add a 'send_email_periodically' preference to every user."
parser = optparse.OptionParser(
usage=usage,
description=textwrap.dedent(description)
)
options, args = parser.parse_args(sys.argv[1:])
if len(args) != 2:
safe_print('You must provide two arguments. '
'The first one is the config file and the '
'second one is the migration name.')
return 2
config_uri = args[0]
migration_name = args[1]
env = bootstrap(config_uri)
settings, closer = env['registry'].settings, env['closer']
try:
db = settings['mongodb'].get_database()
if migration_name in migration_registry:
migration = migration_registry[migration_name]
migration(db)
else:
safe_print('The migration "%s" does not exist.' % migration_name)
return 3
finally:
closer()
def send_email(request, email_template, user, preferences_link):
safe_print('Sending email to %s' % get_user_display_name(user))
context = {'user': user, 'preferences_link': preferences_link}
return create_message(
request,
'yithlibraryserver.scripts:templates/%s' % email_template,
context,
"Yith Library announcement",
[user['email']],
)
def send_email(request, email_template, user, preferences_link):
safe_print('Sending email to %s' % get_user_display_name(user))
context = {'user': user, 'preferences_link': preferences_link}
return create_message(
request,
'yithlibraryserver.scripts:templates/%s' % email_template,
context,
"Yith Library announcement",
[user.email],
)
def send_backups_via_email():
result = setup_simple_command(
"send_backups_via_email",
"Send a password backup to users.",
)
if isinstance(result, int):
return result
else:
settings, closer, env, args = result
try:
request = env['request']
if len(args) == 0:
now = datetime.datetime.utcnow()
if now.day == 1:
user_iterator = get_all_users(request.db, now)
else:
user_iterator = tuple()
else:
user_iterator = get_selected_users(request.db, *args)
tx = transaction.begin()
public_url_root = settings['public_url_root']
preferences_link = urlparse.urljoin(
public_url_root,
request.route_path('user_preferences'))
backups_link = urlparse.urljoin(
public_url_root,
request.route_path('backups_index'))
for user in user_iterator:
if user['email']:
sent = send_passwords(request, user,
preferences_link, backups_link)
if sent:
safe_print('Passwords sent to %s' %
get_user_display_name(user))
tx.commit()
finally:
closer()
def send_backups_via_email():
result = setup_simple_command(
"send_backups_via_email",
"Send a password backup to users.",
)
if isinstance(result, int):
return result
else:
settings, closer, env, args = result
try:
request = env['request']
if len(args) == 0:
now = datetime.datetime.utcnow()
if now.day == 1:
user_iterator = get_all_users(now)
else:
user_iterator = tuple()
else:
user_iterator = get_selected_users(*args)
tx = transaction.begin()
public_url_root = settings['public_url_root']
preferences_link = urlparse.urljoin(
public_url_root, request.route_path('user_preferences'))
backups_link = urlparse.urljoin(public_url_root,
request.route_path('backups_index'))
for user in user_iterator:
if user.email:
sent = send_passwords(request, user, preferences_link,
backups_link)
if sent:
safe_print('Passwords sent to %s' %
get_user_display_name(user))
tx.commit()
finally:
closer()
def announce():
usage = "migrate: %prog config_uri migration_name"
description = "Add a 'send_email_periodically' preference to every user."
parser = optparse.OptionParser(
usage=usage,
description=textwrap.dedent(description)
)
options, args = parser.parse_args(sys.argv[1:])
if len(args) != 2:
safe_print('You must provide two arguments. '
'The first one is the config file and the '
'second one is the email template.')
return 2
config_uri = args[0]
email_template = args[1]
env = bootstrap(config_uri)
settings, closer = env['registry'].settings, env['closer']
try:
db = settings['mongodb'].get_database()
request = env['request']
public_url_root = settings['public_url_root']
preferences_link = urlparse.urljoin(
public_url_root,
request.route_path('user_preferences'))
tx = transaction.begin()
mailer = get_mailer(request)
for user in get_all_users_with_passwords_and_email(db):
message = send_email(request, email_template, user,
preferences_link)
mailer.send(message)
tx.commit()
finally:
closer()
def send_email(request, email_template, user, preferences_link):
safe_print('Sending email to %s' % get_user_display_name(user))
text_body = render(
'yithlibraryserver.scripts:templates/%s.txt' % email_template,
{'user': user, 'preferences_link': preferences_link},
request=request,
)
# chamaleon txt templates are rendered as utf-8 bytestrings
text_body = text_body.decode('utf-8')
html_body = render(
'yithlibraryserver.scripts:templates/%s.pt' % email_template,
{'user': user, 'preferences_link': preferences_link},
request=request,
)
message = Message(subject="Yith Library announcement",
recipients=[user['email']],
body=text_body,
html=html_body)
return message
def announce():
usage = "announce: %prog config_uri email_template"
description = "Send an announce to every user with a verified email that has at least one password."
parser = optparse.OptionParser(usage=usage,
description=textwrap.dedent(description))
options, args = parser.parse_args(sys.argv[1:])
if len(args) != 2:
safe_print('You must provide two arguments. '
'The first one is the config file and the '
'second one is the email template.')
return 2
config_uri = args[0]
email_template = args[1]
env = bootstrap(config_uri)
settings, closer = env['registry'].settings, env['closer']
try:
request = env['request']
public_url_root = settings['public_url_root']
preferences_link = urlparse.urljoin(
public_url_root, request.route_path('user_preferences'))
tx = transaction.begin()
mailer = get_mailer(request)
for user in get_all_users_with_passwords_and_email():
message = send_email(request, email_template, user,
preferences_link)
mailer.send(message)
tx.commit()
finally:
closer()
def statistics():
result = setup_simple_command(
"statistics",
"Report several different statistics.",
)
if isinstance(result, int):
return result
else:
settings, closer, env, args = result
try:
# Get the number of users and passwords
n_users = Session.query(User).count()
if n_users == 0:
return
n_passwords = Session.query(Password).count()
# How many users are verified
n_verified = Session.query(User).filter(
User.email_verified == true()).count()
# How many users allow the analytics cookie
n_allow_cookie = Session.query(User).filter(
User.allow_google_analytics == true()).count()
# Identity providers
by_identity = Session.query(
ExternalIdentity.provider,
func.count(ExternalIdentity.provider).label('provider_count')
).select_from(
ExternalIdentity
).group_by(ExternalIdentity.provider).order_by(desc('provider_count'))
# Email providers
domains_with_counts = select([
func.substring(User.email, r'.*@(.*)').label('domain'),
func.count('*').label('count'),
]).where(User.email != '').group_by('domain').order_by(desc('count'))
aliased = domains_with_counts.alias()
by_email = Session.query(aliased).filter(aliased.c.count > 1)
without_email = Session.query(User).filter(User.email == '').count()
with_email = n_users - without_email
# Top ten users
most_active_users = Session.query(
User, func.count(User.id).label('password_count'),
).join(
Password
).group_by(User.id).order_by(desc('password_count'))
users_with_passwords = most_active_users.count()
most_active_users = most_active_users.limit(10)
# print the statistics
safe_print('Number of users: %d' % n_users)
safe_print('Number of passwords: %d' % n_passwords)
safe_print('Verified users: %.2f%% (%d)' % (
(100.0 * n_verified) / n_users, n_verified))
safe_print('Users that allow Google Analytics cookie: %.2f%% (%d)' % (
(100.0 * n_allow_cookie) / n_users, n_allow_cookie))
safe_print('Identity providers:')
for provider, amount in by_identity:
safe_print('\t%s: %.2f%% (%d)' % (
provider, (100.0 * amount) / n_users, amount))
safe_print('Email providers:')
others = with_email
for provider, amount in by_email:
safe_print('\t%s: %.2f%% (%d)' % (
provider, (100.0 * amount) / with_email, amount))
others -= amount
safe_print('\tOthers: %.2f%% (%d)' % (
(100.0 * others) / with_email, others))
safe_print('Users without email: %.2f%% (%d)' % (
(100.0 * without_email) / n_users, without_email))
safe_print('Most active users:')
for user, n_passwords in most_active_users:
safe_print('\t%s: %s' % (get_user_display_name(user), n_passwords))
users_no_passwords = n_users - users_with_passwords
safe_print('Users without passwords: %.2f%% (%d)' % (
(100 * users_no_passwords) / n_users, users_no_passwords))
finally:
closer()
def statistics():
result = setup_simple_command(
"statistics",
"Report several different statistics.",
)
if isinstance(result, int):
return result
else:
settings, closer, env, args = result
try:
db = settings['mongodb'].get_database()
# Get the number of users and passwords
n_users = db.users.count()
if n_users == 0:
return
n_passwords = db.passwords.count()
# How many users are verified
n_verified = db.users.find({'email_verified': True}).count()
# How many users allow the analytics cookie
n_allow_cookie = db.users.find({'allow_google_analytics': True}).count()
all_users = list(db.users.find())
# Identity providers
by_identity = group_by_identity_provider(all_users)
# Email providers
by_email, without_email = group_by_email_provider(all_users, 1)
with_email = n_users - without_email
# Top ten users
all_passwords = list(db.passwords.find())
most_active_users, users_with_passwords = users_with_most_passwords(
all_users, all_passwords, 10)
# print the statistics
safe_print('Number of users: %d' % n_users)
safe_print('Number of passwords: %d' % n_passwords)
safe_print('Verified users: %.2f%% (%d)' % (
(100.0 * n_verified) / n_users, n_verified))
safe_print('Users that allow Google Analytics cookie: %.2f%% (%d)' % (
(100.0 * n_allow_cookie) / n_users, n_allow_cookie))
safe_print('Identity providers:')
for provider, amount in by_identity:
safe_print('\t%s: %.2f%% (%d)' % (
provider, (100.0 * amount) / n_users, amount))
safe_print('Email providers:')
others = with_email
for provider, amount in by_email:
safe_print('\t%s: %.2f%% (%d)' % (
provider, (100.0 * amount) / with_email, amount))
others -= amount
safe_print('\tOthers: %.2f%% (%d)' % (
(100.0 * others) / with_email, others))
safe_print('Users without email: %.2f%% (%d)' % (
(100.0 * without_email) / n_users, without_email))
safe_print('Most active users:')
for user, n_passwords in most_active_users:
safe_print('\t%s: %s' % (get_user_display_name(user), n_passwords))
users_no_passwords = n_users - users_with_passwords
safe_print('Users without passwords: %.2f%% (%d)' % (
(100 * users_no_passwords) / n_users, users_no_passwords))
finally:
closer()
