def test_admin_scoped_token_can_create_and_send_email(self, mock_auth, mock_mail):
token = ApiOAuth2PersonalToken(
owner=self.user,
name='Admin Token',
scopes='osf.admin'
)
mock_cas_resp = CasResponse(
authenticated=True,
user=self.user._id,
attributes={
'accessToken': token.token_id,
'accessTokenScope': [s for s in token.scopes.split(' ')]
}
)
mock_auth.return_value = self.user, mock_cas_resp
assert_equal(User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 0)
res = self.app.post_json_api(
'{}?send_email=true'.format(self.base_url),
self.data,
headers={'Authorization': 'Bearer {}'.format(token.token_id)}
)
assert_equal(res.status_code, 201)
assert_equal(res.json['data']['attributes']['username'], self.unconfirmed_email)
assert_equal(User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 1)
assert_equal(mock_mail.call_count, 1)
def test_admin_scoped_token_can_create_and_send_email(
self, mock_auth, mock_mail):
token = ApiOAuth2PersonalToken(owner=self.user,
name='Admin Token',
scopes='osf.admin')
mock_cas_resp = CasResponse(authenticated=True,
user=self.user._id,
attributes={
'accessToken':
token.token_id,
'accessTokenScope':
[s for s in token.scopes.split(' ')]
})
mock_auth.return_value = self.user, mock_cas_resp
assert_equal(
User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 0)
res = self.app.post_json_api(
'{}?send_email=true'.format(self.base_url),
self.data,
headers={'Authorization': 'Bearer {}'.format(token.token_id)})
assert_equal(res.status_code, 201)
assert_equal(res.json['data']['attributes']['username'],
self.unconfirmed_email)
assert_equal(
User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 1)
assert_equal(mock_mail.call_count, 1)
def get_events(self, date):
super(UserSummary, self).get_events(date)
# Convert to a datetime at midnight for queries and the timestamp
timestamp_datetime = datetime(date.year, date.month,
date.day).replace(tzinfo=pytz.UTC)
query_datetime = timestamp_datetime + timedelta(1)
counts = {
'keen': {
'timestamp': timestamp_datetime.isoformat()
},
'status': {
'active':
User.find(
Q('is_registered', 'eq', True) & Q('password', 'ne', None)
& Q('merged_by', 'eq', None)
& Q('date_disabled', 'eq', None)
& Q('date_confirmed', 'ne', None)
& Q('date_confirmed', 'lt', query_datetime)).count(),
'unconfirmed':
User.find(
Q('date_registered', 'lt', query_datetime)
& Q('date_confirmed', 'eq', None)).count(),
'deactivated':
User.find(
Q('date_disabled', 'ne', None)
& Q('date_disabled', 'lt', query_datetime)).count()
}
}
logger.info(
'Users counted. Active: {}, Unconfirmed: {}, Deactivated: {}'.
format(counts['status']['active'], counts['status']['unconfirmed'],
counts['status']['deactivated']))
return [counts]
def test_properly_scoped_token_can_create_without_username_but_not_send_email(
self, mock_auth, mock_mail):
token = ApiOAuth2PersonalToken(owner=self.user,
name='Authorized Token',
scopes='osf.users.create')
mock_cas_resp = CasResponse(authenticated=True,
user=self.user._id,
attributes={
'accessToken':
token.token_id,
'accessTokenScope':
[s for s in token.scopes.split(' ')]
})
mock_auth.return_value = self.user, mock_cas_resp
self.data['data']['attributes'] = {'full_name': 'No Email'}
assert_equal(User.find(Q('fullname', 'eq', 'No Email')).count(), 0)
res = self.app.post_json_api(
'{}?send_email=true'.format(self.base_url),
self.data,
headers={'Authorization': 'Bearer {}'.format(token.token_id)})
assert_equal(res.status_code, 201)
assert_equal(res.json['data']['attributes']['username'], None)
assert_equal(User.find(Q('fullname', 'eq', 'No Email')).count(), 1)
assert_equal(mock_mail.call_count, 0)
def test_improperly_scoped_token_can_not_create_or_email(
self, mock_auth, mock_mail):
token = ApiOAuth2PersonalToken(owner=self.user,
name='Unauthorized Token',
scopes='osf.full_write')
mock_cas_resp = CasResponse(authenticated=True,
user=self.user._id,
attributes={
'accessToken':
token.token_id,
'accessTokenScope':
[s for s in token.scopes.split(' ')]
})
mock_auth.return_value = self.user, mock_cas_resp
assert_equal(
User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 0)
res = self.app.post_json_api(
'{}?send_email=true'.format(self.base_url),
self.data,
headers={'Authorization': 'Bearer {}'.format(token.token_id)},
expect_errors=True)
assert_equal(res.status_code, 403)
assert_equal(
User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 0)
assert_equal(mock_mail.call_count, 0)
def get_events(self, date):
super(UserSummary, self).get_events(date)
# Convert to a datetime at midnight for queries and the timestamp
timestamp_datetime = datetime(date.year, date.month,
date.day).replace(tzinfo=pytz.UTC)
query_datetime = timestamp_datetime + timedelta(1)
active_user_query = (Q('is_registered', 'eq', True)
& Q('password', 'ne', None)
& Q('merged_by', 'eq', None)
& Q('date_disabled', 'eq', None)
& Q('date_confirmed', 'ne', None)
& Q('date_confirmed', 'lt', query_datetime))
active_users = 0
depth_users = 0
profile_edited = 0
user_pages = paginated(User, query=active_user_query)
for user in user_pages:
active_users += 1
log_count = count_user_logs(user)
if log_count >= LOG_THRESHOLD:
depth_users += 1
if user.social or user.schools or user.jobs:
profile_edited += 1
counts = {
'keen': {
'timestamp': timestamp_datetime.isoformat()
},
'status': {
'active':
active_users,
'depth':
depth_users,
'unconfirmed':
User.find(
Q('date_registered', 'lt', query_datetime)
& Q('date_confirmed', 'eq', None)).count(),
'deactivated':
User.find(
Q('date_disabled', 'ne', None)
& Q('date_disabled', 'lt', query_datetime)).count(),
'merged':
User.find(
Q('date_registered', 'lt', query_datetime)
& Q('merged_by', 'ne', None)).count(),
'profile_edited':
profile_edited
}
}
logger.info(
'Users counted. Active: {}, Depth: {}, Unconfirmed: {}, Deactivated: {}, Merged: {}, Profile Edited: {}'
.format(counts['status']['active'], counts['status']['depth'],
counts['status']['unconfirmed'],
counts['status']['deactivated'],
counts['status']['merged'],
counts['status']['profile_edited']))
return [counts]
def test_properly_scoped_token_can_create_without_username_but_not_send_email(self, mock_auth, mock_mail):
token = ApiOAuth2PersonalToken(
owner=self.user,
name='Authorized Token',
scopes='osf.users.create'
)
mock_cas_resp = CasResponse(
authenticated=True,
user=self.user._id,
attributes={
'accessToken': token.token_id,
'accessTokenScope': [s for s in token.scopes.split(' ')]
}
)
mock_auth.return_value = self.user, mock_cas_resp
self.data['data']['attributes'] = {'full_name': 'No Email'}
assert_equal(User.find(Q('fullname', 'eq', 'No Email')).count(), 0)
res = self.app.post_json_api(
'{}?send_email=true'.format(self.base_url),
self.data,
headers={'Authorization': 'Bearer {}'.format(token.token_id)}
)
assert_equal(res.status_code, 201)
assert_equal(res.json['data']['attributes']['username'], None)
assert_equal(User.find(Q('fullname', 'eq', 'No Email')).count(), 1)
assert_equal(mock_mail.call_count, 0)
def test_improperly_scoped_token_can_not_create_or_email(self, mock_auth, mock_mail):
token = ApiOAuth2PersonalToken(
owner=self.user,
name='Unauthorized Token',
scopes='osf.full_write'
)
mock_cas_resp = CasResponse(
authenticated=True,
user=self.user._id,
attributes={
'accessToken': token.token_id,
'accessTokenScope': [s for s in token.scopes.split(' ')]
}
)
mock_auth.return_value = self.user, mock_cas_resp
assert_equal(User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 0)
res = self.app.post_json_api(
'{}?send_email=true'.format(self.base_url),
self.data,
headers={'Authorization': 'Bearer {}'.format(token.token_id)},
expect_errors=True
)
assert_equal(res.status_code, 403)
assert_equal(User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 0)
assert_equal(mock_mail.call_count, 0)
def test_logged_out_user_cannot_create_other_user_or_send_mail(self, mock_mail):
assert_equal(User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 0)
res = self.app.post_json_api(
'{}?send_email=true'.format(self.base_url),
self.data,
expect_errors=True
)
assert_equal(res.status_code, 401)
assert_equal(User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 0)
assert_equal(mock_mail.call_count, 0)
def test_logged_out_user_cannot_create_other_user_or_send_mail(self, mock_mail):
assert_equal(User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 0)
res = self.app.post_json_api(
'{}?send_email=true'.format(self.base_url),
self.data,
expect_errors=True
)
assert_equal(res.status_code, 401)
assert_equal(User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 0)
assert_equal(mock_mail.call_count, 0)
def test_cookied_requests_can_create_and_email(self, mock_mail):
session = Session(data={'auth_user_id': self.user._id})
session.save()
cookie = itsdangerous.Signer(settings.SECRET_KEY).sign(session._id)
self.app.set_cookie(settings.COOKIE_NAME, str(cookie))
assert_equal(User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 0)
res = self.app.post_json_api(
'{}?send_email=true'.format(self.base_url),
self.data
)
assert_equal(res.status_code, 201)
assert_equal(User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 1)
assert_equal(mock_mail.call_count, 1)
def test_cookied_requests_can_create_and_email(self, mock_mail):
session = Session(data={'auth_user_id': self.user._id})
session.save()
cookie = itsdangerous.Signer(settings.SECRET_KEY).sign(session._id)
self.app.set_cookie(settings.COOKIE_NAME, str(cookie))
assert_equal(
User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 0)
res = self.app.post_json_api(
'{}?send_email=true'.format(self.base_url), self.data)
assert_equal(res.status_code, 201)
assert_equal(
User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 1)
assert_equal(mock_mail.call_count, 1)
def redirect_to_twitter(twitter_handle):
"""Redirect GET requests for /@TwitterHandle/ to respective the OSF user
account if it associated with an active account
:param uid: uid for requested User
:return: Redirect to User's Twitter account page
"""
try:
user = User.find_one(Q('social.twitter', 'iexact', twitter_handle))
except NoResultsFound:
raise HTTPError(
http.NOT_FOUND,
data={
'message_short':
'User Not Found',
'message_long':
'There is no active user associated with the Twitter handle: {0}.'
.format(twitter_handle)
})
except MultipleResultsFound:
users = User.find(Q('social.twitter', 'iexact', twitter_handle))
message_long = 'There are multiple OSF accounts associated with the ' \
'Twitter handle: <strong>{0}</strong>. <br /> Please ' \
'select from the accounts below. <br /><ul>'.format(twitter_handle)
for user in users:
message_long += '<li><a href="{0}">{1}</a></li>'.format(
user.url, user.fullname)
message_long += '</ul>'
raise HTTPError(http.MULTIPLE_CHOICES,
data={
'message_short': 'Multiple Users Found',
'message_long': message_long
})
return redirect(user.url)
def get_events(self, date):
""" Get all node logs from a given date for a 24 hour period,
ending at the date given.
"""
super(UserDomainEvents, self).get_events(date)
# In the end, turn the date back into a datetime at midnight for queries
date = datetime(date.year, date.month, date.day).replace(tzinfo=pytz.UTC)
logger.info('Gathering user domains between {} and {}'.format(
date, (date + timedelta(1)).isoformat()
))
user_query = Q('date_confirmed', 'lt', date + timedelta(1)) & Q('date_confirmed', 'gte', date)
users = User.find(user_query)
user_domain_events = []
for user in users:
user_date = user.date_confirmed.replace(tzinfo=pytz.UTC)
event = {
'keen': {'timestamp': user_date.isoformat()},
'date': user_date.isoformat(),
'domain': user.username.split('@')[-1]
}
user_domain_events.append(event)
logger.info('User domains collected. {} users and their email domains.'.format(len(user_domain_events)))
return user_domain_events
def test_cookied_requests_do_not_create_or_email(self, mock_mail):
session = Session(data={'auth_user_id': self.user._id})
session.save()
cookie = itsdangerous.Signer(settings.SECRET_KEY).sign(session._id)
self.app.set_cookie(settings.COOKIE_NAME, str(cookie))
assert_equal(User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 0)
res = self.app.post_json_api(
self.base_url,
self.data,
expect_errors=True
)
assert_equal(res.status_code, 403)
assert_equal(User.find(Q('username', 'eq', self.unconfirmed_email)).count(), 0)
assert_equal(mock_mail.call_count, 0)
def redirect_to_twitter(twitter_handle):
"""Redirect GET requests for /@TwitterHandle/ to respective the OSF user
account if it associated with an active account
:param uid: uid for requested User
:return: Redirect to User's Twitter account page
"""
try:
user = User.find_one(Q('social.twitter', 'iexact', twitter_handle))
except NoResultsFound:
raise HTTPError(http.NOT_FOUND, data={
'message_short': 'User Not Found',
'message_long': 'There is no active user associated with the Twitter handle: {0}.'.format(twitter_handle)
})
except MultipleResultsFound:
users = User.find(Q('social.twitter', 'iexact', twitter_handle))
message_long = 'There are multiple OSF accounts associated with the ' \
'Twitter handle: <strong>{0}</strong>. <br /> Please ' \
'select from the accounts below. <br /><ul>'.format(twitter_handle)
for user in users:
message_long += '<li><a href="{0}">{1}</a></li>'.format(user.url, user.fullname)
message_long += '</ul>'
raise HTTPError(http.MULTIPLE_CHOICES, data={
'message_short': 'Multiple Users Found',
'message_long': message_long
})
return redirect(user.url)
def redirect_to_twitter(twitter_handle):
"""Redirect GET requests for /@TwitterHandle/ to respective the OSF user
account if it associated with an active account
:param uid: uid for requested User
:return: Redirect to User's Twitter account page
"""
try:
user = User.find_one(Q("social.twitter", "iexact", twitter_handle))
except NoResultsFound:
raise HTTPError(
http.NOT_FOUND,
data={
"message_short": "User Not Found",
"message_long": "There is no active user associated with the Twitter handle: {0}.".format(
twitter_handle
),
},
)
except MultipleResultsFound:
users = User.find(Q("social.twitter", "iexact", twitter_handle))
message_long = (
"There are multiple OSF accounts associated with the "
"Twitter handle: <strong>{0}</strong>. <br /> Please "
"select from the accounts below. <br /><ul>".format(markupsafe.escape(twitter_handle))
)
for user in users:
message_long += '<li><a href="{0}">{1}</a></li>'.format(user.url, markupsafe.escape(user.fullname))
message_long += "</ul>"
raise HTTPError(
http.MULTIPLE_CHOICES, data={"message_short": "Multiple Users Found", "message_long": message_long}
)
return redirect(user.url)
def get_active_users():
return User.find(
Q('is_registered', 'eq', True) &
Q('password', 'ne', None) &
Q('is_merged', 'ne', True) &
Q('date_confirmed', 'ne', None)
)
def main(dry=True):
count = 0
for user in User.find(Q('username', 'eq', None)):
logger.info('Setting username for {}'.format(user._id))
if not dry:
user.username = user._id
user.save()
count += 1
logger.info('Migrated {} users.'.format(count))
def get_active_users(extra=None):
query = (
Q('is_registered', 'eq', True) &
Q('password', 'ne', None) &
Q('is_merged', 'ne', True) &
Q('date_confirmed', 'ne', None)
)
query = query & extra if extra else query
return User.find(query)
def get_active_users(extra=None):
query = (
Q('is_registered', 'eq', True) &
Q('password', 'ne', None) &
Q('is_merged', 'ne', True) &
Q('date_confirmed', 'ne', None)
)
query = query & extra if extra else query
return User.find(query)
def do_migration():
users = User.find(Q('email_verifications', 'eq', None))
migrated = 0
for user in users:
logger.info('Setting email_verifications for user {} to {{}}'.format(user._id))
user.email_verifications = {}
user.save()
migrated += 1
logger.info('Migrated {} users'.format(migrated))
def main(dry=True):
count = 0
for user in User.find(Q('username', 'eq', None)):
logger.info('Setting username for {}'.format(user._id))
if not dry:
user.username = user._id
user.save()
count += 1
logger.info('Migrated {} users.'.format(count))
def test_integration(self, mock_upload, mock_send_mail):
fullname = 'John Deacon'
username = '******'
title = 'good songs'
conference = ConferenceFactory()
body = 'dragon on my back'
content = 'dragon attack'
recipient = '{0}{1}[email protected]'.format(
'test-' if settings.DEV_MODE else '',
conference.endpoint,
)
self.app.post(
api_url_for('meeting_hook'),
{
'X-Mailgun-Sscore':
0,
'timestamp':
'123',
'token':
'secret',
'signature':
hmac.new(
key=settings.MAILGUN_API_KEY,
msg='{}{}'.format('123', 'secret'),
digestmod=hashlib.sha256,
).hexdigest(),
'attachment-count':
'1',
'X-Mailgun-Sscore':
0,
'from':
'{0} <{1}>'.format(fullname, username),
'recipient':
recipient,
'subject':
title,
'stripped-text':
body,
},
upload_files=[
('attachment-1', 'attachment-1', content),
],
)
assert_true(mock_upload.called)
users = User.find(Q('username', 'eq', username))
assert_equal(users.count(), 1)
nodes = Node.find(Q('title', 'eq', title))
assert_equal(nodes.count(), 1)
node = nodes[0]
assert_equal(node.get_wiki_page('home').content, body)
assert_true(mock_send_mail.called)
call_args, call_kwargs = mock_send_mail.call_args
assert_absolute(call_kwargs['conf_view_url'])
assert_absolute(call_kwargs['set_password_url'])
assert_absolute(call_kwargs['profile_url'])
assert_absolute(call_kwargs['file_url'])
assert_absolute(call_kwargs['node_url'])
def do_migration():
users = User.find(Q('email_verifications', 'eq', None))
migrated = 0
for user in users:
logger.info('Setting email_verifications for user {} to {{}}'.format(
user._id))
user.email_verifications = {}
user.save()
migrated += 1
logger.info('Migrated {} users'.format(migrated))
def main(dry=True):
init_app(set_backends=True, routes=False)
count = 0
for user in User.find(Q("is_claimed", "eq", None)):
is_claimed = bool(user.date_confirmed)
logger.info("User {}: setting is_claimed to {}".format(user._id, is_claimed))
user.is_claimed = is_claimed
count += 1
if not dry:
user.save()
logger.info("Migrated {} users.".format(count))
def log_duplicate_acount(dry):
duplicate_emails = get_duplicate_email()
count = 0
if duplicate_emails:
for email in duplicate_emails:
users = User.find(Q('emails', 'eq', email) & Q('merged_by', 'eq', None) & Q('username', 'ne', None))
for user in users:
count += 1
logger.info("User {}, username {}, id {}, email {} is a duplicate"
.format(user.fullname, user.username, user._id, user.emails))
logger.info("Found {} users with duplicate emails".format(count))
def main(dry=True):
init_app(set_backends=True, routes=False)
count = 0
for user in User.find(Q('is_claimed', 'eq', None)):
is_claimed = bool(user.date_confirmed)
logger.info('User {}: setting is_claimed to {}'.format(
user._id, is_claimed))
user.is_claimed = is_claimed
count += 1
if not dry:
user.save()
logger.info('Migrated {} users.'.format(count))
def main(dry=True):
duplicates = database.user.aggregate([{
"$group": {
"_id": "$username",
"ids": {
"$addToSet": "$_id"
},
"count": {
"$sum": 1
}
}
}, {
"$match": {
"count": {
"$gt": 1
},
"_id": {
"$ne": None
}
}
}, {
"$sort": {
"count": -1
}
}]).get('result')
# [
# {
# 'count': 5,
# '_id': '*****@*****.**',
# 'ids': [
# 'listo','fidst','hatma','tchth','euser','name!'
# ]
# }
# ]
logger.info('Found {} duplicate usernames.'.format(len(duplicates)))
for duplicate in duplicates:
logger.info('Found {} copies of {}: {}'.format(
len(duplicate.get('ids')), duplicate.get('_id'),
', '.join(duplicate['ids'])))
users = list(User.find(Q('_id', 'in', duplicate.get('ids'))))
primary, secondaries = find_primary_and_secondaries(users)
for secondary in secondaries:
logger.info('Merging user {} into user {}'.format(
secondary._id, primary._id))
# don't just rely on the toku txn and prevent a call to merge_user
# when doing a dry run because merge_user does more than just
# db updateds (mailchimp calls, elasticsearch, etc.)
if not dry:
with TokuTransaction():
primary.merge_user(secondary)
primary.save()
secondary.save()
logger.info('Finished migrating {} usernames'.format(len(duplicates)))
def get_enabled_authorized_linked(user_settings_list, has_external_account,
short_name):
""" Gather the number of users who have at least one node in each of the stages for an addon
:param user_settings_list: list of user_settings for a particualr addon
:param has_external_account: where addon is derrived from, determines method to load node settings
:param short_name: short name of addon to get correct node_settings
:return: dict with number of users that have at least one project at each stage
"""
from addons.forward.models import NodeSettings as ForwardNodeSettings
num_enabled = 0 # of users w/ 1+ addon account connected
num_authorized = 0 # of users w/ 1+ addon account connected to 1+ node
num_linked = 0 # of users w/ 1+ addon account connected to 1+ node and configured
# osfstorage and wiki don't have user_settings, so always assume they're enabled, authorized, linked
if short_name == 'osfstorage' or short_name == 'wiki':
num_enabled = num_authorized = num_linked = User.find(
Q('is_registered', 'eq', True) & Q('password', 'ne', None)
& Q('merged_by', 'eq', None) & Q('date_disabled', 'eq', None)
& Q('date_confirmed', 'ne', None)).count()
elif short_name == 'forward':
num_enabled = num_authorized = ForwardNodeSettings.find().count()
num_linked = ForwardNodeSettings.find(Q('url', 'ne', None)).count()
else:
for user_settings in paginated(user_settings_list):
node_settings_list = []
if has_external_account:
if user_settings.has_auth:
num_enabled += 1
node_settings_list = [
Node.load(guid).get_addon(short_name)
for guid in user_settings.oauth_grants.keys()
]
else:
num_enabled += 1
node_settings_list = [
Node.load(guid).get_addon(short_name)
for guid in user_settings.nodes_authorized
]
if any([ns.has_auth for ns in node_settings_list if ns]):
num_authorized += 1
if any([(ns.complete and ns.configured)
for ns in node_settings_list if ns]):
num_linked += 1
return {
'enabled': num_enabled,
'authorized': num_authorized,
'linked': num_linked
}
def main(dry=True):
duplicates = database.user.aggregate([
{
"$group": {
"_id": "$username",
"ids": {"$addToSet": "$_id"},
"count": {"$sum": 1}
}
},
{
"$match": {
"count": {"$gt": 1},
"_id": {"$ne": None}
}
},
{
"$sort": {
"count": -1
}
}
]).get('result')
# [
# {
# 'count': 5,
# '_id': '*****@*****.**',
# 'ids': [
# 'listo','fidst','hatma','tchth','euser','name!'
# ]
# }
# ]
logger.info('Found {} duplicate usernames.'.format(len(duplicates)))
for duplicate in duplicates:
logger.info(
'Found {} copies of {}: {}'.format(
len(duplicate.get('ids')),
duplicate.get('_id'),
', '.join(duplicate['ids'])
)
)
users = list(User.find(Q('_id', 'in', duplicate.get('ids'))))
primary, secondaries = find_primary_and_secondaries(users)
for secondary in secondaries:
logger.info('Merging user {} into user {}'.format(secondary._id, primary._id))
# don't just rely on the toku txn and prevent a call to merge_user
# when doing a dry run because merge_user does more than just
# db updateds (mailchimp calls, elasticsearch, etc.)
if not dry:
with TokuTransaction():
primary.merge_user(secondary)
primary.save()
secondary.save()
logger.info('Finished migrating {} usernames'.format(len(duplicates)))
def get_events(self, date):
super(UserSummary, self).get_events(date)
# Convert to a datetime at midnight for queries and the timestamp
timestamp_datetime = datetime(date.year, date.month, date.day).replace(tzinfo=pytz.UTC)
query_datetime = timestamp_datetime + timedelta(1)
counts = {
'keen': {
'timestamp': timestamp_datetime.isoformat()
},
'status': {
'active': User.find(
Q('is_registered', 'eq', True) &
Q('password', 'ne', None) &
Q('merged_by', 'eq', None) &
Q('date_disabled', 'eq', None) &
Q('date_confirmed', 'ne', None) &
Q('date_confirmed', 'lt', query_datetime)
).count(),
'unconfirmed': User.find(
Q('date_registered', 'lt', query_datetime) &
Q('date_confirmed', 'eq', None)
).count(),
'deactivated': User.find(
Q('date_disabled', 'ne', None) &
Q('date_disabled', 'lt', query_datetime)
).count()
}
}
logger.info(
'Users counted. Active: {}, Unconfirmed: {}, Deactivated: {}'.format(
counts['status']['active'],
counts['status']['unconfirmed'],
counts['status']['deactivated']
)
)
return [counts]
def test_do_migration(self):
today = timezone.now()
user1 = UserFactory.build(date_confirmed=None, date_last_login=today, is_registered=False)
user2 = UserFactory.build(date_confirmed=None, date_last_login=today, is_registered=True)
user1.save()
user2.save()
user_list = User.find(Q('_id', 'eq', user1._id) | Q('_id', 'eq', user2._id))
do_migration(user_list)
assert user1.date_confirmed is today
assert user1.is_registered
assert user2.date_confirmed is today
assert user2.is_registered
def test_creates_user(self):
username = '******'
assert_equal(User.find(Q('username', 'eq', username)).count(), 0)
with capture_signals() as mock_signals:
res = self.app.post(self.url, self.build_payload(username))
assert_equal(res.status_code, 204)
assert_equal(mock_signals.signals_sent(), set([signals.user_confirmed]))
user = User.find_one(Q('username', 'eq', username))
assert_true(user)
assert_in(self.institution, user.affiliated_institutions)
def log_duplicate_acount(dry):
duplicate_emails = get_duplicate_email()
count = 0
if duplicate_emails:
for email in duplicate_emails:
users = User.find(
Q('emails', 'eq', email) & Q('merged_by', 'eq', None)
& Q('username', 'ne', None))
for user in users:
count += 1
logger.info(
"User {}, username {}, id {}, email {} is a duplicate".
format(user.fullname, user.username, user._id,
user.emails))
logger.info("Found {} users with duplicate emails".format(count))
def test_integration(self, mock_upload, mock_send_mail):
fullname = 'John Deacon'
username = '******'
title = 'good songs'
conference = ConferenceFactory()
body = 'dragon on my back'
content = 'dragon attack'
recipient = '{0}{1}[email protected]'.format(
'test-' if settings.DEV_MODE else '',
conference.endpoint,
)
self.app.post(
api_url_for('meeting_hook'),
{
'X-Mailgun-Sscore': 0,
'timestamp': '123',
'token': 'secret',
'signature': hmac.new(
key=settings.MAILGUN_API_KEY,
msg='{}{}'.format('123', 'secret'),
digestmod=hashlib.sha256,
).hexdigest(),
'attachment-count': '1',
'X-Mailgun-Sscore': 0,
'from': '{0} <{1}>'.format(fullname, username),
'recipient': recipient,
'subject': title,
'stripped-text': body,
},
upload_files=[
('attachment-1', 'attachment-1', content),
],
)
assert_true(mock_upload.called)
users = User.find(Q('username', 'eq', username))
assert_equal(users.count(), 1)
nodes = Node.find(Q('title', 'eq', title))
assert_equal(nodes.count(), 1)
node = nodes[0]
assert_equal(node.get_wiki_page('home').content, body)
assert_true(mock_send_mail.called)
call_args, call_kwargs = mock_send_mail.call_args
assert_absolute(call_kwargs['conf_view_url'])
assert_absolute(call_kwargs['set_password_url'])
assert_absolute(call_kwargs['profile_url'])
assert_absolute(call_kwargs['file_url'])
assert_absolute(call_kwargs['node_url'])
def get_or_create_user(fullname, address, is_spam):
"""Get or create user by email address.
"""
user = User.find(Q('username', 'iexact', address))
user = user[0] if user.count() else None
user_created = False
if user is None:
password = str(uuid.uuid4())
user = User.create_confirmed(address, password, fullname)
user.verification_key = security.random_string(20)
# Flag as potential spam account if Mailgun detected spam
if is_spam:
user.system_tags.append('is_spam')
user.save()
user_created = True
return user, user_created
def get_or_create_user(fullname, address, is_spam):
"""Get or create user by email address.
"""
user = User.find(Q('username', 'iexact', address))
user = user[0] if user.count() else None
user_created = False
if user is None:
password = str(uuid.uuid4())
user = User.create_confirmed(address, password, fullname)
user.verification_key = security.random_string(20)
# Flag as potential spam account if Mailgun detected spam
if is_spam:
user.system_tags.append('is_spam')
user.save()
user_created = True
return user, user_created
def test_do_migration(self):
today = dt.datetime.utcnow()
user1 = UserFactory.build(date_confirmed=None,
date_last_login=today,
is_registered=False)
user2 = UserFactory.build(date_confirmed=None,
date_last_login=today,
is_registered=True)
user1.save()
user2.save()
user_list = User.find(
Q('_id', 'eq', user1._id) | Q('_id', 'eq', user2._id))
do_migration(user_list)
assert user1.date_confirmed is today
assert user1.is_registered
assert user2.date_confirmed is today
assert user2.is_registered
def get_enabled_authorized_linked(user_settings_list, has_external_account, short_name):
""" Gather the number of users who have at least one node in each of the stages for an addon
:param user_settings_list: list of user_settings for a particualr addon
:param has_external_account: where addon is derrived from, determines method to load node settings
:param short_name: short name of addon to get correct node_settings
:return: dict with number of users that have at least one project at each stage
"""
num_enabled = 0 # of users w/ 1+ addon account connected
num_authorized = 0 # of users w/ 1+ addon account connected to 1+ node
num_linked = 0 # of users w/ 1+ addon account connected to 1+ node and configured
# osfstorage and wiki don't have user_settings, so always assume they're enabled, authorized, linked
if short_name == 'osfstorage' or short_name == 'wiki':
active = User.find(
Q('is_registered', 'eq', True) &
Q('password', 'ne', None) &
Q('merged_by', 'eq', None) &
Q('date_disabled', 'eq', None) &
Q('date_confirmed', 'ne', None)
).count()
for user in xrange(active):
num_enabled += 1
num_authorized += 1
num_linked += 1
for user_settings in user_settings_list:
if has_external_account:
if user_settings.has_auth:
num_enabled += 1
node_settings_list = [Node.load(guid).get_addon(short_name) for guid in user_settings.oauth_grants.keys()]
else:
num_enabled += 1
node_settings_list = [Node.load(guid).get_addon(short_name) for guid in user_settings.nodes_authorized]
if any([ns.has_auth for ns in node_settings_list if ns]):
num_authorized += 1
if any([(ns.complete and ns.configured) for ns in node_settings_list if ns]):
num_linked += 1
return {
'enabled': num_enabled,
'authorized': num_authorized,
'linked': num_linked
}
def find_by_name(name):
try:
parts = re.split(r'\s+', name.strip())
except:
return None
if len(parts) < 2:
return None
users = User.find(
reduce(
lambda acc, value: acc & value,
[
Q('fullname', 'icontains', part.decode('utf-8', 'ignore'))
for part in parts
]
)
).sort('-date_created')
if not users:
return None
if len(users) > 1:
logger.warn('Multiple users found for name {}'.format(name))
return users[0]
def get_queryset(self):
# TODO: sort
query = self.get_query_from_request()
return User.find(query)
def main():
number_users = User.find().count()
projects = get_projects()
projects_forked = get_projects_forked()
projects_registered = get_projects_registered()
number_projects = projects.count()
projects_public = get_projects_public()
number_projects_public = projects_public.count()
number_projects_forked = projects_forked.count()
number_projects_registered = projects_registered.count()
number_downloads_unique, number_downloads_total = get_number_downloads_unique_and_total(projects=projects)
active_users = get_active_users()
active_users_invited = get_active_users(Q('is_invited', 'eq', True))
dropbox_metrics = get_dropbox_metrics()
extended_profile_counts = profile.get_profile_counts()
private_links = get_private_links()
folders = get_folders()
downloads_unique, downloads_total = count_file_downloads()
node_counts = count_user_nodes(active_users)
nodes_at_least_1 = count_at_least(node_counts, 1)
nodes_at_least_3 = count_at_least(node_counts, 3)
rows = [
['number_users', number_users],
['number_projects', number_projects],
['number_projects_public', number_projects_public],
['number_projects_forked', number_projects_forked],
['number_projects_registered', number_projects_registered],
['number_downloads_total', number_downloads_total],
['number_downloads_unique', number_downloads_unique],
['active-users', active_users.count()],
['active-users-invited', active_users_invited.count()],
['dropbox-users-enabled', dropbox_metrics['enabled']],
['dropbox-users-authorized', dropbox_metrics['authorized']],
['dropbox-users-linked', dropbox_metrics['linked']],
['profile-edits', extended_profile_counts['any']],
['view-only-links', private_links.count()],
['folders', folders.count()],
['downloads-unique', downloads_unique],
['downloads-total', downloads_total],
['nodes-gte-1', nodes_at_least_1],
['nodes-gte-3', nodes_at_least_3],
]
rows.extend(get_log_counts(active_users))
table = tabulate.tabulate(
rows,
headers=['label', 'value'],
)
with open(os.path.join(settings.ANALYTICS_PATH, 'main.txt'), 'w') as fp:
fp.write(table)
tabulate_emails.main()
tabulate_logs.main()
def get_events(self, date):
super(UserSummary, self).get_events(date)
# Convert to a datetime at midnight for queries and the timestamp
timestamp_datetime = datetime(date.year, date.month, date.day).replace(tzinfo=pytz.UTC)
query_datetime = timestamp_datetime + timedelta(1)
active_user_query = (
Q('is_registered', 'eq', True) &
Q('password', 'ne', None) &
Q('merged_by', 'eq', None) &
Q('date_disabled', 'eq', None) &
Q('date_confirmed', 'ne', None) &
Q('date_confirmed', 'lt', query_datetime)
)
active_users = 0
depth_users = 0
profile_edited = 0
user_pages = paginated(User, query=active_user_query)
for user in user_pages:
active_users += 1
log_count = count_user_logs(user)
if log_count >= LOG_THRESHOLD:
depth_users += 1
if user.social or user.schools or user.jobs:
profile_edited += 1
counts = {
'keen': {
'timestamp': timestamp_datetime.isoformat()
},
'status': {
'active': active_users,
'depth': depth_users,
'unconfirmed': User.find(
Q('date_registered', 'lt', query_datetime) &
Q('date_confirmed', 'eq', None)
).count(),
'deactivated': User.find(
Q('date_disabled', 'ne', None) &
Q('date_disabled', 'lt', query_datetime)
).count(),
'merged': User.find(
Q('date_registered', 'lt', query_datetime) &
Q('merged_by', 'ne', None)
).count(),
'profile_edited': profile_edited
}
}
logger.info(
'Users counted. Active: {}, Depth: {}, Unconfirmed: {}, Deactivated: {}, Merged: {}, Profile Edited: {}'.format(
counts['status']['active'],
counts['status']['depth'],
counts['status']['unconfirmed'],
counts['status']['deactivated'],
counts['status']['merged'],
counts['status']['profile_edited']
)
)
return [counts]
from website.app import init_app
from website.models import Node, User
from framework import Q
from framework.analytics import piwik
app = init_app('website.settings', set_backends=True)
# NOTE: This is a naive implementation for migration, requiring a POST request
# for every user and every node. It is possible to bundle these together in a
# single request, but it would require duplication of logic and strict error
# checking of the result. Doing it this way is idempotent, and allows any
# exceptions raised to halt the process with a usable error message.
for user in User.find():
if user.piwik_token:
continue
piwik.create_user(user)
for node in Node.find(
Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False)):
if node.piwik_site_id:
continue
piwik._provision_node(node._id)
def get_targets():
return User.find(Q('merged_by', 'ne', None) & Q('username', 'ne', None))
def get_targets():
return User.find(Q('date_confirmed', 'eq', None) & Q('date_last_login', 'ne', None))
from website.app import init_app
from website.models import Node, User
from framework import Q
from framework.analytics import piwik
app = init_app('website.settings', set_backends=True)
# NOTE: This is a naive implementation for migration, requiring a POST request
# for every user and every node. It is possible to bundle these together in a
# single request, but it would require duplication of logic and strict error
# checking of the result. Doing it this way is idempotent, and allows any
# exceptions raised to halt the process with a usable error message.
for user in User.find():
if user.piwik_token:
continue
piwik.create_user(user)
for node in Node.find(Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False)):
if node.piwik_site_id:
continue
piwik._provision_node(node._id)
def get_count_by_institutions():
institutions = get_institutions()
counts = []
for institution in institutions:
user_query = Q('_affiliated_institutions', 'eq', institution.node)
node_query = (Q('is_deleted', 'ne', True) & Q('is_folder', 'ne', True))
registration_query = node_query & Q('is_registration', 'eq', True)
non_registration_query = node_query & Q('is_registration', 'eq', False)
project_query = non_registration_query & Q('parent_node', 'eq', None)
registered_project_query = registration_query & Q(
'parent_node', 'eq', None)
public_query = Q('is_public', 'eq', True)
private_query = Q('is_public', 'eq', False)
node_public_query = non_registration_query & public_query
node_private_query = non_registration_query & private_query
project_public_query = project_query & public_query
project_private_query = project_query & private_query
registered_node_public_query = registration_query & public_query
registered_node_private_query = registration_query & private_query
registered_project_public_query = registered_project_query & public_query
registered_project_private_query = registered_project_query & private_query
count = {
'institution': {
'id': institution._id,
'name': institution.name,
},
'users': {
'total': User.find(user_query).count(),
},
'nodes': {
'total':
Node.find_by_institutions(institution, node_query).count(),
'public':
Node.find_by_institutions(institution,
node_public_query).count(),
'private':
Node.find_by_institutions(institution,
node_private_query).count(),
},
'projects': {
'total':
Node.find_by_institutions(institution, project_query).count(),
'public':
Node.find_by_institutions(institution,
project_public_query).count(),
'private':
Node.find_by_institutions(institution,
project_private_query).count(),
},
'registered_nodes': {
'total':
Node.find_by_institutions(institution,
registration_query).count(),
'public':
Node.find_by_institutions(
institution, registered_node_public_query).count(),
'embargoed':
Node.find_by_institutions(
institution, registered_node_private_query).count(),
},
'registered_projects': {
'total':
Node.find_by_institutions(institution,
registered_project_query).count(),
'public':
Node.find_by_institutions(
institution, registered_project_public_query).count(),
'embargoed':
Node.find_by_institutions(
institution, registered_project_private_query).count(),
},
}
counts.append(count)
keen_payload = {'institution_analytics': counts}
return keen_payload
def get_targets():
return User.find(
Q('date_confirmed', 'eq', None) & Q('date_last_login', 'ne', None))
def test_get_targets(self):
test = User.find(
Q('date_confirmed', 'ne', None) & Q('date_last_login', 'ne', None))
assert test is not None
def get_events(self, date):
super(InstitutionSummary, self).get_events(date)
institutions = self.get_institutions()
counts = []
# Convert to a datetime at midnight for queries and the timestamp
timestamp_datetime = datetime(date.year, date.month,
date.day).replace(tzinfo=pytz.UTC)
query_datetime = timestamp_datetime + timedelta(1)
for institution in institutions:
user_query = Q('_affiliated_institutions', 'eq', institution.node)
node_query = (Q('is_deleted', 'ne', True)
& Q('is_folder', 'ne', True)
& Q('date_created', 'lt', query_datetime))
registration_query = node_query & Q('is_registration', 'eq', True)
non_registration_query = node_query & Q('is_registration', 'eq',
False)
project_query = non_registration_query & Q('parent_node', 'eq',
None)
registered_project_query = registration_query & Q(
'parent_node', 'eq', None)
public_query = Q('is_public', 'eq', True)
private_query = Q('is_public', 'eq', False)
node_public_query = non_registration_query & public_query
node_private_query = non_registration_query & private_query
project_public_query = project_query & public_query
project_private_query = project_query & private_query
registered_node_public_query = registration_query & public_query
registered_node_private_query = registration_query & private_query
registered_project_public_query = registered_project_query & public_query
registered_project_private_query = registered_project_query & private_query
count = {
'institution': {
'id': institution._id,
'name': institution.name,
},
'users': {
'total': User.find(user_query).count(),
},
'nodes': {
'total':
Node.find_by_institutions(institution, node_query).count(),
'public':
Node.find_by_institutions(institution,
node_public_query).count(),
'private':
Node.find_by_institutions(institution,
node_private_query).count(),
},
'projects': {
'total':
Node.find_by_institutions(institution,
project_query).count(),
'public':
Node.find_by_institutions(institution,
project_public_query).count(),
'private':
Node.find_by_institutions(institution,
project_private_query).count(),
},
'registered_nodes': {
'total':
Node.find_by_institutions(institution,
registration_query).count(),
'public':
Node.find_by_institutions(
institution, registered_node_public_query).count(),
'embargoed':
Node.find_by_institutions(
institution, registered_node_private_query).count(),
},
'registered_projects': {
'total':
Node.find_by_institutions(
institution, registered_project_query).count(),
'public':
Node.find_by_institutions(
institution, registered_project_public_query).count(),
'embargoed':
Node.find_by_institutions(
institution, registered_project_private_query).count(),
},
'keen': {
'timestamp': timestamp_datetime.isoformat()
}
}
logger.info(
'{} Nodes counted. Nodes: {}, Projects: {}, Registered Nodes: {}, Registered Projects: {}'
.format(count['institution']['name'], count['nodes']['total'],
count['projects']['total'],
count['registered_nodes']['total'],
count['registered_projects']['total']))
counts.append(count)
return counts
def main():
number_users = User.find().count()
projects = get_projects()
projects_forked = get_projects_forked()
projects_registered = get_projects_registered()
number_projects = len(projects)
projects_public = get_projects_public()
number_projects_public = projects_public.count()
number_projects_forked = len(projects_forked)
number_projects_registered = len(projects_registered)
number_downloads_unique, number_downloads_total = get_number_downloads_unique_and_total(
)
active_users = get_active_users()
active_users_invited = get_active_users(Q('is_invited', 'eq', True))
dropbox_metrics = get_dropbox_metrics()
extended_profile_counts = profile.get_profile_counts()
private_links = get_private_links()
folders = get_folders()
downloads_unique, downloads_total = count_file_downloads()
node_counts = count_user_nodes(active_users)
nodes_at_least_1 = count_at_least(node_counts, 1)
nodes_at_least_3 = count_at_least(node_counts, 3)
rows = [
['number_users', number_users],
['number_projects', number_projects],
['number_projects_public', number_projects_public],
['number_projects_forked', number_projects_forked],
['number_projects_registered', number_projects_registered],
['number_downloads_total', number_downloads_total],
['number_downloads_unique', number_downloads_unique],
['active-users', active_users.count()],
['active-users-invited',
active_users_invited.count()],
['dropbox-users-enabled', dropbox_metrics['enabled']],
['dropbox-users-authorized', dropbox_metrics['authorized']],
['dropbox-users-linked', dropbox_metrics['linked']],
['profile-edits', extended_profile_counts['any']],
['view-only-links', private_links.count()],
['folders', folders.count()],
['downloads-unique', downloads_unique],
['downloads-total', downloads_total],
['nodes-gte-1', nodes_at_least_1],
['nodes-gte-3', nodes_at_least_3],
]
rows.extend(get_log_counts(active_users))
table = tabulate.tabulate(
rows,
headers=['label', 'value'],
)
with open(os.path.join(settings.ANALYTICS_PATH, 'main.txt'), 'w') as fp:
fp.write(table)
tabulate_emails.main()
tabulate_logs.main()
def get_events(self, date):
super(InstitutionSummary, self).get_events(date)
institutions = self.get_institutions()
counts = []
# Convert to a datetime at midnight for queries and the timestamp
timestamp_datetime = datetime(date.year, date.month, date.day).replace(tzinfo=pytz.UTC)
query_datetime = timestamp_datetime + timedelta(1)
for institution in institutions:
user_query = Q('_affiliated_institutions', 'eq', institution.node)
node_query = (
Q('is_deleted', 'ne', True) &
Q('is_folder', 'ne', True) &
Q('date_created', 'lt', query_datetime)
)
registration_query = node_query & Q('is_registration', 'eq', True)
non_registration_query = node_query & Q('is_registration', 'eq', False)
project_query = non_registration_query & Q('parent_node', 'eq', None)
registered_project_query = registration_query & Q('parent_node', 'eq', None)
public_query = Q('is_public', 'eq', True)
private_query = Q('is_public', 'eq', False)
node_public_query = non_registration_query & public_query
node_private_query = non_registration_query & private_query
project_public_query = project_query & public_query
project_private_query = project_query & private_query
registered_node_public_query = registration_query & public_query
registered_node_private_query = registration_query & private_query
registered_project_public_query = registered_project_query & public_query
registered_project_private_query = registered_project_query & private_query
count = {
'institution':{
'id': institution._id,
'name': institution.name,
},
'users': {
'total': User.find(user_query).count(),
},
'nodes': {
'total':Node.find_by_institutions(institution, node_query).count(),
'public': Node.find_by_institutions(institution, node_public_query).count(),
'private': Node.find_by_institutions(institution, node_private_query).count(),
},
'projects': {
'total': Node.find_by_institutions(institution, project_query).count(),
'public': Node.find_by_institutions(institution, project_public_query).count(),
'private': Node.find_by_institutions(institution, project_private_query).count(),
},
'registered_nodes': {
'total': Node.find_by_institutions(institution, registration_query).count(),
'public': Node.find_by_institutions(institution, registered_node_public_query).count(),
'embargoed': Node.find_by_institutions(institution, registered_node_private_query).count(),
},
'registered_projects': {
'total': Node.find_by_institutions(institution, registered_project_query).count(),
'public': Node.find_by_institutions(institution, registered_project_public_query).count(),
'embargoed': Node.find_by_institutions(institution, registered_project_private_query).count(),
},
'keen': {
'timestamp': timestamp_datetime.isoformat()
}
}
logger.info(
'{} Nodes counted. Nodes: {}, Projects: {}, Registered Nodes: {}, Registered Projects: {}'.format(
count['institution']['name'],
count['nodes']['total'],
count['projects']['total'],
count['registered_nodes']['total'],
count['registered_projects']['total']
)
)
counts.append(count)
return counts
def get_targets():
return User.find(Q('merged_by', 'ne', None) & Q('username', 'ne', None))
def get_active_users():
return User.find(
Q('is_registered', 'eq', True) & Q('password', 'ne', None)
& Q('is_merged', 'ne', True) & Q('date_confirmed', 'ne', None))
