def setUp(self):
super(TestNodeLogAnalytics, self).setUp()
self.user_one = UserFactory()
self.user_two = UserFactory()
# Two node logs for user one
self.node_log_node_created = NodeLogFactory(action='node_created',
user=self.user_one)
self.node_log_file_added = NodeLogFactory(action='file_added',
user=self.user_one)
# Two node logs for user two
self.node_log_wiki_updated = NodeLogFactory(action='wiki_updated',
user=self.user_two)
self.node_log_project_created = NodeLogFactory(
action='project_created', user=self.user_two)
self.end_date = datetime.datetime.utcnow() - datetime.timedelta(1)
for node_log in NodeLog.find():
node_log.date = self.end_date - datetime.timedelta(0.1)
node_log.save()
self.results = NodeLogEvents().get_events(self.end_date.date())
self.node_log_node_created.reload()
self.node_log_file_added.reload()
self.node_log_wiki_updated.reload()
self.node_log_project_created.reload()
def setUp(self):
super(TestNodeLogAnalytics, self).setUp()
self.user_one = UserFactory()
self.user_two = UserFactory()
# Two node logs for user one
self.node_log_node_created = NodeLogFactory(action='node_created', user=self.user_one)
self.node_log_file_added = NodeLogFactory(action='file_added', user=self.user_one)
# Two node logs for user two
self.node_log_wiki_updated = NodeLogFactory(action='wiki_updated', user=self.user_two)
self.node_log_project_created = NodeLogFactory(action='project_created', user=self.user_two)
self.end_date = datetime.datetime.utcnow() - datetime.timedelta(1)
for node_log in NodeLog.find():
node_log.date = self.end_date - datetime.timedelta(0.1)
node_log.save()
self.results = NodeLogEvents().get_events(self.end_date.date())
self.node_log_node_created.reload()
self.node_log_file_added.reload()
self.node_log_wiki_updated.reload()
self.node_log_project_created.reload()
def get_events(self, date):
""" Get all node logs from a given date for a 24 hour period,
ending at the date given.
"""
super(NodeLogEvents, self).get_events(date)
# In the end, turn the date back into a datetime at midnight for queries
date = datetime(date.year, date.month, date.day).replace(tzinfo=pytz.UTC)
logger.info("Gathering node logs between {} and {}".format(date, (date + timedelta(1)).isoformat()))
node_log_query = Q("date", "lt", date + timedelta(1)) & Q("date", "gte", date)
node_logs = NodeLog.find(node_log_query)
node_log_events = []
for node_log in node_logs:
log_date = node_log.date.replace(tzinfo=pytz.UTC)
event = {
"keen": {"timestamp": log_date.isoformat()},
"date": log_date.isoformat(),
"action": node_log.action,
}
if node_log.user:
event.update({"user_id": node_log.user._id})
node_log_events.append(event)
logger.info("NodeLogs counted. {} NodeLogs.".format(len(node_log_events)))
return node_log_events
def migrate_logs(node, children, dry=True):
for log in NodeLog.find(Q('params.node', 'eq', node._id)):
if log.action not in LOG_ACTIONS:
continue
if log.params.get('_path') is not None and log.params.get('_urls'):
logger.warning('Log for file {} has already been migrated'.format(log.params['path']))
continue
if dry:
logger.debug('{!r} {} -> {}'.format(log, log.params['path'], 'New path'))
continue
try:
new = children[log.params['path']]
except KeyError:
if not log.params['path'].startswith('/'):
logger.warning('Failed to migrate log with path {}'.format(log.params['path']))
continue
mpath = new.materialized_path()
url = '/{}/files/osfstorage/{}/'.format(node._id, new._id)
logger.debug('{!r} {} -> {}'.format(log, log.params['path'], mpath))
log.params['_path'] = mpath
log.params['_urls'] = {
'view': url,
'download': url + '?action=download'
}
log.save()
NodeLog._cache.clear()
def do_migration():
dupe_nodes = [n for n in Node.find(Q('_id', 'in', list(set([l.node._id for l in NodeLog.find(Q('action', 'eq', 'preprint_license_updated'))])))) if NodeLog.find(Q('action', 'eq', 'preprint_license_updated') & Q('node', 'eq', n._id)).count() > 1]
logger.info('Found {} nodes with multiple preprint_license_updated logs'.format(len(dupe_nodes)))
for node in dupe_nodes:
preprint_license_updated_logs = [log for log in node.logs if log.action == 'preprint_license_updated']
log = preprint_license_updated_logs.pop()
while(preprint_license_updated_logs):
next_log = preprint_license_updated_logs.pop()
timedelta = log.date - next_log.date
if timedelta.seconds < 60:
logger.info(
'Hiding duplicate preprint_license_updated log with ID {} from node {}, timedelta was {}'.format(
log._id, node._id, timedelta
)
)
log.should_hide = True
log.save()
else:
logger.info(
'Skipping preprint_license_updated log with ID {} from node {}, timedelta was {}'.format(
log._id, node._id, timedelta
)
)
log = next_log
def migrate_logs(node, children, dry=True):
for log in NodeLog.find(Q('params.node', 'eq', node._id)):
if log.action not in LOG_ACTIONS:
continue
if log.params.get('_path') is not None and log.params.get('_urls'):
logger.warning('Log for file {} has already been migrated'.format(log.params['path']))
continue
if dry:
logger.debug('{!r} {} -> {}'.format(log, log.params['path'], 'New path'))
continue
try:
new = children[log.params['path']]
except KeyError:
if not log.params['path'].startswith('/'):
logger.warning('Failed to migrate log with path {}'.format(log.params['path']))
continue
mpath = new.materialized_path()
url = '/{}/files/osfstorage/{}/'.format(node._id, new._id)
logger.debug('{!r} {} -> {}'.format(log, log.params['path'], mpath))
log.params['_path'] = mpath
log.params['_urls'] = {
'view': url,
'download': url + '?action=download'
}
log.save()
NodeLog._cache.clear()
def do_migration():
dupe_nodes = [
n for n in Node.find(
Q(
'_id', 'in',
list(
set([
l.node._id for l in NodeLog.find(
Q('action', 'eq', 'preprint_license_updated'))
])))) if NodeLog.find(
Q('action', 'eq', 'preprint_license_updated')
& Q('node', 'eq', n._id)).count() > 1
]
logger.info(
'Found {} nodes with multiple preprint_license_updated logs'.format(
len(dupe_nodes)))
for node in dupe_nodes:
preprint_license_updated_logs = [
log for log in node.logs
if log.action == 'preprint_license_updated'
]
log = preprint_license_updated_logs.pop()
while (preprint_license_updated_logs):
next_log = preprint_license_updated_logs.pop()
timedelta = log.date - next_log.date
if timedelta.seconds < 60:
logger.info(
'Hiding duplicate preprint_license_updated log with ID {} from node {}, timedelta was {}'
.format(log._id, node._id, timedelta))
log.should_hide = True
log.save()
else:
logger.info(
'Skipping preprint_license_updated log with ID {} from node {}, timedelta was {}'
.format(log._id, node._id, timedelta))
log = next_log
def get_nodes():
"""Return a set of node with log events for today"""
today = datetime.date.today()
midnight = datetime.datetime(
year=today.year,
month=today.month,
day=today.day,
)
return set(
(
log.node
for log in NodeLog.find(Q('date', 'gt', midnight))
)
)
def get_user_logs_since_workshop(user, workshop_date):
query_date = workshop_date + timedelta(days=1)
query = Q('user', 'eq', user._id) & Q('date', 'gt', query_date)
return list(NodeLog.find(query=query))
def get_user_logs_since_workshop(user, workshop_date):
query_date = workshop_date + timedelta(days=1)
query = Q('user', 'eq', user._id) & Q('date', 'gt', query_date)
return list(NodeLog.find(query=query))