def test_update_counters_different_files(self):
# Regression test for https://github.com/CenterForOpenScience/osf.io/issues/2281
@analytics.update_counters('download:{target_id}:{fid}')
def download_file_(**kwargs):
return kwargs.get('node') or kwargs.get('project')
fid1 = 'test.analytics.py'
fid2 = 'test_analytics.py'
download_file_(node=self.node, fid=fid1)
count = analytics.get_basic_counters('download:{0}:{1}'.format(
self.node._id, fid1))
assert_equal(count, (1, 1))
count = analytics.get_basic_counters('download:{0}:{1}'.format(
self.node._id, fid2))
assert_equal(count, (None, None))
page = 'download:{0}:{1}'.format(self.node._id, fid1)
session.data['visited'].append(page)
download_file_(node=self.node, fid=fid1)
download_file_(node=self.node, fid=fid2)
count = analytics.get_basic_counters('download:{0}:{1}'.format(
self.node._id, fid1))
assert_equal(count, (1, 2))
count = analytics.get_basic_counters('download:{0}:{1}'.format(
self.node._id, fid2))
assert_equal(count, (1, 1))
def test_update_counters_file_user_is_contributor(self):
@analytics.update_counters('download:{target_id}:{fid}',
node_info=self.node_info)
def download_file_(**kwargs):
return kwargs.get('node') or kwargs.get('project')
count = analytics.get_basic_counters('download:{0}:{1}'.format(
self.node._id, self.fid))
assert_equal(count, (None, None))
download_file_(node=self.node, fid=self.fid)
count = analytics.get_basic_counters('download:{0}:{1}'.format(
self.node._id, self.fid))
assert_equal(count, (1, 1))
page = 'download:{0}:{1}'.format(self.node._id, self.fid)
session.data['visited'].append(page)
session.data['auth_user_id'] = self.userid
download_file_(node=self.node, fid=self.fid)
count = analytics.get_basic_counters('download:{0}:{1}'.format(
self.node._id, self.fid))
assert_equal(count, (1, 1))
def get_number_downloads_unique_and_total():
number_downloads_unique = 0
number_downloads_total = 0
projects = get_projects()
for project in projects:
for filenode in OsfStorageFile.find(Q('node', 'eq', project)):
for idx, version in enumerate(filenode.versions):
page = ':'.join(
['download', project._id, filenode._id,
str(idx)])
unique, total = get_basic_counters(page)
number_downloads_unique += unique or 0
number_downloads_total += total or 0
for filenode in TrashedFileNode.find(
Q('provider', 'eq', 'osfstorage') & Q('node', 'eq', project)
& Q('is_file', 'eq', True)):
for idx, version in enumerate(filenode.versions):
page = ':'.join(
['download', project._id, filenode._id,
str(idx)])
unique, total = get_basic_counters(page)
number_downloads_total += total or 0
number_downloads_unique += unique or 0
return number_downloads_unique, number_downloads_total
def get_number_downloads_unique_and_total():
number_downloads_unique = 0
number_downloads_total = 0
projects = get_projects()
for project in projects:
addon = project.get_addon('osfstorage')
for filenode in OsfStorageFileNode.find(
Q('node_settings', 'eq', addon) & Q('kind', 'eq', 'file')):
for idx, version in enumerate(filenode.versions):
page = ':'.join(
['download', project._id, filenode._id,
str(idx)])
unique, total = get_basic_counters(page)
number_downloads_unique += unique or 0
number_downloads_total += total or 0
for filenode in OsfStorageTrashedFileNode.find(
Q('node_settings', 'eq', addon) & Q('kind', 'eq', 'file')):
for idx, version in enumerate(filenode.versions):
page = ':'.join(
['download', project._id, filenode._id,
str(idx)])
unique, total = get_basic_counters(page)
number_downloads_total += total or 0
number_downloads_unique += unique or 0
return number_downloads_unique, number_downloads_total
def test_update_counters_different_files(self):
# Regression test for https://github.com/CenterForOpenScience/osf.io/issues/2281
@analytics.update_counters('download:{target_id}:{fid}', db=None)
def download_file_(**kwargs):
return kwargs.get('node') or kwargs.get('project')
fid1 = 'test.analytics.py'
fid2 = 'test_analytics.py'
download_file_(node=self.node, fid=fid1)
count = analytics.get_basic_counters('download:{0}:{1}'.format(self.node._id, fid1), db=None)
assert_equal(count, (1, 1))
count = analytics.get_basic_counters('download:{0}:{1}'.format(self.node._id, fid2), db=None)
assert_equal(count, (None, None))
page = 'download:{0}:{1}'.format(self.node._id, fid1)
session.data['visited'].append(page)
download_file_(node=self.node, fid=fid1)
download_file_(node=self.node, fid=fid2)
count = analytics.get_basic_counters('download:{0}:{1}'.format(self.node._id, fid1), db=None)
assert_equal(count, (1, 2))
count = analytics.get_basic_counters('download:{0}:{1}'.format(self.node._id, fid2), db=None)
assert_equal(count, (1, 1))
def get_number_downloads_unique_and_total():
number_downloads_unique = 0
number_downloads_total = 0
projects = get_projects()
for project in projects:
for filenode in OsfStorageFile.find(Q('node', 'eq', project)):
for idx, version in enumerate(filenode.versions):
page = ':'.join(['download', project._id, filenode._id, str(idx)])
unique, total = get_basic_counters(page)
number_downloads_unique += unique or 0
number_downloads_total += total or 0
for filenode in TrashedFileNode.find(Q('provider', 'eq', 'osfstorage') & Q('node', 'eq', project) & Q('is_file', 'eq', True)):
for idx, version in enumerate(filenode.versions):
page = ':'.join(['download', project._id, filenode._id, str(idx)])
unique, total = get_basic_counters(page)
number_downloads_total += total or 0
number_downloads_unique += unique or 0
clear_modm_cache()
return number_downloads_unique, number_downloads_total
def test_update_counters_file(self):
@analytics.update_counters('download:{target_id}:{fid}', db=self.db)
def download_file_(**kwargs):
return kwargs.get('node') or kwargs.get('project')
count = analytics.get_basic_counters('download:{0}:{1}'.format(
self.node, self.fid),
db=self.db)
assert_equal(count, (None, None))
download_file_(node=self.node, fid=self.fid)
count = analytics.get_basic_counters('download:{0}:{1}'.format(
self.node, self.fid),
db=self.db)
assert_equal(count, (1, 1))
page = 'download:{0}:{1}'.format(self.node, self.fid)
session.data['visited'].append(page)
download_file_(node=self.node, fid=self.fid)
count = analytics.get_basic_counters('download:{0}:{1}'.format(
self.node, self.fid),
db=self.db)
assert_equal(count, (1, 2))
def get_page_counter_count(self, count_type, version=None):
"""Assembles a string to retrieve the correct file data from the pagecounter collection,
then calls get_basic_counters to retrieve the total count. Limit to version if specified.
"""
_, count = get_basic_counters(self.target.guids.first(), self, version=version, action=count_type)
return count or 0
def count_file_downloads():
downloads_unique, downloads_total = 0, 0
for record in OsfStorageFileNode.find():
page = ':'.join(['download', record.node._id, record._id])
unique, total = get_basic_counters(page)
downloads_unique += unique or 0
downloads_total += total or 0
return downloads_unique, downloads_total
def count_file_downloads():
downloads_unique, downloads_total = 0, 0
for record in OsfStorageFileRecord.find():
page = ':'.join(['download', record.node._id, record.path])
unique, total = get_basic_counters(page)
downloads_unique += unique or 0
downloads_total += total or 0
return downloads_unique, downloads_total
def count_file_downloads():
downloads_unique, downloads_total = 0, 0
for record in paginated(OsfStorageFile):
page = ':'.join(['download', record.node._id, record._id])
unique, total = get_basic_counters(page)
downloads_unique += unique or 0
downloads_total += total or 0
clear_modm_cache()
return downloads_unique, downloads_total
def count_file_downloads():
downloads_unique, downloads_total = 0, 0
for record in paginated(OsfStorageFile):
page = ':'.join(['download', record.node._id, record._id])
unique, total = get_basic_counters(page)
downloads_unique += unique or 0
downloads_total += total or 0
clear_modm_cache()
return downloads_unique, downloads_total
def get_download_count(self, version=None):
"""
:param int version: Optional one-based version index
"""
parts = ['download', self.node._id, self.path]
if version is not None:
parts.append(version)
page = ':'.join([format(part) for part in parts])
_, count = get_basic_counters(page)
return count or 0
def get_download_count(self, version=None):
"""
:param int version: Optional one-based version index
"""
parts = ['download', self.node_settings.owner._id, self._id]
if version is not None:
parts.append(version)
page = ':'.join([format(part) for part in parts])
_, count = get_basic_counters(page)
return count or 0
def get_page_counter_count(self, count_type, version=None):
"""Assembles a string to retrieve the correct file data from the pagecounter collection,
then calls get_basic_counters to retrieve the total count. Limit to version if specified.
"""
parts = [count_type, self.target._id, self._id]
if version is not None:
parts.append(version)
page = ':'.join([format(part) for part in parts])
_, count = get_basic_counters(page)
return count or 0
def get_download_count(self, version=None):
if self.is_folder:
return None
parts = ['download', self.node._id, self._id]
if version is not None:
parts.append(version)
page = ':'.join([format(part) for part in parts])
_, count = get_basic_counters(page)
return count or 0
def get_download_count(self, version=None):
if self.is_folder:
return None
parts = ['download', self.node._id, self._id]
if version is not None:
parts.append(version)
page = ':'.join([format(part) for part in parts])
_, count = get_basic_counters(page)
return count or 0
def test_update_counters_file(self):
@analytics.update_counters('download:{target_id}:{fid}', db=None)
def download_file_(**kwargs):
return kwargs.get('node') or kwargs.get('project')
count = analytics.get_basic_counters('download:{0}:{1}'.format(self.node._id, self.fid), db=None)
assert_equal(count, (None, None))
download_file_(node=self.node, fid=self.fid)
count = analytics.get_basic_counters('download:{0}:{1}'.format(self.node._id, self.fid), db=None)
assert_equal(count, (1, 1))
page = 'download:{0}:{1}'.format(self.node._id, self.fid)
session.data['visited'].append(page)
download_file_(node=self.node, fid=self.fid)
count = analytics.get_basic_counters('download:{0}:{1}'.format(self.node._id, self.fid), db=None)
assert_equal(count, (1, 2))
def get_page_counter_count(self, count_type, version=None):
"""Assembles a string to retrieve the correct file data from the pagecounter collection,
then calls get_basic_counters to retrieve the total count. Limit to version if specified.
"""
parts = [count_type, self.target._id, self._id]
if version is not None:
parts.append(version)
page = ':'.join([format(part) for part in parts])
_, count = get_basic_counters(page)
return count or 0
def test_get_basic_counters(self):
page = 'node:' + str(self.node._id)
d = {'$inc': {}}
d['$inc']['total'] = 5
d['$inc']['unique'] = 3
collection = self.db['pagecounters']
collection.update({'_id': page}, d, True, False)
count = analytics.get_basic_counters(page, db=self.db)
assert_equal(count, (3, 5))
def file_info(auth, fid, **kwargs):
versions = []
node = kwargs['node'] or kwargs['project']
file_name = fid
file_name_clean = urlsafe_filename(file_name)
files_page_url = node.web_url_for('collect_file_trees')
latest_download_url = None
api_url = None
anonymous = has_anonymous_link(node, auth)
try:
files_versions = node.files_versions[file_name_clean]
except KeyError:
raise HTTPError(http.NOT_FOUND)
latest_version_number = get_latest_version_number(file_name_clean, node) + 1
for idx, version in enumerate(list(reversed(files_versions))):
node_file = NodeFile.load(version)
number = len(files_versions) - idx
unique, total = get_basic_counters('download:{}:{}:{}'.format(
node._primary_key,
file_name_clean,
number,
))
download_url = node_file.download_url(node)
api_url = node_file.api_url(node)
versions.append({
'file_name': file_name,
'download_url': download_url,
'version_number': number,
'display_number': number if idx > 0 else 'current',
'modified_date': node_file.date_uploaded.strftime('%Y/%m/%d %I:%M %p'),
'downloads': total if total else 0,
'committer_name': privacy_info_handle(
node_file.uploader.fullname, anonymous, name=True
),
'committer_url': privacy_info_handle(node_file.uploader.url, anonymous),
})
if number == latest_version_number:
latest_download_url = download_url
return {
'node_title': node.title,
'file_name': file_name,
'versions': versions,
'registered': node.is_registration,
'urls': {
'api': api_url,
'files': files_page_url,
'latest': {
'download': latest_download_url,
},
}
}
def file_info(auth, fid, **kwargs):
versions = []
node = kwargs['node'] or kwargs['project']
file_name = fid
file_name_clean = urlsafe_filename(file_name)
files_page_url = node.web_url_for('collect_file_trees')
latest_download_url = None
api_url = None
anonymous = has_anonymous_link(node, auth)
try:
files_versions = node.files_versions[file_name_clean]
except KeyError:
raise HTTPError(http.NOT_FOUND)
latest_version_number = get_latest_version_number(file_name_clean, node) + 1
for idx, version in enumerate(list(reversed(files_versions))):
node_file = NodeFile.load(version)
number = len(files_versions) - idx
unique, total = get_basic_counters('download:{}:{}:{}'.format(
node._primary_key,
file_name_clean,
number,
))
download_url = node_file.download_url(node)
api_url = node_file.api_url(node)
versions.append({
'file_name': file_name,
'download_url': download_url,
'version_number': number,
'display_number': number if idx > 0 else 'current',
'modified_date': node_file.date_uploaded.strftime('%Y/%m/%d %I:%M %p'),
'downloads': total if total else 0,
'committer_name': privacy_info_handle(
node_file.uploader.fullname, anonymous, name=True
),
'committer_url': privacy_info_handle(node_file.uploader.url, anonymous),
})
if number == latest_version_number:
latest_download_url = download_url
return {
'node_title': node.title,
'file_name': file_name,
'versions': versions,
'registered': node.is_registration,
'urls': {
'api': api_url,
'files': files_page_url,
'latest': {
'download': latest_download_url,
},
}
}
def test_update_counters_file_user_is_not_contributor(self):
@analytics.update_counters('download:{target_id}:{fid}', db=None, node_info=self.node_info)
def download_file_(**kwargs):
return kwargs.get('node') or kwargs.get('project')
count = analytics.get_basic_counters('download:{0}:{1}'.format(self.node._id, self.fid), db=None)
assert_equal(count, (None, None))
download_file_(node=self.node, fid=self.fid)
count = analytics.get_basic_counters('download:{0}:{1}'.format(self.node._id, self.fid), db=None)
assert_equal(count, (1, 1))
page = 'download:{0}:{1}'.format(self.node._id, self.fid)
session.data['visited'].append(page)
session.data['auth_user_id'] = "asv12uey821vavshl"
download_file_(node=self.node, fid=self.fid)
count = analytics.get_basic_counters('download:{0}:{1}'.format(self.node._id, self.fid), db=None)
assert_equal(count, (1, 2))
def get_download_count(self, version=None):
"""Pull the download count from the pagecounter collection
Limit to version if specified.
Currently only useful for OsfStorage
"""
parts = ['download', self.node._id, self._id]
if version is not None:
parts.append(version)
page = ':'.join([format(part) for part in parts])
_, count = get_basic_counters(page)
return count or 0
def get_download_count(self, version=None):
"""Pull the download count from the pagecounter collection
Limit to version if specified.
Currently only useful for OsfStorage
"""
parts = ['download', self.node._id, self._id]
if version is not None:
parts.append(version)
page = ':'.join([format(part) for part in parts])
_, count = get_basic_counters(page)
return count or 0
def get_number_downloads_unique_and_total():
number_downloads_unique = 0
number_downloads_total = 0
projects = get_projects()
for project in projects:
addon = project.get_addon('osfstorage')
for filenode in OsfStorageFileNode.find(Q('node_settings', 'eq', addon) & Q('kind', 'eq', 'file')):
for idx, version in enumerate(filenode.versions):
page = ':'.join(['download', project._id, filenode._id, str(idx)])
unique, total = get_basic_counters(page)
number_downloads_unique += unique or 0
number_downloads_total += total or 0
for filenode in OsfStorageTrashedFileNode.find(Q('node_settings', 'eq', addon) & Q('kind', 'eq', 'file')):
for idx, version in enumerate(filenode.versions):
page = ':'.join(['download', project._id, filenode._id, str(idx)])
unique, total = get_basic_counters(page)
number_downloads_total += total or 0
number_downloads_unique += unique or 0
return number_downloads_unique, number_downloads_total
def main():
number_users = models.User.find().count()
projects = models.Node.find(
Q('category', 'eq', 'project') &
Q('is_deleted', 'eq', False) &
Q('is_folder', 'ne', True)
)
projects_forked = list(models.Node.find(
Q('category', 'eq', 'project') &
Q('is_deleted', 'eq', False) &
Q('is_folder', 'ne', True) &
Q('is_fork', 'eq', True)
))
projects_registered = models.Node.find(
Q('category', 'eq', 'project') &
Q('is_deleted', 'eq', False) &
Q('is_folder', 'ne', True) &
Q('is_registration', 'eq', True)
)
pf = []
for p in projects_forked:
if not p.contributors[0]:
continue
name = p.contributors[0].fullname
if unicode(name) not in [u'Jeffres R. Spies', 'Brian A. Nosek']:
pf.append(p)
pr = []
for p in projects_registered:
name = p.contributors[0].fullname
if not p.contributors[0]:
continue
if not unicode(name)==u'Jeffrey R. Spies' and not unicode(name)==u'Brian A. Nosek':
pr.append(p)
number_projects = len(projects)
number_projects_public = models.Node.find(
Q('category', 'eq', 'project') &
Q('is_deleted', 'eq', False) &
Q('is_folder', 'ne', True) &
Q('is_public', 'eq', True)
).count()
number_projects_forked = len(pf)
number_projects_registered = len(pr)
##############
number_downloads_total = 0
number_downloads_unique = 0
contributors_per_project = []
contrib = {}
for project in projects:
contributors_per_project.append(len(project.contributors))
for person in project.contributors:
if not person:
continue
if person._id not in contrib:
contrib[person._id] = []
for neighbor in project.contributors:
if not neighbor:
continue
if neighbor._id not in contrib[person._id]:
contrib[person._id].append(neighbor._id)
addon = project.get_addon('osfstorage')
records = addon.file_tree.children if addon.file_tree else []
for record in records:
for idx, version in enumerate(record.versions):
page = ':'.join(['download', project._id, record.path, str(idx + 1)])
unique, total = get_basic_counters(page)
number_downloads_total += total or 0
number_downloads_unique += unique or 0
table = tabulate.tabulate(
[
['number_users', number_users],
['number_projects', number_projects],
['number_projects_public', number_projects_public],
['number_projects_forked', number_projects_forked],
['number_projects_registered', number_projects_registered],
['number_downloads_total', number_downloads_total],
['number_downloads_unique', number_downloads_unique],
],
headers=['label', 'value'],
)
with open(os.path.join(settings.ANALYTICS_PATH, 'legacy.txt'), 'w') as fp:
fp.write(table)
def download_count(self, node):
_, total = get_basic_counters('download:{0}:{1}'.format(
node._id, self.path.replace('.', '_')))
return total or 0
contrib = {}
for project in projects:
#project = Node.load(p['_id'])
contributors_per_project.append(len(project.contributors))
for person in project.contributors:
if not person:
continue
if person._id not in contrib:
contrib[person._id] = []
for neighbor in project.contributors:
if not neighbor:
continue
if neighbor._id not in contrib[person._id]:
contrib[person._id].append(neighbor._id)
unique, total = get_basic_counters('node:' + str(project._id))
if total:
number_views_total += total
number_views_unique += unique
for k, v in project.files_versions.iteritems():
for i, f in enumerate(v):
fi = NodeFile.load(f)
unique, total = get_basic_counters('download:' + str(project._id) +
':' + fi.path.replace('.', '_'))
if total:
number_downloads_total += total
number_downloads_unique += unique
print "number_users", number_users
print "number_projects", number_projects
print "number_projects_public", number_projects_public
def test_get_basic_counters(self):
page = 'node:' + str(self.node._id)
PageCounter.objects.create(_id=page, total=5, unique=3)
count = analytics.get_basic_counters(page, db=None)
assert_equal(count, (3, 5))
def upload_file_public(auth, node_addon, **kwargs):
node = kwargs['node'] or kwargs['project']
do_redirect = request.form.get('redirect', False)
name, content, content_type, size = prepare_file(request.files['file'])
if size > (node_addon.config.max_file_size * MEGABYTE):
raise HTTPError(
http.BAD_REQUEST,
data={
'message_short': 'File too large.',
'message_long': 'The file you are trying to upload exceeds '
'the maximum file size limit.',
},
)
try:
fobj = node.add_file(
auth,
name,
content,
size,
content_type
)
except FileNotModified:
return {
'actionTaken': None,
'name': name,
}
# existing file was updated?
was_updated = node.logs[-1].action == NodeLog.FILE_UPDATED
unique, total = get_basic_counters(
'download:{0}:{1}'.format(
node._id,
fobj.path.replace('.', '_')
)
)
file_info = {
'name': name,
'size': [
float(size),
rubeus.format_filesize(size),
],
# URLs
'urls': {
'view': fobj.url(node),
'download': fobj.download_url(node),
'delete': fobj.api_url(node),
},
rubeus.KIND: rubeus.FILE,
'permissions': {
'view': True,
'edit': True,
},
'dates': {
'uploaded': [
time.mktime(fobj.date_uploaded.timetuple()),
fobj.date_uploaded.strftime('%Y/%m/%d %I:%M %p'),
],
},
'downloads': total or 0,
'actionTaken': NodeLog.FILE_UPDATED if was_updated else NodeLog.FILE_ADDED,
}
if do_redirect:
return redirect(request.referrer)
return file_info, 201
contrib = {}
for project in projects:
#project = Node.load(p['_id'])
contributors_per_project.append(len(project.contributors))
for person in project.contributors:
if not person:
continue
if person._id not in contrib:
contrib[person._id] = []
for neighbor in project.contributors:
if not neighbor:
continue
if neighbor._id not in contrib[person._id]:
contrib[person._id].append(neighbor._id)
unique, total = get_basic_counters('node:' + str(project._id))
if total:
number_views_total += total
number_views_unique += unique
for k,v in project.files_versions.iteritems():
for i, f in enumerate(v):
fi = NodeFile.load(f)
unique, total = get_basic_counters('download:' + str(project._id) + ':' + fi.path.replace('.', '_'))
if total:
number_downloads_total += total
number_downloads_unique += unique
print "number_users" , number_users
print "number_projects" , number_projects
print "number_projects_public" , number_projects_public
print "number_projects_forked" , number_projects_forked
def main():
number_users = models.User.find().count()
projects = models.Node.find(
Q("category", "eq", "project") & Q("is_deleted", "eq", False) & Q("is_folder", "ne", True)
)
projects_forked = list(
models.Node.find(
Q("category", "eq", "project")
& Q("is_deleted", "eq", False)
& Q("is_folder", "ne", True)
& Q("is_fork", "eq", True)
)
)
projects_registered = models.Node.find(
Q("category", "eq", "project")
& Q("is_deleted", "eq", False)
& Q("is_folder", "ne", True)
& Q("is_registration", "eq", True)
)
pf = []
for p in projects_forked:
if not p.contributors[0]:
continue
name = p.contributors[0].fullname
if unicode(name) not in [u"Jeffres R. Spies", "Brian A. Nosek"]:
pf.append(p)
pr = []
for p in projects_registered:
name = p.contributors[0].fullname
if not p.contributors[0]:
continue
if not unicode(name) == u"Jeffrey R. Spies" and not unicode(name) == u"Brian A. Nosek":
pr.append(p)
number_projects = len(projects)
number_projects_public = models.Node.find(
Q("category", "eq", "project")
& Q("is_deleted", "eq", False)
& Q("is_folder", "ne", True)
& Q("is_public", "eq", True)
).count()
number_projects_forked = len(pf)
number_projects_registered = len(pr)
##############
number_downloads_total = 0
number_downloads_unique = 0
contributors_per_project = []
contrib = {}
for project in projects:
contributors_per_project.append(len(project.contributors))
for person in project.contributors:
if not person:
continue
if person._id not in contrib:
contrib[person._id] = []
for neighbor in project.contributors:
if not neighbor:
continue
if neighbor._id not in contrib[person._id]:
contrib[person._id].append(neighbor._id)
addon = project.get_addon("osfstorage")
for filenode in OsfStorageFileNode.find(Q("node_settings", "eq", addon) & Q("kind", "eq", "file")):
for idx, version in enumerate(filenode.versions):
page = ":".join(["download", project._id, filenode._id, str(idx)])
unique, total = get_basic_counters(page)
number_downloads_total += total or 0
number_downloads_unique += unique or 0
for filenode in OsfStorageTrashedFileNode.find(Q("node_settings", "eq", addon) & Q("kind", "eq", "file")):
for idx, version in enumerate(filenode.versions):
page = ":".join(["download", project._id, filenode._id, str(idx)])
unique, total = get_basic_counters(page)
number_downloads_total += total or 0
number_downloads_unique += unique or 0
table = tabulate.tabulate(
[
["number_users", number_users],
["number_projects", number_projects],
["number_projects_public", number_projects_public],
["number_projects_forked", number_projects_forked],
["number_projects_registered", number_projects_registered],
["number_downloads_total", number_downloads_total],
["number_downloads_unique", number_downloads_unique],
],
headers=["label", "value"],
)
with open(os.path.join(settings.ANALYTICS_PATH, "legacy.txt"), "w") as fp:
fp.write(table)
def main():
number_users = models.User.find().count()
projects = models.Node.find(
Q('category', 'eq', 'project') &
Q('is_deleted', 'eq', False) &
Q('is_folder', 'ne', True)
)
projects_forked = list(models.Node.find(
Q('category', 'eq', 'project') &
Q('is_deleted', 'eq', False) &
Q('is_folder', 'ne', True) &
Q('is_fork', 'eq', True)
))
projects_registered = models.Node.find(
Q('category', 'eq', 'project') &
Q('is_deleted', 'eq', False) &
Q('is_folder', 'ne', True) &
Q('is_registration', 'eq', True)
)
pf = []
for p in projects_forked:
if not p.contributors[0]:
continue
name = p.contributors[0].fullname
if unicode(name) not in [u'Jeffres R. Spies', 'Brian A. Nosek']:
pf.append(p)
pr = []
for p in projects_registered:
name = p.contributors[0].fullname
if not p.contributors[0]:
continue
if not unicode(name)==u'Jeffrey R. Spies' and not unicode(name)==u'Brian A. Nosek':
pr.append(p)
number_projects = len(projects)
number_projects_public = models.Node.find(
Q('category', 'eq', 'project') &
Q('is_deleted', 'eq', False) &
Q('is_folder', 'ne', True) &
Q('is_public', 'eq', True)
).count()
number_projects_forked = len(pf)
number_projects_registered = len(pr)
##############
number_downloads_total = 0
number_downloads_unique = 0
contributors_per_project = []
contrib = {}
for project in projects:
contributors_per_project.append(len(project.contributors))
for person in project.contributors:
if not person:
continue
if person._id not in contrib:
contrib[person._id] = []
for neighbor in project.contributors:
if not neighbor:
continue
if neighbor._id not in contrib[person._id]:
contrib[person._id].append(neighbor._id)
addon = project.get_addon('osfstorage')
for filenode in OsfStorageFileNode.find(Q('node_settings', 'eq', addon) & Q('kind', 'eq', 'file')):
for idx, version in enumerate(filenode.versions):
page = ':'.join(['download', project._id, filenode._id, str(idx)])
unique, total = get_basic_counters(page)
number_downloads_total += total or 0
number_downloads_unique += unique or 0
for filenode in OsfStorageTrashedFileNode.find(Q('node_settings', 'eq', addon) & Q('kind', 'eq', 'file')):
for idx, version in enumerate(filenode.versions):
page = ':'.join(['download', project._id, filenode._id, str(idx)])
unique, total = get_basic_counters(page)
number_downloads_total += total or 0
number_downloads_unique += unique or 0
table = tabulate.tabulate(
[
['number_users', number_users],
['number_projects', number_projects],
['number_projects_public', number_projects_public],
['number_projects_forked', number_projects_forked],
['number_projects_registered', number_projects_registered],
['number_downloads_total', number_downloads_total],
['number_downloads_unique', number_downloads_unique],
],
headers=['label', 'value'],
)
with open(os.path.join(settings.ANALYTICS_PATH, 'legacy.txt'), 'w') as fp:
fp.write(table)
def test_get_basic_counters(self):
page = 'node:' + str(self.node._id)
PageCounter.objects.create(_id=page, total=5, unique=3)
count = analytics.get_basic_counters(page)
assert_equal(count, (3, 5))
def upload_file_public(auth, node_addon, **kwargs):
node = kwargs['node'] or kwargs['project']
do_redirect = request.form.get('redirect', False)
name, content, content_type, size = prepare_file(request.files['file'])
if size > (node_addon.config.max_file_size * MEGABYTE):
raise HTTPError(
http.BAD_REQUEST,
data={
'message_short': 'File too large.',
'message_long': 'The file you are trying to upload exceeds '
'the maximum file size limit.',
},
)
try:
fobj = node.add_file(
auth,
name,
content,
size,
content_type
)
except FileNotModified:
return {
'actionTaken': None,
'name': name,
}
# existing file was updated?
was_updated = node.logs[-1].action == NodeLog.FILE_UPDATED
unique, total = get_basic_counters(
'download:{0}:{1}'.format(
node._id,
fobj.path.replace('.', '_')
)
)
file_info = {
'name': name,
'size': [
float(size),
rubeus.format_filesize(size),
],
# URLs
'urls': {
'view': fobj.url(node),
'download': fobj.download_url(node),
'delete': fobj.api_url(node),
},
rubeus.KIND: rubeus.FILE,
'permissions': {
'view': True,
'edit': True,
},
'dates': {
'uploaded': [
time.mktime(fobj.date_uploaded.timetuple()),
fobj.date_uploaded.strftime('%Y/%m/%d %I:%M %p'),
],
},
'downloads': total or 0,
'actionTaken': NodeLog.FILE_UPDATED if was_updated else NodeLog.FILE_ADDED,
}
if do_redirect:
return redirect(request.referrer)
return file_info, 201
