def get_default_odm_query(self):
user = self.get_user()
query = Q('contributors', 'eq', user) & default_node_list_query()
if user != self.request.user:
query &= default_node_permission_query(self.request.user)
return query
def send_confirm_email(user, email, renew=False, external_id_provider=None, external_id=None, destination=None):
"""
Sends `user` a confirmation to the given `email`.
:param user: the user
:param email: the email
:param renew: refresh the token
:param external_id_provider: user's external id provider
:param external_id: user's external id
:param destination: the destination page to redirect after confirmation
:return:
:raises: KeyError if user does not have a confirmation token for the given email.
"""
confirmation_url = user.get_confirmation_url(
email,
external=True,
force=True,
renew=renew,
external_id_provider=external_id_provider,
destination=destination
)
try:
merge_target = User.find_one(Q('emails', 'eq', email))
except NoResultsFound:
merge_target = None
campaign = campaigns.campaign_for_user(user)
branded_preprints_provider = None
# Choose the appropriate email template to use and add existing_user flag if a merge or adding an email.
if external_id_provider and external_id:
# First time login through external identity provider, link or create an OSF account confirmation
if user.external_identity[external_id_provider][external_id] == 'CREATE':
mail_template = mails.EXTERNAL_LOGIN_CONFIRM_EMAIL_CREATE
elif user.external_identity[external_id_provider][external_id] == 'LINK':
mail_template = mails.EXTERNAL_LOGIN_CONFIRM_EMAIL_LINK
elif merge_target:
# Merge account confirmation
mail_template = mails.CONFIRM_MERGE
confirmation_url = '{}?logout=1'.format(confirmation_url)
elif user.is_active:
# Add email confirmation
mail_template = mails.CONFIRM_EMAIL
confirmation_url = '{}?logout=1'.format(confirmation_url)
elif campaign:
# Account creation confirmation: from campaign
mail_template = campaigns.email_template_for_campaign(campaign)
if campaigns.is_proxy_login(campaign) and campaigns.get_service_provider(campaign) != 'OSF':
branded_preprints_provider = campaigns.get_service_provider(campaign)
else:
# Account creation confirmation: from OSF
mail_template = mails.INITIAL_CONFIRM_EMAIL
mails.send_mail(
email,
mail_template,
'plain',
user=user,
confirmation_url=confirmation_url,
email=email,
merge_target=merge_target,
external_id_provider=external_id_provider,
branded_preprints_provider=branded_preprints_provider
)
def get_targets():
return NodeLog.find(
Q('action', 'eq', NodeLog.EMBARGO_APPROVED)
& Q('params.user', 'eq', None))
def get_targets(date):
return models.FileVersion.find(
Q('date_created', 'lt', date - DELTA_DATE)
& Q('status', 'ne', 'cached') & Q('metadata.archive', 'exists', True)
& Q('location', 'ne', None))
def glacier_targets():
return models.FileVersion.find(
Q('status', 'ne', 'cached') & Q('location.object', 'exists', True)
& Q('metadata.archive', 'eq', None))
def get_has_children(self, obj):
return Comment.find(Q('target', 'eq', Guid.load(obj._id))).count() > 0
def get_targets():
return model.OsfStorageFileVersion.find(
Q('status', 'ne', 'cached') & Q('location.object', 'exists', True))
def main(dry_run=True):
init_app(set_backend=True, routes=False)
staff_user = models.User.find_one(Q('username', 'eq', STAFF_EMAIL))
for conference in models.Conference.find():
migrate_conference(conference, staff_user, PERSONAL_ACCOUNTS, dry_run=dry_run)
def get_active_users(extra=None):
query = (Q('is_registered', 'eq', True) & Q('password', 'ne', None)
& Q('merged_by', 'eq', None) & Q('date_confirmed', 'ne', None)
& Q('date_disabled', ' eq', None))
query = query & extra if extra else query
return User.find(query)
def main(env):
INSTITUTIONS = []
if env == 'prod':
INSTITUTIONS = [
{
'_id': 'busara',
'name': 'Busara Center for Behavioral Economics',
'description':
'The <a href="http://www.busaracenter.org/">Busara Center</a> for Behavioral Economics',
'banner_name': 'busara-banner.png',
'logo_name': 'busara-shield.png',
'auth_url': None,
'domains': [],
'email_domains': ['busaracenter.org'],
},
{
'_id': 'cos',
'name': 'Center For Open Science',
'description':
'COS is a non-profit technology company providing free and open services to increase inclusivity and transparency of research. Find out more at <a href="https://cos.io">cos.io</a>.',
'banner_name': 'cos-banner.png',
'logo_name': 'cos-shield.png',
'auth_url': None,
'domains': ['osf.cos.io'],
'email_domains': ['cos.io'],
},
{
'_id': 'esip',
'name':
'Federation of Earth Science Information Partners (ESIP)',
'description':
'<a href="http://www.esipfed.org/">ESIP\'s</a> mission is to support the networking and data dissemination needs of our members and the global Earth science data community by linking the functional sectors of observation, research, application, education and use of Earth science.',
'banner_name': 'esip-banner.png',
'logo_name': 'esip-shield.png',
'auth_url': None,
'domains': [],
'email_domains': ['esipfed.org'],
},
{
'_id':
'nd',
'name':
'University of Notre Dame',
'description':
'In <a href="https://research.nd.edu/news/64035-notre-dame-center-for-open-science-partner-to-advance-open-science-initiatives/">partnership</a> with the <a href="https://crc.nd.edu">Center for Research Computing</a>, <a href="http://esc.nd.edu">Engineering & Science Computing</a>, and the <a href="https://library.nd.edu">Hesburgh Libraries</a>',
'banner_name':
'nd-banner.png',
'logo_name':
'nd-shield.png',
'auth_url':
SHIBBOLETH_SP.format(
encode_uri_component(
'https://login.nd.edu/idp/shibboleth')),
'domains': ['osf.nd.edu'],
'email_domains': [],
},
{
'_id':
'ucr',
'name':
'University of California Riverside',
'description':
'Policy prohibits storing PII or HIPAA data on this site, please see C&C\'s <a href="http://cnc.ucr.edu/security/researchers.html">security site</a> for more information.',
'banner_name':
'ucr-banner.png',
'logo_name':
'ucr-shield.png',
'auth_url':
SHIBBOLETH_SP.format(
encode_uri_component('urn:mace:incommon:ucr.edu')),
'domains': ['osf.ucr.edu'],
'email_domains': [],
},
# {
# '_id': 'ugent',
# 'name': 'Universiteit Gent [Test]',
# 'description': None,
# 'banner_name': 'ugent-banner.png',
# 'logo_name': 'ugent-shield.png',
# 'auth_url': SHIBBOLETH_SP.format(encode_uri_component('https://identity.ugent.be/simplesaml/saml2/idp/metadata.php')),
# 'domains': ['osf.ugent.be'],
# 'email_domains': [],
# },
{
'_id':
'usc',
'name':
'University of Southern California',
'description':
'Projects must abide by <a href="http://policy.usc.edu/info-security/">USC\'s Information Security Policy</a>. Data stored for human subject research repositories must abide by <a href="http://policy.usc.edu/biorepositories/">USC\'s Biorepository Policy</a>. The OSF may not be used for storage of Personal Health Information that is subject to <a href="http://policy.usc.edu/hipaa/">HIPPA regulations</a>.',
'banner_name':
'usc-banner.png',
'logo_name':
'usc-shield.png',
'auth_url':
SHIBBOLETH_SP.format(
encode_uri_component('urn:mace:incommon:usc.edu')),
'domains': ['osf.usc.edu'],
'email_domains': [],
},
{
'_id':
'uva',
'name':
'University of Virginia',
'description':
'Projects must abide by the University <a href="http://www.virginia.edu/informationpolicy/security.html">Security and Data Protection Policies</a>',
'banner_name':
'uva-banner.png',
'logo_name':
'uva-shield.png',
'auth_url':
SHIBBOLETH_SP.format(
encode_uri_component('urn:mace:incommon:virginia.edu')),
'domains': ['osf.virginia.edu'],
'email_domains': [],
},
]
if env == 'stage':
INSTITUTIONS = [
{
'_id': 'cos',
'name': 'Center For Open Science [Stage]',
'description': 'Center for Open Science [Stage]',
'banner_name': 'cos-banner.png',
'logo_name': 'cos-shield.png',
'auth_url': None,
'domains': ['staging-osf.cos.io'],
'email_domains': ['cos.io'],
},
{
'_id':
'nd',
'name':
'University of Notre Dame [Stage]',
'description':
'University of Notre Dame [Stage]',
'banner_name':
'nd-banner.png',
'logo_name':
'nd-shield.png',
'auth_url':
SHIBBOLETH_SP.format(
encode_uri_component(
'https://login-test.cc.nd.edu/idp/shibboleth')),
'domains': ['staging-osf-nd.cos.io'],
'email_domains': [],
},
{
'_id': 'google',
'name': 'Google [Stage]',
'description': 'Google [Stage]',
'banner_name': 'google-banner.png',
'logo_name': 'google-shield.png',
'auth_url': None,
'domains': [],
'email_domains': ['gmail.com'],
},
{
'_id': 'yahoo',
'name': 'Yahoo [Stage]',
'description': 'Yahoo [Stage]',
'banner_name': 'yahoo-banner.png',
'logo_name': 'yahoo-shield.png',
'auth_url': None,
'domains': [],
'email_domains': ['yahoo.com'],
},
]
if env == 'stage2':
INSTITUTIONS = [
{
'_id': 'cos',
'name': 'Center For Open Science [Stage2]',
'description': 'Center for Open Science [Stage2]',
'banner_name': 'cos-banner.png',
'logo_name': 'cos-shield.png',
'auth_url': None,
'domains': ['staging2-osf.cos.io'],
'email_domains': ['cos.io'],
},
]
elif env == 'test':
INSTITUTIONS = [
{
'_id': 'busara',
'name': 'Busara Center for Behavioral Economics [Test]',
'description':
'The <a href="http://www.busaracenter.org/">Busara Center</a> for Behavioral Economics',
'banner_name': 'busara-banner.png',
'logo_name': 'busara-shield.png',
'auth_url': None,
'domains': [],
'email_domains': ['busaracenter.org'],
},
{
'_id': 'cos',
'name': 'Center For Open Science [Test]',
'description':
'COS is a non-profit technology company providing free and open services to increase inclusivity and transparency of research. Find out more at <a href="https://cos.io">cos.io</a>.',
'banner_name': 'cos-banner.png',
'logo_name': 'cos-shield.png',
'auth_url': None,
'domains': ['test-osf.cos.io'],
'email_domains': ['cos.io'],
},
{
'_id': 'esip',
'name':
'Federation of Earth Science Information Partners (ESIP) [Test]',
'description':
'<a href="http://www.esipfed.org/">ESIP\'s</a> mission is to support the networking and data dissemination needs of our members and the global Earth science data community by linking the functional sectors of observation, research, application, education and use of Earth science.',
'banner_name': 'esip-banner.png',
'logo_name': 'esip-shield.png',
'auth_url': None,
'domains': [],
'email_domains': ['esipfed.org'],
},
{
'_id':
'nd',
'name':
'University of Notre Dame [Test]',
'description':
'In <a href="https://research.nd.edu/news/64035-notre-dame-center-for-open-science-partner-to-advance-open-science-initiatives/">partnership</a> with the <a href="https://crc.nd.edu">Center for Research Computing</a>, <a href="http://esc.nd.edu">Engineering & Science Computing</a>, and the <a href="https://library.nd.edu">Hesburgh Libraries</a>',
'banner_name':
'nd-banner.png',
'logo_name':
'nd-shield.png',
'auth_url':
SHIBBOLETH_SP.format(
encode_uri_component(
'https://login-test.cc.nd.edu/idp/shibboleth')),
'domains': ['test-osf-nd.cos.io'],
'email_domains': [],
},
{
'_id':
'ucr',
'name':
'University of California Riverside [Test]',
'description':
'Policy prohibits storing PII or HIPAA data on this site, please see C&C\'s <a href="http://cnc.ucr.edu/security/researchers.html">security site</a> for more information.',
'banner_name':
'ucr-banner.png',
'logo_name':
'ucr-shield.png',
'auth_url':
SHIBBOLETH_SP.format(
encode_uri_component('urn:mace:incommon:ucr.edu')),
'domains': ['test-osf-ucr.cos.io'],
'email_domains': [],
},
{
'_id':
'ugent',
'name':
'Universiteit Gent [Test]',
'description':
'Universiteit Gent [Test]',
'banner_name':
'ugent-banner.png',
'logo_name':
'ugent-shield.png',
'auth_url':
SHIBBOLETH_SP.format(
encode_uri_component(
'https://identity.ugent.be/simplesaml/saml2/idp/metadata.php'
)),
'domains': ['test-osf-ugent.cos.io'],
'email_domains': [],
},
{
'_id':
'usc',
'name':
'University of Southern California [Test]',
'description':
'Projects must abide by <a href="http://policy.usc.edu/info-security/">USC\'s Information Security Policy</a>. Data stored for human subject research repositories must abide by <a href="http://policy.usc.edu/biorepositories/">USC\'s Biorepository Policy</a>. The OSF may not be used for storage of Personal Health Information that is subject to <a href="http://policy.usc.edu/hipaa/">HIPPA regulations</a>.',
'banner_name':
'usc-banner.png',
'logo_name':
'usc-shield.png',
'auth_url':
SHIBBOLETH_SP.format(
encode_uri_component('urn:mace:incommon:usc.edu')),
'domains': ['test-osf-usc.cos.io'],
'email_domains': [],
},
{
'_id':
'uva',
'name':
'University of Virginia [Test]',
'description':
'Projects must abide by the University <a href="http://www.virginia.edu/informationpolicy/security.html">Security and Data Protection Policies</a>',
'banner_name':
'uva-banner.png',
'logo_name':
'uva-shield.png',
'auth_url':
SHIBBOLETH_SP.format(
encode_uri_component(
'https://shibidp-test.its.virginia.edu/idp/shibboleth')
),
'domains': ['test-osf-virginia.cos.io'],
'email_domains': [],
},
]
init_app(routes=False)
with TokuTransaction():
for inst_data in INSTITUTIONS:
new_inst, inst_created = update_or_create(inst_data)
# update the nodes elastic docs, to have current names of institutions. This will
# only work properly if this file is the only thing changing institution attributes
if not inst_created:
nodes = Node.find_by_institutions(new_inst,
query=Q(
'is_deleted', 'ne',
True))
for node in nodes:
update_node(node, async=False)
for extra_inst in Institution.find(
Q('_id', 'nin', [x['_id'] for x in INSTITUTIONS])):
logger.warn('Extra Institution : {} - {}'.format(
extra_inst._id, extra_inst.name))
def main(env):
INSTITUTIONS = []
if env == 'prod':
INSTITUTIONS = [
{
'_id': 'busara',
'name': 'Busara Center for Behavioral Economics',
'description':
'The <a href="http://www.busaracenter.org/">Busara Center</a> for Behavioral Economics',
'banner_name': 'busara-banner.png',
'logo_name': 'busara-shield.png',
'auth_url': None,
'logout_url': None,
'domains': [],
'email_domains': ['busaracenter.org'],
},
{
'_id': 'cos',
'name': 'Center For Open Science',
'description':
'COS is a non-profit technology company providing free and open services to increase inclusivity and transparency of research. Find out more at <a href="https://cos.io">cos.io</a>.',
'banner_name': 'cos-banner.png',
'logo_name': 'cos-shield.png',
'auth_url': None,
'logout_url': None,
'domains': ['osf.cos.io'],
'email_domains': ['cos.io'],
},
{
'_id': 'esip',
'name':
'Federation of Earth Science Information Partners (ESIP)',
'description':
'<a href="http://www.esipfed.org/">ESIP\'s</a> mission is to support the networking and data dissemination needs of our members and the global Earth science data community by linking the functional sectors of observation, research, application, education and use of Earth science.',
'banner_name': 'esip-banner.png',
'logo_name': 'esip-shield.png',
'auth_url': None,
'logout_url': None,
'domains': [],
'email_domains': ['esipfed.org'],
},
{
'_id': 'ljaf',
'name': 'Laura and John Arnold Foundation',
'description':
'Projects listed below are for grants awarded by the Foundation. Please see the <a href="http://www.arnoldfoundation.org/wp-content/uploads/Guidelines-for-Investments-in-Research.pdf">LJAF Guidelines for Investments in Research</a> for more information and requirements.',
'banner_name': 'ljaf-banner.png',
'logo_name': 'ljaf-shield.png',
'auth_url': None,
'logout_url': None,
'domains': [],
'email_domains': ['arnoldfoundation.org'],
},
{
'_id':
'nd',
'name':
'University of Notre Dame',
'description':
'In <a href="https://research.nd.edu/news/64035-notre-dame-center-for-open-science-partner-to-advance-open-science-initiatives/">partnership</a> with the <a href="https://crc.nd.edu">Center for Research Computing</a>, <a href="http://esc.nd.edu">Engineering & Science Computing</a>, and the <a href="https://library.nd.edu">Hesburgh Libraries</a>',
'banner_name':
'nd-banner.png',
'logo_name':
'nd-shield.png',
'auth_url':
SHIBBOLETH_SP_LOGIN.format(
encode_uri_component(
'https://login.nd.edu/idp/shibboleth')),
'logout_url':
SHIBBOLETH_SP_LOGOUT.format(
encode_uri_component('https://osf.io/goodbye')),
'domains': ['osf.nd.edu'],
'email_domains': [],
},
{
'_id':
'nyu',
'name':
'New York University',
'description':
'A Research Project and File Management Tool for the NYU Community: <a href="https://www.nyu.edu/research.html">Research at NYU</a> | <a href="http://guides.nyu.edu/data_management">Research Data Management Planning</a> | <a href="https://library.nyu.edu/services/research/">NYU Library Research Services</a> | <a href="https://nyu.qualtrics.com/jfe6/form/SV_8dFc5TpA1FgLUMd">Get Help</a>',
'banner_name':
'nyu-banner.png',
'logo_name':
'nyu-shield.png',
'auth_url':
SHIBBOLETH_SP_LOGIN.format(
encode_uri_component('urn:mace:incommon:nyu.edu')),
'logout_url':
SHIBBOLETH_SP_LOGOUT.format(
encode_uri_component(
'https://shibboleth.nyu.edu/idp/profile/Logout')),
'domains': ['osf.nyu.edu'],
'email_domains': [],
},
{
'_id':
'ucsd',
'name':
'University of California San Diego',
'description':
'This service is supported on campus by the UC San Diego Library for our research community. Do not use this service to store or transfer personally identifiable information, personal health information, or any other controlled unclassified information. For assistance please contact the Library\'s Research Data Curation Program at <a href="mailto:[email protected]">[email protected]</a>.',
'banner_name':
'ucsd-banner.png',
'logo_name':
'ucsd-shield.png',
'auth_url':
SHIBBOLETH_SP_LOGIN.format(
encode_uri_component('urn:mace:incommon:ucsd.edu')),
'logout_url':
SHIBBOLETH_SP_LOGOUT.format(
encode_uri_component('https://osf.io/goodbye')),
'domains': ['osf.ucsd.edu'],
'email_domains': [],
},
{
'_id':
'ucr',
'name':
'University of California Riverside',
'description':
'Policy prohibits storing PII or HIPAA data on this site, please see C&C\'s <a href="http://cnc.ucr.edu/security/researchers.html">security site</a> for more information.',
'banner_name':
'ucr-banner.png',
'logo_name':
'ucr-shield.png',
'auth_url':
SHIBBOLETH_SP_LOGIN.format(
encode_uri_component('urn:mace:incommon:ucr.edu')),
'logout_url':
SHIBBOLETH_SP_LOGOUT.format(
encode_uri_component('https://osf.io/goodbye')),
'domains': ['osf.ucr.edu'],
'email_domains': [],
},
{
'_id':
'ugent',
'name':
'Universiteit Gent',
'description':
None,
'banner_name':
'ugent-banner.png',
'logo_name':
'ugent-shield.png',
'auth_url':
SHIBBOLETH_SP_LOGIN.format(
encode_uri_component(
'https://identity.ugent.be/simplesaml/saml2/idp/metadata.php'
)),
'logout_url':
SHIBBOLETH_SP_LOGOUT.format(
encode_uri_component('https://osf.io/goodbye')),
'domains': ['osf.ugent.be'],
'email_domains': [],
},
{
'_id':
'usc',
'name':
'University of Southern California',
'description':
'Projects must abide by <a href="http://policy.usc.edu/info-security/">USC\'s Information Security Policy</a>. Data stored for human subject research repositories must abide by <a href="http://policy.usc.edu/biorepositories/">USC\'s Biorepository Policy</a>. The OSF may not be used for storage of Personal Health Information that is subject to <a href="http://policy.usc.edu/hipaa/">HIPPA regulations</a>.',
'banner_name':
'usc-banner.png',
'logo_name':
'usc-shield.png',
'auth_url':
SHIBBOLETH_SP_LOGIN.format(
encode_uri_component('urn:mace:incommon:usc.edu')),
'logout_url':
SHIBBOLETH_SP_LOGOUT.format(
encode_uri_component('https://osf.io/goodbye')),
'domains': ['osf.usc.edu'],
'email_domains': [],
},
{
'_id':
'uva',
'name':
'University of Virginia',
'description':
'In partnership with the <a href="http://www.virginia.edu/vpr/">Vice President for Research</a>, <a href="http://dsi.virginia.edu">Data Science Institute</a>, <a href="https://www.hsl.virginia.edu">Health Sciences Library</a>, and <a href="http://data.library.virginia.edu">University Library</a>. Learn more about <a href="http://cadre.virginia.edu">UVA resources for computational and data-driven research</a>. Projects must abide by the <a href="http://www.virginia.edu/informationpolicy/security.html">University Security and Data Protection Policies</a>.',
'banner_name':
'uva-banner.png',
'logo_name':
'uva-shield.png',
'auth_url':
SHIBBOLETH_SP_LOGIN.format(
encode_uri_component('urn:mace:incommon:virginia.edu')),
'logout_url':
SHIBBOLETH_SP_LOGOUT.format(
encode_uri_component('https://osf.io/goodbye')),
'domains': ['osf.virginia.edu'],
'email_domains': [],
},
{
'_id':
'vt',
'name':
'Virginia Tech',
'description':
None,
'banner_name':
'vt-banner.png',
'logo_name':
'vt-shield.png',
'auth_url':
SHIBBOLETH_SP_LOGIN.format(
encode_uri_component('urn:mace:incommon:vt.edu')),
'logout_url':
SHIBBOLETH_SP_LOGOUT.format(
encode_uri_component('https://osf.io/goodbye')),
'domains': ['osf.vt.edu'],
'email_domains': [],
},
]
if env == 'stage':
INSTITUTIONS = [
{
'_id': 'cos',
'name': 'Center For Open Science [Stage]',
'description': 'Center for Open Science [Stage]',
'banner_name': 'cos-banner.png',
'logo_name': 'cos-shield.png',
'auth_url': None,
'logout_url': None,
'domains': ['staging-osf.cos.io'],
'email_domains': ['cos.io'],
},
{
'_id':
'nd',
'name':
'University of Notre Dame [Stage]',
'description':
'University of Notre Dame [Stage]',
'banner_name':
'nd-banner.png',
'logo_name':
'nd-shield.png',
'auth_url':
SHIBBOLETH_SP_LOGIN.format(
encode_uri_component(
'https://login-test.cc.nd.edu/idp/shibboleth')),
'logout_url':
SHIBBOLETH_SP_LOGOUT.format(
encode_uri_component('https://staging.osf.io/goodbye')),
'domains': ['staging-osf-nd.cos.io'],
'email_domains': [],
},
{
'_id': 'google',
'name': 'Google [Stage]',
'description': 'Google [Stage]',
'banner_name': 'google-banner.png',
'logo_name': 'google-shield.png',
'auth_url': None,
'logout_url': None,
'domains': [],
'email_domains': ['gmail.com'],
},
{
'_id': 'yahoo',
'name': 'Yahoo [Stage]',
'description': 'Yahoo [Stage]',
'banner_name': 'yahoo-banner.png',
'logo_name': 'yahoo-shield.png',
'auth_url': None,
'domains': [],
'email_domains': ['yahoo.com'],
},
]
if env == 'stage2':
INSTITUTIONS = [
{
'_id': 'cos',
'name': 'Center For Open Science [Stage2]',
'description': 'Center for Open Science [Stage2]',
'banner_name': 'cos-banner.png',
'logo_name': 'cos-shield.png',
'auth_url': None,
'logout_url': None,
'domains': ['staging2-osf.cos.io'],
'email_domains': ['cos.io'],
},
]
elif env == 'test':
INSTITUTIONS = [
{
'_id': 'busara',
'name': 'Busara Center for Behavioral Economics [Test]',
'description':
'The <a href="http://www.busaracenter.org/">Busara Center</a> for Behavioral Economics',
'banner_name': 'busara-banner.png',
'logo_name': 'busara-shield.png',
'auth_url': None,
'logout_url': None,
'domains': [],
'email_domains': ['busaracenter.org'],
},
{
'_id': 'cos',
'name': 'Center For Open Science [Test]',
'description':
'COS is a non-profit technology company providing free and open services to increase inclusivity and transparency of research. Find out more at <a href="https://cos.io">cos.io</a>.',
'banner_name': 'cos-banner.png',
'logo_name': 'cos-shield.png',
'auth_url': None,
'logout_url': None,
'domains': ['test-osf.cos.io'],
'email_domains': ['cos.io'],
},
{
'_id': 'esip',
'name':
'Federation of Earth Science Information Partners (ESIP) [Test]',
'description':
'<a href="http://www.esipfed.org/">ESIP\'s</a> mission is to support the networking and data dissemination needs of our members and the global Earth science data community by linking the functional sectors of observation, research, application, education and use of Earth science.',
'banner_name': 'esip-banner.png',
'logo_name': 'esip-shield.png',
'auth_url': None,
'logout_url': None,
'domains': [],
'email_domains': ['esipfed.org'],
},
{
'_id': 'ljaf',
'name': 'Laura and John Arnold Foundation [Test]',
'description':
'Projects listed below are for grants awarded by the Foundation. Please see the <a href="http://www.arnoldfoundation.org/wp-content/uploads/Guidelines-for-Investments-in-Research.pdf">LJAF Guidelines for Investments in Research</a> for more information and requirements.',
'banner_name': 'ljaf-banner.png',
'logo_name': 'ljaf-shield.png',
'auth_url': None,
'logout_url': None,
'domains': [],
'email_domains': ['arnoldfoundation.org'],
},
{
'_id':
'nd',
'name':
'University of Notre Dame [Test]',
'description':
'In <a href="https://research.nd.edu/news/64035-notre-dame-center-for-open-science-partner-to-advance-open-science-initiatives/">partnership</a> with the <a href="https://crc.nd.edu">Center for Research Computing</a>, <a href="http://esc.nd.edu">Engineering & Science Computing</a>, and the <a href="https://library.nd.edu">Hesburgh Libraries</a>',
'banner_name':
'nd-banner.png',
'logo_name':
'nd-shield.png',
'auth_url':
SHIBBOLETH_SP_LOGIN.format(
encode_uri_component(
'https://login-test.cc.nd.edu/idp/shibboleth')),
'logout_url':
SHIBBOLETH_SP_LOGOUT.format(
encode_uri_component('https://test.osf.io/goodbye')),
'domains': ['test-osf-nd.cos.io'],
'email_domains': [],
},
{
'_id':
'nyu',
'name':
'New York University [Test]',
'description':
'A Research Project and File Management Tool for the NYU Community: <a href="https://www.nyu.edu/research.html">Research at NYU</a> | <a href="http://guides.nyu.edu/data_management">Research Data Management Planning</a> | <a href="https://library.nyu.edu/services/research/">NYU Library Research Services</a> | <a href="https://nyu.qualtrics.com/jfe6/form/SV_8dFc5TpA1FgLUMd">Get Help</a>',
'banner_name':
'nyu-banner.png',
'logo_name':
'nyu-shield.png',
'auth_url':
SHIBBOLETH_SP_LOGIN.format(
encode_uri_component(
'https://shibbolethqa.es.its.nyu.edu/idp/shibboleth')),
'logout_url':
SHIBBOLETH_SP_LOGOUT.format(
encode_uri_component(
'https://shibbolethqa.es.its.nyu.edu/idp/profile/Logout'
)),
'domains': ['test-osf-nyu.cos.io'],
'email_domains': [],
},
{
'_id':
'ucsd',
'name':
'University of California San Diego [Test]',
'description':
'This service is supported on campus by the UC San Diego Library for our research community. Do not use this service to store or transfer personally identifiable information, personal health information, or any other controlled unclassified information. For assistance please contact the Library\'s Research Data Curation Program at <a href="mailto:[email protected]">[email protected]</a>.',
'banner_name':
'ucsd-banner.png',
'logo_name':
'ucsd-shield.png',
'auth_url':
SHIBBOLETH_SP_LOGIN.format(
encode_uri_component('urn:mace:incommon:ucsd.edu')),
'logout_url':
SHIBBOLETH_SP_LOGOUT.format(
encode_uri_component('https://osf.io/goodbye')),
'domains': ['test-osf-ucsd.cos.io'],
'email_domains': [],
},
{
'_id':
'ucr',
'name':
'University of California Riverside [Test]',
'description':
'Policy prohibits storing PII or HIPAA data on this site, please see C&C\'s <a href="http://cnc.ucr.edu/security/researchers.html">security site</a> for more information.',
'banner_name':
'ucr-banner.png',
'logo_name':
'ucr-shield.png',
'auth_url':
SHIBBOLETH_SP_LOGIN.format(
encode_uri_component('urn:mace:incommon:ucr.edu')),
'logout_url':
SHIBBOLETH_SP_LOGOUT.format(
encode_uri_component('https://test.osf.io/goodbye')),
'domains': ['test-osf-ucr.cos.io'],
'email_domains': [],
},
{
'_id':
'ugent',
'name':
'Universiteit Gent [Test]',
'description':
'Universiteit Gent [Test]',
'banner_name':
'ugent-banner.png',
'logo_name':
'ugent-shield.png',
'auth_url':
SHIBBOLETH_SP_LOGIN.format(
encode_uri_component(
'https://identity.ugent.be/simplesaml/saml2/idp/metadata.php'
)),
'logout_url':
SHIBBOLETH_SP_LOGOUT.format(
encode_uri_component('https://test.osf.io/goodbye')),
'domains': ['test-osf-ugent.cos.io'],
'email_domains': [],
},
{
'_id':
'usc',
'name':
'University of Southern California [Test]',
'description':
'Projects must abide by <a href="http://policy.usc.edu/info-security/">USC\'s Information Security Policy</a>. Data stored for human subject research repositories must abide by <a href="http://policy.usc.edu/biorepositories/">USC\'s Biorepository Policy</a>. The OSF may not be used for storage of Personal Health Information that is subject to <a href="http://policy.usc.edu/hipaa/">HIPPA regulations</a>.',
'banner_name':
'usc-banner.png',
'logo_name':
'usc-shield.png',
'auth_url':
SHIBBOLETH_SP_LOGIN.format(
encode_uri_component('urn:mace:incommon:usc.edu')),
'logout_url':
SHIBBOLETH_SP_LOGOUT.format(
encode_uri_component('https://test.osf.io/goodbye')),
'domains': ['test-osf-usc.cos.io'],
'email_domains': [],
},
{
'_id':
'uva',
'name':
'University of Virginia [Test]',
'description':
'In partnership with the <a href="http://www.virginia.edu/vpr/">Vice President for Research</a>, <a href="http://dsi.virginia.edu">Data Science Institute</a>, <a href="https://www.hsl.virginia.edu">Health Sciences Library</a>, and <a href="http://data.library.virginia.edu">University Library</a>. Learn more about <a href="http://cadre.virginia.edu">UVA resources for computational and data-driven research</a>. Projects must abide by the <a href="http://www.virginia.edu/informationpolicy/security.html">University Security and Data Protection Policies</a>.',
'banner_name':
'uva-banner.png',
'logo_name':
'uva-shield.png',
'auth_url':
SHIBBOLETH_SP_LOGIN.format(
encode_uri_component(
'https://shibidp-test.its.virginia.edu/idp/shibboleth')
),
'logout_url':
SHIBBOLETH_SP_LOGOUT.format(
encode_uri_component('https://test.osf.io/goodbye')),
'domains': ['test-osf-virginia.cos.io'],
'email_domains': [],
},
{
'_id':
'vt',
'name':
'Virginia Tech [Test]',
'description':
None,
'banner_name':
'vt-banner.png',
'logo_name':
'vt-shield.png',
'auth_url':
SHIBBOLETH_SP_LOGIN.format(
encode_uri_component(
'https://shib-pprd.middleware.vt.edu')),
'logout_url':
SHIBBOLETH_SP_LOGOUT.format(
encode_uri_component('https://test.osf.io/goodbye')),
'domains': ['osf.vt.edu'],
'email_domains': [],
},
]
init_app(routes=False)
with TokuTransaction():
for inst_data in INSTITUTIONS:
new_inst, inst_created = update_or_create(inst_data)
# update the nodes elastic docs, to have current names of institutions. This will
# only work properly if this file is the only thing changing institution attributes
if not inst_created:
nodes = Node.find_by_institutions(new_inst,
query=Q(
'is_deleted', 'ne',
True))
for node in nodes:
update_node(node, async=False)
for extra_inst in Institution.find(
Q('_id', 'nin', [x['_id'] for x in INSTITUTIONS])):
logger.warn('Extra Institution : {} - {}'.format(
extra_inst._id, extra_inst.name))
def test_querying_on_id():
# Test that queries on _id on models that inherit from ObjectIDMixin are
# translated properly
archive_target = ArchiveTarget(name='s3', stat_result={}, errors=[])
archive_target.save()
assert archive_target in ArchiveTarget.find(Q('_id', 'eq', archive_target._id))
import sys
import logging
from modularodm import Q
from nose.tools import * # noqa (PEP8 asserts)
from website.app import init_app
from framework.auth.core import User
from tests.base import OsfTestCase
from tests.factories import UserFactory, ProjectFactory
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
QUERY = Q('date_confirmed', 'ne', None) & Q('unclaimed_records', 'ne', {})
def do_migration(dry=False):
"""Clear unclaimed_records for confirmed users."""
n_migrated = 0
for user in get_targets():
n_migrated += 1
logger.info('Clearing unclaimed records for {0!r}'.format(user))
if not dry:
user.unclaimed_records = {}
user.save()
logger.info('Migrated {0} records.'.format(n_migrated))
return n_migrated
def find_queued_mails_ready_to_be_sent():
return QueuedMail.find(
Q('send_at', 'lt', timezone.now()) &
Q('sent_at', 'eq', None)
)
def update_node(node, index=None, bulk=False):
index = index or INDEX
from website.addons.wiki.model import NodeWikiPage
category = get_doctype_from_node(node)
elastic_document_id = node._id
parent_id = node.parent_id
from website.files.models.osfstorage import OsfStorageFile
for file_ in paginated(OsfStorageFile, Q('node', 'eq', node)):
update_file(file_, index=index)
if node.is_deleted or not node.is_public or node.archiving:
delete_doc(elastic_document_id, node)
else:
try:
normalized_title = six.u(node.title)
except TypeError:
normalized_title = node.title
normalized_title = unicodedata.normalize('NFKD',
normalized_title).encode(
'ascii', 'ignore')
elastic_document = {
'id':
elastic_document_id,
'contributors': [{
'fullname': x.fullname,
'url': x.profile_url if x.is_active else None
} for x in node.visible_contributors if x is not None],
'title':
node.title,
'normalized_title':
normalized_title,
'category':
category,
'public':
node.is_public,
'tags': [tag._id for tag in node.tags if tag],
'description':
node.description,
'url':
node.url,
'is_registration':
node.is_registration,
'is_pending_registration':
node.is_pending_registration,
'is_retracted':
node.is_retracted,
'is_pending_retraction':
node.is_pending_retraction,
'embargo_end_date':
node.embargo_end_date.strftime("%A, %b. %d, %Y")
if node.embargo_end_date else False,
'is_pending_embargo':
node.is_pending_embargo,
'registered_date':
node.registered_date,
'wikis': {},
'parent_id':
parent_id,
'date_created':
node.date_created,
'license':
serialize_node_license_record(node.license),
'affiliated_institutions':
[inst.name for inst in node.affiliated_institutions],
'boost':
int(not node.is_registration) +
1, # This is for making registered projects less relevant
}
if not node.is_retracted:
for wiki in [
NodeWikiPage.load(x)
for x in node.wiki_pages_current.values()
]:
elastic_document['wikis'][wiki.page_name] = wiki.raw_text(node)
if bulk:
return elastic_document
else:
es.index(index=index,
doc_type=category,
id=elastic_document_id,
body=elastic_document,
refresh=True)
def find_candidate_parents(node):
return Node.find(
Q('logs', 'eq', node.logs[0]._id) &
Q('is_fork', 'eq', node.is_fork) &
Q('is_registration', 'eq', node.is_registration)
)
def test_filter_build(self):
qs = TestFile._filter(Q('test', 'eq', 'test'))
_, is_file, provider = qs.nodes
assert_equal(is_file.__dict__, Q('is_file', 'eq', True).__dict__)
assert_equal(provider.__dict__, Q('provider', 'eq', 'test').__dict__)
def find_templated_orphans():
return Node.find(
Q('template_node', 'ne', None) &
Q('category', 'ne', 'project') &
Q('__backrefs.parent.node.nodes.0', 'exists', False)
)
def find_nested_projects():
return Node.find(
Q('__backrefs.parent.node.nodes.0', 'exists', True)
& Q('category', 'eq', 'project') & Q('is_deleted', 'eq', False))
def search_projects_by_title(**kwargs):
""" Search for nodes by title. Can pass in arguments from the URL to modify the search
:arg term: The substring of the title.
:arg category: Category of the node.
:arg isDeleted: yes, no, or either. Either will not add a qualifier for that argument in the search.
:arg isFolder: yes, no, or either. Either will not add a qualifier for that argument in the search.
:arg isRegistration: yes, no, or either. Either will not add a qualifier for that argument in the search.
:arg includePublic: yes or no. Whether the projects listed should include public projects.
:arg includeContributed: yes or no. Whether the search should include projects the current user has
contributed to.
:arg ignoreNode: a list of nodes that should not be included in the search.
:return: a list of dictionaries of projects
"""
# TODO(fabianvf): At some point, it would be nice to do this with elastic search
user = kwargs['auth'].user
term = request.args.get('term', '')
max_results = int(request.args.get('maxResults', '10'))
category = request.args.get('category', 'project').lower()
is_deleted = request.args.get('isDeleted', 'no').lower()
is_collection = request.args.get('isFolder', 'no').lower()
is_registration = request.args.get('isRegistration', 'no').lower()
include_public = request.args.get('includePublic', 'yes').lower()
include_contributed = request.args.get('includeContributed', 'yes').lower()
ignore_nodes = request.args.getlist('ignoreNode', [])
matching_title = (
Q('title', 'icontains', term) & # search term (case insensitive)
Q('category', 'eq', category) # is a project
)
matching_title = conditionally_add_query_item(matching_title, 'is_deleted',
is_deleted)
matching_title = conditionally_add_query_item(matching_title,
'is_collection',
is_collection)
matching_title = conditionally_add_query_item(matching_title,
'is_registration',
is_registration)
if len(ignore_nodes) > 0:
for node_id in ignore_nodes:
matching_title = matching_title & Q('_id', 'ne', node_id)
my_projects = []
my_project_count = 0
public_projects = []
if include_contributed == 'yes':
my_projects = Node.find(
matching_title
& Q('contributors', 'eq', user._id) # user is a contributor
).limit(max_results)
my_project_count = my_project_count
if my_project_count < max_results and include_public == 'yes':
public_projects = Node.find(matching_title
& Q('is_public', 'eq', True) # is public
).limit(max_results - my_project_count)
results = list(my_projects) + list(public_projects)
ret = process_project_search_results(results, **kwargs)
return ret
def default_node_list_query():
return (Q('is_deleted', 'ne', True) & Q('type', 'eq', 'osf.node'))
def get_events(self, date):
super(UserSummary, self).get_events(date)
# Convert to a datetime at midnight for queries and the timestamp
timestamp_datetime = datetime(date.year, date.month,
date.day).replace(tzinfo=pytz.UTC)
query_datetime = timestamp_datetime + timedelta(1)
active_user_query = (Q('is_registered', 'eq', True)
& Q('password', 'ne', None)
& Q('merged_by', 'eq', None)
& Q('date_disabled', 'eq', None)
& Q('date_confirmed', 'ne', None)
& Q('date_confirmed', 'lt', query_datetime))
active_users = 0
depth_users = 0
profile_edited = 0
user_pages = paginated(User, query=active_user_query)
for user in user_pages:
active_users += 1
log_count = count_user_logs(user)
if log_count >= LOG_THRESHOLD:
depth_users += 1
if user.social or user.schools or user.jobs:
profile_edited += 1
counts = {
'keen': {
'timestamp': timestamp_datetime.isoformat()
},
'status': {
'active':
active_users,
'depth':
depth_users,
'unconfirmed':
User.find(
Q('date_registered', 'lt', query_datetime)
& Q('date_confirmed', 'eq', None)).count(),
'deactivated':
User.find(
Q('date_disabled', 'ne', None)
& Q('date_disabled', 'lt', query_datetime)).count(),
'merged':
User.find(
Q('date_registered', 'lt', query_datetime)
& Q('merged_by', 'ne', None)).count(),
'profile_edited':
profile_edited
}
}
logger.info(
'Users counted. Active: {}, Depth: {}, Unconfirmed: {}, Deactivated: {}, Merged: {}, Profile Edited: {}'
.format(counts['status']['active'], counts['status']['depth'],
counts['status']['unconfirmed'],
counts['status']['deactivated'],
counts['status']['merged'],
counts['status']['profile_edited']))
return [counts]
def figshare_view_file(*args, **kwargs):
auth = kwargs['auth']
node = kwargs['node'] or kwargs['project']
node_settings = kwargs['node_addon']
article_id = kwargs.get('aid') or None
file_id = kwargs.get('fid') or None
anonymous = has_anonymous_link(node, auth)
if not article_id or not file_id:
raise HTTPError(http.NOT_FOUND)
connect = Figshare.from_settings(node_settings.user_settings)
if node_settings.figshare_type == 'project':
item = connect.project(node_settings, node_settings.figshare_id)
else:
item = connect.article(node_settings, node_settings.figshare_id)
if article_id not in str(item):
raise HTTPError(http.NOT_FOUND)
article = connect.article(node_settings, article_id)
found = False
for f in article['items'][0]['files']:
if f['id'] == int(file_id):
found = f
break
if not found:
raise HTTPError(http.NOT_FOUND)
try:
# If GUID has already been created, we won't redirect, and can check
# whether the file exists below
guid = FigShareGuidFile.find_one(
Q('node', 'eq', node) & Q('article_id', 'eq', article_id)
& Q('file_id', 'eq', file_id))
except:
guid = FigShareGuidFile(node=node,
article_id=article_id,
file_id=file_id)
guid.save()
redirect_url = check_file_guid(guid)
if redirect_url:
return redirect(redirect_url)
private = not (article['items'][0]['status'] == 'Public')
figshare_url = 'http://figshare.com/'
if private:
figshare_url += 'preview/_preview/{0}'.format(
article['items'][0]['article_id'])
else:
figshare_url += 'articles/{0}/{1}'.format(
article['items'][0]['title'].replace(' ', '_'),
article['items'][0]['article_id'])
version_url = "http://figshare.com/articles/{filename}/{file_id}".format(
filename=article['items'][0]['title'],
file_id=article['items'][0]['article_id'])
download_url = node.api_url + 'figshare/download/article/{aid}/file/{fid}'.format(
aid=article_id, fid=file_id)
render_url = node.api_url + \
'figshare/render/article/{aid}/file/{fid}'.format(aid=article_id, fid=file_id)
delete_url = node.api_url + 'figshare/article/{aid}/file/{fid}/'.format(
aid=article_id, fid=file_id)
filename = found['name']
cache_file_name = get_cache_file(article_id, file_id)
rendered = get_cache_content(node_settings, cache_file_name)
if private:
rendered = messages.FIGSHARE_VIEW_FILE_PRIVATE.format(
url='http://figshare.com/')
elif rendered is None:
filename, size, filedata = connect.get_file(node_settings, found)
if figshare_settings.MAX_RENDER_SIZE is not None and size > figshare_settings.MAX_RENDER_SIZE:
rendered = messages.FIGSHARE_VIEW_FILE_OVERSIZED.format(
url=found.get('download_url'))
else:
rendered = get_cache_content(
node_settings,
cache_file_name,
start_render=True,
remote_path=filename,
file_content=filedata,
download_url=download_url,
)
# categories = connect.categories()['items'] # TODO Cache this
# categories = ''.join(
# ["<option value='{val}'>{label}</option>".format(val=i['id'], label=i['name']) for i in categories])
rv = {
'node': {
'id': node._id,
'title': node.title
},
'file_name':
filename,
'rendered':
rendered,
'file_status':
article['items'][0]['status'],
'file_version':
article['items'][0]['version'],
'doi':
'http://dx.doi.org/10.6084/m9.figshare.{0}'.format(
article['items'][0]['article_id']),
'parent_type':
'fileset'
if article['items'][0]['defined_type'] == 'fileset' else 'singlefile',
'parent_id':
article['items'][0]['article_id'],
# 'figshare_categories': categories,
'figshare_title':
article['items'][0]['title'],
'figshare_desc':
article['items'][0]['description'],
'render_url':
render_url,
'urls': {
'render': render_url,
'download': found.get('download_url'),
'version': version_url,
'figshare': privacy_info_handle(figshare_url, anonymous),
'delete': delete_url,
'files': node.web_url_for('collect_file_trees')
}
}
rv.update(_view_project(node, auth, primary=True))
return rv
def get_default_odm_query(self):
owner = self.request.user
return (Q('owner', 'eq', owner) & Q('is_active', 'eq', True))
def parity_targets():
# TODO: Add metadata.parity information from wb so we do not need to check remote services
return models.FileVersion.find(
Q('status', 'ne', 'cached') & Q('location.object', 'exists', True)
# & Q('metadata.parity', 'eq', None)
)
def get_default_odm_query(self):
return Q('is_deleted', 'ne', True)
def get_targets():
return Node.find(Q('is_deleted', 'ne', True))
def get_default_odm_query(self):
user_id = self.request.user._id
return (Q('owner', 'eq', user_id) & Q('is_active', 'eq', True))
def auth_callback(self, user):
"""Exchange temporary credentials for permanent credentials
This is called in the view that handles the user once they are returned
to the OSF after authenticating on the external service.
"""
session = get_session()
# make sure the user has temporary credentials for this provider
try:
cached_credentials = session.data['oauth_states'][self.short_name]
except KeyError:
raise PermissionsError("OAuth flow not recognized.")
if self._oauth_version == OAUTH1:
request_token = request.args.get('oauth_token')
# make sure this is the same user that started the flow
if cached_credentials.get('token') != request_token:
raise PermissionsError("Request token does not match")
response = OAuth1Session(
client_key=self.client_id,
client_secret=self.client_secret,
resource_owner_key=cached_credentials.get('token'),
resource_owner_secret=cached_credentials.get('secret'),
verifier=request.args.get('oauth_verifier'),
).fetch_access_token(self.callback_url)
elif self._oauth_version == OAUTH2:
state = request.args.get('state')
# make sure this is the same user that started the flow
if cached_credentials.get('state') != state:
raise PermissionsError("Request token does not match")
try:
response = OAuth2Session(
self.client_id,
redirect_uri=web_url_for(
'oauth_callback',
service_name=self.short_name,
_absolute=True
),
).fetch_token(
self.callback_url,
client_secret=self.client_secret,
code=request.args.get('code'),
)
except (MissingTokenError, RequestsHTTPError):
raise HTTPError(http.SERVICE_UNAVAILABLE)
# pre-set as many values as possible for the ``ExternalAccount``
info = self._default_handle_callback(response)
# call the hook for subclasses to parse values from the response
info.update(self.handle_callback(response))
try:
# create a new ``ExternalAccount`` ...
self.account = ExternalAccount(
provider=self.short_name,
provider_id=info['provider_id'],
provider_name=self.name,
)
self.account.save()
except KeyExistsException:
# ... or get the old one
self.account = ExternalAccount.find_one(
Q('provider', 'eq', self.short_name) &
Q('provider_id', 'eq', info['provider_id'])
)
assert self.account is not None
# ensure that provider_name is correct
self.account.provider_name = self.name
# required
self.account.oauth_key = info['key']
# only for OAuth1
self.account.oauth_secret = info.get('secret')
# only for OAuth2
self.account.expires_at = info.get('expires_at')
self.account.refresh_token = info.get('refresh_token')
# additional information
self.account.display_name = info.get('display_name')
self.account.profile_url = info.get('profile_url')
self.account.save()
# add it to the user's list of ``ExternalAccounts``
if self.account not in user.external_accounts:
user.external_accounts.append(self.account)
user.save()
def get_default_odm_query(self):
return (Q('is_registered', 'eq', True) & Q('is_merged', 'ne', True)
& Q('date_disabled', 'eq', None))
