def create_user_tags():
logger.info('create user_tags')
user_tags_by_name = {}
user = User.query.filter_by(email="*****@*****.**").one()
tag = Tag.query.filter_by(text="coral").one()
user_tags_by_name["reviewer 0 / coral"] = UserTag(
user=user,
tag=tag
)
user = User.query.filter_by(email="*****@*****.**").one()
tag = Tag.query.filter_by(text="immunology").one()
user_tags_by_name["reviewer 2 / immunology"] = UserTag(
user=user,
tag=tag
)
ApiHandler.save(*user_tags_by_name.values())
logger.info('created {} user_tags_by_name'.format(len(user_tags_by_name)))
return user_tags_by_name
def create_roles():
logger.info('create_roles')
roles_by_name = {}
for user in User.query.all():
regexp = r'{}test.(.[a-z]+).(.*)@{}.{}'.format(COMMAND_NAME, APP_NAME,
TLD)
user_type = re.match(regexp, user.email).group(1)
if user_type == "master":
for role_type in RoleType:
roles_by_name['{} {}'.format(user.email, role_type)] = Role(
type=role_type.value, user=user)
elif user_type != 'user':
roles_by_name['{} {}'.format(user.email,
user_type)] = Role(type=getattr(
RoleType, user_type).value,
user=user)
ApiHandler.save(*roles_by_name.values())
logger.info('created {} roles'.format(len(roles_by_name)))
return roles_by_name
def get_articles_from_climate_feedback_feedbacks_scrap(articles_max=3,
editor_user=None,
store=None):
# store is an object that sets in it
# all the currently being created objects
# as { articles: [], ..., users: [] }
# because we don't want to create a new user object
# when we scrap a review article made by an already created user one.
if store is None:
store = {"users": []}
result = requests.get('https://climatefeedback.org/feedbacks')
soup = BeautifulSoup(result.text, 'html.parser')
articles = []
evaluation_rows = soup.find("main").find_all("div", class_="row")
logger.info('{} evaluation rows and we pick {} max'.format(
len(evaluation_rows), articles_max))
for (evaluation_row_index,
evaluation_row) in enumerate(evaluation_rows[:articles_max]):
logger.info('article {}...'.format(evaluation_row_index))
evaluation_media_body = evaluation_row.find("div", class_="media-body")
article = Article()
evaluation_url = evaluation_media_body.find("a")['href']
set_article_from_climate_feedback_evaluation_scrap(
article, evaluation_url, editor_user, store)
articles.append(article)
return articles
def create_scopes():
logger.info('create_scopes')
scopes_by_name = {}
for article_tag in ARTICLE_TAGS:
tag = Tag.query.filter_by(text=article_tag['text']).one()
scopes_by_name["article {}".format(article_tag)] = Scope(
tag=tag, type=ScopeType.article.value)
for review_verdict_tag in REVIEW_VERDICT_TAGS:
tag = Tag.query.filter_by(text=review_verdict_tag['text']).one()
scopes_by_name["review {}".format(review_verdict_tag)] = Scope(
tag=tag, type=ScopeType.review.value)
scopes_by_name["verdict {}".format(review_verdict_tag)] = Scope(
tag=tag, type=ScopeType.verdict.value)
for user_tag in USER_TAGS:
tag = Tag.query.filter_by(text=user_tag['text']).one()
scopes_by_name["review {}".format(user_tag)] = Scope(
tag=tag,
type=ScopeType.user.value,
)
ApiHandler.save(*scopes_by_name.values())
logger.info('created {} scopes'.format(len(scopes_by_name)))
return scopes_by_name
def create_verdicts():
logger.info('create_verdicts')
verdicts_by_name = {}
articles = Article.query.all()
user = User.query.filter_by(
email="*****@*****.**").one()
article = Article.query.filter_by(
url=
"https://www.breitbart.com/big-government/2017/03/20/delingpole-great-barrier-reef-still-not-dying-whatever-washington-post-says"
).one()
evaluation_value = -2
evaluation = Evaluation.query.filter_by(type="article",
value=evaluation_value).one()
verdicts_by_name[article.id] = Verdict(
article=article,
comment=
'{"blocks":[{"key":"2l86g","text":"C\'est abusé, voici mon verdict lol","type":"unstyled","depth":0,"inlineStyleRanges":[],"entityRanges":[],"data":{}}],"entityMap":{}}',
evaluation=evaluation,
rating=evaluation_value,
user=user)
ApiHandler.save(*verdicts_by_name.values())
logger.info('created {} verdicts'.format(len(verdicts_by_name)))
return verdicts_by_name
def create_user_tags():
logger.info('create user_tags')
user_tags_by_name = {}
user = User.query.filter_by(email="{}test.reviewer0@{}.{}".format(COMMAND_NAME, APP_NAME, TLD)).one()
tag = Tag.query.filter_by(text="coral").one()
user_tags_by_name["reviewer 0 / coral"] = UserTag(
user=user,
tag=tag
)
user = User.query.filter_by(email="{}test.reviewer2@{}.{}".format(COMMAND_NAME, APP_NAME, TLD)).one()
tag = Tag.query.filter_by(text="immunology").one()
user_tags_by_name["reviewer 2 / immunology"] = UserTag(
user=user,
tag=tag
)
ApiHandler.save(*user_tags_by_name.values())
logger.info('created {} user_tags_by_name'.format(len(user_tags_by_name)))
return user_tags_by_name
def clean_all_database():
""" Order of deletions matters because of foreign key constraints """
logger.info("clean all the database...")
Scope.query.delete()
Appearance.query.delete()
ArticleTag.query.delete()
ReviewTag.query.delete()
UserTag.query.delete()
VerdictTag.query.delete()
Tag.query.delete()
ReviewTag.query.delete()
Review.query.delete()
Evaluation.query.delete()
VerdictUser.query.delete()
Verdict.query.delete()
UserArticle.query.delete()
Article.query.delete()
Role.query.delete()
UserSession.query.delete()
User.query.delete()
ClaimClaim.query.delete()
Claim.query.delete()
versioning_manager.activity_cls.query.delete()
db.session.commit()
logger.info("clean all the database...Done.")
def sync(max_records=None, sync_to_airtable=False):
logger.info('sync science feedback airtable data...')
create_or_modify_sf_organization_and_media()
for name in NAME_TO_AIRTABLE:
sync_for(name, max_records=max_records, sync_to_airtable=sync_to_airtable)
logger.info('sync science feedback airtable data...Done.')
def create_scopes():
logger.info('create_scopes')
scopes = []
ApiHandler.save(*scopes)
logger.info('created {} scopes'.format(len(scopes)))
def clean():
logger.info('clean all the database...')
for table in reversed(db.metadata.sorted_tables):
print("Clearing table {table_name}...".format(table_name=table))
db.session.execute(table.delete())
db.session.commit()
logger.info('clean all the database...Done.')
def create_user_articles():
logger.info('create_user_articles')
article_users_by_name = {}
logger.info('created {} article_users_by_name'.format(
len(article_users_by_name)))
return article_users_by_name
def create_users():
logger.info('create_users')
hashed_default_password = get_hashed_default_password()
users_by_name = {}
user_types = [role_type.value
for role_type in RoleType] + ['master', 'user']
count = 0
for user_type in user_types:
for role_index in range(USERS_BY_TYPE_COUNT):
user_dict = {
"email": get_sandbox_role_email(user_type, role_index),
"firstName": pick_one(FIRST_NAMES, count),
"lastName": pick_one(LAST_NAMES, count),
}
if user_type == 'reviewer':
user_dict.update({
"academicWebsite":
"https://fr.wikipedia.org/wiki/Christiane_Faure",
"affiliation":
pick_one(AFFILIATIONS, count),
"expertise":
"{}, {}".format(pick_one(EXPERTISE, count),
pick_one(EXPERTISE, count + 1)),
"orcidId":
"0000-0003-3164-2391",
"title":
pick_one(TITLES, count)
})
if user_type == 'editor':
user_dict.update({
"affiliation": "Climate Feedback",
"title": "Science Editor"
})
user = User(**user_dict)
user.password = hashed_default_password
users_by_name['{} {}'.format(user_type, role_index)] = user
count += 1
ApiHandler.save(*users_by_name.values())
for user_type in user_types:
for role_index in range(USERS_BY_TYPE_COUNT):
store_public_object_from_sandbox_assets(
"thumbs", users_by_name['{} {}'.format(user_type, role_index)],
"{}_{}".format(user_type, role_index))
logger.info('created {} users'.format(len(users_by_name)))
return users_by_name
def assert_created_counts(**counts_by_model_name):
models = import_models()
for model in models:
model_name = model.__name__
if model_name not in counts_by_model_name:
logger.info('{} is not in counts_by_model_name, ... is it normal ?'.format(model_name))
continue
print(model_name, model.query.count(), counts_by_model_name[model_name])
assert model.query.count() == counts_by_model_name[model_name]
def claim_verdicts_from_airtable(verdicts_to_sync=None,
max_verdicts=None,
sync_async=False):
if verdicts_to_sync is None:
query = Verdict.query.filter(Verdict.scienceFeedbackUrl != None)
if max_verdicts is not None:
query = query.limit(max_verdicts)
verdicts = query.all()
else:
verdicts = verdicts_to_sync
if max_verdicts is not None:
max_verdicts = len(verdicts)
urls = [verdict.scienceFeedbackUrl for verdict in verdicts][:max_verdicts]
if sync_async:
claim_reviews = map_asynchronous(claim_review_from_url, urls)
else:
claim_reviews = [claim_review_from_url(url) for url in urls]
for (index, verdict) in enumerate(verdicts):
claim_review = claim_reviews[index]
if not claim_review:
continue
for conclusion in claim_review['conclusions']:
try:
tag = Tag.create_or_modify({
'__SEARCH_BY__': ['label', 'type'],
'label': conclusion,
'type': TagType.CONCLUSION
})
if tag.id is None:
logger.info('Saving tag {}'.format(as_dict(tag)))
ApiHandler.save(tag)
verdict_tag = VerdictTag.create_or_modify({
'__SEARCH_BY__': ['tagId', 'verdictId'],
'tagId':
humanize(tag.id),
'verdictId':
humanize(verdict.id)
})
verdict.verdictTags = verdict.verdictTags + [verdict_tag]
except IntegrityError as e:
logger.error('IntegrityError: {}, Conclusion: {}'.format(
e, conclusion))
except InvalidRequestError as e:
logger.error('InvalidRequestError: {}, Conclusion: {}'.format(
e, conclusion))
except NotNullViolation as violation:
logger.error('NotNullViolation: {}, Conclusion: {}'.format(
violation, conclusion))
return verdicts
def create_sandbox():
from repository.tags import sync as sync_tags
from repository.science_feedback import sync as sync_science_feedback
from repository.users import sync as sync_users
logger.info('create_sf_sandbox...')
sync_tags()
sync_science_feedback()
sync_users()
logger.info('create_sf_sandbox...Done.')
def sync_outdated_rows(max_records=None):
logger.info('sync science feedback outdated airtable data...')
for name in NAME_TO_AIRTABLE:
sync_for(
name,
formula='FIND("Out of sync", {Sync status})>0',
max_records=max_records,
sync_to_airtable=True
)
logger.info('sync science feedback outdated airtable data...Done.')
def create_reviews():
logger.info('create_reviews')
reviews_by_name = {}
article = Article.query.filter_by(
url=
'https://www.breitbart.com/big-government/2017/03/20/delingpole-great-barrier-reef-still-not-dying-whatever-washington-post-says'
).one()
evaluation = Evaluation.query.filter_by(type='article', value=1).one()
user = User.query.filter_by(email='{}test.reviewer0@{}.{}'.format(
COMMAND_NAME, APP_NAME, TLD)).one()
reviews_by_name['Great Barrier / reviewer 0'] = Review(
article=article,
comment=
'{"blocks":[{"key":"2l86g","text":"C\'est neutre mais pas tout à fait","type":"unstyled","depth":0,"inlineStyleRanges":[],"entityRanges":[],"data":{}}],"entityMap":{}}',
evaluation=evaluation,
rating=1,
user=user)
article = Article.query.filter_by(
url=
'http://www.dailymail.co.uk/sciencetech/article-4192182/World-leaders-duped-manipulated-global-warming-data.html'
).one()
evaluation = Evaluation.query.filter_by(type='article', value=-1).one()
user = User.query.filter_by(email='{}test.reviewer0@{}.{}'.format(
COMMAND_NAME, APP_NAME, TLD)).one()
reviews_by_name['Daily Mail inflates disagreement / reviewer 0'] = Review(
article=article,
comment=
'{"blocks":[{"key":"2l86g","text":"C\'est pas très précis","type":"unstyled","depth":0,"inlineStyleRanges":[],"entityRanges":[],"data":{}}],"entityMap":{}}',
evaluation=evaluation,
rating=-1,
user=user)
article = Article.query.filter_by(
url=
'http://www.dailymail.co.uk/sciencetech/article-4192182/World-leaders-duped-manipulated-global-warming-data.html'
).one()
evaluation = Evaluation.query.filter_by(type='article', value=-2).one()
user = User.query.filter_by(email='{}test.reviewer1@{}.{}'.format(
COMMAND_NAME, APP_NAME, TLD)).one()
reviews_by_name['Daily Mail inflates disagreement / reviewer 1'] = Review(
article=article,
comment=
'{"blocks":[{"key":"2l86g","text":"On peut dire que c\'est pourri.","type":"unstyled","depth":0,"inlineStyleRanges":[],"entityRanges":[],"data":{}}],"entityMap":{}}',
evaluation=evaluation,
rating=-2,
user=user)
ApiHandler.save(*reviews_by_name.values())
logger.info('created {} reviews'.format(len(reviews_by_name)))
return reviews_by_name
def create_tags():
logger.info('create_tags')
tags = []
for tag in TAGS:
tags.append(Tag.create_or_modify(tag))
ApiHandler.save(*tags)
logger.info('created {} tags'.format(len(tags)))
def sync():
logger.info('sync tags data...')
tags = []
for tag_dict in TAGS:
tag = Tag.create_or_modify({
'__SEARCH_BY__': ['label', 'type'],
**tag_dict
})
tags.append(tag)
ApiHandler.save(*tags)
logger.info('sync tags data...Done.')
def create_users():
logger.info('create_users')
users_by_name = {}
default_user = create_default_user()
user_types = [role_type.value for role_type in RoleType] + ['master', 'user']
count = 0
for user_type in user_types:
for role_index in range(USERS_BY_TYPE_COUNT):
user_dict = {
'email': get_sandbox_role_email(user_type, role_index),
'firstName': pick_one(FIRST_NAMES, count),
'lastName': pick_one(LAST_NAMES, count),
}
if user_type == 'reviewer':
user_dict.update({
'academicWebsite': 'https://fr.wikipedia.org/wiki/Christiane_Faure',
'affiliation': pick_one(AFFILIATIONS, count),
'expertise': '{}, {}'.format(
pick_one(EXPERTISE, count),
pick_one(EXPERTISE, count + 1)
),
'orcidId': '0000-0003-3164-2391',
'title': pick_one(TITLES, count)
})
if user_type == 'editor':
user_dict.update({
'affiliation': 'Climate Feedback',
'title': 'Science Editor'
})
user = User(**user_dict)
user.password = default_user.password
users_by_name['{} {}'.format(user_type, role_index)] = user
count += 1
ApiHandler.save(*users_by_name.values())
for user_type in user_types:
for role_index in range(USERS_BY_TYPE_COUNT):
store_public_object_from_sandbox_assets(
'thumbs',
users_by_name['{} {}'.format(user_type, role_index)],
'{}_{}'.format(user_type, role_index)
)
logger.info('created {} users'.format(len(users_by_name)))
def create_arguments():
logger.info('create_arguments')
arguments_by_name = {}
arguments_by_name["Great Barrier"] = Argument(comment="")
ApiHandler.save(*arguments_by_name.values())
logger.info('created {} arguments'.format(len(arguments_by_name)))
return arguments_by_name
def create_claims():
claims = []
claims.append(Claim(text='global warming is caused by solar cycle'))
claims.append(Claim(text='clem is the best parapentiste boy'))
ApiHandler.save(*claims)
logger.info('created {} claims'.format(len(claims)))
return claims
def create_article_tags():
logger.info('create_article_tags')
article_tags_by_name = {}
article = Article.query.filter_by(
url=
"https://www.breitbart.com/big-government/2017/03/20/delingpole-great-barrier-reef-still-not-dying-whatever-washington-post-says"
).one()
tag = Tag.query.filter_by(text="climate").one()
article_tags_by_name["Great Barrier / climate"] = ArticleTag(
article=article, tag=tag)
article = Article.query.filter_by(
url=
"http://www.dailymail.co.uk/sciencetech/article-4192182/World-leaders-duped-manipulated-global-warming-data.html"
).one()
tag = Tag.query.filter_by(text="climate").one()
article_tags_by_name[
"Daily Mail inflates disagreement / climate"] = ArticleTag(
article=article, tag=tag)
article = Article.query.filter_by(
url=
"https://www.washingtonpost.com/news/energy-environment/wp/2017/02/15/its-official-the-oceans-are-losing-oxygen-posing-growing-threats-to-marine-life"
).one()
tag = Tag.query.filter_by(text="climate").one()
article_tags_by_name[
"Decline in the amount of dissolved oxygen / climate"] = ArticleTag(
article=article, tag=tag)
article = Article.query.filter_by(
url=
"https://www.lemonde.fr/sciences/article/2018/07/24/maladie-de-lyme-fronde-contre-la-haute-autorite-de-sante_5335369_1650684.html"
).one()
tag = Tag.query.filter_by(text="health").one()
article_tags_by_name["Infection bactérienne / health"] = ArticleTag(
article=article, tag=tag)
article = Article.query.filter_by(
url="https://www.earth-syst-sci-data.net/7/47/2015/essd-7-47-2015.html"
).one()
tag = Tag.query.filter_by(text="climate").one()
article_tags_by_name["Global carbon budget 2014 / health"] = ArticleTag(
article=article, tag=tag)
ApiHandler.save(*article_tags_by_name.values())
logger.info('created {} article_tags_by_name'.format(
len(article_tags_by_name)))
return article_tags_by_name
def create_tags():
logger.info('create_tags')
tags_by_text = {}
for tag in ALL_TAGS:
tags_by_text[tag['text']] = Tag(**tag)
ApiHandler.save(*tags_by_text.values())
logger.info('created {} tags'.format(len(tags_by_text)))
return tags_by_text
def create_sandbox(**kwargs):
logger.info('create_cf_sandbox...')
editor_user = User(
email="*****@*****.**",
firstName="Emmanuel Vincent",
lastName="Vincent"
)
editor_user.set_password("emmanuel.Vincent.0")
role = Role(editor_user, role_type="editor")
ApiHandler.save(editor_user)
articles = get_articles_from_climate_feedback_feedbacks_scrap(10, editor_user)
ApiHandler.save(*articles)
logger.info('create_cf_sandbox...done')
def create_reviews():
logger.info('create_reviews')
reviews = []
content = Content.query.filter_by(
url=
'https://www.breitbart.com/big-government/2017/03/20/delingpole-great-barrier-reef-still-not-dying-whatever-washington-post-says'
).one()
reviewer = User.query.filter_by(email='{}test.reviewer0@{}.{}'.format(
COMMAND_NAME, APP_NAME, TLD)).one()
reviews.append(
Review(
comment=
'{"blocks":[{"key":"2l86g","text":"C\'est neutre mais pas tout à fait","type":"unstyled","depth":0,"inlineStyleRanges":[],"entityRanges":[],"data":{}}],"entityMap":{}}',
content=content,
reviewer=reviewer))
content = Content.query.filter_by(
url=
'http://www.dailymail.co.uk/sciencetech/article-4192182/World-leaders-duped-manipulated-global-warming-data.html'
).one()
reviewer = User.query.filter_by(email='{}test.reviewer0@{}.{}'.format(
COMMAND_NAME, APP_NAME, TLD)).one()
reviews.append(
Review(
comment=
'{"blocks":[{"key":"2l86g","text":"C\'est pas très précis","type":"unstyled","depth":0,"inlineStyleRanges":[],"entityRanges":[],"data":{}}],"entityMap":{}}',
content=content,
reviewer=reviewer))
content = Content.query.filter_by(
url=
'http://www.dailymail.co.uk/sciencetech/article-4192182/World-leaders-duped-manipulated-global-warming-data.html'
).one()
reviewer = User.query.filter_by(email='{}test.reviewer1@{}.{}'.format(
COMMAND_NAME, APP_NAME, TLD)).one()
reviews.append(
Review(
comment=
'{"blocks":[{"key":"2l86g","text":"On peut dire que c\'est pourri.","type":"unstyled","depth":0,"inlineStyleRanges":[],"entityRanges":[],"data":{}}],"entityMap":{}}',
content=content,
reviewer=reviewer))
ApiHandler.save(*reviews)
logger.info('created {} reviews'.format(len(reviews)))
def create_review_tags():
logger.info('create_review_tags')
review_tags_by_name = {}
article = Article.query.filter_by(
url=
"https://www.breitbart.com/big-government/2017/03/20/delingpole-great-barrier-reef-still-not-dying-whatever-washington-post-says"
).one()
user = User.query.filter_by(
email="*****@*****.**").one()
review = Review.query.filter_by(article=article, user=user).one()
tag = Tag.query.filter_by(text="accurate").one()
review_tags_by_name["Great Barrier / reviewer 1 / accurate"] = ReviewTag(
review=review, tag=tag)
article = Article.query.filter_by(
url=
"http://www.dailymail.co.uk/sciencetech/article-4192182/World-leaders-duped-manipulated-global-warming-data.html"
).one()
user = User.query.filter_by(
email="*****@*****.**").one()
review = Review.query.filter_by(article=article, user=user).one()
tag = Tag.query.filter_by(text="imprecise / unclear").one()
review_tags_by_name[
"Daily Mail inflates disagreement / reviewer 0 / imprecise"] = ReviewTag(
review=review, tag=tag)
article = Article.query.filter_by(
url=
"http://www.dailymail.co.uk/sciencetech/article-4192182/World-leaders-duped-manipulated-global-warming-data.html"
).one()
user = User.query.filter_by(
email="*****@*****.**").one()
review = Review.query.filter_by(article=article, user=user).one()
tag = Tag.query.filter_by(text="imprecise / unclear").one()
review_tags_by_name[
"Daily Mail inflates disagreement / reviewer 1 / imprecise"] = ReviewTag(
review=review, tag=tag)
ApiHandler.save(*review_tags_by_name.values())
logger.info('created {} review_tags_by_name'.format(
len(review_tags_by_name)))
return review_tags_by_name
def create_sandbox(with_capture=False):
logger.info('create_ci_sandbox...')
create_claims()
create_tags()
create_scopes()
create_users()
create_appearances()
create_user_tags()
create_roles()
create_articles(with_capture=with_capture)
create_article_tags()
create_evaluations()
create_reviews()
create_review_tags()
create_verdicts()
create_verdict_tags()
create_verdict_users()
logger.info('create_ci_sandbox...Done.')
def create_user_tags():
logger.info('create user_tags')
user_tags = []
user = User.query.filter_by(email='{}test.reviewer0@{}.{}'.format(
COMMAND_NAME, APP_NAME, TLD)).one()
tag = Tag.query.filter_by(label='Coral').one()
user_tags.append(UserTag(user=user, tag=tag))
user = User.query.filter_by(email='{}test.reviewer2@{}.{}'.format(
COMMAND_NAME, APP_NAME, TLD)).one()
tag = Tag.query.filter_by(label='Immunology').one()
user_tags.append(UserTag(user=user, tag=tag))
ApiHandler.save(*user_tags)
logger.info('created {} user_tags'.format(len(user_tags)))
def create_sandbox(with_capture=False):
logger.info('create_ci_sandbox...')
create_claims()
create_tags()
create_scopes()
create_users()
create_user_tags()
create_roles()
create_contents(with_capture=with_capture)
create_content_tags()
create_author_contents()
create_appearances()
create_reviews()
create_review_tags()
create_verdicts()
create_verdict_tags()
create_verdict_reviewers()
logger.info('create_ci_sandbox...Done.')
