def store_public_object_from_sandbox_assets(folder,
public_object,
thumb_id,
index=0):
dir_path = Path(os.path.dirname(os.path.realpath(__file__)))
plural_model_name = get_model_plural_name(public_object)
thumb_path = dir_path\
/ '..' / '..' / folder / plural_model_name\
/ str(thumb_id)
with open(thumb_path, mode='rb') as thumb_file:
if folder == "thumbs":
save_thumb(public_object,
thumb_file.read(),
index,
convert=False,
symlink_path=thumb_path)
else:
store_public_object(folder,
plural_model_name + '/' +
humanize(public_object.id),
thumb_file.read(),
MIMES_BY_FOLDER[folder],
symlink_path=thumb_path)
ApiHandler.save(public_object)
def create_user_tags():
logger.info('create user_tags')
user_tags_by_name = {}
user = User.query.filter_by(email="{}test.reviewer0@{}.{}".format(COMMAND_NAME, APP_NAME, TLD)).one()
tag = Tag.query.filter_by(text="coral").one()
user_tags_by_name["reviewer 0 / coral"] = UserTag(
user=user,
tag=tag
)
user = User.query.filter_by(email="{}test.reviewer2@{}.{}".format(COMMAND_NAME, APP_NAME, TLD)).one()
tag = Tag.query.filter_by(text="immunology").one()
user_tags_by_name["reviewer 2 / immunology"] = UserTag(
user=user,
tag=tag
)
ApiHandler.save(*user_tags_by_name.values())
logger.info('created {} user_tags_by_name'.format(len(user_tags_by_name)))
return user_tags_by_name
def signup():
check_thumb_in_request(files=request.files, form=request.form)
user_dict = dict({}, **request.form)
if 'thumb' in user_dict:
del user_dict['thumb']
publications = []
for index in [1, 2, 3]:
publication_key = 'publication{}'.format(index)
if publication_key in request.form:
publication_url = request.form[publication_key]
publication = Publication.query.filter_by(url=publication_url).first()
if not publication:
publication = Publication(url=publication_url)
publications.append(publication)
thumb = read_thumb(files=request.files, form=request.form)
check_thumb_quality(thumb)
new_user = User(**user_dict)
ApiHandler.save(new_user, *publications)
save_thumb(new_user, thumb, 0, crop=get_crop(request.form))
login_user(new_user)
stamp_session(new_user)
return jsonify(as_dict(new_user, includes=USER_INCLUDES)), 201
def signup():
check_thumb_in_request(files=request.files, form=request.form)
new_user = User()
new_user_dict = {**request.form}
if 'thumb' in new_user_dict:
del new_user_dict['thumb']
if 'publications' in new_user_dict:
publications = json.loads(new_user_dict['publications'])
new_user_dict['userPublications'] = []
for publication in publications:
new_user_dict['userPublications'].append({
'publication': publication,
'user': new_user
})
thumb = read_thumb(files=request.files, form=request.form)
check_thumb_quality(thumb)
new_user.modify(new_user_dict)
ApiHandler.save(new_user)
save_thumb(new_user, thumb, 0, crop=get_crop(request.form))
login_user(new_user)
stamp_session(new_user)
return jsonify(as_dict(new_user, includes=USER_INCLUDES)), 201
def create_roles():
logger.info('create_roles')
roles_by_name = {}
for user in User.query.all():
regexp = r'{}test.(.[a-z]+).(.*)@{}.{}'.format(COMMAND_NAME, APP_NAME,
TLD)
user_type = re.match(regexp, user.email).group(1)
if user_type == "master":
for role_type in RoleType:
roles_by_name['{} {}'.format(user.email, role_type)] = Role(
type=role_type.value, user=user)
elif user_type != 'user':
roles_by_name['{} {}'.format(user.email,
user_type)] = Role(type=getattr(
RoleType, user_type).value,
user=user)
ApiHandler.save(*roles_by_name.values())
logger.info('created {} roles'.format(len(roles_by_name)))
return roles_by_name
def setup(flask_app,
with_login_manager=False):
flask_app.config['SECRET_KEY'] = '@##&6cweafhv3426445'
flask_app.config['SQLALCHEMY_DATABASE_URI'] = POSTGRES_URL
flask_app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
flask_app.config['TESTING'] = True
db.init_app(flask_app)
@flask_app.teardown_request
def remove_db_session(exc):
try:
db.session.remove()
except AttributeError:
pass
flask_app.app_context().push()
import_models()
if with_login_manager:
from flask_login import LoginManager
flask_app.config['SESSION_COOKIE_HTTPONLY'] = not flask_app.config['TESTING']
flask_app.config['SESSION_COOKIE_SECURE'] = False
flask_app.config['REMEMBER_COOKIE_HTTPONLY'] = not flask_app.config['TESTING']
if not flask_app.config['TESTING']:
flask_app.config['PERMANENT_SESSION_LIFETIME'] = 90 * 24 * 3600
flask_app.config['REMEMBER_COOKIE_DURATION'] = 90 * 24 * 3600
flask_app.config['REMEMBER_COOKIE_SECURE'] = True
login_manager = LoginManager()
login_manager.init_app(flask_app)
import api.utils.login_manager
if IS_APP or IS_WORKER:
from api.utils.celery import CELERY_APP
ApiHandler.set_celery(CELERY_APP, flask_app)
def test_modify_offer_saves_an_update_activity(self, app):
# Given
offer_dict = {'name': 'bar', 'type': 'foo'}
offer = Offer(**offer_dict)
ApiHandler.save(offer)
modify_dict = {'name': 'bor'}
offer.modify(modify_dict)
# When
ApiHandler.save(offer)
# Then
all_activities = Activity.query.all()
offer_activities = offer.__activities__
update_offer_activity = offer_activities[1]
assert len(all_activities) == 2
assert len(offer_activities) == 2
assert update_offer_activity.entityIdentifier == offer.activityIdentifier
assert update_offer_activity.verb == 'update'
assert {
**offer_dict,
**modify_dict
}.items() <= update_offer_activity.datum.items()
assert modify_dict.items() == update_offer_activity.patch.items()
assert offer_dict.items() <= update_offer_activity.oldDatum.items()
def create_verdicts():
logger.info('create_verdicts')
verdicts_by_name = {}
articles = Article.query.all()
user = User.query.filter_by(
email="*****@*****.**").one()
article = Article.query.filter_by(
url=
"https://www.breitbart.com/big-government/2017/03/20/delingpole-great-barrier-reef-still-not-dying-whatever-washington-post-says"
).one()
evaluation_value = -2
evaluation = Evaluation.query.filter_by(type="article",
value=evaluation_value).one()
verdicts_by_name[article.id] = Verdict(
article=article,
comment=
'{"blocks":[{"key":"2l86g","text":"C\'est abusé, voici mon verdict lol","type":"unstyled","depth":0,"inlineStyleRanges":[],"entityRanges":[],"data":{}}],"entityMap":{}}',
evaluation=evaluation,
rating=evaluation_value,
user=user)
ApiHandler.save(*verdicts_by_name.values())
logger.info('created {} verdicts'.format(len(verdicts_by_name)))
return verdicts_by_name
def test_same_insert_activity_should_not_be_recorded_twice_from_dict_edition(
self, app):
# Given
offer_activity_identifier = uuid4()
offer_patch = {'name': 'bar', 'type': 'foo'}
offer_activity1 = Activity(
**{
'modelName': 'Offer',
'dateCreated': '2021-03-31T14:18:52.618Z',
'localIdentifier':
'ec510dea-1087-4770-83be-91c2b91a7d05/2021-03-30T12:21:54.540Z',
'localDossierId': 'AEE8EHCP',
'entityIdentifier': str(offer_activity_identifier),
'patch': offer_patch
})
ApiHandler.activate(offer_activity1)
duplicate_offer_activity = Activity(
**{
'modelName': 'Offer',
'dateCreated': '2021-03-31T14:18:52.618Z',
'localIdentifier':
'ec510dea-1087-4770-83be-91c2b91a7d05/2021-03-30T12:21:54.540Z',
'localDossierId': 'AEE8EHCP',
'entityIdentifier': str(offer_activity_identifier),
'patch': offer_patch
})
# When
ApiHandler.activate(duplicate_offer_activity)
# Then
offer = Offer.query.filter_by(
activityIdentifier=offer_activity_identifier).one()
assert offer.activityIdentifier == offer_activity1.entityIdentifier
assert len(offer.__activities__) == 1
def test_create_delete_activity_with_previous_updated_activities(
self, app):
# Given
offer = Offer(name='bar', type='foo')
ApiHandler.save(offer)
update_activity = Activity(dateCreated=datetime.utcnow(),
entityIdentifier=offer.activityIdentifier,
modelName='Offer',
patch={'name': 'bor'})
delete_activity = Activity(dateCreated=datetime.utcnow(),
entityIdentifier=offer.activityIdentifier,
modelName='Offer',
verb='delete')
# When
ApiHandler.activate(update_activity, delete_activity)
# Then
query_filter = (Activity.data['id'].astext.cast(Integer) == offer.id) & \
(Activity.verb == 'delete')
activity = Activity.query.filter(query_filter).one()
offers = Offer.query.all()
assert len(offers) == 0
assert activity.entityIdentifier == offer.activityIdentifier
def test_create_two_related_entities_with_good_date_created_order(
self, app):
# Given
offer_activity_identifier = uuid4()
offer_patch = {'name': 'bar', 'type': 'foo'}
offer_activity = Activity(dateCreated=datetime.utcnow(),
entityIdentifier=offer_activity_identifier,
patch=offer_patch,
tableName='offer')
stock_activity_identifier = uuid4()
stock_patch = {
'offerActivityIdentifier': offer_activity_identifier,
'price': 3
}
stock_activity = Activity(dateCreated=datetime.utcnow(),
entityIdentifier=stock_activity_identifier,
patch=stock_patch,
tableName='stock')
# When
ApiHandler.activate(stock_activity, offer_activity)
# Then
offer = offer_activity.entity
stock = stock_activity.entity
assert offer == Offer.query.filter_by(
activityIdentifier=offer_activity_identifier).one()
assert stock == Stock.query.filter_by(
activityIdentifier=stock_activity_identifier).one()
assert offer.activityIdentifier == offer_activity.entityIdentifier
assert stock.activityIdentifier == stock_activity.entityIdentifier
assert stock.offerId == offer.id
def test_create_activity_on_not_existing_offer_saves_an_insert_activity(
self, app):
# Given
offer_activity_identifier = uuid4()
patch = {'name': 'bar', 'type': 'foo'}
activity = Activity(dateCreated=datetime.utcnow(),
entityIdentifier=offer_activity_identifier,
patch=patch,
tableName='offer')
# When
ApiHandler.activate(activity)
# Then
all_activities = Activity.query.all()
offer = Offer.query.filter_by(
activityIdentifier=offer_activity_identifier).one()
offer_activities = offer.__activities__
insert_offer_activity = offer_activities[0]
assert len(all_activities) == 1
assert len(offer_activities) == 1
assert insert_offer_activity.entityIdentifier == offer.activityIdentifier
assert insert_offer_activity.verb == 'insert'
assert patch.items() <= insert_offer_activity.datum.items()
assert patch.items() <= insert_offer_activity.patch.items()
assert insert_offer_activity.datum['id'] == humanize(offer.id)
assert insert_offer_activity.patch['id'] == humanize(offer.id)
def create_user_tags():
logger.info('create user_tags')
user_tags_by_name = {}
user = User.query.filter_by(email="*****@*****.**").one()
tag = Tag.query.filter_by(text="coral").one()
user_tags_by_name["reviewer 0 / coral"] = UserTag(
user=user,
tag=tag
)
user = User.query.filter_by(email="*****@*****.**").one()
tag = Tag.query.filter_by(text="immunology").one()
user_tags_by_name["reviewer 2 / immunology"] = UserTag(
user=user,
tag=tag
)
ApiHandler.save(*user_tags_by_name.values())
logger.info('created {} user_tags_by_name'.format(len(user_tags_by_name)))
return user_tags_by_name
def test_create_offer_saves_an_insert_activity(self, app):
# Given
offer_dict = {'name': 'bar', 'type': 'foo'}
offer = Offer(**offer_dict)
# When
ApiHandler.save(offer)
# Then
all_activities = Activity.query.all()
offer_activities = offer.__activities__
insert_offer_activity = offer_activities[0]
insert_offer_activity_from_query = Activity.query \
.filter_by(entityIdentifier=offer.activityIdentifier) \
.one()
assert len(all_activities) == 1
assert len(offer_activities) == 1
assert insert_offer_activity_from_query == insert_offer_activity
assert offer.activityIdentifier == insert_offer_activity.entityIdentifier
assert insert_offer_activity.entityInsertedAt == offer.dateCreated
assert insert_offer_activity.oldDatum == None
assert insert_offer_activity.transaction == None
assert insert_offer_activity.verb == 'insert'
assert offer_dict.items() <= insert_offer_activity.patch.items()
assert offer_dict.items() <= insert_offer_activity.datum.items()
assert insert_offer_activity.datum['id'] == humanize(offer.id)
assert insert_offer_activity.patch['id'] == humanize(offer.id)
def test_returns_total_data_count_with_has_more(self, app):
# Given
offer = Offer(name="foo", type="ThingType.JEUX_ABO")
ApiHandler.save(offer)
page = 4
paginate = 10
prices_length = 35
stocks = []
for price in range(prices_length):
stock = Stock(price=price)
stock.offer = offer
stocks.append(stock)
ApiHandler.save(*stocks)
# When
result = get_result(
Stock,
page=page,
paginate=paginate,
with_total_data_count=True
)
# Then
assert len(result['data']) == page * paginate - prices_length
assert result['has_more'] == False
assert result['total_data_count'] == prices_length
def create_scopes():
logger.info('create_scopes')
scopes_by_name = {}
for article_tag in ARTICLE_TAGS:
tag = Tag.query.filter_by(text=article_tag['text']).one()
scopes_by_name["article {}".format(article_tag)] = Scope(
tag=tag, type=ScopeType.article.value)
for review_verdict_tag in REVIEW_VERDICT_TAGS:
tag = Tag.query.filter_by(text=review_verdict_tag['text']).one()
scopes_by_name["review {}".format(review_verdict_tag)] = Scope(
tag=tag, type=ScopeType.review.value)
scopes_by_name["verdict {}".format(review_verdict_tag)] = Scope(
tag=tag, type=ScopeType.verdict.value)
for user_tag in USER_TAGS:
tag = Tag.query.filter_by(text=user_tag['text']).one()
scopes_by_name["review {}".format(user_tag)] = Scope(
tag=tag,
type=ScopeType.user.value,
)
ApiHandler.save(*scopes_by_name.values())
logger.info('created {} scopes'.format(len(scopes_by_name)))
return scopes_by_name
def save_thumb(model_with_thumb,
thumb,
image_index,
image_type=None,
convert=True,
crop=None,
symlink_path=None,
need_save=True,
store_thumb: Callable = store_public_object):
new_thumb = thumb
if convert:
crop_params = crop if crop is not None else DO_NOT_CROP
new_thumb = standardize_image(thumb, crop_params)
store_thumb('thumbs',
model_with_thumb.get_thumb_storage_id(image_index),
new_thumb,
'image/' + (image_type or 'jpeg'),
symlink_path=symlink_path)
model_with_thumb.thumbCount = model_with_thumb.thumbCount + 1
if need_save:
ApiHandler.save(model_with_thumb)
def change_password(user, password):
User = get_model_with_table_name('user')
if type(user) != User:
user = User.query.filter_by(email=user).one()
user.set_password(password)
user = db.session.merge(user)
ApiHandler.save(user)
def test_404_with_not_existing_id(self, app):
# Given
offer1 = Offer(name='foo', type='bar')
ApiHandler.save(offer1)
# When
with pytest.raises(NotFound):
load_or_404(Offer, humanize(offer1.id + 1))
def edit_verdict(verdict_id):
check_has_role(current_user, 'EDITOR')
verdict = load_or_404(Verdict, verdict_id)
verdict.modify(request.json)
ApiHandler.save(verdict)
return jsonify(as_dict(verdict, includes=VERDICT_INCLUDES)), 201
def create_verdict_reviewer():
check_has_role(current_user, 'EDITOR')
verdict_reviewer = VerdictReviewer()
verdict_reviewer.modify(request.json)
ApiHandler.save(verdict_reviewer)
return jsonify(as_dict(verdict_reviewer)), 201
def create_verdict_user():
check_has_role(current_user, 'editor')
verdict_user = VerdictUser()
verdict_user.populate_from_dict(request.json)
ApiHandler.save(verdict_user)
return jsonify(as_dict(verdict_user)), 201
def delete():
logger.info('Delete all the tables...')
for table in reversed(db.metadata.sorted_tables):
print('Deleting table {table_name}...'.format(table_name=table))
db.session.execute(table.delete())
ApiHandler.get_activity().query.delete()
db.session.commit()
logger.info('Delete all the database...Done.')
def sync_articles(from_date, to_date):
articles = filter_by_activity_date_and_verb(Article.query,
from_date=from_date,
to_date=to_date,
verb='insert').all()
for article in articles:
update_article(article)
ApiHandler.save(*articles)
def create_scopes():
logger.info('create_scopes')
scopes = []
ApiHandler.save(*scopes)
logger.info('created {} scopes'.format(len(scopes)))
def claim_verdicts_from_airtable(verdicts_to_sync=None,
max_verdicts=None,
sync_async=False):
if verdicts_to_sync is None:
query = Verdict.query.filter(Verdict.scienceFeedbackUrl != None)
if max_verdicts is not None:
query = query.limit(max_verdicts)
verdicts = query.all()
else:
verdicts = verdicts_to_sync
if max_verdicts is not None:
max_verdicts = len(verdicts)
urls = [verdict.scienceFeedbackUrl for verdict in verdicts][:max_verdicts]
if sync_async:
claim_reviews = map_asynchronous(claim_review_from_url, urls)
else:
claim_reviews = [claim_review_from_url(url) for url in urls]
for (index, verdict) in enumerate(verdicts):
claim_review = claim_reviews[index]
if not claim_review:
continue
for conclusion in claim_review['conclusions']:
try:
tag = Tag.create_or_modify({
'__SEARCH_BY__': ['label', 'type'],
'label': conclusion,
'type': TagType.CONCLUSION
})
if tag.id is None:
logger.info('Saving tag {}'.format(as_dict(tag)))
ApiHandler.save(tag)
verdict_tag = VerdictTag.create_or_modify({
'__SEARCH_BY__': ['tagId', 'verdictId'],
'tagId':
humanize(tag.id),
'verdictId':
humanize(verdict.id)
})
verdict.verdictTags = verdict.verdictTags + [verdict_tag]
except IntegrityError as e:
logger.error('IntegrityError: {}, Conclusion: {}'.format(
e, conclusion))
except InvalidRequestError as e:
logger.error('InvalidRequestError: {}, Conclusion: {}'.format(
e, conclusion))
except NotNullViolation as violation:
logger.error('NotNullViolation: {}, Conclusion: {}'.format(
violation, conclusion))
return verdicts
def create_verdict():
check_has_role(current_user, 'EDITOR')
verdict = Verdict()
verdict.modify(request.json)
verdict.user = current_user
ApiHandler.save(verdict)
return jsonify(as_dict(verdict, includes=VERDICT_INCLUDES)), 201
def test_404_with_wrong_base_id(self, app):
# Given
offer1 = Offer(name='foo', type='bar')
offer1.id = 6
ApiHandler.save(offer1)
# When
with pytest.raises(NotFound):
load_or_404(Offer, 'AZ')
def create_organization():
content = dict(wikidata_from(request.json['name']))
organization = Organization.create_or_modify(content, 'entity')
ApiHandler.save(organization)
return jsonify(as_dict(organization)), 201
def create_users():
logger.info('create_users')
hashed_default_password = get_hashed_default_password()
users_by_name = {}
user_types = [role_type.value
for role_type in RoleType] + ['master', 'user']
count = 0
for user_type in user_types:
for role_index in range(USERS_BY_TYPE_COUNT):
user_dict = {
"email": get_sandbox_role_email(user_type, role_index),
"firstName": pick_one(FIRST_NAMES, count),
"lastName": pick_one(LAST_NAMES, count),
}
if user_type == 'reviewer':
user_dict.update({
"academicWebsite":
"https://fr.wikipedia.org/wiki/Christiane_Faure",
"affiliation":
pick_one(AFFILIATIONS, count),
"expertise":
"{}, {}".format(pick_one(EXPERTISE, count),
pick_one(EXPERTISE, count + 1)),
"orcidId":
"0000-0003-3164-2391",
"title":
pick_one(TITLES, count)
})
if user_type == 'editor':
user_dict.update({
"affiliation": "Climate Feedback",
"title": "Science Editor"
})
user = User(**user_dict)
user.password = hashed_default_password
users_by_name['{} {}'.format(user_type, role_index)] = user
count += 1
ApiHandler.save(*users_by_name.values())
for user_type in user_types:
for role_index in range(USERS_BY_TYPE_COUNT):
store_public_object_from_sandbox_assets(
"thumbs", users_by_name['{} {}'.format(user_type, role_index)],
"{}_{}".format(user_type, role_index))
logger.info('created {} users'.format(len(users_by_name)))
return users_by_name