def reindex_all_claims(orcidid, since=None, ignore_errors=False):
"""
Procedure that will re-play all claims
that were modified since a given starting point.
"""
last_check = get_date(since or '1974-11-09T22:56:52.518001Z')
recs_modified = set()
with app.session_scope() as session:
author = matcher.retrieve_orcid(orcidid)
claimed = set()
removed = set()
for claim in session.query(ClaimsLog).filter(
and_(ClaimsLog.orcidid == orcidid, ClaimsLog.created > last_check)
).all():
if claim.status in ('claimed', 'updated', 'forced'):
claimed.add(claim.bibcode)
elif claim.status == 'removed':
removed.add(claim.bibcode)
with app.session_scope() as session:
for bibcode in removed:
r = session.query(Records).filter_by(bibcode=bibcode).first()
if r is None:
continue
rec = r.toJSON()
if _remove_orcid(rec, orcidid):
r.claims = json.dumps(rec.get('claims'))
r.updated = get_date()
recs_modified.add(bibcode)
for bibcode in claimed:
r = session.query(Records).filter_by(bibcode=bibcode).first()
if r is None:
continue
rec = r.toJSON()
claim = {'bibcode': bibcode, 'orcidid': orcidid}
claim.update(author.get('facts', {}))
try:
_claims = update_record(rec, claim)
if _claims:
r.claims = json.dumps(rec.get('claims', {}))
r.updated = get_date()
recs_modified.add(bibcode)
except Exception, e:
if ignore_errors:
app.logger.error(u'Error processing {0} {1}'.format(bibcode, orcidid))
else:
raise e
session.commit()
return list(recs_modified)
def sync_buffs():
with open(os.path.join(root_dir, "app/database/data/buffs.json"),
"r") as file:
data = json.load(file)
while True:
response = input(
"Enter a class name (e.g. 'Eliotrope'), type 'update all' to update all classes, or type 'q' to move onto item buffs: "
)
if response == "q":
break
with session_scope() as db_session:
if response == "update all":
for class_name in data["spells"]:
sync_spell_buffs(db_session, class_name,
data["spells"][class_name])
break
elif response in data["spells"]:
all_spell_data = data["spells"][response]
sync_spell_buffs(db_session, response, all_spell_data)
else:
print(
"'{}' class does not exist, please try again.".format(
response))
name_to_item_record_map = {}
for r in data["items"]:
name_to_item_record_map[r["name"]] = r
while True:
response = input(
"Enter an item name (e.g. 'Crimson Dofus') to update its buffs, type 'update all' to update all buffs for items in the file, 'add missing buffs' to only add buffs to items that are missing them, or type 'q' to quit: "
)
if response == "q":
break
with session_scope() as db_session:
if response == "update all":
for record in data["items"]:
update_or_create_item_buff(db_session, record["name"],
record, False)
break
elif response == "add missing buffs":
for record in data["items"]:
update_or_create_item_buff(db_session, record["name"],
record, True)
break
elif response in name_to_item_record_map:
record = name_to_item_record_map[response]
update_or_create_item_buff(db_session, response, record,
False)
else:
print(
"That item does not have a buff or a typo was made, please try again."
)
def sync_set():
with open(os.path.join(app_root, "app/database/data/sets.json"), "r") as file:
data = json.load(file)
name_to_record_map = {}
for set_data in data:
name_to_record_map[set_data["name"]["en"]] = set_data
while True:
response = input(
"Enter a set name, type 'update all' to update all sets, type 'add missing sets' to only add sets that are missing, or type 'q' to quit: "
)
if response == "q":
break
with session_scope() as db_session:
if response == "update all":
for set_data in data:
create_or_update_set(
db_session, set_data["name"]["en"], set_data, False
)
break
elif response == "add missing sets":
for set_data in data:
create_or_update_set(
db_session, set_data["name"]["en"], set_data, True
)
break
elif response in name_to_record_map:
record = name_to_record_map[response]
create_or_update_set(db_session, response, record, False)
else:
print(
"The set {} does not exist. Please try again.".format(response)
)
def sync_custom_set_tag():
print("Loading and processing file...")
with open(
os.path.join(app_root, "app/database/data/custom_set_tags.json"), "r",
) as file:
data = json.load(file)
name_to_record_map = {}
for r in data:
name_to_record_map[r["name"]["en"]] = r
should_prompt = True
while should_prompt:
tag_name = input(
"Enter tag name, e.g. 'Str', type 'update all' to update all tags, or 'q' to quit: "
)
if tag_name == "q":
return
with session_scope() as db_session:
if tag_name == "update all":
should_prompt = False
create_or_update_all_custom_set_tags(db_session, data)
elif tag_name in name_to_record_map:
record = name_to_record_map[tag_name]
create_or_update_custom_set_tag(db_session, tag_name, record)
def mutate(self, info, **kwargs):
with session_scope() as db_session:
equipped_item_id = kwargs.get("equipped_item_id")
weapon_element_mage = kwargs.get("weapon_element_mage")
stats = kwargs.get("stats")
equipped_item = db_session.query(ModelEquippedItem).get(
equipped_item_id)
check_owner(equipped_item.custom_set)
db_session.query(ModelEquippedItemExo).filter_by(
equipped_item_id=equipped_item_id).delete(
synchronize_session=False)
exo_models = map(
lambda stat_line: ModelEquippedItemExo(
stat=Stat(stat_line.stat),
value=stat_line.value,
equipped_item_id=equipped_item_id,
),
stats,
)
if stats:
db_session.add_all(exo_models)
slot_name = (db_session.query(ModelItemSlotTranslation).filter(
ModelItemSlotTranslation.locale == "en").filter(
ModelItemSlotTranslation.item_slot_id ==
equipped_item.slot.uuid).one().name)
if weapon_element_mage and slot_name != "Weapon":
raise GraphQLError(
_("Invalid element mage on non-weapon item."))
equipped_item.weapon_element_mage = (
WeaponElementMage(weapon_element_mage)
if weapon_element_mage else None)
equipped_item.custom_set.last_modified = datetime.utcnow()
return MageEquippedItem(equipped_item=equipped_item)
def unequip_item(self, item_slot_id):
with session_scope() as session:
equipped_item = (session.query(ModelEquippedItem).filter_by(
custom_set_id=self.uuid,
item_slot_id=item_slot_id).one_or_none())
if equipped_item:
session.delete(equipped_item)
def sync_spell_buffs(db_session, class_name, spells):
print(class_name)
name_to_spell_record_map = {}
for spell in spells:
name_to_spell_record_map[spell["name"]] = spell
while True:
response = input(
"Enter a spell name, type 'update all' to update all spell buffs for this class, 'add missing buffs' to only add buffs to spells that are missing them, or type 'q' to quit: "
)
if response == "q":
break
with session_scope() as db_session:
if response == "update all":
for spell in spells:
update_or_create_spell_buff(db_session, spell["name"],
spell, False)
break
elif response == "add missing buffs":
for spell in spells:
update_or_create_spell_buff(db_session, spell["name"],
spell, True)
break
elif response in name_to_spell_record_map:
spell_data = name_to_spell_record_map[response]
update_or_create_spell_buff(db_session, response, spell_data,
False)
else:
print(
"That spell either does not have a buff or a typo was made, please try again."
)
def sync_item_type():
print("Loading and processing file...")
with open(
os.path.join(app_root, "app/database/data/item_types.json"),
"r",
) as file:
data = json.load(file)
name_to_record_map = {}
for r in data:
name_to_record_map[r["en"]] = r
should_prompt_item = True
while should_prompt_item:
item_name = input(
"Enter item type name, e.g. 'Prysmaradite', type 'update all' to update all item types, or 'q' to quit: "
)
if item_name == "q":
return
with session_scope() as db_session:
if item_name == "update all":
should_prompt_item = False
for record in data:
recreate_item_type_translations(
db_session, record["en"], record)
elif item_name in name_to_record_map:
record = name_to_record_map[item_name]
recreate_item_type_translations(db_session, item_name,
record)
def record_claims(bibcode, claims, authors=None):
"""
Stores results of the processing in the database.
:param: bibcode
:type: string
:param: claims
:type: dict
"""
if not isinstance(claims, basestring):
claims = json.dumps(claims)
if authors and not isinstance(authors, basestring):
authors = json.dumps(authors)
with app.session_scope() as session:
if not isinstance(claims, basestring):
claims = json.dumps(claims)
r = session.query(Records).filter_by(bibcode=bibcode).first()
if r is None:
t = get_date()
r = Records(bibcode=bibcode,
claims=claims,
created=t,
updated=t,
authors=authors
)
session.add(r)
else:
r.updated = get_date()
r.claims = claims
if authors:
r.authors = authors
session.merge(r)
session.commit()
def sync_spell():
print("Loading and processing file...")
with open(os.path.join(app_root, "app/database/data/spells.json"), "r") as file:
data = json.load(file)
name_to_record_map = {}
for r in data:
class_name = r["names"]["en"]
for pair in r["spells"]:
for spell in pair:
name_to_record_map[spell["name"]["en"]] = spell
should_prompt_spell = True
while should_prompt_spell:
response = input(
"Enter spell name, e.g. 'Blindness' or multiple spell names separated "
"by commas, type 'update all' to update all spells in file, or 'q' to quit: "
)
if response == "q":
return
with session_scope() as db_session:
if response == "update all":
should_prompt_spell = False
for record in data:
update_spell(db_session, record["name"]["en"], record)
else:
spell_names = response.split(",")
print(spell_names)
for name in spell_names:
spell_name = name.strip()
if spell_name in name_to_record_map:
record = name_to_record_map[spell_name]
update_spell(db_session, spell_name, record)
def mutate(self, info, **kwargs):
with session_scope() as db_session:
username = kwargs.get("username")
email = kwargs.get("email")
password = kwargs.get("password")
validation.validate_registration(username, email, password)
try:
if current_user.is_authenticated:
raise GraphQLError(_("You are already logged in."))
user = ModelUserAccount(
username=username,
email=email,
password=ModelUserAccount.generate_hash(password),
locale=str(get_locale()),
)
token = encode_token(user.email, verify_email_salt)
verify_url = generate_url("verify_email.verify_email", token)
template = template_env.get_template("verify_email.html")
content = template.render(display_name=username,
verify_url=verify_url)
db_session.add(user)
db_session.flush()
q.enqueue(send_email, user.email,
_("Verify your DofusLab account"), content)
login_user(user)
save_custom_sets(db_session)
except Exception as e:
raise GraphQLError(_("An error occurred while registering."))
return RegisterUser(user=user, ok=True)
def mutate(self, info, **kwargs):
custom_set_id = kwargs.get("custom_set_id")
with session_scope() as db_session:
old_custom_set = db_session.query(ModelCustomSet).get(
custom_set_id)
custom_set = get_or_create_custom_set(None, db_session)
custom_set.name = (_(
"%(old_name)s copy",
old_name=old_custom_set.name)[:MAX_NAME_LENGTH]
if old_custom_set.name else _("Copy"))
custom_set.level = old_custom_set.level
custom_set.parent_custom_set_id = old_custom_set.uuid
for stat in base_stat_list + scrolled_stat_list:
setattr(custom_set.stats, stat,
getattr(old_custom_set.stats, stat))
for old_equipped_item in old_custom_set.equipped_items:
equipped_item = ModelEquippedItem(
custom_set_id=custom_set.uuid,
item_slot_id=old_equipped_item.item_slot_id,
item_id=old_equipped_item.item_id,
weapon_element_mage=old_equipped_item.weapon_element_mage,
)
db_session.add(equipped_item)
db_session.flush()
for old_exo in old_equipped_item.exos:
exo = ModelEquippedItemExo(
stat=old_exo.stat,
value=old_exo.value,
equipped_item_id=equipped_item.uuid,
)
db_session.add(exo)
return CopyCustomSet(custom_set=custom_set)
def test_db():
init_db()
info_test = Info(content='test', link='link.test')
with session_scope() as session:
assert info_test == session.query(Info).filter(
Info.content == 'test').all()
def send_email(email, subject, content):
with session_scope() as db_session:
ses = boto3.client(
"ses",
region_name=os.getenv("SES_REGION"),
aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
)
ses.send_email(
Source=os.getenv("SES_EMAIL_SOURCE"),
Destination={"ToAddresses": [email]},
Message={
"Subject": {
"Data": subject
},
"Body": {
"Html": {
"Data": content
}
},
},
)
user = db_session.query(ModelUserAccount).filter_by(email=email).one()
user.email_sent = True
return True
def mutate(self, info, **kwargs):
with session_scope() as db_session:
custom_set_id = kwargs.get("custom_set_id")
stats = kwargs.get("stats")
custom_set = get_or_create_custom_set(custom_set_id, db_session)
edit_custom_set_stats(custom_set, stats)
return EditCustomSetStats(custom_set=custom_set)
def mutate(self, info, **kwargs):
custom_set_id = kwargs.get("custom_set_id")
with session_scope() as db_session:
custom_set = db_session.query(ModelCustomSet).get(custom_set_id)
check_owner(custom_set)
db_session.delete(custom_set)
return DeleteCustomSet(ok=True)
def add_weapons():
print("Adding weapons to database")
with open(os.path.join(app_root, "app/database/data/weapons.json"), "r") as file:
with session_scope() as db_session:
data = json.load(file)
for record in data:
oneoff.sync_item.create_item(db_session, record)
def add_sets_and_items():
print("Adding sets to database")
with open(os.path.join(app_root, "app/database/data/sets.json"), "r") as file:
with session_scope() as db_session:
data = json.load(file)
for record in data:
create_set(db_session, record)
print("Adding items to database")
with open(os.path.join(app_root, "app/database/data/items.json"), "r") as file:
with session_scope() as db_session:
data = json.load(file)
for record in data:
if record["itemType"] == "Living object":
continue
oneoff.sync_item.create_item(db_session, record)
def mutate(self, info, **kwargs):
with session_scope() as db_session:
custom_set_id = kwargs.get("custom_set_id")
item_slot_id = kwargs.get("item_slot_id")
custom_set = db_session.query(ModelCustomSet).get(custom_set_id)
check_owner(custom_set)
custom_set.unequip_item(item_slot_id)
return DeleteCustomSetItem(custom_set=custom_set)
def mutate(self, info, **kwargs):
with session_scope() as db_session:
custom_set_id = kwargs.get("custom_set_id")
set_id = kwargs.get("set_id")
custom_set = get_or_create_custom_set(custom_set_id, db_session)
set_obj = db_session.query(ModelSet).get(set_id)
custom_set.equip_set(set_obj, db_session)
return EquipSet(custom_set=custom_set)
def mutate(self, info, **kwargs):
with session_scope() as db_session:
custom_set_id = kwargs.get("custom_set_id")
item_slot_id = kwargs.get("item_slot_id")
item_id = kwargs.get("item_id")
custom_set = get_or_create_custom_set(custom_set_id, db_session)
custom_set.equip_item(item_id, item_slot_id, db_session)
return UpdateCustomSetItem(custom_set=custom_set)
def mutate(self, info, **kwargs):
with session_scope():
user = current_user._get_current_object()
classic = kwargs.get("classic")
if current_user.is_authenticated:
user.settings.classic = classic
session["classic"] = classic
refresh()
return ChangeClassic(ok=True)
def mutate(self, info, **kwargs):
with session_scope() as db_session:
custom_set_id = kwargs.get("custom_set_id")
name = kwargs.get("name")
level = kwargs.get("level")
custom_set = get_or_create_custom_set(custom_set_id, db_session)
edit_custom_set_metadata(custom_set, name, level)
return EditCustomSetMetadata(custom_set=custom_set)
def load_data(date_time):
"""Query data from database"""
# extract function to extract day, month, year in date_added of database
# a simple query data if the day of query is the same of Info.date_added
# use context manager of session_scope()
with session_scope() as session:
return session.query(Info).filter(
extract('day', Info.date_added) == date_time.day,
extract('month', Info.date_added) == date_time.month,
extract('year', Info.date_added) == date_time.year).all()
def mutate(self, info, **kwargs):
with session_scope() as db_session:
custom_set_id = kwargs.get("custom_set_id")
item_ids = kwargs.get("item_ids")
custom_set = get_or_create_custom_set(custom_set_id, db_session)
items = db_session.query(ModelItem).filter(
ModelItem.uuid.in_(item_ids))
custom_set.equip_items(items, db_session)
return EquipMultipleItems(custom_set=custom_set)
def mutate(self, info, **kwargs):
if not current_user.is_authenticated:
raise GraphQLError(_("You are not logged in."))
item_id = kwargs.get("item_id")
is_favorite = kwargs.get("is_favorite")
user_account_id = current_user.get_id()
with session_scope() as db_session:
ModelFavoriteItem.toggle_favorite(db_session, user_account_id,
item_id, is_favorite)
return ToggleFavoriteItem(user=current_user._get_current_object())
def save_data(url, items):
"""Save data crawled to database"""
data = json.loads(extract_data(url, items))
# data = extract_data(url, items)
with session_scope() as session:
for link, content in data.items():
info = Info(link=link, content=content)
session.add(info)
session.commit()
def mutate(self, info, **kwargs):
locale = kwargs.get("locale")
if not locale in supported_languages:
raise GraphQLError(_("Received unsupported locale."))
with session_scope():
user = current_user._get_current_object()
if current_user.is_authenticated:
user.locale = locale
session["locale"] = locale
refresh()
return ChangeLocale(ok=True)
def clean_data():
"""Clean old data from database"""
# use context manager of session_scope()
with session_scope() as session:
old_data = session.query(Info).filter(
extract('day', Info.date_added) < datetime.today().day,
extract('month', Info.date_added) <= datetime.today().month,
extract('year', Info.date_added) <= datetime.today().year).all()
for data in old_data:
session.delete(data)
session.commit()
def sync_name():
while True:
file_name = input(
"Enter JSON file name without extension ({}) or q to quit: ".
format(", ".join(allowed_file_names)))
if file_name == "q":
break
if file_name in allowed_file_names:
with session_scope() as db_session:
update_name(db_session, file_name)
else:
print("File name not allowed or is incorrect. Please try again.")
def verify_email(token):
with session_scope() as db_session:
email = decode_token(token, verify_email_salt)
if not email:
return redirect(REDIRECT_URL_FORMAT.format(REDIRECT_URL,
"invalid"))
user = db_session.query(ModelUserAccount).filter_by(email=email).one()
if user.verified:
return redirect(
REDIRECT_URL_FORMAT.format(REDIRECT_URL, "already_verified"))
user.verified = True
db_session.add(user)
return redirect(REDIRECT_URL_FORMAT.format(REDIRECT_URL, "success"))
def add_classes_and_spells():
print("Adding classes to database")
with open(os.path.join(app_root, "app/database/data/spells.json"), "r") as file:
with session_scope() as db_session:
data = json.load(file)
for record in data:
en_name = record["names"]["en"]
class_object = ModelClass(
face_image_url=face_url_base.format(en_name),
male_sprite_image_url=male_sprite_url_base.format(en_name),
female_sprite_image_url=female_sprite_url_base.format(en_name),
)
db_session.add(class_object)
db_session.flush()
for locale in record["names"]:
class_translation = ModelClassTranslation(
class_id=class_object.uuid,
locale=locale,
name=record["names"][locale],
)
db_session.add(class_translation)
class_object.name.append(class_translation)
for spell_pair in record["spells"]:
spell_pair_object = ModelSpellVariantPair(
class_id=class_object.uuid
)
db_session.add(spell_pair_object)
db_session.flush()
for spell in spell_pair:
spell_object = ModelSpell(
spell_variant_pair_id=spell_pair_object.uuid,
image_url=spell["imageUrl"],
is_trap=spell.get("isTrap", False),
)
db_session.add(spell_object)
db_session.flush()
for locale in spell["name"]:
spell_translation = ModelSpellTranslation(
spell_id=spell_object.uuid,
locale=locale,
name=spell["name"][locale],
description=spell["description"][locale],
)
db_session.add(spell_translation)
spell_object.spell_translation.append(spell_translation)
create_spell_stats(db_session, spell, spell_object)
def mark_processed(bibcode):
"""Updates the date on which the record has been processed (i.e.
something has consumed it
:param: bibcode
:type: str
:return: None
"""
with app.session_scope() as session:
r = session.query(Records).filter_by(bibcode=bibcode).first()
if r is None:
raise IgnorableException('Nonexistant record for {0}'.format(bibcode))
r.processed = get_date()
session.commit()
return True
def update_author(author):
"""Updates existing AuthorInfo records.
It will check for new information. If there is a difference,
updates the record and also records the old values.
:param: author - AuthorInfo instance
:return: AuthorInfo object
:sideeffect: Will insert new records (ChangeLog) and also update
the author instance
"""
try:
new_facts = harvest_author_info(author.orcidid)
except:
return author.toJSON()
info = author.toJSON()
with session_scope() as session:
old_facts = info['facts']
attrs = set(new_facts.keys())
attrs = attrs.union(old_facts.keys())
is_dirty = False
for attname in attrs:
if old_facts.get(attname, None) != new_facts.get(attname, None):
session.add(ChangeLog(key=u'{0}:update:{1}'.format(author.orcidid, attname),
oldvalue=json.dumps(old_facts.get(attname, None)),
newvalue=json.dumps(new_facts.get(attname, None))))
is_dirty = True
if bool(author.account_id) != bool(new_facts.get('authorized', False)):
author.account_id = new_facts.get('authorized', False) and 1 or None
if is_dirty:
author.facts = json.dumps(new_facts)
author.name = new_facts.get('name', author.name)
aid=author.id
session.commit()
return session.query(AuthorInfo).filter_by(id=aid).first().toJSON()
else:
return info
def retrieve_orcid(orcid):
"""
Finds (or creates and returns) model of ORCID
from the dbase. It will automatically update our
knowledge about the author every time it gets
called.
:param orcid - String (orcid id)
:return - OrcidModel datastructure
"""
with session_scope() as session:
u = session.query(AuthorInfo).filter_by(orcidid=orcid).first()
if u is not None:
return update_author(u)
u = create_orcid(orcid)
session.add(u)
session.commit()
return session.query(AuthorInfo).filter_by(orcidid=orcid).first().toJSON()
def retrieve_record(bibcode):
"""
Gets a record from the database (creates one if necessary)
"""
with app.session_scope() as session:
r = session.query(Records).filter_by(bibcode=bibcode).first()
if r is None:
r = Records(bibcode=bibcode)
session.add(r)
out = r.toJSON()
metadata = retrieve_metadata(bibcode)
authors = metadata.get('author', [])
if out.get('authors') != authors:
r.authors = json.dumps(authors)
out['authors'] = authors
session.commit()
return out
