def test_batch_delete_all_documents():
keys = []
u = DeleteAllModel.collection.create(name="Azeem1")
keys.append(u.key)
u = DeleteAllModel.collection.create(name="Arfan1")
keys.append(u.key)
u = DeleteAllModel.collection.create(name="Haider1")
keys.append(u.key)
users = DeleteAllModel.collection.get_all(keys)
for user in users:
assert user.name in ['Azeem1', 'Arfan1', 'Haider1']
batch = fireo.batch()
DeleteAllModel.collection.delete_all(keys, batch)
for user in users:
assert user.name in ['Azeem1', 'Arfan1', 'Haider1']
batch.commit()
users = DeleteAllModel.collection.get_all(keys)
for user in users:
assert user is None
def test_simple_batch_write():
batch = fireo.batch()
BatchCity.collection.create(batch=batch, name='CA', population=100)
c = BatchCity()
c.name = 'LA'
c.population = 200
c.save(batch=batch)
cities = BatchCity.collection.fetch()
index = 0
for city in cities:
index += 1
assert index == 0
batch.commit()
cities = BatchCity.collection.fetch()
index = 0
for city in cities:
assert city.name in ['CA', 'LA']
index += 1
assert index >= 2
def test_issue_128_in_document_creation():
batch = fireo.batch()
create_ref = City.collection.create(batch=batch,
state='NYC',
population=500000)
assert isinstance(create_ref, DocumentReference)
def test_issue_128_in_document_update():
batch = fireo.batch()
city = City.collection.create(state='NYC', population=500000)
city = City()
city.population = 1000000
upadte_ref = city.update(key=city.key, batch=batch)
assert isinstance(upadte_ref, DocumentReference)
def test_batch_delete_with_filter():
batch = fireo.batch()
BatchCity.collection.create(name='LA', population=100)
BatchCity.collection.filter('name', '==', 'LA').batch(batch).delete()
city = BatchCity.collection.filter('name', '==', 'LA').get()
assert city.name == 'LA'
batch.commit()
city = BatchCity.collection.filter('name', '==', 'LA').get()
assert city is None
def test_batch_update():
batch = fireo.batch()
c = BatchCity.collection.create(name='CA', population=300)
city_key = c.key
c.population = 400
c.update(batch=batch)
city = BatchCity.collection.get(city_key)
assert city.population == 300
batch.commit()
city = BatchCity.collection.get(city_key)
assert city.population == 400
def test_batch_delete():
batch = fireo.batch()
BatchCity.collection.batch(batch).delete()
cities = BatchCity.collection.fetch()
index = 0
for city in cities:
index += 1
assert index >= 2
batch.commit()
cities = BatchCity.collection.fetch()
index = 0
for city in cities:
index += 1
assert index == 0
def store_n_gram_chunk(
n_gram_chunk: List[AlmostCompleteIndexedEventGram],
credentials_file: str,
) -> None:
"""
Write all IndexedEventGrams in a single batch.
This isn't about an atomic batch but reducing the total upload time.
"""
# Init batch
batch = fireo.batch()
# Trigger upserts for all items
event_lut: Dict[str, db_models.Event] = {}
for almost_complete_ieg in n_gram_chunk:
if almost_complete_ieg.event_id not in event_lut:
event_lut[
almost_complete_ieg.event_id] = db_models.Event.collection.get(
f"{db_models.Event.collection_name}/{almost_complete_ieg.event_id}"
)
# Construct true ieg
ieg = db_models.IndexedEventGram()
ieg.event_ref = event_lut[almost_complete_ieg.event_id]
ieg.unstemmed_gram = almost_complete_ieg.unstemmed_gram
ieg.stemmed_gram = almost_complete_ieg.stemmed_gram
ieg.context_span = almost_complete_ieg.context_span
ieg.value = almost_complete_ieg.value
ieg.datetime_weighted_value = almost_complete_ieg.datetime_weighted_value
db_functions.upload_db_model(
db_model=ieg,
credentials_file=credentials_file,
batch=batch,
)
# Commit
batch.commit()
######### UCI ###########
# start number: 373652
# end number: 377993
# Book No: 6
#########################
import fireo
import json
from uci import UCI
from models.hadiths.ibnemaja import IbneMaja
ibnemaja_file = open('./data/hadiths/ibne_maja.json', "r", encoding="utf8")
ibnemaja = json.load(ibnemaja_file)
hadith_batch = fireo.batch()
count = 0
overall_count = 0
uci = UCI(373652)
for hadith in ibnemaja:
ibnemaja = IbneMaja()
ibnemaja.id = str(hadith['hadees_number'])
ibnemaja.hadith_number = int(hadith['hadees_number'])
ibnemaja.book_number = int(hadith['Kitab_ID'])
ibnemaja.book_name = {
"urdu": hadith['Kitab'],
"english": hadith['Kitab_Eng']
}
ibnemaja.chapter = {
"urdu": hadith['Baab'],
"english": hadith['Baab_Eng']
}
######### UCI ###########
# start number: 337185
# end number: 343534
#########################
import fireo
from uci import UCI
from models.quran.ayah import Ayah
ar_quran = open('./data/quran/ar.quran_rtl.txt', "r", encoding="utf8")
en_quran = open('./data/quran/en.sahih_ltr.txt', "r", encoding="utf8")
ur_quran = open('./data/quran/ur.maududi_rtl.txt', "r", encoding="utf8")
ayah_batch = fireo.batch()
count = 0
line_count = 0
uci = UCI(337185)
for ar, en, ur in zip(ar_quran, en_quran, ur_quran):
if not en.strip():
print('English file end')
break
surah_number, ayah_number, en_text = en.strip().split('|')
_, _, ur_text = ur.strip().split('|')
_, _, ar_text = ar.strip().split('|')
print(' writing line...' + str(line_count) + ' ...Surah ' + surah_number +
' ...Ayah ' + ayah_number)
line_count += 1
}, {
"id": "fr.leclerc",
"language": "fr",
"name": "Youssouf Leclerc",
"english_name": "Youssouf Leclerc",
"format": "audio",
"type": "versebyverse",
"direction": None
}, {
"id": "ar.aymanswoaid",
"language": "ar",
"name": "أيمن سويد",
"english_name": "Ayman Sowaid",
"format": "audio",
"type": "versebyverse",
"direction": None
}]
edition_batch = fireo.batch()
for e in data:
edition = Edition()
edition.id = e["id"]
edition.language = e["language"]
edition.name = e["name"]
edition.english_name = e["english_name"]
edition.type = 'Audio'
edition.save(batch=edition_batch)
edition_batch.commit()
import requests
import fireo
from models.ayah import Ayah
from models.surah import Surah
surah_batch = fireo.batch()
surah_count = 0
init_surah_number = 0
resp = requests.get('http://api.alquran.cloud/v1/surah')
if resp.status_code == 200:
data = resp.json()['data']
for s in data:
surah = Surah()
surah.id = str(s["number"])
surah.number = s["number"]
surah.name = s["name"]
surah.english_name = s["englishName"]
surah.english_name_translation = s["englishNameTranslation"]
surah.number_of_ayahs = s["numberOfAyahs"]
surah.revelation_type = s["revelationType"]
surah.save(batch=surah_batch)
surah_count += 1
print('Surah complete ', surah_count)
for single_file in onlyfiles:
detail = single_file.replace('.txt', '')
# print(detail)
edition_id, language, name, translator, direction = detail.split('_')
edition = Edition()
edition.id = edition_id
edition.language = language
edition.name = name
edition.translator = translator
edition.save()
file_location = './data/' + single_file
f = open(file_location, "r", encoding="utf8")
trans_batch = fireo.batch()
count = 0
line_count = 0
for line in f:
if not line.strip():
print('file end')
break
# print(line)
surah_number, ayat_number, *line_text = line.strip().split('|')
text = ' '.join(line_text)
print('For File ' + edition_id + ' writing line...' + str(line_count))
trans_id = surah_number + '-' + ayat_number
