def test_batch_delete_all_documents(): keys = [] u = DeleteAllModel.collection.create(name="Azeem1") keys.append(u.key) u = DeleteAllModel.collection.create(name="Arfan1") keys.append(u.key) u = DeleteAllModel.collection.create(name="Haider1") keys.append(u.key) users = DeleteAllModel.collection.get_all(keys) for user in users: assert user.name in ['Azeem1', 'Arfan1', 'Haider1'] batch = fireo.batch() DeleteAllModel.collection.delete_all(keys, batch) for user in users: assert user.name in ['Azeem1', 'Arfan1', 'Haider1'] batch.commit() users = DeleteAllModel.collection.get_all(keys) for user in users: assert user is None
def test_simple_batch_write(): batch = fireo.batch() BatchCity.collection.create(batch=batch, name='CA', population=100) c = BatchCity() c.name = 'LA' c.population = 200 c.save(batch=batch) cities = BatchCity.collection.fetch() index = 0 for city in cities: index += 1 assert index == 0 batch.commit() cities = BatchCity.collection.fetch() index = 0 for city in cities: assert city.name in ['CA', 'LA'] index += 1 assert index >= 2
def test_issue_128_in_document_creation(): batch = fireo.batch() create_ref = City.collection.create(batch=batch, state='NYC', population=500000) assert isinstance(create_ref, DocumentReference)
def test_issue_128_in_document_update(): batch = fireo.batch() city = City.collection.create(state='NYC', population=500000) city = City() city.population = 1000000 upadte_ref = city.update(key=city.key, batch=batch) assert isinstance(upadte_ref, DocumentReference)
def test_batch_delete_with_filter(): batch = fireo.batch() BatchCity.collection.create(name='LA', population=100) BatchCity.collection.filter('name', '==', 'LA').batch(batch).delete() city = BatchCity.collection.filter('name', '==', 'LA').get() assert city.name == 'LA' batch.commit() city = BatchCity.collection.filter('name', '==', 'LA').get() assert city is None
def test_batch_update(): batch = fireo.batch() c = BatchCity.collection.create(name='CA', population=300) city_key = c.key c.population = 400 c.update(batch=batch) city = BatchCity.collection.get(city_key) assert city.population == 300 batch.commit() city = BatchCity.collection.get(city_key) assert city.population == 400
def test_batch_delete(): batch = fireo.batch() BatchCity.collection.batch(batch).delete() cities = BatchCity.collection.fetch() index = 0 for city in cities: index += 1 assert index >= 2 batch.commit() cities = BatchCity.collection.fetch() index = 0 for city in cities: index += 1 assert index == 0
def store_n_gram_chunk( n_gram_chunk: List[AlmostCompleteIndexedEventGram], credentials_file: str, ) -> None: """ Write all IndexedEventGrams in a single batch. This isn't about an atomic batch but reducing the total upload time. """ # Init batch batch = fireo.batch() # Trigger upserts for all items event_lut: Dict[str, db_models.Event] = {} for almost_complete_ieg in n_gram_chunk: if almost_complete_ieg.event_id not in event_lut: event_lut[ almost_complete_ieg.event_id] = db_models.Event.collection.get( f"{db_models.Event.collection_name}/{almost_complete_ieg.event_id}" ) # Construct true ieg ieg = db_models.IndexedEventGram() ieg.event_ref = event_lut[almost_complete_ieg.event_id] ieg.unstemmed_gram = almost_complete_ieg.unstemmed_gram ieg.stemmed_gram = almost_complete_ieg.stemmed_gram ieg.context_span = almost_complete_ieg.context_span ieg.value = almost_complete_ieg.value ieg.datetime_weighted_value = almost_complete_ieg.datetime_weighted_value db_functions.upload_db_model( db_model=ieg, credentials_file=credentials_file, batch=batch, ) # Commit batch.commit()
######### UCI ########### # start number: 373652 # end number: 377993 # Book No: 6 ######################### import fireo import json from uci import UCI from models.hadiths.ibnemaja import IbneMaja ibnemaja_file = open('./data/hadiths/ibne_maja.json', "r", encoding="utf8") ibnemaja = json.load(ibnemaja_file) hadith_batch = fireo.batch() count = 0 overall_count = 0 uci = UCI(373652) for hadith in ibnemaja: ibnemaja = IbneMaja() ibnemaja.id = str(hadith['hadees_number']) ibnemaja.hadith_number = int(hadith['hadees_number']) ibnemaja.book_number = int(hadith['Kitab_ID']) ibnemaja.book_name = { "urdu": hadith['Kitab'], "english": hadith['Kitab_Eng'] } ibnemaja.chapter = { "urdu": hadith['Baab'], "english": hadith['Baab_Eng'] }
######### UCI ########### # start number: 337185 # end number: 343534 ######################### import fireo from uci import UCI from models.quran.ayah import Ayah ar_quran = open('./data/quran/ar.quran_rtl.txt', "r", encoding="utf8") en_quran = open('./data/quran/en.sahih_ltr.txt', "r", encoding="utf8") ur_quran = open('./data/quran/ur.maududi_rtl.txt', "r", encoding="utf8") ayah_batch = fireo.batch() count = 0 line_count = 0 uci = UCI(337185) for ar, en, ur in zip(ar_quran, en_quran, ur_quran): if not en.strip(): print('English file end') break surah_number, ayah_number, en_text = en.strip().split('|') _, _, ur_text = ur.strip().split('|') _, _, ar_text = ar.strip().split('|') print(' writing line...' + str(line_count) + ' ...Surah ' + surah_number + ' ...Ayah ' + ayah_number) line_count += 1
}, { "id": "fr.leclerc", "language": "fr", "name": "Youssouf Leclerc", "english_name": "Youssouf Leclerc", "format": "audio", "type": "versebyverse", "direction": None }, { "id": "ar.aymanswoaid", "language": "ar", "name": "أيمن سويد", "english_name": "Ayman Sowaid", "format": "audio", "type": "versebyverse", "direction": None }] edition_batch = fireo.batch() for e in data: edition = Edition() edition.id = e["id"] edition.language = e["language"] edition.name = e["name"] edition.english_name = e["english_name"] edition.type = 'Audio' edition.save(batch=edition_batch) edition_batch.commit()
import requests import fireo from models.ayah import Ayah from models.surah import Surah surah_batch = fireo.batch() surah_count = 0 init_surah_number = 0 resp = requests.get('http://api.alquran.cloud/v1/surah') if resp.status_code == 200: data = resp.json()['data'] for s in data: surah = Surah() surah.id = str(s["number"]) surah.number = s["number"] surah.name = s["name"] surah.english_name = s["englishName"] surah.english_name_translation = s["englishNameTranslation"] surah.number_of_ayahs = s["numberOfAyahs"] surah.revelation_type = s["revelationType"] surah.save(batch=surah_batch) surah_count += 1 print('Surah complete ', surah_count)
for single_file in onlyfiles: detail = single_file.replace('.txt', '') # print(detail) edition_id, language, name, translator, direction = detail.split('_') edition = Edition() edition.id = edition_id edition.language = language edition.name = name edition.translator = translator edition.save() file_location = './data/' + single_file f = open(file_location, "r", encoding="utf8") trans_batch = fireo.batch() count = 0 line_count = 0 for line in f: if not line.strip(): print('file end') break # print(line) surah_number, ayat_number, *line_text = line.strip().split('|') text = ' '.join(line_text) print('For File ' + edition_id + ' writing line...' + str(line_count)) trans_id = surah_number + '-' + ayat_number