def update_tags(self):
latest_tag = self.tag_repository.get_latest_tag_id()
log.info(f'Starting tag updated. Latest tag {latest_tag}')
counter = 0
for batch in util.batch(10000, self.api.iterate_tags(start=latest_tag)):
counter += len(batch)
log.info(f'Saving {len(batch)} tags')
self.tag_repository.persist_bulk(batch)
log.info(
f'Finished updating tags. {counter} tags were added to the database')
def update_database(self):
id = self.database.latest_post_id()
log.info("starting database update. latest post {}", id)
counter = 0
for batch in util.batch(1000, iterate_posts(id)):
counter += len(batch)
self.database.get_session().bulk_save_objects(batch)
self.database.get_session().commit()
log.info("finished database update. added {} posts to database",
counter)
def update_posts(self):
latest_post = self.post_repository.get_latest_post_id()
counter = 0
log.info(f'Starting post update. Latest post {latest_post}')
for posts in util.batch(100,
self.api.iterate_posts(start=latest_post)):
self._process_posts(posts)
counter += len(posts)
log.info(
f'Finished updating posts. {counter} posts were added to the database'
)
def update_database(self):
id = self.database.latest_post_id()
log.info("starting database update. latest post {}", id)
session = self.database.DBSession()
counter = 0
for b in batch(1000, iterate_posts(id)):
counter += len(b)
session.bulk_save_objects(b)
session.commit()
session.close()
log.info("finished database update. added {} posts to database",
counter)
from logbook import Logger
import config
from rep0st import util, api
from rep0st.rep0st import get_rep0st
config.load()
log = Logger('tags')
if __name__ == '__main__':
rep = get_rep0st()
counter = 0
for batch in util.batch(
10000, api.iterate_tags(start=rep.database.latest_tag_id())):
rep.database.session.bulk_save_objects(batch)
rep.database.get_session().commit()
rep.close()
