def main(config: CommandConfig): influxdb_config = config.influxdb client = InfluxDBClient(influxdb_config.host, influxdb_config.port, influxdb_config.username, influxdb_config.password, influxdb_config.database) client.create_database(influxdb_config.database) db = SqliteExtDatabase(config.database.filename) try: db_model.database_proxy.initialize(db) db.create_tables([db_model.InfluxdbPostPosition]) except IntegrityError as ex: logger.error("database open error.", ex) db.rollback() exit(1) while True: db_influx_last_position = read_influx_post_position(db) last_post_position = db_influx_last_position.post_log_position logger.info(f'db_influx_last_position={last_post_position}') with db.transaction() as tran: db_result = db_model.EnvSensorLog.select() \ .where(db_model.EnvSensorLog.id > last_post_position) \ .order_by(db_model.EnvSensorLog.id) \ .limit(1000) logger.info(F"db select count={db_result.count()}") if db_result.count() == 0: break influx_data = [] for env_sensor_log in db_result: logger.debug( f"log_id={env_sensor_log.id}, data={env_sensor_log.data}") db_influx_last_position.post_log_position = env_sensor_log.id db_influx_last_position.timestamp = datetime.now() influx_data.append(convert(env_sensor_log.data)) logger.debug(f"write_data={influx_data}") logger.info( f"processed position={db_influx_last_position.post_log_position}" ) client.write_points(influx_data) db_influx_last_position.save() logger.info(F"process complete.")
Journal, PLOSArticle, ArticleType, CoAuthorPLOSArticle, CorrespondingAuthor, JATSType, Affiliations, Country, SubjectsPLOSArticle, Subjects ]) corpus_dir = starterdir if args.starter else None all_files = Corpus(corpus_dir) num_files = len(all_files) if args.random is None else args.random for article in tqdm(islice(all_files, args.random), total=num_files): journal_name = journal_title_dict[article.journal.upper()] with db.atomic() as atomic: try: journal = Journal.create(journal=journal_name) except IntegrityError: db.rollback() journal = Journal.get(Journal.journal == journal_name) with db.atomic() as atomic: try: article_type = ArticleType.create(article_type=article.plostype) except IntegrityError: db.rollback() article_type = ArticleType.get( ArticleType.article_type == article.plostype) with db.atomic() as atomic: try: j_type = JATSType.create(jats_type=article.type_) except IntegrityError: db.rollback() j_type = JATSType.get(JATSType.jats_type == article.type_) p_art = PLOSArticle.create(