def main(config: CommandConfig):
influxdb_config = config.influxdb
client = InfluxDBClient(influxdb_config.host, influxdb_config.port,
influxdb_config.username, influxdb_config.password,
influxdb_config.database)
client.create_database(influxdb_config.database)
db = SqliteExtDatabase(config.database.filename)
try:
db_model.database_proxy.initialize(db)
db.create_tables([db_model.InfluxdbPostPosition])
except IntegrityError as ex:
logger.error("database open error.", ex)
db.rollback()
exit(1)
while True:
db_influx_last_position = read_influx_post_position(db)
last_post_position = db_influx_last_position.post_log_position
logger.info(f'db_influx_last_position={last_post_position}')
with db.transaction() as tran:
db_result = db_model.EnvSensorLog.select() \
.where(db_model.EnvSensorLog.id > last_post_position) \
.order_by(db_model.EnvSensorLog.id) \
.limit(1000)
logger.info(F"db select count={db_result.count()}")
if db_result.count() == 0:
break
influx_data = []
for env_sensor_log in db_result:
logger.debug(
f"log_id={env_sensor_log.id}, data={env_sensor_log.data}")
db_influx_last_position.post_log_position = env_sensor_log.id
db_influx_last_position.timestamp = datetime.now()
influx_data.append(convert(env_sensor_log.data))
logger.debug(f"write_data={influx_data}")
logger.info(
f"processed position={db_influx_last_position.post_log_position}"
)
client.write_points(influx_data)
db_influx_last_position.save()
logger.info(F"process complete.")
Journal, PLOSArticle, ArticleType, CoAuthorPLOSArticle,
CorrespondingAuthor, JATSType, Affiliations, Country, SubjectsPLOSArticle,
Subjects
])
corpus_dir = starterdir if args.starter else None
all_files = Corpus(corpus_dir)
num_files = len(all_files) if args.random is None else args.random
for article in tqdm(islice(all_files, args.random), total=num_files):
journal_name = journal_title_dict[article.journal.upper()]
with db.atomic() as atomic:
try:
journal = Journal.create(journal=journal_name)
except IntegrityError:
db.rollback()
journal = Journal.get(Journal.journal == journal_name)
with db.atomic() as atomic:
try:
article_type = ArticleType.create(article_type=article.plostype)
except IntegrityError:
db.rollback()
article_type = ArticleType.get(
ArticleType.article_type == article.plostype)
with db.atomic() as atomic:
try:
j_type = JATSType.create(jats_type=article.type_)
except IntegrityError:
db.rollback()
j_type = JATSType.get(JATSType.jats_type == article.type_)
p_art = PLOSArticle.create(
