def conf_corpus(args):
corpus = args.corpus
if not corpus:
corpus = default_corpus
logging.info("corpus: {}".format(corpus))
corpus_conf_dir_name = args.corpus_conf_dir_name
logging.info("corpus_conf_dir_name: {}".format(corpus_conf_dir_name))
corpus_service.conf_corpus(corpus, corpus_conf_dir_name)
def conf_corpus(args):
corpus = args.corpus
if not corpus:
corpus = default_corpus
logging.info("corpus: {}".format(corpus))
corpus_conf_dir_name = args.corpus_conf_dir_name
logging.info("corpus_conf_dir_name: {}".format(corpus_conf_dir_name))
corpus_service.conf_corpus(corpus, corpus_conf_dir_name)
# conf params
corpus = "num"
source = "FNSP"
rec_id_prefix = "sc_"
input_dir_path = os.path.join("data", "num", "input")
input_format = constants.FORMAT_UNIMARC
output_dir_path = os.path.join("data", "num", "output")
if not os.path.exists(output_dir_path):
os.mkdir(output_dir_path)
error_file_name = "".join(["validation-", corpus, ".txt"])
error_file_path = os.path.join(output_dir_path, error_file_name)
#logging.debug("error_file_path: {}".format(error_file_path))
# conf corpus
corpus_service.clean_corpus(corpus)
corpus_service.conf_corpus(corpus, "aime")
date_clean = datetime.datetime.now()
chrono.chrono_trace("Clean and conf corpus", date_begin, date_clean, None)
# import
input_file_paths = io_service.get_relevant_file_list_by_format(
input_dir_path, input_format)
results = corpus_service.import_metadata_files(corpus, input_file_paths,
input_format, source,
rec_id_prefix, True, None)
date_import = datetime.datetime.now()
chrono.chrono_trace("Import corpus", date_clean, date_import, None)
# Validate
corpus_service.validate_corpus(corpus, error_file_path)
date_validate = datetime.datetime.now()
if __name__ == "__main__":
date_begin = datetime.datetime.now()
# conf params
corpus = "perio"
source = "Sciences Po | la bibliothèque"
rec_id_prefix = ""
input_file_path = os.path.join("data", "unimarc", "periouni.mrc")
input_format = constants.FORMAT_UNIMARC
csv_file_name = "".join(["validation-", corpus, ".csv"])
csv_file_path = os.path.join("data", "result", csv_file_name)
# conf corpus
corpus_service.clean_corpus(corpus)
corpus_service.conf_corpus(corpus, "aime")
date_clean = datetime.datetime.now()
chrono.chrono_trace("Clean and conf corpus", date_begin, date_clean, None)
# import
corpus_service.import_metadata_file(corpus, input_file_path, input_format, source, rec_id_prefix, True, None)
date_import = datetime.datetime.now()
chrono.chrono_trace("Import corpus", date_clean, date_import, None)
# Validate perio
documents = repository_service.get_documents(corpus)
validate_perios(documents, csv_file_path)
date_validate = datetime.datetime.now()
chrono.chrono_trace("Validate perio", date_import, date_validate, None)
