def test_update_workbook():
dataset_csv = Path(TEST_DATA_DIR) / TEST_DATASET_DIR / TEST_DATASET_CSV
num_records = 2
data = CsvParser.get_records(dataset_csv, num_records, True)
gcp_sheets.update_workbook(data, WORKBOOK_ID)
def test_get_all_records_with_header():
dataset_csv = Path(TEST_DATA_DIR) / TEST_DATASET_DIR / TEST_DATASET_CSV
assert len(CsvParser.get_records(dataset_csv, 0, True)) == ROW_COUNT_EXPECTED_PRE_WITH_HEADER
def test_get_partial_records_with_header():
dataset_csv = Path(TEST_DATA_DIR) / TEST_DATASET_DIR / TEST_DATASET_CSV
num_records = 5
assert len(CsvParser.get_records(dataset_csv, num_records, True)) == num_records
def main():
print(str(sys.argv))
parser = argparse.ArgumentParser(PROGRAM_NAME)
parser.add_argument("-d",
"--data-dir",
dest="data_dir",
required=True,
help="The absolute path to the data directory.")
parser.add_argument("-m",
"--mode",
dest="mode",
required=True,
help="The valid operating modes: " + str(MODES) + ".")
parser.add_argument("-sid",
"--sheet-id",
dest="sheet_id",
required=True,
help="The id of the Google Sheet.")
args = parser.parse_args()
# validate the logic of the provided arguments
validate_arguments(args)
# Instantiate the Constants class
const = Constants(args.data_dir, args.mode)
# generate the environment and manifest in all MODE cases except MODE_PUBLISH_DATASET_ONLY
if not const.mode == MODE_PUBLISH_DATASET_ONLY:
# verify the provided path and generate underlying substructure
generate_manifest(args.data_dir, args.mode)
# if the mode is MODE_CREATE_MANIFEST_ONLY then exit
if const.mode == MODE_CREATE_MANIFEST_ONLY:
logging.info(
f"Manifest ha sido creado con exito en {str(const.get_path_manifest_json().resolve())}"
)
logging.info(f"Operating mode es {const.mode}. Exiting.")
sys.exit(0)
# generate the environment and manifest in all MODE cases except:
# MODE_CREATE_MANIFEST_ONLY or MODE_PUBLISH_DATASET_ONLY
if not const.mode == MODE_CREATE_MANIFEST_ONLY and not const.mode == MODE_PUBLISH_DATASET_ONLY:
build_dataset(str(const.get_path_manifest_json().resolve()))
if const.mode == MODE_BUILD_DATASET_ONLY:
logging.info(
f"Dataset ha sido creado con exito en {str(const.get_path_data_set().resolve())}"
)
logging.info(f"Operating mode es {const.mode}. Exiting.")
sys.exit(0)
# publish the dataset if MODE MODE_PUBLISH_DATASET_ONLY or MODE_CREATE_BUILD_PUBLISH
if const.mode == MODE_PUBLISH_DATASET_ONLY or const.mode == MODE_CREATE_BUILD_PUBLISH:
with open(str(const.get_path_manifest_json().resolve())) as json_file:
data = json.load(json_file)
dataset_csv = data['dataset']['path']
num_updates = len(data['calculators'])
logging.info(
f"Number of records to update in google sheet: {num_updates}")
records_to_update = CsvParser.get_records(dataset_csv, num_updates,
False)
for record in records_to_update:
logging.info(f"Updating record for client: {record['client']}")
logging.debug(f"Record to update: {record}")
gcp_sheets.update_workbook(records_to_update, args.sheet_id)
if const.mode == MODE_PUBLISH_DATASET_ONLY:
logging.info(
f"Dataset ha sido publicado con exito en {GOOGLE_SHEET_URL}{args.sheet_id}"
)
logging.info(f"Operating mode es {const.mode}. Exiting.")
sys.exit(0)
logging.info("Program exiting correctly.")