def test_simple(token_restore):
assert not GCSFileSystem.tokens
gcs = GCSFileSystem(TEST_PROJECT, token=GOOGLE_TOKEN)
assert gcs.ls('')
# token is now cached
gcs = GCSFileSystem(TEST_PROJECT)
assert gcs.ls('')
def test_simple():
assert not GCSFileSystem.tokens
gcs = GCSFileSystem(TEST_PROJECT, token=GOOGLE_TOKEN)
gcs.ls(TEST_BUCKET) # no error
gcs.ls("/" + TEST_BUCKET) # OK to lead with '/'
# all other tables: store the original PK to the new column, removing the original column
else:
return row.pop(self.pk_name)
# Process the known (hardcoded) tables
for table_name in table_names:
log.info(f"PROCESSING {table_name}")
primary_key_gen = PrimaryKeyGenerator(table_name, args.firecloud)
# set to hold all columns for this table, list to hold all the rows
column_set = set()
row_list = []
# generated PK column headers for Firecloud compatibility
entity_name = primary_key_gen.generate_entity_name()
# read json data
for path in gcs.ls(os.path.join(args.input_dir, table_name)):
log.info(f"...Opening {path}")
with gcs.open(path, 'r') as json_file:
print(path, json_file)
for line in json_file:
row = json.loads(line)
if not row:
raise RuntimeError(f'Encountered invalid JSON "{line}", aborting.')
row[entity_name] = primary_key_gen.generate_primary_key(row)
column_set.update(row.keys())
row_list.append(row)