def test_simple(token_restore): assert not GCSFileSystem.tokens gcs = GCSFileSystem(TEST_PROJECT, token=GOOGLE_TOKEN) assert gcs.ls('') # token is now cached gcs = GCSFileSystem(TEST_PROJECT) assert gcs.ls('')
def test_simple(): assert not GCSFileSystem.tokens gcs = GCSFileSystem(TEST_PROJECT, token=GOOGLE_TOKEN) gcs.ls(TEST_BUCKET) # no error gcs.ls("/" + TEST_BUCKET) # OK to lead with '/'
# all other tables: store the original PK to the new column, removing the original column else: return row.pop(self.pk_name) # Process the known (hardcoded) tables for table_name in table_names: log.info(f"PROCESSING {table_name}") primary_key_gen = PrimaryKeyGenerator(table_name, args.firecloud) # set to hold all columns for this table, list to hold all the rows column_set = set() row_list = [] # generated PK column headers for Firecloud compatibility entity_name = primary_key_gen.generate_entity_name() # read json data for path in gcs.ls(os.path.join(args.input_dir, table_name)): log.info(f"...Opening {path}") with gcs.open(path, 'r') as json_file: print(path, json_file) for line in json_file: row = json.loads(line) if not row: raise RuntimeError(f'Encountered invalid JSON "{line}", aborting.') row[entity_name] = primary_key_gen.generate_primary_key(row) column_set.update(row.keys()) row_list.append(row)