def up(client):
migration = BigQueryMigration(client)
dataset = migration.create_dataset(dataset_name)
parent_dataset = migration.dataset('pyr_bluesun_{}'.format(client.env))
tbls = migration.client.list_tables(parent_dataset)
clusters = migration.default_clustering_fields
clusters.insert(0, 'icentris_client:STRING')
for item in tbls:
tbl = migration.client.get_table(item.reference)
orig = tbl.schema
new = orig.copy()
migration.create_table(name=tbl.table_id,
project=migration.client.project,
schema=new,
dataset=dataset,
partition={'type': 'time'},
clustering_fields=clusters)
return dataset
def up(client):
migration = BigQueryMigration(client)
global dataset_name
dataset_name = dataset_name.format(client.env)
wv_ds = migration.create_dataset(dataset_name)
bs_ds = migration.dataset('pyr_bluesun_{}'.format(client.env))
ls = client.list_tables(bs_ds)
for tbl in ls:
tbl_ref = bs_ds.table(tbl.table_id)
tbl = client.get_table(tbl_ref)
migration.create_table(tbl_ref.table_id,
tbl_ref.project,
wv_ds,
schema=tbl.schema)
return wv_ds
def up(client):
client = BigQueryMigration(client)
dataset = client.create_dataset('staging')
for tbl, schema in schemas.items():
clusters = ['leo_eid:STRING', 'ingestion_timestamp:TIMESTAMP']
client.create_table(name=tbl,
project=client.client.project, schema=schema, dataset=dataset,
partition={
'type': 'range',
'field':
'client_partition_id',
'start': 1,
'end': 100,
'interval': 1},
clustering_fields=clusters)
return dataset
def up(client):
migration = BigQueryMigration(client)
name = dataset_name + '_{}'.format(client.env)
with PosixPath('/workspace/bigquery/migrations/bluesun_schema.json').open(
mode='r') as f:
tbls = json.loads(f.read())
dataset = migration.create_dataset(name)
for tbl, raw in tbls.items():
schema = []
for f in raw['fields']:
schema.append(bigquery.SchemaField(f['name'], f['type'],
f['mode']))
migration.create_table(name=tbl,
project=migration.client.project,
schema=schema,
dataset=dataset)
return dataset
def up(client):
migration = BigQueryMigration(client)
dataset = migration.create_dataset(
dataset_name
) # use me if you are creating a new dataset. -- ndg 2/5/20
for tbl, schema in schemas.items():
migration.create_table(
name=tbl,
project=migration.client.project,
schema=schema,
dataset=dataset,
partition={
'type': 'range',
'field': 'client_partition_id',
'start': 1,
'end': 100,
'interval': 1
},
clustering_fields=migration.default_clustering_fields)
return dataset
def up(client):
client = BigQueryMigration(client)
dataset = client.create_dataset('pii')
pii_schema = [
bigquery.SchemaField('client_partition_id', 'INTEGER',
mode='REQUIRED'),
bigquery.SchemaField('client_wrench_id', 'STRING', mode='REQUIRED'),
bigquery.SchemaField("icentris_client", "STRING", mode="REQUIRED"),
bigquery.SchemaField("tree_user_id", "INTEGER"),
bigquery.SchemaField("first_name", "STRING"),
bigquery.SchemaField("last_name", "STRING"),
bigquery.SchemaField("company_name", "STRING"),
bigquery.SchemaField("email", "STRING"),
bigquery.SchemaField("phone", "STRING"),
bigquery.SchemaField("mobile_phone", "STRING"),
bigquery.SchemaField("street", "STRING"),
bigquery.SchemaField("city", "STRING"),
bigquery.SchemaField("state", "STRING"),
bigquery.SchemaField("country", "STRING"),
bigquery.SchemaField("birth_date", "DATE"),
bigquery.SchemaField("gender", "STRING")
]
client.create_table(
name='users',
project=client.client.project,
schema=pii_schema,
dataset=dataset,
partition={
'type': 'range',
'field': 'client_partition_id',
'start': 1,
'end': 100,
'interval': 1
},
clustering_fields=['leo_eid:STRING', 'ingestion_timestamp:TIMESTAMP'])
return dataset
def up(client):
migration = BigQueryMigration(client)
dataset = migration.create_dataset(dataset_name)
return dataset