Python DAG_Factory Examples

Example #1

File: bulkdozer_dag.py Project: quan/starthinker

        }
      },
      'timezone': {
        'field': {
          'kind': 'timezone',
          'name': 'recipe_timezone',
          'description': 'Timezone for report dates.',
          'default': 'America/Chicago'
        }
      },
      'auth': 'user',
      'dcm_profile_id': {
        'field': {
          'order': 1,
          'kind': 'string',
          'name': 'dcm_profile_id',
          'description': 'Campaign Manager Profile ID (optional if account id provided)',
          'default': ''
        }
      }
    }
  }
]

DAG_FACTORY = DAG_Factory('bulkdozer', { 'tasks':TASKS }, INPUTS)
DAG_FACTORY.apply_credentails(USER_CONN_ID, GCP_CONN_ID)
DAG = DAG_FACTORY.execute()

if __name__ == "__main__":
  DAG_FACTORY.print_commandline()

Example #2

                    'recipe_slug',
                    'description':
                    'Place where tables will be created in BigQuery.'
                }
            }],
            'legacy':
            False
        },
        'to': {
            'view': 'DV360_Audience_Analysis',
            'dataset': {
                'field': {
                    'kind':
                    'string',
                    'name':
                    'recipe_slug',
                    'description':
                    'Place where tables will be created in BigQuery.'
                }
            }
        }
    }
}]

DAG_FACTORY = DAG_Factory('audience_analysis', {'tasks': TASKS}, INPUTS)
DAG_FACTORY.apply_credentails(USER_CONN_ID, GCP_CONN_ID)
DAG = DAG_FACTORY.execute()

if __name__ == "__main__":
    DAG_FACTORY.print_commandline()

Example #3

File: dcm_replicate_to_bigquery_dag.py Project: quan/starthinker

                'is_incremental_load': {
                    'field': {
                        'order': 7,
                        'kind': 'boolean',
                        'name': 'Aggregate',
                        'description':
                        'Append report data to existing table, requires Date column.',
                        'default': False
                    }
                },
                'dataset': {
                    'field': {
                        'order': 4,
                        'kind': 'string',
                        'name': 'recipe_slug',
                        'default': ''
                    }
                }
            }
        }
    }
}]

DAG_FACTORY = DAG_Factory('dcm_replicate_to_bigquery', {'tasks': TASKS},
                          INPUTS)
DAG_FACTORY.apply_credentails(USER_CONN_ID, GCP_CONN_ID)
DAG = DAG_FACTORY.execute()

if __name__ == "__main__":
    DAG_FACTORY.print_commandline()

Example #4

File: conversion_upload_from_sheets_dag.py Project: hernanperalta/starthinker

                    'default': ''
                }
            },
            'tab': {
                'field': {
                    'name': 'sheet_tab',
                    'kind': 'string',
                    'order': 10,
                    'default': ''
                }
            },
            'range': {
                'field': {
                    'name': 'sheet_range',
                    'kind': 'string',
                    'order': 11,
                    'default': ''
                }
            }
        }
    }
}]

DAG_FACTORY = DAG_Factory('conversion_upload_from_sheets', {'tasks': TASKS},
                          INPUTS)
DAG_FACTORY.apply_credentails(USER_CONN_ID, GCP_CONN_ID)
DAG = DAG_FACTORY.execute()

if __name__ == "__main__":
    DAG_FACTORY.print_commandline()

Example #5

File: trends_places_to_sheets_via_value_dag.py Project: hernanperalta/starthinker

            'sheets': {
                'sheet': {
                    'field': {
                        'name': 'destination_sheet',
                        'kind': 'string',
                        'order': 6,
                        'default': ''
                    }
                },
                'tab': {
                    'field': {
                        'name': 'destination_tab',
                        'kind': 'string',
                        'order': 7,
                        'default': ''
                    }
                },
                'range': 'A1'
            }
        }
    }
}]

DAG_FACTORY = DAG_Factory('trends_places_to_sheets_via_value',
                          {'tasks': TASKS}, INPUTS)
DAG_FACTORY.apply_credentails(USER_CONN_ID, GCP_CONN_ID)
DAG = DAG_FACTORY.execute()

if __name__ == "__main__":
    DAG_FACTORY.print_commandline()

Example #6

File: bigquery_census_normalize_dag.py Project: viohman/starthinker

            'dataset': {
                'field': {
                    'name': 'dataset',
                    'kind': 'string',
                    'order': 4,
                    'default': '',
                    'description': 'Existing BigQuery dataset.'
                }
            },
            'type': {
                'field': {
                    'name': 'type',
                    'kind': 'choice',
                    'order': 5,
                    'default': 'table',
                    'description': 'Write Census_Percent as table or view.',
                    'choices': ['table', 'view']
                }
            }
        }
    }
}]

DAG_FACTORY = DAG_Factory('bigquery_census_normalize', {'tasks': TASKS},
                          INPUTS)
DAG_FACTORY.apply_credentails(USER_CONN_ID, GCP_CONN_ID)
DAG = DAG_FACTORY.execute()

if __name__ == "__main__":
    DAG_FACTORY.print_commandline()

Example #7

File: lineitem_read_to_bigquery_via_value_dag.py Project: quan/starthinker

                'bigquery': {
                    'table': {
                        'field': {
                            'order': 3,
                            'kind': 'string',
                            'name': 'destination_table',
                            'default': ''
                        }
                    },
                    'dataset': {
                        'field': {
                            'order': 2,
                            'kind': 'string',
                            'name': 'destination_dataset',
                            'default': ''
                        }
                    }
                }
            }
        }
    }
}]

DAG_FACTORY = DAG_Factory('lineitem_read_to_bigquery_via_value',
                          {'tasks': TASKS}, INPUTS)
DAG_FACTORY.apply_credentails(USER_CONN_ID, GCP_CONN_ID)
DAG = DAG_FACTORY.execute()

if __name__ == "__main__":
    DAG_FACTORY.print_commandline()

Example #8

File: dv360_feature_adoption_dag.py Project: viohman/starthinker

                    'string',
                    'description':
                    'Place where tables will be created in BigQuery.'
                }
            }],
            'legacy':
            False
        },
        'to': {
            'dataset': {
                'field': {
                    'name':
                    'recipe_slug',
                    'kind':
                    'string',
                    'description':
                    'Place where tables will be created in BigQuery.'
                }
            },
            'view': 'DV360_Feature_Adoption_Analysis'
        }
    }
}]

DAG_FACTORY = DAG_Factory('dv360_feature_adoption', {'tasks': TASKS}, INPUTS)
DAG_FACTORY.apply_credentails(USER_CONN_ID, GCP_CONN_ID)
DAG = DAG_FACTORY.execute()

if __name__ == "__main__":
    DAG_FACTORY.print_commandline()

Example #9

File: bigquery_pearson_significance_test_dag.py Project: quan/starthinker

        'field': {
          'order': 0,
          'kind': 'authentication',
          'name': 'auth',
          'description': 'Credentials used for writing function.',
          'default': 'service'
        }
      },
      'function': 'pearson_significance_test',
      'to': {
        'dataset': {
          'field': {
            'order': 1,
            'kind': 'string',
            'name': 'dataset',
            'description': 'Existing BigQuery dataset.',
            'default': ''
          }
        }
      }
    }
  }
]

DAG_FACTORY = DAG_Factory('bigquery_pearson_significance_test', { 'tasks':TASKS }, INPUTS)
DAG_FACTORY.apply_credentails(USER_CONN_ID, GCP_CONN_ID)
DAG = DAG_FACTORY.execute()

if __name__ == "__main__":
  DAG_FACTORY.print_commandline()

Example #10

            }
          },
          'query': {
            'field': {
              'name': 'query',
              'kind': 'string',
              'order': 2,
              'default': 'SELECT * FROM `Dataset.Table`;'
            }
          },
          'legacy': {
            'field': {
              'name': 'legacy',
              'kind': 'boolean',
              'order': 3,
              'default': False
            }
          }
        }
      }
    }
  }
]

DAG_FACTORY = DAG_Factory('lineitem_write_from_bigquery', { 'tasks':TASKS }, INPUTS)
DAG_FACTORY.apply_credentails(USER_CONN_ID, GCP_CONN_ID)
DAG = DAG_FACTORY.execute()

if __name__ == "__main__":
  DAG_FACTORY.print_commandline()

Example #11

    'recipe_name': '',  # Name of document to deploy to.
}

TASKS = [{
    'drive': {
        'auth': 'user',
        'hour': [],
        'copy': {
            'source':
            'https://docs.google.com/spreadsheets/d/1tt597dMsAaxYXaJdifwKYNVzJrIl6E9Pe8GysfVrWOs/',
            'destination': {
                'field': {
                    'name': 'recipe_name',
                    'prefix': 'CM User Editor For ',
                    'kind': 'string',
                    'order': 1,
                    'description': 'Name of document to deploy to.',
                    'default': ''
                }
            }
        }
    }
}]

DAG_FACTORY = DAG_Factory('cm_campaign_audit', {'tasks': TASKS}, INPUTS)
DAG_FACTORY.apply_credentails(USER_CONN_ID, GCP_CONN_ID)
DAG = DAG_FACTORY.execute()

if __name__ == "__main__":
    DAG_FACTORY.print_commandline()

Example #12

                'kind': 'authentication',
                'name': 'auth_read',
                'description': 'Credentials used for reading data.',
                'default': 'user'
            }
        },
        'read': {
            'sheet': {
                'range': 'A1:D',
                'sheet': {
                    'field': {
                        'order': 2,
                        'kind': 'string',
                        'name': 'sheet_url',
                        'description': 'Feed Sheet URL',
                        'default': ''
                    }
                },
                'tab': 'Rules'
            }
        }
    }
}]

DAG_FACTORY = DAG_Factory('dv3po_custom_signals', {'tasks': TASKS}, INPUTS)
DAG_FACTORY.apply_credentails(USER_CONN_ID, GCP_CONN_ID)
DAG = DAG_FACTORY.execute()

if __name__ == "__main__":
    DAG_FACTORY.print_commandline()