Python run_pipeline示例

示例#1

patch(data_integration.config.system_statistics_collection_period)(lambda: 15)


@patch(data_integration.config.root_pipeline)
def root_pipeline():
    return bc_reg_root_pipeline()


mara_host = os.environ.get('MARA_DB_HOST', 'bcregdb')
mara_database = os.environ.get('MARA_DB_DATABASE', 'mara_db')
mara_port = os.environ.get('MARA_DB_PORT', '5432')
mara_user = os.environ.get('MARA_DB_USER', 'mara_db')
mara_password = os.environ.get('MARA_DB_PASSWORD')

mara_db.config.databases \
    = lambda: {'mara': mara_db.dbs.PostgreSQLDB(user=mara_user, password=mara_password, host=mara_host, database=mara_database, port=mara_port)}

(initial_load_pipeline, success) = data_integration.pipelines.find_node(
    ['initialization_and_load_tasks', 'bc_reg_corp_loader'])
if success:
    corps_ct = 1
    prev_corps_ct = 0
    while 0 < corps_ct and corps_ct != prev_corps_ct:
        # run at least once to get an initial data load
        run_pipeline(initial_load_pipeline)
        with EventProcessor() as eventprocessor:
            prev_corps_ct = corps_ct
            corps_ct = eventprocessor.get_outstanding_corps_record_count()
else:
    print("Pipeline not found")

示例#2

import mara_db.config
import mara_db.dbs
import data_integration
import data_integration.config
from mara_app.monkey_patch import patch
from bcreg.bcreg_pipelines import bc_reg_root_pipeline

patch(data_integration.config.system_statistics_collection_period)(lambda: 15)


@patch(data_integration.config.root_pipeline)
def root_pipeline():
    return bc_reg_root_pipeline()


mara_host = os.environ.get('MARA_DB_HOST', 'bcregdb')
mara_database = os.environ.get('MARA_DB_DATABASE', 'mara_db')
mara_port = os.environ.get('MARA_DB_PORT', '5432')
mara_user = os.environ.get('MARA_DB_USER', 'mara_db')
mara_password = os.environ.get('MARA_DB_PASSWORD')

mara_db.config.databases \
    = lambda: {'mara': mara_db.dbs.PostgreSQLDB(user=mara_user, password=mara_password, host=mara_host, database=mara_database, port=mara_port)}

(child_pipeline,
 success) = data_integration.pipelines.find_node(['bc_reg_event_processor'])
if success:
    run_pipeline(child_pipeline)
else:
    print("Pipeline not found")

示例#3

文件： bc_reg_populate_audit_table.py 项目： weiiv/von-bc-registries-agent

import mara_db.dbs
import data_integration
import data_integration.config
from mara_app.monkey_patch import patch
from bcreg.bcreg_pipelines import bc_reg_root_pipeline

patch(data_integration.config.system_statistics_collection_period)(lambda: 15)


@patch(data_integration.config.root_pipeline)
def root_pipeline():
    return bc_reg_root_pipeline()


mara_host = os.environ.get('MARA_DB_HOST', 'bcregdb')
mara_database = os.environ.get('MARA_DB_DATABASE', 'mara_db')
mara_port = os.environ.get('MARA_DB_PORT', '5432')
mara_user = os.environ.get('MARA_DB_USER', 'mara_db')
mara_password = os.environ.get('MARA_DB_PASSWORD')

mara_db.config.databases \
    = lambda: {'mara': mara_db.dbs.PostgreSQLDB(user=mara_user, password=mara_password, host=mara_host, database=mara_database, port=mara_port)}

(populate_audit_table_pipeline,
 success) = data_integration.pipelines.find_node(
     ['bc_reg_populate_audit_table'])
if success:
    run_pipeline(populate_audit_table_pipeline)
else:
    print("Pipeline not found")

示例#4

文件： bc_reg_migrate.py 项目： nrempel/von-bc-registries-agent

import os
from data_integration.commands.bash import RunBash
from data_integration.commands.python import ExecutePython
from data_integration.pipelines import Pipeline, Task
from data_integration.ui.cli import run_pipeline, run_interactively
from data_integration.ui.cli import run_pipeline
import mara_db.auto_migration
import mara_db.config
import mara_db.dbs
import data_integration
from bcreg.bcreg_pipelines import db_init_pipeline

mara_host = os.environ.get('MARA_DB_HOST', 'bcregdb')
mara_database = os.environ.get('MARA_DB_DATABASE', 'mara_db')
mara_port = os.environ.get('MARA_DB_PORT', '5432')
mara_user = os.environ.get('MARA_DB_USER', 'mara_db')
mara_password = os.environ.get('MARA_DB_PASSWORD')

mara_db.config.databases \
    = lambda: {'mara': mara_db.dbs.PostgreSQLDB(user=mara_user, password=mara_password, host=mara_host, database=mara_database, port=mara_port)}

parent_pipeline = Pipeline(
    id = 'holder_for_pipeline_versions',
    description = 'Holder for the different versions of the BC Registries pipeline.')

parent_pipeline.add(db_init_pipeline())

run_pipeline(parent_pipeline)

示例#5

from data_integration.ui.cli import run_pipeline
import mara_db.auto_migration
import mara_db.config
import mara_db.dbs
import data_integration
import data_integration.config
from mara_app.monkey_patch import patch
from von_pipeline.von_data_pipelines import von_data_root_pipeline


patch(data_integration.config.system_statistics_collection_period)(lambda: 15)

@patch(data_integration.config.root_pipeline)
def root_pipeline():
    return von_data_root_pipeline()

mara_host = os.environ.get('MARA_DB_HOST', 'von-pipeline-db')
mara_database = os.environ.get('MARA_DB_DATABASE', 'mara_db')
mara_port = os.environ.get('MARA_DB_PORT', '5432')
mara_user = os.environ.get('MARA_DB_USER', 'mara_db')
mara_password = os.environ.get('MARA_DB_PASSWORD')

mara_db.config.databases \
    = lambda: {'mara': mara_db.dbs.PostgreSQLDB(user=mara_user, password=mara_password, host=mara_host, database=mara_database, port=mara_port)}

(post_credential_pipeline, success) = data_integration.pipelines.find_node(['initialization_and_load_tasks','von_data_credential_poster']) 
if success:
	run_pipeline(post_credential_pipeline)
else:
	print("Pipeline not found")

示例#6

文件： von_data_pipeline_status.py 项目： seanadipose/indy-catalyst-issuer-controller

import mara_db.config
import mara_db.dbs
import data_integration
import data_integration.config
from mara_app.monkey_patch import patch
from von_pipeline.von_data_pipelines import von_data_root_pipeline

patch(data_integration.config.system_statistics_collection_period)(lambda: 15)


@patch(data_integration.config.root_pipeline)
def root_pipeline():
    return von_data_root_pipeline()


mara_host = os.environ.get('MARA_DB_HOST', 'von-pipeline-db')
mara_database = os.environ.get('MARA_DB_DATABASE', 'mara_db')
mara_port = os.environ.get('MARA_DB_PORT', '5432')
mara_user = os.environ.get('MARA_DB_USER', 'mara_db')
mara_password = os.environ.get('MARA_DB_PASSWORD')

mara_db.config.databases \
    = lambda: {'mara': mara_db.dbs.PostgreSQLDB(user=mara_user, password=mara_password, host=mara_host, database=mara_database, port=mara_port)}

(status_pipeline,
 success) = data_integration.pipelines.find_node(['von_data_pipeline_status'])
if success:
    run_pipeline(status_pipeline)
else:
    print("Pipeline not found")

示例#7

文件： bc_reg_migrate.py 项目： ised-isde-canada/von-gc-corp-canada-agent

import mara_db.config
import mara_db.dbs
import data_integration
import data_integration.config
from mara_app.monkey_patch import patch
from bcreg.bcreg_pipelines import bc_reg_root_pipeline

patch(data_integration.config.system_statistics_collection_period)(lambda: 15)


@patch(data_integration.config.root_pipeline)
def root_pipeline():
    return bc_reg_root_pipeline()


mara_host = os.environ.get('MARA_DB_HOST', 'bcregdb')
mara_database = os.environ.get('MARA_DB_DATABASE', 'mara_db')
mara_port = os.environ.get('MARA_DB_PORT', '5432')
mara_user = os.environ.get('MARA_DB_USER', 'mara_db')
mara_password = os.environ.get('MARA_DB_PASSWORD')

mara_db.config.databases \
    = lambda: {'mara': mara_db.dbs.PostgreSQLDB(user=mara_user, password=mara_password, host=mara_host, database=mara_database, port=mara_port)}

(init_pipeline, success) = data_integration.pipelines.find_node(
    ['initialization_and_load_tasks', 'bc_reg_db_init'])
if success:
    run_pipeline(init_pipeline)
else:
    print("Pipeline not found")