Python dataFlowProcessingLoop示例

示例#1

        dataProfile = dataFlow.get_profile()

        # Now generate column and data flow inventories
        columnInventory = getColumnStats(dataProfile, dataName,
                                         thisStageNumber, operatorToUse,
                                         operationFlag)
        dataFlowInventory = getDataFlowStats(dataFlow, dataProfile, dataName,
                                             thisStageNumber, operatorToUse,
                                             operationFlag)

        # Finally save the data flow so it can be passed onto the next stage of the process...
        targetPackagePath = saveDataFlowPackage(dataFlow, dataName,
                                                thisStageNumber, qualityFlag)
        print('{0}: saved package to {1}'.format(dataName, targetPackagePath))

        return dataFlow, columnInventory, dataFlowInventory

    else:
        print('{0}: no package file found at location {1}'.format(
            dataName, fullPackagePath))
        return None, None, None


#%%
dataFlowInventoryAll = dataFlowProcessingLoop(previousStageNumber,
                                              thisStageNumber, 'A',
                                              'RemoveDuplicates',
                                              removeDuplicates)

#%%
dataFlowInventoryAll

示例#2

文件： newstage_2_23_automaticallyDetectTypes.py 项目： liamehenderson/dataprep

        # Now generate column and data flow inventories
        columnInventory = getColumnStats(dataProfile, dataName,
                                         thisStageNumber, operatorToUse,
                                         operationFlag)
        dataFlowInventory = getDataFlowStats(dataFlow, dataProfile, dataName,
                                             thisStageNumber, operatorToUse,
                                             operationFlag)

        # Finally save the data flow so it can be passed onto the next stage of the process...
        targetPackagePath = saveDataFlowPackage(dataFlow, dataName,
                                                thisStageNumber, 'A')
        print('{0}: saved package to {1}'.format(dataName, targetPackagePath))

        return dataFlow, columnInventory, dataFlowInventory

    else:
        print('{0}: no package file found at location {1}'.format(
            dataName, fullPackagePath))
        return None, None, None


#%%
dataFlowInventoryAll = dataFlowProcessingLoop(previousStageNumber,
                                              thisStageNumber, 'A',
                                              'AutoDetectTypes',
                                              automaticallyDetectColumnTypes)

#%%
dataFlowInventoryAll

示例#3

文件： newstage_5_50_createUPMDataflow.py 项目： liamehenderson/dataprep

        dataProfile = dataFlow.get_profile()

        # Now generate column and data flow inventories
        columnInventory = getColumnStats(dataProfile, dataName,
                                         thisStageNumber, operatorToUse,
                                         operationFlag)
        dataFlowInventory = getDataFlowStats(dataFlow, dataProfile, dataName,
                                             thisStageNumber, operatorToUse,
                                             operationFlag)

        # Finally save the data flow so it can be passed onto the next stage of the process...
        targetPackagePath = saveDataFlowPackage(dataFlow, dataName,
                                                thisStageNumber, qualityFlag)
        print('{0}: saved package to {1}'.format(dataName, targetPackagePath))

        return dataFlow, columnInventory, dataFlowInventory

    else:
        print('{0}: no package file found at location {1}'.format(
            dataName, fullPackagePath))
        return None, None, None


#%%
dataFlowInventoryAll = dataFlowProcessingLoop(previousStageNumber,
                                              thisStageNumber, 'A',
                                              'UPMDataFlowMapping',
                                              createUPMDataflow)

#%%
dataFlowInventoryAll

示例#4

文件： newstage_3_33_mapLookups.py 项目： liamehenderson/dataprep

        dataProfile = dataFlow.get_profile()

        # Now generate column and data flow inventories
        columnInventory = getColumnStats(dataProfile, dataName,
                                         thisStageNumber, operatorToUse,
                                         operationFlag)
        dataFlowInventory = getDataFlowStats(dataFlow, dataProfile, dataName,
                                             thisStageNumber, operatorToUse,
                                             operationFlag)

        # Finally save the data flow so it can be passed onto the next stage of the process...
        targetPackagePath = saveDataFlowPackage(dataFlow, dataName,
                                                thisStageNumber, qualityFlag)
        print('{0}: saved package to {1}'.format(dataName, targetPackagePath))

        return dataFlow, columnInventory, dataFlowInventory

    else:
        print('{0}: no package file found at location {1}'.format(
            dataName, fullPackagePath))
        return None, None, None


#%%
dataFlowInventoryAll = dataFlowProcessingLoop(previousStageNumber,
                                              thisStageNumber, 'A',
                                              'MapLookups', mapLookups)

#%%
dataFlowInventoryAll

示例#5

文件： newstage_3_34_cleanUpAddress.py 项目： liamehenderson/dataprep

        dataProfile = dataFlow.get_profile()

        # Now generate column and data flow inventories
        columnInventory = getColumnStats(dataProfile, dataName,
                                         thisStageNumber, operatorToUse,
                                         operationFlag)
        dataFlowInventory = getDataFlowStats(dataFlow, dataProfile, dataName,
                                             thisStageNumber, operatorToUse,
                                             operationFlag)

        # Finally save the data flow so it can be passed onto the next stage of the process...
        targetPackagePath = saveDataFlowPackage(dataFlow, dataName,
                                                thisStageNumber, 'A')
        print('{0}: saved package to {1}'.format(dataName, targetPackagePath))

        return dataFlow, columnInventory, dataFlowInventory

    else:
        print('{0}: no package file found at location {1}'.format(
            dataName, fullPackagePath))
        return None, None, None


#%%
dataFlowInventoryAll = dataFlowProcessingLoop(previousStageNumber,
                                              thisStageNumber, 'A',
                                              'CleanUpAddress', cleanUpAddress)

#%%
dataFlowInventoryAll

示例#6

        dataProfile = dataFlow.get_profile()

        # Now generate column and data flow inventories
        columnInventory = getColumnStats(dataProfile, dataName,
                                         thisStageNumber, operatorToUse,
                                         operationFlag)
        dataFlowInventory = getDataFlowStats(dataFlow, dataProfile, dataName,
                                             thisStageNumber, operatorToUse,
                                             operationFlag)

        # Finally save the data flow so it can be passed onto the next stage of the process...
        targetPackagePath = saveDataFlowPackage(dataFlow, dataName,
                                                thisStageNumber, 'A')
        print('{0}: saved package to {1}'.format(dataName, targetPackagePath))

        return dataFlow, columnInventory, dataFlowInventory

    else:
        print('{0}: no package file found at location {1}'.format(
            dataName, fullPackagePath))
        return None, None, None


#%%
dataFlowInventoryAll = dataFlowProcessingLoop(previousStageNumber,
                                              thisStageNumber, 'A',
                                              'RenameColumns', renameColumns)

#%%
dataFlowInventoryAll

示例#7

文件： newstage_2_20_parseNulls.py 项目： liamehenderson/dataprep

        dataProfile = dataFlow.get_profile()

        # Now generate column and data flow inventories
        columnInventory = getColumnStats(dataProfile, dataName,
                                         thisStageNumber, operatorToUse,
                                         operationFlag)
        dataFlowInventory = getDataFlowStats(dataFlow, dataProfile, dataName,
                                             thisStageNumber, operatorToUse,
                                             operationFlag)

        # Finally save the data flow so it can be passed onto the next stage of the process...
        targetPackagePath = saveDataFlowPackage(dataFlow, dataName,
                                                thisStageNumber, 'A')
        print('{0}: saved package to {1}'.format(dataName, targetPackagePath))

        return dataFlow, columnInventory, dataFlowInventory

    else:
        print('{0}: no package file found at location {1}'.format(
            dataName, fullPackagePath))
        return None, None, None


#%%
dataFlowInventoryAll = dataFlowProcessingLoop(previousStageNumber,
                                              thisStageNumber, 'A',
                                              'ParseNullString', parseNulls)

#%%
dataFlowInventoryAll

示例#8

#%% [markdown]
# # Join Tables - Pass 2
# To join two tables:
# - Define left and right dataflows
# - Define left and right join keys
# NOTE - future enhancement would be to quarantine orphaned rows from left and right dataflows

#%%
# Import all of the libraries we need to use...
import pandas as pd
import azureml.dataprep as dprep
import os as os
import re as re
import collections
from azureml.dataprep import col
from azureml.dataprep import Dataflow
from commonDataFlowProcessingLoop import dataFlowProcessingLoop
from commonInventoryCreation import getColumnStats, getDataFlowStats
from commonPackageHandling import openDataFlowPackage, saveDataFlowPackage
from commonJoinHandling import joinTables

# Let's also set up global variables and common functions...
previousStageNumber = '40'
thisStageNumber = '41'

#%%
dataFlowInventoryAll = dataFlowProcessingLoop(previousStageNumber, thisStageNumber, 'A', 'JoinTablesPass2', joinTables)

#%%
dataFlowInventoryAll

示例#9

        columnInventoryIntermediate = columnInventoryIntermediate.append(
            columnInventory)

        dataFlowInventory = getDataFlowStats(dataFlow, dataProfile, dataName,
                                             thisStageNumber, operatorToUse,
                                             operationFlag)
        dataFlowInventoryIntermediate = dataFlowInventoryIntermediate.append(
            dataFlowInventory)

        # Finally save the data flow so it can be passed onto the next stage of the process...
        targetPackagePath = saveDataFlowPackage(dataFlow, dataName,
                                                thisStageNumber, qualityFlag)
        print('{0}: saved package to {1}'.format(dataName, targetPackagePath))

        # Now return all of the components badk to the main loop...
        return dataFlow, columnInventoryIntermediate, dataFlowInventoryIntermediate

    else:
        print('{0}: no package file found at location {1}'.format(
            dataName, fullPackagePath))
        return None, None, None


#%%
dataFlowInventoryAll = dataFlowProcessingLoop(previousStageNumber,
                                              thisStageNumber, 'A',
                                              'SplitTable',
                                              splitTableBasedOnSingleColumn)

#%%
dataFlowInventoryAll

示例#10

文件： newstage_2_22_quarantineExtraColumns.py 项目： liamehenderson/dataprep

        # Now generate column and data flow inventories
        columnInventory = getColumnStats(dataProfile, dataName,
                                         thisStageNumber, operatorToUse,
                                         operationFlag)
        dataFlowInventory = getDataFlowStats(dataFlow, dataProfile, dataName,
                                             thisStageNumber, operatorToUse,
                                             operationFlag)

        # Finally save the data flow so it can be passed onto the next stage of the process...
        targetPackagePath = saveDataFlowPackage(dataFlow, dataName,
                                                thisStageNumber, qualityFlag)
        print('{0}: saved package to {1}'.format(dataName, targetPackagePath))

        # Now return all of the components badk to the main loop...
        return dataFlow, columnInventory, dataFlowInventory

    else:
        print('{0}: no package file found at location {1}'.format(
            dataName, fullPackagePath))
        return None, None, None


#%%
dataFlowInventoryAll = dataFlowProcessingLoop(previousStageNumber,
                                              thisStageNumber, 'A',
                                              'QuarantineExtraColumns',
                                              quarantineRows)

#%%
dataFlowInventoryAll

示例#11

文件： newstage_2_21_removeFirstRows.py 项目： liamehenderson/dataprep

        # Now generate column and data flow inventories
        columnInventory = getColumnStats(dataProfile, dataName,
                                         thisStageNumber, operatorToUse,
                                         operationFlag)
        dataFlowInventory = getDataFlowStats(dataFlow, dataProfile, dataName,
                                             thisStageNumber, operatorToUse,
                                             operationFlag)

        # Finally save the data flow so it can be passed onto the next stage of the process...
        targetPackagePath = saveDataFlowPackage(dataFlow, dataName,
                                                thisStageNumber, qualityFlag)
        print('{0}: saved package to {1}'.format(dataName, targetPackagePath))

        # Now return all of the components badk to the main loop...
        return dataFlow, columnInventory, dataFlowInventory

    else:
        print('{0}: no package file found at location {1}'.format(
            dataName, fullPackagePath))
        return None, None, None


#%%
dataFlowInventoryAll = dataFlowProcessingLoop(previousStageNumber,
                                              thisStageNumber, 'A',
                                              'RemoveRowsFromTop',
                                              removeRowsFromTop)

#%%
dataFlowInventoryAll