# See the License for the specific language governing permissions and # limitations under the License. from rdc.etl.extra.example import Harness, build_producer, build_simple_transform, run from rdc.etl.io import STDOUT, STDOUT2 from rdc.etl.transform.flow.split import Split from rdc.etl.transform.util import Log print('################') print('# Linear shape #') print('################') print print('Producer -> SimpleTransform -> Log') h = Harness() p1 = build_producer('Producer 1') h.add_chain(p1, build_simple_transform(), Log()) run(h) print('#####################################') print('# Split shape (2 different outputs) #') print('#####################################') print print('Producer -> Split ---(stdout)--> SimpleTransform1 -> Log1') print(' `-(stdout2)-> SimpleTransform2 -> Log2') h = Harness() producer = build_producer('Producer 1', 10) split = Split(output_selector = lambda h: h.get('id') % 2 and STDOUT2 or STDOUT)
# limitations under the License. from rdc.etl.extra.example import Harness, build_producer, run from rdc.etl.io import STDIN2 from rdc.etl.status.console import ConsoleStatus from rdc.etl.transform.flow.sort import Sort from rdc.etl.transform.flow.sortedjoin import SortedJoin from rdc.etl.transform.util import Log print('#################') print('# Software sort #') print('#################') print print('Producer -> Sort -> Log') h = Harness() h.status.append(ConsoleStatus()) p1 = build_producer('Producer 1') h.add_chain(p1, Sort(key=('id', )), Log()) run(h) print('###############') print('# Sorted Join #') print('###############') print print("Producer1 -> Sort -(stdin)---> SortedJoin --> Log") print("Producer2 -> Sort -(stdin2)-'") h = Harness() h.status.append(ConsoleStatus()) p1 = build_producer('Producer 1')
from rdc.etl.extra.example import Harness, build_producer, run from rdc.etl.extra.simple import SimpleTransform from rdc.etl.transform.util import Log print('########################') print('# Simplest ETL process #') print('########################') print print('Producer -> Transform -> Log') h = Harness() p = build_producer('Producer') t = SimpleTransform() h.add_chain(p, t, Log()) run(h) print '\n'.join(map(repr, h.get_threads()))
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from rdc.etl.extra.example import Harness, build_producer, build_simple_transform, run from rdc.etl.io import STDOUT, STDOUT2 from rdc.etl.transform.flow.split import Split from rdc.etl.transform.util import Log print('################') print('# Linear shape #') print('################') print print('Producer -> SimpleTransform -> Log') h = Harness() p1 = build_producer('Producer 1') h.add_chain(p1, build_simple_transform(), Log()) run(h) print('#####################################') print('# Split shape (2 different outputs) #') print('#####################################') print print('Producer -> Split ---(stdout)--> SimpleTransform1 -> Log1') print(' `-(stdout2)-> SimpleTransform2 -> Log2') h = Harness() producer = build_producer('Producer 1', 10) split = Split(output_selector=lambda h: h.get('id') % 2 and STDOUT2 or STDOUT) h.add_chain(producer, split, build_simple_transform(), Log())
# limitations under the License. from rdc.etl.extra.example import Harness, build_producer, run from rdc.etl.io import STDIN2 from rdc.etl.status.console import ConsoleStatus from rdc.etl.transform.flow.sort import Sort from rdc.etl.transform.flow.sortedjoin import SortedJoin from rdc.etl.transform.util import Log print('#################') print('# Software sort #') print('#################') print print('Producer -> Sort -> Log') h = Harness() h.status.append(ConsoleStatus()) p1 = build_producer('Producer 1') h.add_chain(p1, Sort(key=('id',)), Log()) run(h) print('###############') print('# Sorted Join #') print('###############') print print("Producer1 -> Sort -(stdin)---> SortedJoin --> Log") print("Producer2 -> Sort -(stdin2)-'") h = Harness() h.status.append(ConsoleStatus()) p1 = build_producer('Producer 1')