Python Harness Examples

Example #1

File: example_shapes.py Project: hartym/rdc.etl

# See the License for the specific language governing permissions and
# limitations under the License.

from rdc.etl.extra.example import Harness, build_producer, build_simple_transform, run
from rdc.etl.io import STDOUT, STDOUT2
from rdc.etl.transform.flow.split import Split
from rdc.etl.transform.util import Log


print('################')
print('# Linear shape #')
print('################')
print
print('Producer -> SimpleTransform -> Log')

h = Harness()
p1 = build_producer('Producer 1')
h.add_chain(p1, build_simple_transform(), Log())
run(h)


print('#####################################')
print('# Split shape (2 different outputs) #')
print('#####################################')
print
print('Producer -> Split ---(stdout)--> SimpleTransform1 -> Log1')
print('                   `-(stdout2)-> SimpleTransform2 -> Log2')

h = Harness()
producer = build_producer('Producer 1', 10)
split = Split(output_selector = lambda h: h.get('id') % 2 and STDOUT2 or STDOUT)

Example #2

File: example_flow.py Project: yelord/rdc.etl

# limitations under the License.

from rdc.etl.extra.example import Harness, build_producer, run
from rdc.etl.io import STDIN2
from rdc.etl.status.console import ConsoleStatus
from rdc.etl.transform.flow.sort import Sort
from rdc.etl.transform.flow.sortedjoin import SortedJoin
from rdc.etl.transform.util import Log

print('#################')
print('# Software sort #')
print('#################')
print
print('Producer -> Sort -> Log')

h = Harness()
h.status.append(ConsoleStatus())
p1 = build_producer('Producer 1')
h.add_chain(p1, Sort(key=('id', )), Log())
run(h)

print('###############')
print('# Sorted Join #')
print('###############')
print
print("Producer1 -> Sort -(stdin)---> SortedJoin --> Log")
print("Producer2 -> Sort -(stdin2)-'")

h = Harness()
h.status.append(ConsoleStatus())
p1 = build_producer('Producer 1')

Example #3

from rdc.etl.extra.example import Harness, build_producer, run
from rdc.etl.extra.simple import SimpleTransform
from rdc.etl.transform.util import Log

print('########################')
print('# Simplest ETL process #')
print('########################')
print
print('Producer -> Transform -> Log')

h = Harness()
p = build_producer('Producer')
t = SimpleTransform()
h.add_chain(p, t, Log())
run(h)

print '\n'.join(map(repr, h.get_threads()))

Example #4

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from rdc.etl.extra.example import Harness, build_producer, build_simple_transform, run
from rdc.etl.io import STDOUT, STDOUT2
from rdc.etl.transform.flow.split import Split
from rdc.etl.transform.util import Log

print('################')
print('# Linear shape #')
print('################')
print
print('Producer -> SimpleTransform -> Log')

h = Harness()
p1 = build_producer('Producer 1')
h.add_chain(p1, build_simple_transform(), Log())
run(h)

print('#####################################')
print('# Split shape (2 different outputs) #')
print('#####################################')
print
print('Producer -> Split ---(stdout)--> SimpleTransform1 -> Log1')
print('                   `-(stdout2)-> SimpleTransform2 -> Log2')

h = Harness()
producer = build_producer('Producer 1', 10)
split = Split(output_selector=lambda h: h.get('id') % 2 and STDOUT2 or STDOUT)
h.add_chain(producer, split, build_simple_transform(), Log())

Example #5

File: example_flow.py Project: hartym/rdc.etl

# limitations under the License.

from rdc.etl.extra.example import Harness, build_producer, run
from rdc.etl.io import STDIN2
from rdc.etl.status.console import ConsoleStatus
from rdc.etl.transform.flow.sort import Sort
from rdc.etl.transform.flow.sortedjoin import SortedJoin
from rdc.etl.transform.util import Log

print('#################')
print('# Software sort #')
print('#################')
print
print('Producer -> Sort -> Log')

h = Harness()
h.status.append(ConsoleStatus())
p1 = build_producer('Producer 1')
h.add_chain(p1, Sort(key=('id',)), Log())
run(h)

print('###############')
print('# Sorted Join #')
print('###############')
print
print("Producer1 -> Sort -(stdin)---> SortedJoin --> Log")
print("Producer2 -> Sort -(stdin2)-'")

h = Harness()
h.status.append(ConsoleStatus())
p1 = build_producer('Producer 1')