def run(self, nodes, master, user, user_shell, volumes): for worker, uri in self.work: log.info("Worker #%d ingesting %s", worker, uri) with master.ssh.remote_file(DEPLOYMENT_PATH, 'r') as descriptor: connection = MyriaConnection(deployment=descriptor, ssl=self.ssl) log.info("MyriaConnection URI: " + connection._url_start) relation = MyriaRelation(self.name, schema=self.schema, connection=connection) query = MyriaQuery.parallel_import( relation, self.work, scan_type=self.scan_type, scan_parameters=self.scan_parameters, insert_type=self.insert_type, insert_parameters=self.insert_parameters, timeout=self.timeout) log.info("Ingesting as query %d", query.query_id) if self.wait_for_completion: query.wait_for_completion() log.info("Ingest complete (%d, %s)", query.query_id, query.status) MyriaInstaller.web_restart(master)
from myria import MyriaConnection, MyriaRelation, MyriaQuery, MyriaSchema import sys connection = MyriaConnection(hostname = sys.argv[1], port=sys.argv[2]) tenant = sys.argv[3] schema = MyriaSchema({"columnNames": ["l_orderkey", "l_linenumber", "l_custkey", "l_partkey", "l_suppkey", "l_orderdate", "l_orderpriority", "l_shippriority", "l_quantity", "l_extendedprice", "l_ordtotalprice", "l_discount", "l_revenue", "l_supplycost", "l_tax", "l_commitdate", "l_shipmode"], "columnTypes": ["LONG_TYPE", "LONG_TYPE", "LONG_TYPE", "LONG_TYPE", "LONG_TYPE", "STRING_TYPE", "STRING_TYPE", "LONG_TYPE", "DOUBLE_TYPE", "DOUBLE_TYPE", "DOUBLE_TYPE", "DOUBLE_TYPE", "LONG_TYPE", "LONG_TYPE", "DOUBLE_TYPE", "LONG_TYPE", "STRING_TYPE"]}) #12 relation = MyriaRelation('public:adhoc10GB12W_tenant'+ str(tenant) + ':lineitem', connection=connection, schema=schema) work = [(1, 'https://s3-us-west-2.amazonaws.com/tpchssb/12Workers/lineitem-part1'), (2, 'https://s3-us-west-2.amazonaws.com/tpchssb/12Workers/lineitem-part2'), (3, 'https://s3-us-west-2.amazonaws.com/tpchssb/12Workers/lineitem-part3'), (4, 'https://s3-us-west-2.amazonaws.com/tpchssb/12Workers/lineitem-part4'), (5, 'https://s3-us-west-2.amazonaws.com/tpchssb/12Workers/lineitem-part5'), (6, 'https://s3-us-west-2.amazonaws.com/tpchssb/12Workers/lineitem-part6'), (7, 'https://s3-us-west-2.amazonaws.com/tpchssb/12Workers/lineitem-part7'), (8, 'https://s3-us-west-2.amazonaws.com/tpchssb/12Workers/lineitem-part8'), (9, 'https://s3-us-west-2.amazonaws.com/tpchssb/12Workers/lineitem-part9'), (10, 'https://s3-us-west-2.amazonaws.com/tpchssb/12Workers/lineitem-part10'), (11, 'https://s3-us-west-2.amazonaws.com/tpchssb/12Workers/lineitem-part11'), (12, 'https://s3-us-west-2.amazonaws.com/tpchssb/12Workers/lineitem-part12')] queryImport = MyriaQuery.parallel_import(relation=relation, work=work, scan_parameters={'delimiter':'|'}, insert_parameters={'argOverwriteTable':True})