def run(self):
self.read_pipe.close()
http_export = E.http_transport(self.shard_id,
config.dsn,
E.HTTP_EXPORT,
compression=True,
encryption=True)
http_import = E.http_transport(self.shard_id,
config.dsn,
E.HTTP_IMPORT,
compression=True,
encryption=True)
# Send list of proxy strings, one element per HTTP transport instance
self.write_pipe.send(
[http_export.get_proxy(),
http_import.get_proxy()])
self.write_pipe.close()
pd = http_export.export_to_callback(cb.export_to_pandas, None)
print(f'EXPORT shard_id:{self.shard_id}, affected_rows:{len(pd)}')
http_import.import_from_callback(cb.import_from_pandas, pd)
print(f'IMPORT shard_id:{self.shard_id}, affected_rows:{len(pd)}')
def run(self):
self.read_pipe.close()
http_export = pyexasol.http_transport(self.node['host'], self.node['port'], pyexasol.HTTP_EXPORT, compression=True, encryption=True)
http_import = pyexasol.http_transport(self.node['host'], self.node['port'], pyexasol.HTTP_IMPORT, compression=True, encryption=True)
# Send list of proxy strings, one element per HTTP transport instance
self.write_pipe.send([http_export.get_proxy(), http_import.get_proxy()])
self.write_pipe.close()
pd = http_export.export_to_callback(cb.export_to_pandas, None)
print(f"EXPORT shard_id:{self.node['idx']}, affected_rows:{len(pd)}")
http_import.import_from_callback(cb.import_from_pandas, pd)
print(f"IMPORT shard_id:{self.node['idx']}, affected_rows:{len(pd)}")
def run(self):
self.read_pipe.close()
http = E.http_transport(self.shard_id, config.dsn, E.HTTP_IMPORT)
self.write_pipe.send(http.get_proxy())
self.write_pipe.close()
data = [
{
'user_id': 1,
'user_name': 'John',
'shard_id': self.shard_id
},
{
'user_id': 2,
'user_name': 'Foo',
'shard_id': self.shard_id
},
{
'user_id': 3,
'user_name': 'Bar',
'shard_id': self.shard_id
},
]
pd = pandas.DataFrame(data,
columns=['user_id', 'user_name', 'shard_id'])
http.import_from_callback(cb.import_from_pandas, pd)
def run(self):
self.read_pipe.close()
http = pyexasol.http_transport(self.node['host'], self.node['port'],
pyexasol.HTTP_IMPORT)
self.write_pipe.send(http.get_proxy())
self.write_pipe.close()
data = [
{
'user_id': 1,
'user_name': 'John',
'shard_id': self.node['idx']
},
{
'user_id': 2,
'user_name': 'Foo',
'shard_id': self.node['idx']
},
{
'user_id': 3,
'user_name': 'Bar',
'shard_id': self.node['idx']
},
]
pd = pandas.DataFrame(data,
columns=['user_id', 'user_name', 'shard_id'])
http.import_from_callback(cb.import_from_pandas, pd)
def run(self):
self.read_pipe.close()
http = E.http_transport(self.shard_id, config.dsn, E.HTTP_EXPORT)
self.write_pipe.send(http.get_proxy())
self.write_pipe.close()
pd = http.export_to_callback(cb.export_to_pandas, None)
print(f'{self.shard_id}:{len(pd)}')
def run(self):
self.read_pipe.close()
http = pyexasol.http_transport(self.node['host'], self.node['port'],
pyexasol.HTTP_EXPORT)
self.write_pipe.send(http.get_proxy())
self.write_pipe.close()
pd = http.export_to_callback(cb.export_to_pandas, None)
print(f"{self.node['idx']}:{len(pd)}")
def run(self):
self.read_pipe.close()
# Init separate HTTP transport connections for EXPORT and IMPORT
http_export = pyexasol.http_transport(self.node['ipaddr'], self.node['port'], compression=True, encryption=True)
http_import = pyexasol.http_transport(self.node['ipaddr'], self.node['port'], compression=True, encryption=True)
# Send pairs of internal Exasol address to parent process
self.write_pipe.send([http_export.exa_address, http_import.exa_address])
self.write_pipe.close()
# Read data from HTTP transport to DataFrame
pd = http_export.export_to_callback(cb.export_to_pandas, None)
print(f"EXPORT child process {self.node['idx']} finished, exported rows:{len(pd)}")
# Modify data set
pd['GROSS_AMT'] = pd['GROSS_AMT'] + 1
# Write data back to HTTP transport
http_import.import_from_callback(cb.import_from_pandas, pd)
print(f"IMPORT child process {self.node['idx']} finished, imported rows:{len(pd)}")
def run(self):
self.read_pipe.close()
# Init HTTP transport connection
http = pyexasol.http_transport(self.node['ipaddr'], self.node['port'])
# Send internal Exasol address to parent process
self.write_pipe.send(http.exa_address)
self.write_pipe.close()
# Read data from HTTP transport to DataFrame
pd = http.export_to_callback(cb.export_to_pandas, None)
print(
f"Child process {self.node['idx']} finished, exported rows: {len(pd)}"
)
def run(self):
self.read_pipe.close()
# Init HTTP transport connection
http = pyexasol.http_transport(self.node['ipaddr'], self.node['port'])
# Send internal Exasol address to parent process
self.write_pipe.send(http.exa_address)
self.write_pipe.close()
data = [
{
'user_id': 1,
'user_name': 'John',
'shard_id': self.node['idx']
},
{
'user_id': 2,
'user_name': 'Foo',
'shard_id': self.node['idx']
},
{
'user_id': 3,
'user_name': 'Bar',
'shard_id': self.node['idx']
},
]
pd = pandas.DataFrame(data,
columns=['user_id', 'user_name', 'shard_id'])
# Send data from DataFrame to HTTP transport
http.import_from_callback(cb.import_from_pandas, pd)
print(
f"Child process {self.node['idx']} finished, imported rows: {len(pd)}"
)