def export(self, array, *args, **kwargs):
workers = utility._get_workers(array.interface)
ports = [random.randint(50000, 60000) for _ in workers]
pool = multiprocessing.Pool(processes=len(workers))
uris = utility._get_worker_uris(workers, None, scheme='tcp',
include_path=False, ports=ports)
threading.Thread(target=self._write_pipes, args = (array, uris)).start()
return namedtuple('Intermediate', ['source', 'uris', 'schema'])(
array, uris, SciDBSchema(array))
def export(cls, array, *args, **kwargs):
filename = utility._get_filename() #path='/home/bhaynes/scidb') #TODO
workers = utility._get_workers(array.interface)
uris = utility._get_worker_uris(workers, filename)
array.interface.query("save(scan({}), '{}', -1, '{}')".format(
array.name, filename, 'csv+'))
return namedtuple('Intermediate',
['source', 'uris', 'schema'])(array, uris,
SciDBSchema(array))
def export(self, array, *args, **kwargs):
pipename = utility._get_filename() #path='/home/bhaynes/scidb') #TODO
workers = utility._get_workers(array.interface)
uris = utility._get_worker_uris(workers, pipename)
pool = multiprocessing.Pool(processes=len(workers))
pool.map(_create_pipe, izip(repeat(pipename), workers))
threading.Thread(target=self._write_pipes,
args=(array, pipename)).start()
return namedtuple('Intermediate',
['source', 'uris', 'schema'])(array, uris,
SciDBSchema(array))
def import_(cls, source, intermediate, *args, **kwargs):
interface = scidbpy.connect(kwargs['url'])
workers = utility._get_workers(interface)
pool = multiprocessing.Pool(processes=len(workers))
name = source.name.replace(':', '_')
schema = SciDBSchema(source.schema).local
try:
result = interface.query("create_array({}, {})".format(name, schema))
except Exception:
pass
pool.map(partial(_load, name, schema, kwargs['url'],
kwargs['hostname'], kwargs['port']), workers)
pool.close()
pool.join()
def export(cls, array, *args, **kwargs):
#TODO
filename = utility._get_filename() #TODO path='/home/bhaynes/scidb')
#filename = utility._get_filename(path=kwargs.pop('path', None))
workers = utility._get_workers(array.interface)
uris = utility._get_worker_uris(workers[:1], filename)
import time
start = time.time()
array.interface.query("save(scan({}), '{}', -2, '{}')"
.format(array.name, filename, 'csv+'))
print time.time() - start
return namedtuple('Intermediate', ['source', 'uris', 'schema'])(
array, uris, SciDBSchema(array))