def export(self, array, *args, **kwargs): workers = utility._get_workers(array.interface) ports = [random.randint(50000, 60000) for _ in workers] pool = multiprocessing.Pool(processes=len(workers)) uris = utility._get_worker_uris(workers, None, scheme='tcp', include_path=False, ports=ports) threading.Thread(target=self._write_pipes, args = (array, uris)).start() return namedtuple('Intermediate', ['source', 'uris', 'schema'])( array, uris, SciDBSchema(array))
def export(cls, array, *args, **kwargs): filename = utility._get_filename() #path='/home/bhaynes/scidb') #TODO workers = utility._get_workers(array.interface) uris = utility._get_worker_uris(workers, filename) array.interface.query("save(scan({}), '{}', -1, '{}')".format( array.name, filename, 'csv+')) return namedtuple('Intermediate', ['source', 'uris', 'schema'])(array, uris, SciDBSchema(array))
def export(self, array, *args, **kwargs): pipename = utility._get_filename() #path='/home/bhaynes/scidb') #TODO workers = utility._get_workers(array.interface) uris = utility._get_worker_uris(workers, pipename) pool = multiprocessing.Pool(processes=len(workers)) pool.map(_create_pipe, izip(repeat(pipename), workers)) threading.Thread(target=self._write_pipes, args=(array, pipename)).start() return namedtuple('Intermediate', ['source', 'uris', 'schema'])(array, uris, SciDBSchema(array))
def import_(cls, source, intermediate, *args, **kwargs): interface = scidbpy.connect(kwargs['url']) workers = utility._get_workers(interface) pool = multiprocessing.Pool(processes=len(workers)) name = source.name.replace(':', '_') schema = SciDBSchema(source.schema).local try: result = interface.query("create_array({}, {})".format(name, schema)) except Exception: pass pool.map(partial(_load, name, schema, kwargs['url'], kwargs['hostname'], kwargs['port']), workers) pool.close() pool.join()
def export(cls, array, *args, **kwargs): #TODO filename = utility._get_filename() #TODO path='/home/bhaynes/scidb') #filename = utility._get_filename(path=kwargs.pop('path', None)) workers = utility._get_workers(array.interface) uris = utility._get_worker_uris(workers[:1], filename) import time start = time.time() array.interface.query("save(scan({}), '{}', -2, '{}')" .format(array.name, filename, 'csv+')) print time.time() - start return namedtuple('Intermediate', ['source', 'uris', 'schema'])( array, uris, SciDBSchema(array))