Beispiel #1
0
    from conf_analysis.meg import artifacts, preprocessing
    from conf_analysis.behavior import empirical, metadata, keymap
    import mne, locale
    import numpy as np
    import pickle
    locale.setlocale(locale.LC_ALL, "en_US")
    result = {}
    raw = mne.io.read_raw_ctf(filename, system_clock='ignore')
    trials = preprocessing.blocks(raw, full_file_cache=True)
    trl, bl = trials['trial'], trials['block']
    bcnt = 0
    for b in np.unique(bl):
        if len(trl[bl == b]) >= 75:
            result[b] = bcnt
            bcnt += 1
        print((b, bcnt))
    return result


block_map = {}
for snum in range(1, 16):
    filenames = [metadata.get_raw_filename(snum, b) for b in range(4)]
    block_map[snum] = {}
    for session, filename in enumerate(filenames):
        block_map[snum][session] = executor.submit(do_one, filename)

diagnostics.progress(block_map)
block_map = executor.gather(block_map)

pickle.dump(block_map, open('blockmap.pickle', 'w'))
Beispiel #2
0
SCHEDULER_PORT = 5678
SCHEDULER_HTTP_PORT = 9786
SCHEDULER_BOKEH_PORT = 12345
SCHEDULER_IP = '127.0.0.1'

HOME_PAGE = 'http://localhost:5050'


def test(one, two):
    return 4


executor = Executor('{}:{}'.format(SCHEDULER_IP, SCHEDULER_PORT))

taskclient.add_user(HOME_PAGE)
for _ in range(5):
    taskclient.add_job(HOME_PAGE)

result_list = []
num_iters = 50
#result = executor.map(taskclient.worker, [HOME_PAGE, HOME_PAGE], [0, 1])
result = executor.map(taskclient.worker,
                      itertools.repeat(HOME_PAGE, num_iters), range(num_iters))
result_list = result

distributed.diagnostics.progress(result_list)
print()
print(executor.who_has(result_list))
sim_results = executor.gather(result_list)
print('---------')
Beispiel #3
0
class DistributedContext(object):
    io_loop = None
    io_thread = None

    def __init__(self,
                 ip="127.0.0.1",
                 port=8787,
                 spawn_workers=0,
                 write_partial_results=None,
                 track_progress=False,
                 time_limit=None,
                 job_observer=None):
        """
        :type ip: string
        :type port: int
        :type spawn_workers: int
        :type write_partial_results: int
        :type track_progress: bool
        :type time_limit: int
        :type job_observer: JobObserver
        """

        self.worker_count = spawn_workers
        self.ip = ip
        self.port = port
        self.active = False
        self.write_partial_results = write_partial_results
        self.track_progress = track_progress
        self.execution_count = 0
        self.timeout = TimeoutManager(time_limit) if time_limit else None
        self.job_observer = job_observer

        if not DistributedContext.io_loop:
            DistributedContext.io_loop = IOLoop()
            DistributedContext.io_thread = Thread(
                target=DistributedContext.io_loop.start)
            DistributedContext.io_thread.daemon = True
            DistributedContext.io_thread.start()

        if spawn_workers > 0:
            self.scheduler = self._create_scheduler()
            self.workers = [self._create_worker()
                            for i in xrange(spawn_workers)]
            time.sleep(0.5)  # wait for workers to spawn

        self.executor = Executor((ip, port))

    def run(self, domain,
            worker_reduce_fn, worker_reduce_init,
            global_reduce_fn, global_reduce_init):
        size = domain.steps
        assert size is not None  # TODO: Iterators without size

        workers = 0
        for name, value in self.executor.ncores().items():
            workers += value

        if workers == 0:
            raise Exception("There are no workers")

        batch_count = workers * 4
        batch_size = max(int(round(size / float(batch_count))), 1)
        batches = self._create_batches(batch_size, size, domain,
                                       worker_reduce_fn, worker_reduce_init)

        logging.info("Qit: starting {} batches with size {}".format(
            batch_count, batch_size))

        if self.job_observer:
            self.job_observer.on_computation_start(batch_count, batch_size)

        futures = self.executor.map(process_batch, batches)

        if self.track_progress:
            distributed.diagnostics.progress(futures)

        if self.write_partial_results is not None:
            result_saver = ResultSaver(self.execution_count,
                                       self.write_partial_results)
        else:
            result_saver = None

        timeouted = False
        results = []

        for future in as_completed(futures):
            job = future.result()
            if result_saver:
                result_saver.handle_result(job.result)
            if self.job_observer:
                self.job_observer.on_job_completed(job)

            results.append(job.result)

            if self.timeout and self.timeout.is_finished():
                logging.info("Qit: timeouted after {} seconds".format(
                    self.timeout.timeout))
                timeouted = True
                break

        # order results
        if not timeouted:
            results = [j.result for j in self.executor.gather(futures)]

        self.execution_count += 1

        if worker_reduce_fn is None:
            results = list(itertools.chain.from_iterable(results))

        logging.info("Qit: finished run with size {} (taking {})".format(
            len(results), domain.size))

        results = results[:domain.size]  # trim results to required size

        if global_reduce_fn is None:
            return results
        else:
            if global_reduce_init is None:
                return reduce(global_reduce_fn, results)
            else:
                return reduce(global_reduce_fn, results, global_reduce_init)

    def _create_scheduler(self):
        scheduler = Scheduler(ip=self.ip)
        scheduler.start(self.port)
        return scheduler

    def _create_worker(self):
        worker = Worker(scheduler_ip=self.ip,
                        scheduler_port=self.port,
                        ncores=1)
        worker.start(0)
        return worker

    def _create_batches(self, batch_size, size,
                        domain,
                        worker_reduce_fn,
                        worker_reduce_init):
        batches = []
        i = 0

        while True:
            new = i + batch_size
            if i + batch_size <= size:
                batches.append((domain, i, batch_size,
                                worker_reduce_fn, worker_reduce_init))
                i = new
                if new == size:
                    break
            else:
                batches.append((domain, i, size - i,
                                worker_reduce_fn, worker_reduce_init))
                break

        return batches
Beispiel #4
0
import argparse


def inject(url):
    import config
    from pymongo import MongoClient
    import os
    #connstring = config.db['host'] + ":" + config.db['port']
    run = "sqlmap -u " + url + ' --batch '
    command = os.popen(run).read()
    data = {'url': url, 'output': command}
    client = MongoClient('192.168.1.14:27017')
    dbn = config.db['dbname']
    db = client[dbn]
    results = db.results
    results.insert(data)
    return command


executor = Executor()
parser = argparse.ArgumentParser()
parser.add_argument("-f")
args = parser.parse_args()
links = []
s = open(args.f, 'r')
for x in s:
    links.append(x.rstrip().replace("\n", ""))

job = executor.map(inject, links)
print executor.gather(job)
Beispiel #5
0
        if (not args.append) and (os.path.exists(out_path)):
            shutil.rmtree(out_path)

        if not os.path.exists(out_path):
            os.makedirs(os.path.join(out_path, 'feat/df'))
            os.makedirs(os.path.join(out_path, 'feat/desc'))

        odDicts = [{
            'flight_hdf': args.uvan,
            'img_num': ii,
            'kp_det_func': kp_type_dict[kp_type],
            'kp_desc_func': kp_type_dict[kp_type],
            'p_meta': tf_meta,
            'o_path': out_path
        } for ii in range(num_imgs)]

        r = executor.map(extract_kp_from_frame, odDicts, pure=False)

        kp_list = executor.gather(r)
        kp_meta = pd.DataFrame(kp_list,
                               columns=[
                                   'num_feat', 'center_lon', 'center_lat',
                                   'df_path', 'desc_path', 'flight', 'img_num'
                               ])

        kp_meta.to_hdf(os.path.join(out_path, 'feat_meta.hdf'),
                       key='feat_meta')

    print('what')