コード例 #1
0
def main():
    SDL_Init(SDL_INIT_VIDEO)
    window = SDL_CreateWindow(b"Bolidozor Snapshot Browser",
                              SDL_WINDOWPOS_CENTERED, SDL_WINDOWPOS_CENTERED,
                              430, 600, SDL_WINDOW_SHOWN)
    windowsurface = SDL_GetWindowSurface(window)

    main_thread_queue = queue.Queue()

    def run_on_main_thread(func):
        main_thread_queue.put(func)

        event = SDL_Event()
        event.type = SDL_USEREVENT
        SDL_PushEvent(event)

    thpool = mpdummy.Pool(processes=3)

    connector = bzpost.HTTPConnector(
        "http://space.astro.cz/bolidozor/OBSUPICE/OBSUPICE-R3/")
    connector.connect()

    drawable_snapshots = []

    def put_up_snapshot(snapshot):
        print("downloading %s..." % snapshot.url)
        x = urllib.request.urlopen(snapshot.url)

        fits = pyfits.open(io.BytesIO(x.read()))
        print("downloading %s... done" % snapshot.url)

        imunit = None
        for unit in fits:
            if unit.data is not None:
                imunit = unit

        img = imunit.data[:, :]
        imin, imax = np.min(img), np.max(img)
        rgbimg = np.repeat(
            ((img - imin) / (imax - imin) * 255).astype(np.uint8), 3)

        def finish():
            h, w = img.shape
            surface = SDL_CreateRGBSurfaceFrom(rgbimg.ctypes.data, w, h, 24,
                                               3 * w, 0, 0, 0, 0)
            drawable_snapshots.append({
                'time': snapshot.time,
                'surface': surface,
                'imgdata': rgbimg
            })

        run_on_main_thread(finish)

    collection = SnapshotCollection(
        connector, lambda a: thpool.apply_async(put_up_snapshot, (a, )))

    time = 1421000000
    running = True
    event = SDL_Event()

    run_on_main_thread(lambda: None)

    while running:
        while SDL_WaitEvent(ctypes.byref(event)) != 0:
            if event.type == SDL_QUIT:
                running = False
                break

            if event.type == SDL_MOUSEWHEEL:
                time -= event.wheel.y * 10 * TIME_PER_PIX
                break

            if event.type == SDL_USEREVENT:
                if not main_thread_queue.empty():
                    while not main_thread_queue.empty():
                        main_thread_queue.get()()

                    break

        collection.cover(time - 800 * TIME_PER_PIX * 4,
                         time + 800 * TIME_PER_PIX * 5)

        sdl2.ext.fill(windowsurface.contents, BLACK)

        for snapshot in drawable_snapshots:
            y = (snapshot['time'] - bzpost.normalize_time(
                int(time))).total_seconds() / TIME_PER_PIX
            SDL_BlitSurface(snapshot['surface'], None, windowsurface,
                            SDL_Rect(10, int(y), 0, 0))

        SDL_UpdateWindowSurface(window)

    SDL_DestroyWindow(window)
    SDL_Quit()

    return 0
コード例 #2
0
    def __init__(self,
                 policies: Sequence[py_policy.PyPolicy],
                 multithreading: bool = True):
        """Batch together multiple (non-batched) py policies.

    The policies can be different but must use the same action and
    observation specs.

    Args:
      policies: List python policies (must be non-batched).
      multithreading: Python bool describing whether interactions with the
        given policies should happen in their own threadpool.  If `False`,
        then all interaction is performed serially in the current thread.

        This may be combined with `TFPyPolicy(..., py_policy_is_batched=True)`
        to ensure that multiple policies are all run in the same thread.

    Raises:
      ValueError: If policies is not a list or tuple, or is zero length, or if
        one of the policies is already batched.
      ValueError: If the action or observation specs don't match.
    """
        if not isinstance(policies, (list, tuple)):
            raise ValueError("policies must be a list or tuple.  Got: %s" %
                             policies)

        self._parallel_execution = multithreading
        self._policies = policies
        self._num_policies = len(policies)
        self._time_step_spec = self._policies[0].time_step_spec
        self._action_spec = self._policies[0].action_spec
        self._policy_state_spec = self._policies[0].policy_state_spec
        self._info_spec = self._policies[0].info_spec
        self._policy_step_spec = self._policies[0].policy_step_spec
        self._trajectory_spec = self._policies[0].trajectory_spec
        self._collect_data_spec = self._policies[0].collect_data_spec
        self._observation_and_action_constraint_splitter = \
            self._policies[0].observation_and_action_constraint_splitter

        self._validate_spec(py_policy.PyPolicy.time_step_spec,
                            self._time_step_spec)
        self._validate_spec(py_policy.PyPolicy.action_spec, self._action_spec)
        self._validate_spec(py_policy.PyPolicy.policy_state_spec,
                            self._policy_state_spec)
        self._validate_spec(py_policy.PyPolicy.info_spec, self._info_spec)
        self._validate_spec(py_policy.PyPolicy.policy_step_spec,
                            self._policy_step_spec)
        self._validate_spec(py_policy.PyPolicy.trajectory_spec,
                            self._trajectory_spec)
        self._validate_spec(py_policy.PyPolicy.collect_data_spec,
                            self._collect_data_spec)
        self._validate_spec(
            py_policy.PyPolicy.observation_and_action_constraint_splitter,
            self._observation_and_action_constraint_splitter)

        # Create a multiprocessing threadpool for execution.
        if multithreading:
            self._pool = mp_threads.Pool(self._num_policies)

        super(BatchedPyPolicy,
              self).__init__(self._time_step_spec, self._action_spec,
                             self._policy_state_spec, self._info_spec,
                             self._observation_and_action_constraint_splitter)
コード例 #3
0
            with open(str(file), "r", encoding="utf8") as f:
                content = f.readlines()
            if not only_sleep1(
                    content) or len(content) <= 5:  #FIXME: that's broken test
                replay_file_list.append(file)
            else:
                print("skipping " + str(file))

    print(replay_file_list)

    if len(replay_file_list) == 0:
        continue

    pool_size = 8

    with mp.Pool(processes=pool_size) as pool:
        task_list = []
        for image_name in image_name_list:
            task = pool.apply_async(run_cases_on_image, (
                apk_path,
                replay_file_list,
                image_name,
            ))
            task_list.append(task)
        for task in task_list:
            try:
                print("[*]", str(task.get(timeout=400)), "DONE!")
            except Exception as e:
                print(e)

    logging.info("ALL DONE!")
コード例 #4
0
ファイル: h2csmuggler.py プロジェクト: rapradab/h2csmuggler-1
def init():
    global MAX_TIMEOUT, UPGRADE_ONLY

    if sys.version_info < (3, 0):
        sys.stdout.write("Sorry, requires Python 3.x, not Python 2.x\n")
        sys.exit(1)

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description="Detect and exploit insecure forwarding of h2c upgrades.",
        epilog="Example Usage:\n"
               + sys.argv[0] + " --scan-list urls.txt --threads 5\n"
               + sys.argv[0] + " -x https://edgeserver http://localhost\n"
               + sys.argv[0] + " -x https://edgeserver -XPOST -d "
               "'{\"data\":1}' -H \"Content-Type: application/json\""
               "-H \"X-ADMIN: true\" http://backend/private/endpoint"
    )
    parser.add_argument("--scan-list",
                        help="list of URLs for scanning")
    parser.add_argument("--threads",
                        type=int,
                        default=5,
                        help="# of threads (for use with --scan-list)")
    parser.add_argument("--upgrade-only",
                        default=False,
                        action="store_true",
                        help="drop HTTP2-Settings from outgoing "
                             "Connection header")
    parser.add_argument("-x", "--proxy",
                        help="proxy server to try to bypass")
    parser.add_argument("-i", "--wordlist",
                        help="list of paths to bruteforce")
    parser.add_argument("-X", "--request",
                        default="GET",
                        help="smuggled verb")
    parser.add_argument("-d", "--data",
                        help="smuggled data")
    parser.add_argument("-H", "--header",
                        action="append",
                        help="smuggled headers")
    parser.add_argument("-m", "--max-time",
                        type=float,
                        default=10,
                        help="socket timeout in seconds "
                             "(type: float; default 10)")
    parser.add_argument("-t", "--test",
                        help="test a single proxy server",
                        action="store_true")
    parser.add_argument("-v", "--verbose",
                        action="store_true")
    parser.add_argument("url", nargs="?")
    args = parser.parse_args()

    MAX_TIMEOUT = args.max_time
    UPGRADE_ONLY = args.upgrade_only

    if args.scan_list:
        lines = []
        with open(args.scan_list) as fd:
            lines = [line.strip() for line in fd.readlines()]

        p = mp.Pool(args.threads)
        p.map(scan, lines)
        p.close()
        p.join()
        sys.exit(1)

    if not args.proxy:
        print("Please provide a server for tunneling ('-x') flag ",
              file=sys.stderr)
        sys.exit(1)

    if not args.test and not args.url:
        print("Please specify the '-t' flag or provide smuggled URL")
        sys.exit(1)

    if args.url and not urlparse(args.url).scheme:
        print("Please specify scheme (e.g., http[s]://) for: " + args.url)
        sys.exit(1)

    if not urlparse(args.proxy).scheme:
        print("Please specify scheme (e.g., http[s]://) for: " + args.proxy)
        sys.exit(1)

    main(args)
def main(test, m):
    p = multiprocessing.Pool()
    results = p.map(lambda f: f(test), [read_RuFactEval, runtest])
    p.close()
    p.join()
    factsRuFactEval = results[0]

    #  runtest(test) #если есть результат, то не нужно запускать повторно
    # factsRuFactEval = read_RuFactEval(test)
    factsPatterns = read_OurFact(test)
    allFactsPatternsCount = 0
    for book in factsPatterns:
        allFactsPatternsCount += len(factsPatterns[book])
    allFactsRuFactEvalCount = 0
    for book in factsRuFactEval:
        allFactsRuFactEvalCount += len(factsRuFactEval[book])

    gaz = cleanBYgaz(factsRuFactEval)
    hard = cleanBYhard(factsRuFactEval)
    sent = cleanBYsent(test, factsRuFactEval)
    all = cleanBYhard(cleanBYgaz((sent)))
    print("Отфильтрованные базы для", test, "получены")

    p = multiprocessing.Pool()
    run = [[testPHexact, "EQ", factsRuFactEval.copy()],
           [testPHabout, "Normalize",
            factsRuFactEval.copy()],
           [testPHexact, "EQ-gaz-filter",
            gaz.copy()], [testPHabout, "Normalize-gaz-filter",
                          gaz.copy()],
           [testPHexact, "EQ-hard-filter",
            hard.copy()], [testPHabout, "Normalize-hard-filter",
                           hard.copy()],
           [testPHexact, "EQ-sent-filter",
            sent.copy()], [testPHabout, "Normalize-sent-filter",
                           sent.copy()],
           [testPHexact, "EQ-all-filter",
            all.copy()], [testPHabout, "Normalize-all-filter",
                          all.copy()]]
    results = p.map(
        lambda f: analyse_results(test, f[2], factsPatterns.copy(), f[0], f[1]
                                  ), run)
    # print(results)
    p.close()
    p.join()

    trueFactsPatternsCount = results[0][0]
    trueFactsPatternsCount2 = results[1][0]
    th1 = results[0][1]
    th2 = results[1][1]

    data = ""
    data += m
    data += ("\n\nВсего извлечено = \t" + str(allFactsPatternsCount))
    data += ("\nВсего должно быть извлечено = \t" +
             str(allFactsRuFactEvalCount))
    data += ("\n\nИзвлечено правильно (точное совпадение) = \t" +
             str(trueFactsPatternsCount))
    data += ("\n\nИз них повышенной сложности = \t" + str(th1))
    if trueFactsPatternsCount > 0:
        data += PRF(trueFactsPatternsCount, allFactsPatternsCount,
                    allFactsRuFactEvalCount)
    if trueFactsPatternsCount2 > 0:
        # print("\nА ещё мы посчитали результаты с учётом форм слов")
        data += ("\n\nИзвлечено правильно, с учётом форм слов = " +
                 str(trueFactsPatternsCount2))
        data += ("\n\nИз них повышенной сложности = \t" + str(th2))
        data += PRF(trueFactsPatternsCount2, allFactsPatternsCount,
                    allFactsRuFactEvalCount)
    k = 1

    k += 1
    ro = results[k][0]
    if ro > 0:
        data += (
            "\n\nРезульаты с очисткой по газеттиру (точное совпадение), правильно = \t"
            + str(ro))
        data += ("\n\nРазмер словаря правильных ответов = \t" + str(len(gaz)))
        data += PRF(ro, allFactsPatternsCount, allFactsRuFactEvalCount)

    k += 1
    ro = results[k][0]
    if ro > 0:
        data += (
            "\n\nРезульаты с очисткой по газеттиру (нормализация), правильно = "
            + str(ro))
        data += PRF(ro, allFactsPatternsCount, allFactsRuFactEvalCount)

    k += 1
    ro = results[k][0]
    if ro > 0:
        data += (
            "\n\nРезульаты с очисткой по сложности (точное совпадение), правильно = "
            + str(ro))
        data += ("\n\nРазмер словаря правильных ответов = \t" + str(len(hard)))
        data += PRF(ro, allFactsPatternsCount, allFactsRuFactEvalCount)

    k += 1
    ro = results[k][0]
    if ro > 0:
        data += (
            "\n\nРезульаты с очисткой по сложности (нормализация), правильно = "
            + str(ro))
        data += PRF(ro, allFactsPatternsCount, allFactsRuFactEvalCount)

    k += 1
    ro = results[k][0]
    if ro > 0:
        data += (
            "\n\nРезульаты с очисткой по предложениям (точное совпадение), правильно = "
            + str(ro))
        data += ("\n\nРазмер словаря правильных ответов = \t" + str(len(sent)))
        data += PRF(ro, allFactsPatternsCount, allFactsRuFactEvalCount)

    k += 1
    ro = results[k][0]
    if ro > 0:
        data += (
            "\n\nРезульаты с очисткой по предложениям (нормализация), правильно = "
            + str(ro))
        data += PRF(ro, allFactsPatternsCount, allFactsRuFactEvalCount)

    k += 1
    ro = results[k][0]
    if ro > 0:
        data += (
            "\n\nРезульаты с полной очисткой (точное совпадение), правильно = "
            + str(ro))
        data += ("\n\nРазмер словаря правильных ответов = \t" + str(len(all)))
        data += PRF(ro, allFactsPatternsCount, allFactsRuFactEvalCount)

    k += 1
    ro = results[k][0]
    if ro > 0:
        data += (
            "\n\nРезульаты с полной очисткой (нормализация), правильно = " +
            str(ro))
        data += PRF(ro, allFactsPatternsCount, allFactsRuFactEvalCount)

    wfile = open("Results.txt", "a", encoding="cp1251")
    wfile.write(data)
    wfile.close()
    print("результаты записаны:", test)
コード例 #6
0
ファイル: database.py プロジェクト: robertcolvin/AudiThor
    def rdsItem(self, current, aconnect, itemlist):
        added = False
        printColor(['_____LISTING RDS [] now....in .%s' % (aconnect._region)])
        threaded = aconnect.getMultiThread()
        nEnv = aconnect._env
        lfound = []
        objs = []
        rows = []

        pools = 2  # 171.35 seconds  #168 seconds
        totalItems = len(itemlist)
        q = None
        if threaded:
            pools = auditMeth.poolThreadNumber(totalItems, mp.cpu_count())

            p = mp.Pool(1 if pools == 0 else pools)

        else:
            p = mp.Pool()
            m = mp.Manager()
            q = m.Queue()
        if totalItems > 0:
            objs.append([
                'Name[%s]' % (current.svc), 'Audit', 'Owner', 'Engine',
                'Size (GB)', 'Instance', 'MutliAZ', 'VPC', 'last_Modified',
                'connections', 'cost'
            ])
        if not threaded:
            for unit in itemlist:
                name = unit['DBInstanceIdentifier']
                #sg=unit['VpcSecurityGroups'][0]['VpcSecurityGroupId']
                #ec2 = aconnect.__get_client__('ec2')
                #sgUnit = ec2.describe_security_groups(GroupIds=[sg])['SecurityGroups'][0]
                #account = sgUnit['OwnerId']
                if nEnv in name or aconnect._useAccounts:
                    if pools == 0:
                        objs, row = rdsDefine(unit, name, q)
                    else:
                        getit = p.apply_async(rdsDefine, (unit, name, q))
                    #getit.get()
                    lfound.append(name)

                    #objs,row = self.rdsDefine(current,aconnect,objs,unit,name)
                    #rows.append(row)
        else:
            #print itemlist
            lfound = [unit['DBInstanceIdentifier'] for unit in itemlist]
            results = p.map(rdsDefine, (unit for unit in itemlist))

        if pools > 0:
            p.close()
            p.join()
            if threaded:
                for que in results:
                    newobjs, row = que
                    #print ' DYNAMO --got[C] result', row
                    objs = objs + newobjs
                    rows.append(row)
            else:
                while not q.empty():
                    newobjs, row = q.get()
                    objs = objs + newobjs
                    # print newobjs
                    rows.append(row)
        print objs
        return (lfound, objs, rows)
コード例 #7
0
def get_multiple_entities(entities):
    import multiprocessing.dummy as multiprocessing
    return dict(multiprocessing.Pool(THREAD_POOL_PROCESSES).map(_get_entity, entities))
コード例 #8
0
ファイル: gui.py プロジェクト: znes/openmod.sh
from .bookkeeping import Job
from openmod.sh.api import (provide_element_api, json_to_db,
                            provide_elements_api, provide_sequence_api,
                            allowed_file, explicate_hubs,
                            delete_element_from_db, results_to_db,
                            get_hub_results, create_transmission,
                            get_flow_results, get_flow_result, get_co2_results)
from openmod.sh.forms import ComputeForm
from openmod.sh.visualization import make_graph_plot
from openmod.sh.web import app, csrf
from openmod.sh import mcbeth
from openmod.sh.config import get_config

# Set up a pool of workers to which jobs can be submitted and a dictionary
# which stores the asynchronous result objects.
app.workers = mpd.Pool(1) if app.debug else mpp.Pool(1)
app.results = OD()

babel = Babel(app)


@babel.localeselector
def get_locale():
    # if a user is logged in, use the locale from the user settings
    user = getattr(flask.g, 'user', None)
    if user is not None:
        return user.locale
    # otherwise try to guess the language from the user accept
    # header the browser transmits. The best match wins.
    return flask.request.accept_languages.best_match(['de', 'en'])
コード例 #9
0

if __name__ == "__main__":

    PARSER_ = argparse.ArgumentParser(description="Parameters")
    PARSER_.add_argument("--annotation",
                         nargs="?",
                         type=str,
                         default=None,
                         help="File annotation path")
    PARSER_.add_argument("--path",
                         nargs="?",
                         type=str,
                         default=None,
                         help="Dataset path")

    ARGS_ = PARSER_.parse_args()
    annotation_file = ARGS_.annotation
    dataset_path = ARGS_.path
    cloud_path = dataset_path + '/cloud'
    pathlib.Path(cloud_path).mkdir(parents=True, exist_ok=True)
    with open(annotation_file, 'r') as f:
        f.readline()
        fx, fy, rx, ry = csv_camera_info(f.readline())
        f.readline()
        f.readline()
        pf = mp.Pool(12)
        for i, _ in enumerate(pf.imap_unordered(thread_cloud, f, 1)):
            #sys.stderr.write('\rdone {0:%}\n'.format(i/scene_.m_total_frames))
            sys.stderr.write('\rdone: {}'.format(i))
コード例 #10
0
ファイル: network.py プロジェクト: albamhp/mcv-m6-2019-team5
    def __init__(self, num_dims: int = 16):
        super().__init__()

        self.embedding_net = EmbeddingNet(num_dims)
        self.pool = mp.Pool(processes=3)
コード例 #11
0
ファイル: construct_bim_dataset.py プロジェクト: ml-lab/bim
def divide_train_val():
    all_masks = []
    for dir_name in ['obj', 'scene', 'scene_only']:
        image_dir = os.path.join(os.getcwd(), 'data', dir_name)
        if tf.io.gfile.exists(image_dir):
            tf.io.gfile.rmtree(image_dir)
        tf.io.gfile.makedirs(image_dir)
        train_dir = os.path.join(image_dir, 'train')
        tf.io.gfile.makedirs(train_dir)
        val_dir = os.path.join(image_dir, 'val')
        tf.io.gfile.makedirs(val_dir)

        loc_table = {}
        lines = tf.io.gfile.GFile(os.path.join(image_dir + '_all',
                                               'loc.txt')).readlines()
        for line in lines:
            loc_table[line.split(' ')[0]] = line.split(' ')[1]

        classes = tf.io.gfile.listdir(image_dir + '_all')
        classes.remove('loc.txt')
        classes.sort()

        pool = multiprocessing.Pool(100)
        val_lines = []
        loc_lines = []
        for i in range(len(classes)):
            a_class = classes[i]
            if dir_name == 'obj':
                all_masks.append(
                    np.load(os.path.join(os.getcwd(), 'data', 'coco',
                                         'segments', a_class, 'val_mask'),
                            allow_pickle=True))

            files = np.array(
                tf.io.gfile.listdir(os.path.join(image_dir + '_all', a_class)))
            train_files = []
            val_files = []
            for f in files:
                if re.search('9[0-9]{2}\.jpg', f):
                    val_files.append(f)
                else:
                    train_files.append(f)
            val_files.sort()
            tf.io.gfile.makedirs(os.path.join(train_dir, a_class))
            pool.map(
                lambda f: tf.io.gfile.copy(
                    os.path.join(image_dir + '_all', a_class, f),
                    os.path.join(train_dir, a_class, f)), train_files)
            pool.map(
                lambda f: tf.io.gfile.copy(
                    os.path.join(image_dir + '_all', a_class, f),
                    os.path.join(val_dir, f)), val_files)
            val_lines += [f + ' ' + str(i) + '\n' for f in val_files]
            loc_lines += [f + ' ' + loc_table[f] for f in val_files]

        val_txt_path = os.path.join(image_dir, 'val.txt')
        with tf.io.gfile.GFile(val_txt_path, 'w') as f:
            f.write(''.join(sorted(val_lines)))

        with tf.io.gfile.GFile(os.path.join(image_dir, 'val_loc.txt'),
                               'w') as f:
            f.write(''.join(sorted(loc_lines)))

        # Copy over the object masks for the validation set
        np.save(tf.io.gfile.GFile(os.path.join(image_dir, 'val_mask'), 'w'),
                np.concatenate(all_masks))
        shutil.copy(os.path.join(image_dir + '_all', 'loc.txt'),
                    os.path.join(image_dir, 'loc.txt'))
        tf.io.gfile.rmtree(image_dir + '_all')
コード例 #12
0
    #     img = cv2.imread(wsi_path + '/' + name)
    #     cv2.imwrite(save_path + '/%f_%s' % (grade , name) , img)


reco_path = r'X:\GXB\SNS\test_result\unet_4down_wsi_reco'

path = r'X:\GXB\SNS\test_result\unet_4down_wsi'
wsi_names = os.listdir(path)

for wsi_name in wsi_names[1:]:
    print(wsi_name)
    img_names = os.listdir(path + '/' + wsi_name)
    img_names = [x for x in img_names if x.find('.tif') != -1]

    mean_value = []
    names = []
    pool = dummy.Pool(cpu_count())

    for img_name in img_names:
        t = pool.apply_async(post_handle,
                             args=(path + '/%s/%s' % (wsi_name, img_name), ))
        mean_value.append(t.get())
        names.append(img_name)
    pool.close()
    pool.join()

    mean_value = np.array(mean_value)
    print(np.shape(mean_value))

    recommand(reco_path + '/' + wsi_name, path + '/' + wsi_name, names,
              mean_value)
コード例 #13
0
 def map(self, func, args):
     with thrd.Pool(processes=self.processes) as pool:
         return pool.map(func, args)
コード例 #14
0
 def _prepare_pool(self, n_jobs=None):
     if n_jobs == 1:
         return dummy_processing.Pool(processes=n_jobs)
     return actual_processing.Pool(processes=n_jobs)
コード例 #15
0
ファイル: control.py プロジェクト: vladtcvs/tempesta
def servers_get_stats(servers):
    threads = __servers_pool_size(len(servers))
    pool = multiprocessing.Pool(threads)
    pool.map(Nginx.get_stats, servers)
コード例 #16
0
def similarity(cd, num_random_exp=None, num_workers=25):
  """Returns cosine similarity of all discovered concepts.

  Args:
    cd: The ConceptDiscovery module for discovered conceps.
    num_random_exp: If None, calculates average similarity using all the class's
      random concepts. If a number, uses that many random counterparts.
    num_workers: If greater than 0, runs the function in parallel.

  Returns:
    A similarity dict in the form of {(concept1, concept2):[list of cosine
    similarities]}
  """

  def concepts_similarity(cd, concepts, rnd, bn):
    """Calcualtes the cosine similarity of concept cavs.

    This function calculates the pairwise cosine similarity of all concept cavs
    versus an specific random concept

    Args:
      cd: The ConceptDiscovery instance
      concepts: List of concepts to calculate similarity for
      rnd: a random counterpart
      bn: bottleneck layer the concepts belong to

    Returns:
      A dictionary of cosine similarities in the form of
      {(concept1, concept2): [list of cosine similarities], ...}
    """
    similarity_dic = {}
    for c1 in concepts:
      cav1 = cd.load_cav_direction(c1, rnd, bn)
      for c2 in concepts:
        if (c1, c2) in similarity_dic.keys():
          continue
        cav2 = cd.load_cav_direction(c2, rnd, bn)
        similarity_dic[(c1, c2)] = cosine_similarity(cav1, cav2)
        similarity_dic[(c2, c1)] = similarity_dic[(c1, c2)]
    return similarity_dic

  similarity_dic = {bn: {} for bn in cd.bottlenecks}
  if num_random_exp is None:
    num_random_exp = cd.num_random_exp
  randoms = ['random500_{}'.format(i) for i in np.arange(num_random_exp)]
  concepts = {}
  for bn in cd.bottlenecks:
    concepts[bn] = [cd.target_class, cd.random_concept] + cd.dic[bn]['concepts']
  for bn in cd.bottlenecks:
    concept_pairs = [(c1, c2) for c1 in concepts[bn] for c2 in concepts[bn]]
    similarity_dic[bn] = {pair: [] for pair in concept_pairs}
    def t_func(rnd):
      return concepts_similarity(cd, concepts[bn], rnd, bn)
    if num_workers:
      pool = multiprocessing.Pool(num_workers)
      sims = pool.map(lambda rnd: t_func(rnd), randoms)
    else:
      sims = [t_func(rnd) for rnd in randoms]
    while sims:
      sim = sims.pop()
      for pair in concept_pairs:
        similarity_dic[bn][pair].append(sim[pair])
  return similarity_dic
コード例 #17
0
ファイル: database.py プロジェクト: robertcolvin/AudiThor
    def dynamoItem(self, current, aconnect, itemlist):
        added = False
        printColor(
            ['_____LISTING DynamoDB [] now....in .%s' % (aconnect._region)])
        threaded = aconnect.getMultiThread()
        nEnv = aconnect._env
        lfound = []
        objs = []
        rows = []

        pools = 2  # 171.35 seconds  #168 seconds
        totalItems = len(itemlist)
        if threaded:
            pools = auditMeth.poolThreadNumber(totalItems, mp.cpu_count())
            if pools > 0:
                p = mp.Pool(pools)
            elif pools == 0:
                threaded = False
        else:
            p = mp.Pool()
            m = mp.Manager()
            q = m.Queue()
        if totalItems > 0:
            objs.append([
                'Name[%s]' % (current.svc), 'Audit', 'Owner', 'Status',
                'PartitionKey', 'indexes', 'totalRead', 'totalWrite'
            ])

        client = aconnect.__get_client__('dynamodb')
        if not threaded:
            for name in itemlist:
                unit = client.describe_table(TableName=name)['Table']
                #account = dunit['TableArn']
                if nEnv in name or aconnect._useAccounts:
                    if pools == 0:
                        objs, row = dynamoDefine(unit, name, q)
                    else:
                        getit = p.apply_async(dynamoDefine, (unit, name, q))
                    #getit.get()
                    lfound.append(name)
                    #objs,row=self.dynamoDefine(current,aconnect,objs,unit,name)
                    #rows.append(row)
        else:
            #print itemlist
            lfound = [name for name in itemlist]
            #print lfound
            #print 'client ready?'
            #dd = client.describe_table(TableName='Tags')['Table']
            #print dd
            results = p.map(dynamoDefine,
                            (client.describe_table(TableName=name)['Table']
                             for name in itemlist))

        if pools > 0:
            p.close()
            p.join()
            if threaded:
                for que in results:
                    newobjs, row = que
                    #print ' RDS --got[C] result', row
                    objs = objs + newobjs
                    rows.append(row)
            else:
                while not q.empty():
                    newobjs, row = q.get()
                    objs = objs + newobjs
                    # print newobjs
                    rows.append(row)
        return (lfound, objs, rows)
コード例 #18
0
def parmap(fun,seq,N=None,Nt=1,chunksize=1,ordered=True,\
                daemon=False,progress=False):
    """
    parmap -- Simple parallel mapper that can split amongst processes (N)
              and threads (Nt) (within the processes).

              Does *NOT* require functions to be pickleable (unlike
              vanilla multiprocess.Pool.map)

    Inputs:
    -------
    fun
        Single input function. Use lambdas or functools.partial
        to enable/exapnd multi-input. See example

    sequence
        Sequence of inputs to map in parallel

    Options:
    --------
    N [None] (integer or None)
        Number of processes to use. If `None`, will use the CPU_COUNT

    Nt [1] (integer)
        Number of threads to use. See notes below on multi-threaded vs
        multi-processes.

    chunksize [1] (int)
        How to be break up the incoming sequence. Useful if also using threads.
        Will be (re)set to max(chunksize,Nt)

    ordered [True] (bool)
        Whether or not to order the results. If False, will return in whatever
        order they finished.

    daemon [False] (bool)
        Sets the multiprocessing `daemon` flag. If  True, can not spawn child
        processes (i.e. cannot nest parmap) but should allow for CTRL+C type
        stopping. Supposedly, there may be issues with CTRL+C with it set to
        False. Use at your own risk

    progress [False] (bool)
        Display a progress bar or counter.
        Warning: Inconsistant in iPython/Jupyter notebooks and may clear
        other printed content. Instead, specify as 'nb' to use a Jupyter 
        Widget progress

    Notes:
    ------

    Performs SEMI-lazy iteration based on chunksize. It will exhaust the input
    iterator but will yield as results are computed (This is similar to the
    `multiprocessing.Pool().imap` behavior)

    Explicitly wrap the parmap call in a list(...) to force immediate
    evaluation

    Threads and/or processes:
    -------------------------
    This tool has the ability to split work amongst python processes
    (via multiprocessing) and python threads (via the multiprocessing.dummy
    module). Python is not very performant in multi-threaded situations
    (due to the GIL) therefore, processes are the usually the best for CPU
    bound tasks and threading is good for those that release the GIL (such
    as IO-bound tasks). Note that many NumPy functions *do* release the GIL
    and can be threaded, but many NumPy functions are, themselves, multi-
    threaded.

    Alternatives:
    -------------

    This tool allows more data types, can split with threads, has an optional
    progress bar, and has fewer pickling issues, but these come at a small cost. 
    For simple needs, the following may be better:

    >>> import multiprocessing as mp
    >>> pool = mp.Pool(N) # Or mp.Pool() for N=None
    >>> results = list( pool.imap(fun,seq) ) # or just pool.map
    >>> pool.close()

    Process Method:
    ---------------
    This code uses iterators/generators to handle and distribute the workload.
    By doing this, it is easy to have all results pass through a common
    counting function for display of the progress without the use of
    global (multiprocessing manager) variables and locks.

    With the exception of when N == 1 (where it falls back to serial methods)
    the code works as follows:

    - A background thread is started the will iterate over the incoming sequence
      and add items to the queue. If the incoming sequence is exhausted, the
      worker sends kill signals into the queue.
        - The items are also chunked and indexed (used later)
    - After the background thread is started a function to pull from the OUTPUT
      queue is created. This counts the number of closed processes but otherwise
      yields the computed result items
    - A pool of workers is created. Each worker will read from the input queue
      and distribute the work amongst threads (if using). It will then
      return the resuts into a queue
    - Now the main work happens. It is done as chain of generators/iterators.
      The background worker has already begin adding items to the queue so
      now we work through the output queue. Note that this is in serial
      since the work was already done in parallel
            - Generator to pull from the result queue
            - Generator to count and display progress (if progress=True).
            - Generator to hold on to and return items in a sorted manner
              if sorting is requested. This can cause itermediate results to be
              stored until they can be returned in order
    - The output generator chain is iterated pulling items through and then
      are yielded.
    - cleanup.

    Last Updated:
    -------------
    2018-07-30
    """
    if N is None:
        N = CPU_COUNT

    chunksize = max(chunksize, Nt)

    try:
        tot = len(seq)
    except TypeError:
        tot = None

    if tqdm is None:
        if   isinstance(progress,(str,unicode))\
         and progress.lower() in ['jupyter','notebook','nb']:
            counter = partial(_counter_nb, tot=tot)
        else:
            counter = partial(_counter, tot=tot)
    else:
        if   isinstance(progress,(str,unicode))\
         and progress.lower() in ['jupyter','notebook','nb']\
         and hasattr(tqdm,'tqdm_notebook'):
            counter = partial(tqdm.tqdm_notebook, total=tot)
        else:
            counter = partial(
                tqdm.tqdm,
                total=tot)  # Set the total since tqdm won't be able to get it.

    if N == 1:
        if Nt == 1:
            out = imap(fun, seq)
        else:
            pool = mpd.Pool(Nt)  # thread pools don't have the pickle issues
            out = pool.imap(fun, seq)

        if progress:
            out = counter(out)
        for item in out:
            yield item

        if Nt > 1:
            pool.close()
        return

    q_in = mp.JoinableQueue()
    q_out = mp.Queue()

    # Start the workers
    workers = [
        mp.Process(target=_worker, args=(fun, q_in, q_out, Nt))
        for _ in xrange(N)
    ]

    for worker in workers:
        worker.daemon = daemon
        worker.start()

    # Create a separate thread to add to the queue in the background
    def add_to_queue():
        for iixs in _iter_chunks(enumerate(seq), chunksize):
            q_in.put(iixs)

        # Once (if ever) it is exhausted, send None to close workers
        for _ in xrange(N):
            q_in.put(None)

    add_to_queue_thread = Thread(target=add_to_queue)
    add_to_queue_thread.start()

    # Generator we use to return
    def queue_getter():
        finished = 0
        count = 0
        while finished < N:
            out = q_out.get()
            if out is None:
                finished += 1
                continue
            yield out

    # Chain generators on output
    out = queue_getter()
    if progress:
        out = counter(out)

    if ordered:
        out = _sort_generator_unique_integers(out, key=lambda a: a[0])

    # Return
    for item in out:
        yield item[1]

    # Clean up
    q_in.join()
    for worker in workers:
        worker.join()
コード例 #19
0

    if pbar:
        pbar.update(1)
    
    return a+b


# make pies
a_list = range(16)
b_list = range(16)
c_list = []

pbar = tqdm(total=len(a_list), position=0, leave=True,
            desc = f"aa: ", )
p=mp.Pool(workers)
c_list = p.starmap(foo, zip(a_list,b_list,repeat(pbar)))
p.close()
pbar.close()
p.join()

print(c_list)





# PIL read 

# https://pillow.readthedocs.io/en/stable/reference/Image.html
from PIL import Image
コード例 #20
0
import multiprocessing.dummy as mp

items = [0] * 10


def do_print(i):
    print(i)
    items[i] = i * 2 + 1


p = mp.Pool(4)
p.map(do_print,
      range(0, 10))  # range(0,1000) if you want to replicate your example
p.close()
p.join()
print(items)
コード例 #21
0
import numpy as np
import multiprocessing.dummy as mp


def dummy_function(x, y):
    return x * y


def dummy_iter(i):
    for i in range(i):
        yield (i, i)


if __name__ == '__main__':
    p = mp.Pool(1)
    itera = dummy_iter(10)
    results = p.starmap(dummy_function, itera)
    p.close()
    p.join()
    print(results)
コード例 #22
0
            new_xml_name = new_filename + '.xml'

            cv2.imwrite(new_img_name, combined_im)
            xml_doc.find('./filename').text = os.path.basename(new_img_name)
            xml_doc.write(new_xml_name, encoding='utf8')
            # cv2.imshow("coin", coin_im)
            # cv2.imshow("bg", bg_im)
            # cv2.imshow(str(uuid.uuid4()), combined_im)


if __name__ == '__main__':
    # for i in range(1):
    #     generate_new_image('T:\emb_fin\images\\190103_005.xml', 2)
    # while True:
    #     cv2.waitKey(500)
    if not os.path.exists(NEW_DATA_DIR):
        os.mkdir(NEW_DATA_DIR)

    xml_files = glob.glob(os.path.join(sys.argv[1], "*.xml"))
    p = mp.Pool()
    p.map(generate_new_image_with_blur, xml_files)
    p.close()
    p.join()
    # for xml_path in xml_files:
    #     try:
    #         generate_new_image(xml_path)
    #     except Exception as e:
    #         print("Error when processing image: " + xml_path)
    #         raise e
    # pass
コード例 #23
0
ファイル: ace.py プロジェクト: justcho5/ACE
  def create_patches(self, class_names, method='slic', discovery_images=None,
                     param_dict=None):
    """Creates a set of image patches using superpixel methods.

    This method takes in the concept discovery images and transforms it to a
    dataset made of the patches of those images.

    Args:
      method: The superpixel method used for creating image patches. One of
        'slic', 'watershed', 'quickshift', 'felzenszwalb'.
      discovery_images: Images used for creating patches. If None, the images in
        the target class folder are used.

      param_dict: Contains parameters of the superpixel method used in the form
                of {'param1':[a,b,...], 'param2':[z,y,x,...], ...}. For instance
                {'n_segments':[15,50,80], 'compactness':[10,10,10]} for slic
                method.
    """
    self.class_names = class_names
    print("START MAKING PATCHES")
    if param_dict is None:
      param_dict = {}
    dataset, image_numbers, patches, segment_class_labels = [], [], [], []
    if discovery_images is None:
        print("target class images")
        discovery_images= self.load_concept_imgs(self.target_class, self.num_discovery_imgs)
        np.save(os.path.join(self.np_dir,"discovery_images.npy"), discovery_images)
    else:
      discovery_images_list = [self.load_concept_imgs(clss, math.floor(self.num_discovery_imgs/len(self.class_names)), return_filenames=True) for clss in self.class_names]
      discovery_images = [i for i,j in discovery_images_list]
      filenames = [j for i,j in discovery_images_list]
      
      with open(os.path.join(self.np_dir,'discovery_filenames.txt'), 'w') as f:
        for i, lst in enumerate(filenames):
          for filename in lst:
            f.write("{} {} {}\n".format(i, self.class_names[i], filename))
      class_labels = [[i]*len(d) for i, d in enumerate(discovery_images)]
      self.discovery_class_labels = np.concatenate(class_labels) # index for class in self.class_names
      discovery_images = np.concatenate(discovery_images)
      filenames=np.concatenate(filenames)
      np.save(os.path.join(self.np_dir,"discovery_images.npy"), discovery_images)
      np.save(os.path.join(self.np_dir,"filenames.npy"),filenames)
    if self.num_workers:
      pool = multiprocessing.Pool(self.num_workers)
      outputs = pool.map(
          lambda img: self._return_superpixels(img, method, param_dict),
          discovery_images)
      for fn, sp_outputs in enumerate(outputs):
        image_superpixels, image_patches = sp_outputs
        for superpixel, patch in zip(image_superpixels, image_patches):
          dataset.append(superpixel)
          patches.append(patch)
          image_numbers.append(fn)
          segment_class_labels.append(self.discovery_class_labels[fn])

    else:
      print("num images: {}".format(len(discovery_images)))
      for fn, img in enumerate(discovery_images):
        print(fn)

        p_dict = param_dict.copy()
        print(p_dict)
        image_superpixels, image_patches = self._return_superpixels(
            img, method, p_dict)

        for superpixel, patch in zip(image_superpixels, image_patches):
          dataset.append(superpixel)
          patches.append(patch)
          image_numbers.append(fn)
          segment_class_labels.append(self.discovery_class_labels[fn])
    image_superpixels.clear()
    image_patches.clear()
    print("starting np loading")
    np.save(os.path.join(self.np_dir, "dataset.npy"),np.array(dataset, dtype=np.float16))
    dataset.clear()
    np.save(os.path.join(self.np_dir, "image_numbers.npy"),np.array(image_numbers, dtype=np.int8))
    image_numbers.clear()
    np.save(os.path.join(self.np_dir, "patches.npy"),np.array(patches, dtype=np.float16))
    patches.clear()
    self.segment_class_labels = np.array(segment_class_labels)
    np.save(os.path.join(self.np_dir, "segment_class_labels.npy"), np.array(segment_class_labels,dtype=np.int8))
    self.discovery_size=len(discovery_images)
    # np_dataset = np.array(dataset, dtype=np.float16)
    # np_image_numbers = np.array(image_numbers, dtype=np.int16)
    # np_patches = np.array(patches, dtype=np.float16)



    #
    # all_objects = muppy.get_objects()
    # sum1 = summary.summarize(all_objects)
    # summary.print_(sum1)


    print("end of np loading")



    print("discover concepts done")
コード例 #24
0
def test():
    reuse = False
    if os.path.isfile('distribution_results/activity.pkl') and reuse:
        with open('distribution_results/activity.pkl', 'rb') as f:
            video_label = pickle.load(f)
    else:
        video_label = {}

    test_video_path = 'dataset/Video/BreakfastII_15fps_qvga_sync'

    with tf.Session(config=config) as sess:
        Network = activity_network(number_of_classes=49)
        IO_tool = IO_manager(Batch_size, frames_per_step, window_size, sess)
        IO_tool.openpose.load_openpose_weights()
        IO_tool.hidden_states_dim = Network.lstm_units
        sess.run(Network.init)
        dataset = IO_tool.dataset.Train_dataset
        untrimmed = IO_tool.dataset.untrimmed_train_dataset

        train_writer = tf.summary.FileWriter("logdir/test", sess.graph)

        tot_second = 0
        for root, dirs, files in os.walk(test_video_path):
            for video_name in files:
                path = root + '/' + video_name
                if path not in untrimmed.keys():
                    continue
                video = cv2.VideoCapture(path)
                video.set(cv2.CAP_PROP_POS_AVI_RATIO, 1)
                max_len = int(video.get(cv2.CAP_PROP_POS_MSEC) / 1000)
                tot_second += max_len

        # Loading initial C3d or presaved network
        use_pretrained_model = True
        if os.path.isfile('./checkpoint/checkpoint') and use_pretrained_model:
            print('new model loaded')
            Network.model_saver.restore(
                sess, tf.train.latest_checkpoint('./checkpoint'))
        elif load_c3d and use_pretrained_model:
            print('original c3d loaded')
            Network.c3d_loader.restore(sess, c3d_saved_weigths)

        with tf.name_scope('whole_saver'):
            whole_saver = tf.train.Saver()
        whole_saver.save(sess, model_filename, global_step=Network.global_step)

        pbar_whole = tqdm(total=(tot_second), desc='Videos')
        for root, dirs, files in os.walk(test_video_path):
            for video_name in files:
                print(video_name)
                path = root + '/' + video_name
                if path not in untrimmed.keys():
                    continue
                video = cv2.VideoCapture(path)
                video.set(cv2.CAP_PROP_POS_AVI_RATIO, 1)
                max_len = video.get(cv2.CAP_PROP_POS_MSEC) / 1000
                fps = video.get(cv2.CAP_PROP_FPS)

                step_times = [v for v in range(1, int(max_len / 1))]

                pbar_video = tqdm(total=(len(step_times)),
                                  desc='Video_completition')
                for t_end in step_times:
                    if video_name in video_label.keys():
                        if t_end in video_label[video_name].keys():
                            pbar_video.update(1)
                            continue

                    segment = [(t_end - 1) * fps + 2, t_end * fps]

                    pbar = tqdm(total=(tasks * Batch_size * frames_per_step +
                                       tasks),
                                leave=False,
                                desc='Batch Generation')

                    def multiprocess_batch(x):
                        X, c, h, label = IO_tool.test_generator(
                            pbar, path, segment)
                        return {'X': X, 'c': c, 'h': h, 'label': label}

                    pool = mp.Pool(processes=tasks)
                    ready_batch = pool.map(multiprocess_batch, range(0, 1))
                    pbar.close()
                    pool.close()
                    pool.join()
                    batch = IO_tool.add_pose(ready_batch, sess,
                                             augment=False)[0]

                    conv3 = sess.run(
                        [Network.conv3],
                        feed_dict={
                            Network.input_batch: batch['X'],
                            Network.c_input: batch['c'],
                            Network.h_input: batch['h']
                        })

                    if video_name not in video_label.keys():
                        video_label[video_name] = {}

                    video_label[video_name][t_end] = {}
                    video_label[video_name][t_end]['tensor'] = conv3
                    video_label[video_name][t_end]['label'] = batch['label']
                    video_label[video_name][t_end]['limb'] = batch['X'][
                        0, :, :, :, 3]
                    video_label[video_name][t_end]['joint'] = batch['X'][
                        0, :, :, :, 4]

                    pbar_video.update(1)
                    pbar_whole.update(1)

                with open(
                        'distribution_results/activity' + video_name + '.pkl',
                        'wb') as f:
                    pickle.dump(video_label, f, pickle.HIGHEST_PROTOCOL)
                pbar_video.close()
コード例 #25
0
def listFilePath():
    query = urllib.urlencode({
        b'path': g_netDirPath.encode('utf-8')
    }).decode('utf-8')
    url = 'http://%s/listdir?%s' % (g_netHost, query)
    for line in urllib2.urlopen(url).readlines():
        g_filePathList.append(line.decode('utf-8').strip())


def userInput():
    global g_netHost, g_netDirPath, g_localDirPath
    i = raw_input('input net host(default=localhost:8080):').decode(
        sys.getfilesystemencoding())
    if i: g_netHost = i
    g_netDirPath = raw_input('input net dir:').decode(
        sys.getfilesystemencoding())
    assert g_netDirPath
    g_localDirPath = raw_input('input local dir:').decode(
        sys.getfilesystemencoding())
    if not g_localDirPath:
        g_localDirPath = os.path.split(g_netDirPath)[1] or 'temp'

    if not g_netDirPath.endswith(os.path.sep): g_netDirPath += os.path.sep
    if not g_localDirPath.endswith(os.path.sep): g_localDirPath += os.path.sep


userInput()
listFilePath()
dummy.Pool(16).map(downloadFile, g_filePathList)
コード例 #26
0
ファイル: datasets.py プロジェクト: msaad1311/Cartoonify
import tqdm
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
from torchvision.datasets import ImageFolder
import cv2
import os
import numpy as np
from .transforms import get_default_transforms
from PIL import Image, ImageOps
import matplotlib.pyplot as plt
import multiprocessing.dummy as mp

n_threads = 16
p = mp.Pool(n_threads)

from numba import njit, jit, prange


@jit
def edge_promoting(root, save):

    #from numba.errors import NumbaDeprecationWarning, NumbaPendingDeprecationWarning, NumbaWarning
    #import warnings

    #warnings.simplefilter('ignore', category=NumbaDeprecationWarning)
    #warnings.simplefilter('ignore', category=NumbaPendingDeprecationWarning)
    #warnings.simplefilter('ignore', category=NumbaWarning)

    img_size = (384, 384)
コード例 #27
0
def main(opts):
    global verbose
    if opts.verbose is not None:
        verbose = opts.verbose

    if verbose>1:opt_zopfli[0].append('-v')

    if not is_exec(opts.nasm):
        eprint("nasm binary not found, doesn't exist or is not executable.")
        return

    if not is_exec(opts.gzip  ): opts.gzip   = None
    if not is_exec(opts.xz    ): opts.xz     = None
    if not is_exec(opts.lzma  ): opts.lzma   = None
    if not is_exec(opts.zopfli): opts.zopfli = None

    comprs = sum([ \
        [(opts.gzip  , opt_gzip  , True )] if opts.gzip   is not None else [],
        [(opts.xz    , opt_xz    , False)] if opts.xz     is not None else [],
        [(opts.lzma  , opt_lzma  , False)] if opts.lzma   is not None else [],
        [(opts.zopfli, opt_zopfli, True )] if opts.zopfli is not None else [],
        [(opts.zstd  , opt_zstd  , 2    )] if opts.zstd   is not None else [] \
    ], [])

    if len(comprs) == 0:
        eprint("No compression methods specified.")
        return

    mpool = multi.Pool(processes=opts.jobs)

    allofthem = list(sum(map(lambda x: run_compr(x, mpool, opts.input_file),
                             comprs),[]))

    if len(allofthem) == 0:
        eprint("No useable results available. See error log.")
        return

    eprint(len(allofthem))
    allofthem_s = sorted(allofthem, key=lambda x:len(x[0]))

    best = allofthem_s[0]
    

    if verbose > 0:
        lbl = "xz"
        if best[1] == 2:
            lbl = "zstd"
        elif best[1]:
            lbl = "gzip"
        eprint(len(best[0]), lbl)

    res, stubbed = best[0], None
    if not opts.nostub:
        stubbed = do_vndh(opts.nasm, opts.vndh, best,
                      opts.vndh_tag, opts.vndh_vfork, opts.vndh_unibin,
                      opts.vndh_cheat,opts.vndh_no_argv)

    if verbose > 0: eprint("final: "+str(len(res)))

    opts.output_file.write(stubbed or res)
    if stubbed is not None and opts.rawout is not None:
        opts.rawout.write(res)
コード例 #28
0
ファイル: control.py プロジェクト: vladtcvs/tempesta
def servers_stop(servers):
    threads = __servers_pool_size(len(servers))
    pool = multiprocessing.Pool(threads)
    pool.map(Nginx.stop, servers)
コード例 #29
0
    def collate(self):

        # Set the stacksize to be unlimited
        res.setrlimit(res.RLIMIT_STACK, (res.RLIM_INFINITY, res.RLIM_INFINITY))

        # Locate the FMS collation tool
        # Check config for collate executable
        mppnc_path = self.expt.config.get('collate_exe')
        if mppnc_path is None:
            for f in os.listdir(self.expt.lab.bin_path):
                if f.startswith('mppnccombine'):
                    mppnc_path = os.path.join(self.expt.lab.bin_path, f)
                    break
        else:
            if not os.path.isabs(mppnc_path):
                mppnc_path = os.path.join(self.expt.lab.bin_path, mppnc_path)

        assert mppnc_path

        # Check config for collate command line options
        collate_flags = self.expt.config.get('collate_flags')
        if collate_flags is None:
            collate_flags = '-n4 -z -m -r'

        # Import list of collated files to ignore
        collate_ignore = self.expt.config.get('collate_ignore')
        if collate_ignore is None:
            collate_ignore = []
        elif type(collate_ignore) != list:
            collate_ignore = [collate_ignore]

        # Generate collated file list and identify the first tile
        tile_fnames = [
            f for f in os.listdir(self.output_path)
            if f[-4:].isdigit() and f[-8:-4] == '.nc.'
        ]

        mnc_tiles = defaultdict(list)
        for t_fname in tile_fnames:
            t_base, t_ext = os.path.splitext(t_fname)
            t_ext = t_ext.lstrip('.')

            # Skip any files listed in the ignore list
            if t_base in collate_ignore:
                continue

            mnc_tiles[t_base].append(t_fname)

        # If this is run interactively NCPUS is set in collate_cmd, otherwise
        # the cpu_count will return the number of CPUs assigned to the PBS job
        count = int(os.environ.get('NCPUS', multiprocessing.cpu_count()))
        pool = multiprocessing.Pool(processes=count)

        # Collate each tileset into a single file
        results = []
        for nc_fname in mnc_tiles:
            nc_path = os.path.join(self.output_path, nc_fname)

            # Remove the collated file if it already exists, since it is
            # probably from a failed collation attempt
            # TODO: Validate this somehow
            if os.path.isfile(nc_path):
                os.remove(nc_path)

            cmd = '{} {} {} {}'.format(mppnc_path, collate_flags, nc_fname,
                                       ' '.join(mnc_tiles[nc_fname]))
            print(cmd)
            result = pool.apply_async(cmdthread, args=(cmd, self.output_path))
            results.append(result.get())

        pool.close()
        pool.join()

        # TODO: Categorise the return codes
        if any(rc is not None for rc in results):
            for p, rc in enumerate(results):
                if rc is not None:
                    print('payu: error: Thread {} crased with error code {}.'
                          ''.format(p, rc),
                          file=sys.stderr)
            sys.exit(-1)
コード例 #30
0
ファイル: main.py プロジェクト: lxq2537664558/DailyProjects
# vim:fileencoding=utf-8

import re, urllib2
import multiprocessing.dummy as dummy

charset_re = re.compile(r'charset=([\w-]+)')
title_re = re.compile(r'<title>([^<]+)</title>')


def captitle(url):
    req = urllib2.Request(url)
    req.add_header('User-Agent',
                   'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)')
    f = urllib2.urlopen(req)
    content = f.read()
    title = (['unkown'] + title_re.findall(content))[-1]
    enc = ''
    enc = enc or ([''] +
                  charset_re.findall(f.headers.dict['content-type']))[-1]
    enc = enc or ([''] + charset_re.findall(content))[-1]
    enc = enc or 'utf-8'
    return url, title.decode(enc)


pool = dummy.Pool(16)
urls = [url.strip() for url in file('1.txt').readlines()]
for url, title in pool.map(captitle, urls):
    print '%s : %s' % (url, title)