コード例 #1
0
ファイル: server.py プロジェクト: drslump/sublime-boo
 def reset_queue(self, queue):
     """ Make sure the queue is empty """
     try:
         while True:
             queue.get_nowait()
     except:
         pass
コード例 #2
0
ファイル: ht_proxy_if.py プロジェクト: ogsts/hometop_HT3
 def remove_client(self, clientID):
     txThread=self._thread.pop(clientID)
     txThread.stop()
     queue=self._rxqueue.pop(clientID)
     while queue.qsize() > 0:
         queue.get_nowait()
     queue=self._txqueue.pop(clientID)
     while queue.qsize() > 0:
         queue.get_nowait()
     self._logger.info("Client-ID:{0}; removed; number of clients:{1}".format(clientID, self._clientcounter))
コード例 #3
0
ファイル: deamon.py プロジェクト: kanekin/wfapi
    def _task(self):
        queue = self.queue
        # queue can be overflow

        with self.transaction_lock:
            while not queue.empty():
                queue.get_nowait()
                queue.task_done()

            queue.put(0)
            # INTERNAL: start counter for task

        while True:
            wait = self.default_execute_wait
            event = queue.get()

            if event is None:
                # STOP EVENT
                while not queue.empty():
                    queue.get_nowait()
                    queue.task_done()
                    # TODO: warning not queued event?
                    # TODO: just new stop flag?

                queue.task_done()
                # for stop event.
                return

            time.sleep(wait)
            # TODO: how to sleep automation?
            # TODO: use some good schuler?

            with self.transaction_lock:
                current_transaction = self.current_transaction
                if current_transaction is None:
                    with self.transaction() as current_transaction:
                        pass

                if not current_transaction.operations:
                    queue.put(event + wait)
                    queue.task_done()
                    continue

                if event >= self.status.default_execute_wait:
                    self.current_transaction = None
                    self.execute_transaction(current_transaction)

                queue.put(0)
                queue.task_done()
コード例 #4
0
    def _task(self):
        queue = self.queue
        # queue can be overflow

        with self.transaction_lock:
            while not queue.empty():
                queue.get_nowait()
                queue.task_done()

            queue.put(0)
            # INTERNAL: start counter for task

        while True:
            wait = self.default_execute_wait
            event = queue.get()

            if event is None:
                # STOP EVENT
                while not queue.empty():
                    queue.get_nowait()
                    queue.task_done()
                    # TODO: warning not queued event?
                    # TODO: just new stop flag?

                queue.task_done()
                # for stop event.
                return

            time.sleep(wait)
            # TODO: how to sleep automation?
            # TODO: use some good schuler?

            with self.transaction_lock:
                current_transaction = self.current_transaction
                if current_transaction is None:
                    with self.transaction() as current_transaction:
                        pass

                if not current_transaction.operations:
                    queue.put(event + wait)
                    queue.task_done()
                    continue

                if event >= self.status.default_execute_wait:
                    self.current_transaction = None
                    self.execute_transaction(current_transaction)

                queue.put(0)
                queue.task_done()
コード例 #5
0
ファイル: run.py プロジェクト: rodrigoalviani/MessageQueue
def fnThreadLoop(i, queue, lock):

    s = requests.Session()

    while True:
        #exit Thread when detect signal to quit.
        while libextra.fnExitNow():
            try:
                r = queue.get_nowait()
                break
            except:
                #libextra.fnQueueEmpty(1,lock)
                time.sleep(0.1)
                continue

        if libextra.fnExitNow() == False:
            break

        id = r[0]
        (status, gw_resp, log_msg) = fnJOB_CallAPI(s, r)
        gw_resp = pymysql.escape_string(gw_resp)

        #if (log_msg != ''):
        #    print(log_msg)

        QueueUpdate.put((id, status, gw_resp))
        queue.task_done()
コード例 #6
0
ファイル: scriptor.py プロジェクト: kretep/video-scriptor
def getFromQueue(queue):
    result = None
    try:
        result = queue.get_nowait()
    except:
        pass
    return result
コード例 #7
0
ファイル: utils.py プロジェクト: lvjinyuan/crawler_project
    def run(self):
        while True:
            self.__event.wait()
            if self.shutdownFlag is True:
                return

            self.__event.clear()
            while self._isEmpty() is False:
                request = None
                currentQueue = None
                for queue in self.__requestQueueList:
                    try:
                        request = queue.get_nowait()
                        currentQueue = queue
                        break
                    except queue.Empty:
                        continue
                if request is None:
                    continue
                try:
                    self._handleRequest(request)
                except Exception:
                    traceInfo = traceback.format_exc()
                    self.logger.error('Fail to handle request: %s', traceInfo)
                currentQueue.task_done()
            if self.__sleep_time > 0:
                time.sleep(self.__sleep_time)
コード例 #8
0
ファイル: run.py プロジェクト: rodrigoalviani/MessageQueue
def fnThreadLoop(i, queue, lock):

    s = requests.Session()

    while True:
        #exit Thread when detect signal to quit.
        while libextra.fnExitNow():
            try:
                r = queue.get_nowait()
                break
            except:
                #libextra.fnQueueEmpty(1,lock)
                time.sleep(0.1)
                continue

        if libextra.fnExitNow() == False:
            break

        id = r[0]
        (status, gw_resp, log_msg) = fnJOB_CallAPI(s,r)
        gw_resp = pymysql.escape_string(gw_resp)

        #if (log_msg != ''):
        #    print(log_msg)

        QueueUpdate.put((id,status,gw_resp))
        queue.task_done()
コード例 #9
0
def climbMain(conn, queue):
    fol = queue.get_nowait()
    # get followers strings
    followers = (getuserlists(username=fol.username, header=myHeader, ftype='followers'))

    for val in followers:
        # get followers info
        userInfo = getpageinfo(username=val, header=myHeader)
        print(userInfo['showname'])
        user = Users(
                username=userInfo['username'],
                showname=userInfo['showname'],
                followees=userInfo['followees'],
                followers=userInfo['followers'],
                focus=userInfo['focus'],
                fr_status=0,
                fe_status=0
                )
        conn.add(user)
        conn.commit()

    conn.query(Users).filter(Users.id == fol.id).\
        update({Users.fr_status:1}, synchronize_session=False)

    print('++++++'+fol.showname+'finished++++')
コード例 #10
0
def worker(queue, handle=None):
    try:
        while True:
            proxy = queue.get_nowait()
            result = ""
            counter[0] += 1
            sys.stdout.write("\r%s\r" % ROTATION_CHARS[counter[0] % len(ROTATION_CHARS)])
            sys.stdout.flush()
            start = time.time()
            candidate = "%s://%s:%s" % (proxy["proto"].replace("https", "http"), proxy["ip"], proxy["port"])
            if not all((proxy["ip"], proxy["port"])) or re.search(r"[^:/\w.]", candidate):
                continue
            if not check_alive(proxy["ip"], proxy["port"]):
                continue
            if not FALLBACK_METHOD:
                process = subprocess.Popen("curl -m %d -A \"%s\" --proxy %s %s" % (timeout, USER_AGENT, candidate, random_ifconfig()), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                result, _ = process.communicate()
            elif proxy["proto"] in ("http", "https"):
                opener = build_opener(ProxyHandler({"http": candidate, "https": candidate}))
                result = retrieve(random_ifconfig(), timeout=timeout, opener=opener)
            if (result or "").strip() == proxy["ip"].encode("utf8"):
                latency = time.time() - start
                if latency < timeout:
                    sys.stdout.write("\r%s%s # latency: %.2f sec; country: %s; anonymity: %s (%s)\n" % (candidate, " " * (32 - len(candidate)), latency, ' '.join(_.capitalize() for _ in (proxy["country"].lower() or '-').split(' ')), proxy["type"], proxy["anonymity"]))
                    sys.stdout.flush()
                    if handle:
                        os.write(handle, "%s%s" % (candidate, os.linesep))
    except:
        pass
コード例 #11
0
    def _dequeue(self, queue):
        """Removes queue entries till an alive reference was found.
        The referenced image holder will be returned in this case.
        Otherwise if there wasn't found any alive reference
        None will be returned.

        Args:
            queue (queue.Queue): the queue to operate on

        Returns:
            tuple of (ImageHolder, tuple of (width: int, height: int),
                      PostLoadImageProcessor):
                an queued image holder or None, upper bound size or None,
                the post load image processor or None
        """
        holder_reference = None
        image_holder = None
        upper_bound_size = None
        post_load_processor = None

        while not queue.empty():
            holder_reference, upper_bound_size, post_load_processor = \
                queue.get_nowait()
            image_holder = holder_reference and holder_reference()
            if (holder_reference is None or image_holder is not None):
                break

        return image_holder, upper_bound_size, post_load_processor
コード例 #12
0
def download_music(queue, folder, name):
    while not queue.empty():
        item = queue.get_nowait()
        content = requests.get(item['url']).content
        with open('%s/%s - %s.mp3' % (folder, item['artist'], item['song']),
                  'wb') as f:
            f.write(content)
            print('[线程%s]下载完成: %s - %s' % (name, item['artist'], item['song']))
コード例 #13
0
 def _get_results(self, queue):
     '''Generator to yield one after the others all items currently
        in the queue, without any waiting
     '''
     try:
         while True:
             yield queue.get_nowait()
     except queue.Empty:
         raise StopIteration
コード例 #14
0
ファイル: main.py プロジェクト: scris/whatnot-ide
 def tailStdOut(self):
     result = []
     if (self.m_scriptMonitor != None):
         queue = self.m_scriptMonitor.getStdoutQueue()
         for x in xrange(min(100, queue.qsize())):
             try:
                 val = queue.get_nowait()
                 result.append(val)
             except Empty:
                 pass
     return {"tail": result}
コード例 #15
0
def scrawler(queue, query):
    time.sleep(1)
    while not queue.empty():
        try:
            print('thread %s is running...' % threading.currentThread().name)
            # 不阻塞的读取队列数据
            url = queue.get_nowait()
            getgroup(url, query)
        except Exception as e:
            print(e)
            continue
コード例 #16
0
    def basic_incognito_algorithm(self, priority_queue):
        self.init_C1_and_E1()
        queue = priority_queue

        marked_nodes = set()

        for i in range(1, len(self.Q) + 1):
            i_str = str(i)
            self.cursor.execute("SELECT * FROM C" + i_str + "")
            Si = set(self.cursor)

            # no edge directed to a node => root
            self.cursor.execute("SELECT C" + i_str + ".* FROM C" + i_str +
                                ", E" + i_str + " WHERE C" + i_str +
                                ".ID = E" + i_str + ".start EXCEPT SELECT C" +
                                i_str + ".* FROM C" + i_str + ", E" + i_str +
                                " WHERE C" + i_str + ".ID = E" + i_str +
                                ".end ")
            roots = set(self.cursor)
            roots_in_queue = set()

            for node in roots:
                height = self.get_height_of_node(node)
                # -height because priority queue shows the lowest first. Syntax: (priority number, data)
                roots_in_queue.add((-height, node))

            for upgraded_node in roots_in_queue:
                queue.put_nowait(upgraded_node)

            while not queue.empty():
                upgraded_node = queue.get_nowait()
                # [1] => pick 'node' in (-height, node);
                node = upgraded_node[1]
                if node[0] not in marked_nodes:
                    if node in roots:
                        frequency_set = self.frequency_set_of_T_wrt_attributes_of_node_using_T(
                            node)
                    else:
                        frequency_set = self.frequency_set_of_T_wrt_attributes_of_node_using_parent_s_frequency_set(
                            node, i)
                    if self.table_is_k_anonymous_wrt_attributes_of_node(
                            frequency_set):
                        self.mark_all_direct_generalizations_of_node(
                            marked_nodes, node, i)
                    else:
                        Si.remove(node)
                        self.insert_direct_generalization_of_node_in_queue(
                            node, queue, i, Si)
                        self.cursor.execute("DELETE FROM C" + str(i) +
                                            " WHERE ID = " + str(node[0]))

            self.graph_generation(Si, i)
            marked_nodes = set()
コード例 #17
0
    def __call__(self, *args, **kwargs):
        procs = []
        queue = mp.Queue(self.n_gpus)
        results = [None] * self.n_gpus
        for dev in range(self.n_gpus):
            p = mp.Process(
                target=_run_wrapped,
                args=(
                    self.func,
                    self.world_size > self.n_gpus,
                    self.master_ip,
                    self.port,
                    self.world_size,
                    dev + self.rank_start,
                    dev,
                    args,
                    kwargs,
                    queue,
                ),
            )
            p.start()
            procs.append(p)

        devs = list(range(self.n_gpus))

        def terminate():
            for dev in devs:
                procs[dev].terminate()
            devs.clear()

        while len(devs) > 0:
            left = []
            # check all processes in one second
            time_to_wait = 1.0 / len(devs)
            for dev in devs:
                procs[dev].join(time_to_wait)
                code = procs[dev].exitcode
                # terminate processes if one of them has failed
                if code != 0 and code != None:
                    terminate()
                assert (
                    code == 0 or code == None
                ), "subprocess {} exit with code {}".format(dev + self.rank_start, code)
                if code == None:
                    left.append(dev)
                elif queue.empty():
                    get_logger().warning(WARN_SUBPROCESS_EXIT_WITHOUT_RETURN)
                else:
                    dev, ret = queue.get_nowait()
                    results[dev] = ret
            devs = left

        return results
コード例 #18
0
def scrawler(queue):
    time.sleep(1)
    threedaysago = int(time.time()) - 3600 * 24 * 30
    while not queue.empty():
        try:
            print('thread %s is running...' % threading.currentThread().name)
            # 不阻塞的读取队列数据
            url = queue.get_nowait()
            getrentinfo(url, threedaysago)
        except Exception as e:
            print(e)
            continue
コード例 #19
0
ファイル: stream.py プロジェクト: vaig-2019/multi-cam
def grab(cam, queue):
    global running
    global isStop

    capture = cv.VideoCapture(cam)
    while (1):
        while (running):
            capture.grab()
            ret, img = capture.read()
            if not ret:
                break
            if not queue.empty():
                try:
                    queue.get_nowait()
                except:
                    pass
            img = cv.cvtColor(img, cv.COLOR_RGB2BGR)
            queue.put(img)
        while not queue.empty():
            queue.get_nowait()
        while isStop:
            sys.exit()
コード例 #20
0
ファイル: cryptobook.py プロジェクト: vulogov/Bitmasher
def cryptobook_get(namespace):
    queue = dpget(namespace, "/cryptobook/queue")
    used = dpget(namespace, "/cryptobook/used")
    try:
        id, key, masher = queue.get_nowait()
        used[id] = (key, masher)
        dpset(namespace, "/key/private", key)
        dpset(namespace, "/key/masher", masher)
        dpset(namespace, "/key/id", id)
    except queue.Empty:
        namespace = cryptobook_add(namespace)
        return cryptobook_get(namespace)
    return namespace
コード例 #21
0
def clap_worker(vc, queue):
    print('creating worker')

    # vc = message.guild.voice_client
    # clap = discord.FFmpegPCMAudio(random.choice(files))
    while vc.is_connected():
        if not vc.is_playing() and not queue.empty():
            try:
                clap = queue.get_nowait()
                vc.play(discord.FFmpegPCMAudio(clap), after=queue.task_done())
            except:
                pass
        else:
            time.sleep(1)
    print('worker destroyed')
コード例 #22
0
ファイル: tmpBatch.py プロジェクト: lilang001/my_python
def _run():
    while 1:
        try:
            row = queue.get_nowait()
            try:
                print(row)
                print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
                      [:-3])
                test_1()
            except Exception as ee:
                print(ee)
                print(row[0], 'error')
        except Exception as e:
            print(e)
            return
コード例 #23
0
ファイル: core.py プロジェクト: Fatal1ty/vkdownloader
 def work(queue):
     while True:
         if queue.empty():
             if self.all_fetched_event.is_set():
                 break
             else:
                 time.sleep(0.001)
         try:
             task = queue.get_nowait()
         except Exception:
             continue
         if not os.path.exists(os.path.dirname(task.destination)):
             os.makedirs(os.path.dirname(task.destination),
                         exist_ok=True)
         with open(task.destination, 'wb') as f:
             f.write(self.__download(task.url))
コード例 #24
0
    def read_sensor(self, gas, thermister, photoresister, tracking, queue):
        data = {}
        while True:
            data.update({"Gas": gas.read()})
            time.sleep(0.1)
            data.update({"Thermister": thermister.read()})
            time.sleep(0.1)
            data.update({"Photoresister": photoresister.read()})
            time.sleep(0.1)
            data.update({"Tracking": tracking.read()})
            time.sleep(0.1)
            queue.put(data)
            #print(queue)
            time.sleep(0.1)
            self.client.publish(self.pubtopic,
                                payload=json.dumps(queue.get_nowait()))

            time.sleep(0.1)
コード例 #25
0
    def pollQueue(self, whatInbox):
        """This method checks all the queues from the outside world, and forwards any waiting data
        to the child component. Returns False if we propogated a shutdown signal, true otherwise."""
        parentSource = self.childInboxMapping[whatInbox]
        queue = self.inQueues[whatInbox]
        while not queue.empty():
            if not self.outboxes[parentSource].isFull():
                msg = queue.get_nowait() # won't fail, we're the only one reading from the queue.
                try:
                    self.send(msg, parentSource)
                except noSpaceInBox as e:
                    raise RuntimeError("Box delivery failed despite box (earlier) reporting being not full. Is more than one thread directly accessing boxes?")
                if isinstance(msg, (Ipc.shutdownMicroprocess, Ipc.producerFinished)):
#                    print ("Quietly dieing?")
                    return False
            else:
                # if the component's inboxes are full, do something here. Preferably not succeed.
                break
        return True
コード例 #26
0
 def pollQueue(self, whatInbox):
     """This method checks all the queues from the outside world, and forwards any waiting data
     to the child component. Returns False if we propogated a shutdown signal, true otherwise."""
     parentSource = self.childInboxMapping[whatInbox]
     queue = self.inQueues[whatInbox]
     while not queue.empty():
         if not self.outboxes[parentSource].isFull():
             msg = queue.get_nowait()  # won't fail, we're the only one reading from the queue.
             try:
                 self.send(msg, parentSource)
             except noSpaceInBox as e:
                 raise RuntimeError(
                     "Box delivery failed despite box (earlier) reporting being not full. Is more than one thread directly accessing boxes?"
                 )
             if isinstance(msg, (Ipc.shutdownMicroprocess, Ipc.producerFinished)):
                 #                    print ("Quietly dieing?")
                 return False
         else:
             # if the component's inboxes are full, do something here. Preferably not succeed.
             break
     return True
コード例 #27
0
ファイル: generic.py プロジェクト: buzzworkers/tl
def waitforqueue(queue, timeout=10000, maxitems=None, bot=None):
    """ wait for results to arrive in a queue. return list of results. """
    #if len(queue) > 1: return list(queue)
    result = []
    counter = 0
    if not maxitems: maxitems = 100
    logging.warn("waiting for queue: %s - %s" % (timeout, maxitems))
    try:
        while not len(queue):
            if len(queue) > maxitems: break
            if counter > timeout: break
            time.sleep(0.001) ; counter += 10
        logging.warn("waitforqueue - result is %s items (%s) - %s" % (len(queue), counter, str(queue)))
        return queue
    except (AttributeError, TypeError):
        q = []
        while 1:
            if counter > timeout: break
            try:
                q.append(queue.get_nowait())
            except queue.Empty: break
            time.sleep(0.001) ;  counter += 10
        logging.warn("waitforqueue - result is %s items (%s) - %s" % (len(q), counter, str(q)))
        return q
コード例 #28
0
async def worker(name, client):
    log = logging.getLogger(name)
    while True:
        try:
            line = queue.get_nowait()
        except Empty as e:
            log.info('Queue is empty.')
            await asyncio.sleep(1)
        else:
            line = line.decode('utf-8')
            match = pattern.match(line)
            if match is None:
                log.info(f"No match found for {line.strip()}")
                await asyncio.sleep(0.001)
            else:
                async with client.post(
                        "http://es01:9200/logs/_doc/",
                        data=json.dumps(match.groupdict()).encode('utf-8'),
                        headers={'Content-Type': 'application/json'}) as resp:
                    if resp.status != 201:
                        err = await resp.text()
                        log.info(f"{resp.status}: {err}")
                    else:
                        log.info("Upload succesful.")
コード例 #29
0
ファイル: launcher.py プロジェクト: MegEngine/MegEngine
    def __call__(self, *args, **kwargs):
        procs = []
        queue = mp.Queue(self.n_gpus)
        results = [None] * self.n_gpus
        for dev in range(self.n_gpus):
            p = mp.Process(
                target=_run_wrapped,
                args=(
                    self.func,
                    self.world_size > self.n_gpus,
                    self.master_ip,
                    self.port,
                    self.world_size,
                    dev + self.rank_start,
                    dev,
                    self.device_type,
                    args,
                    kwargs,
                    queue,
                ),
            )
            p.start()
            procs.append(p)

        devs = list(range(self.n_gpus))

        def terminate():
            for dev in devs:
                procs[dev].terminate()
            devs.clear()

        result_count = 0
        while len(devs) > 0:
            left = []
            # check all processes in one second
            time_to_wait = 1.0 / len(devs)
            for dev in devs:
                procs[dev].join(time_to_wait)
                code = procs[dev].exitcode
                # terminate processes if one of them has failed
                if code != 0 and code != None:
                    terminate()
                assert (code == 0 or code
                        == None), "subprocess {} exit with code {}".format(
                            dev + self.rank_start, code)
                if code == None:
                    left.append(dev)

                # DO NOT delete it, multiprocess.Queue has small buffer
                # fetch data early to avoid dead lock
                if not queue.empty():
                    result_count += 1
                    dev, ret = queue.get_nowait()
                    results[dev] = ret
            devs = left

        while not queue.empty():
            result_count += 1
            dev, ret = queue.get_nowait()
            results[dev] = ret

        if result_count < self.n_gpus:
            get_logger().warning(WARN_SUBPROCESS_EXIT_WITHOUT_RETURN)

        return results
コード例 #30
0
ファイル: transport.py プロジェクト: thgcode/orca-remote
def clear_queue(queue):
    try:
        while True:
            queue.get_nowait()
    except Exception:
        pass
コード例 #31
0

class PriorityQueue(queue.Queue):
    "Thread-safe priority queue"

    def _put(self, item):
        # insert in order
        bisect.insort(self.queue, item)


#
# try it

queue = PriorityQueue(0)

# add items out of order
queue.put((20, "second"))
queue.put((10, "first"))
queue.put((30, "third"))

# print queue contents
try:
    while 1:
        print(queue.get_nowait())
except Empty:
    pass

## third
## second
## first