예제 #1
0
파일: parser.py 프로젝트: aq1/WillBeams
    def get_threads(self, fetch_tool):
        alive_threads = set()
        result = []
        status, data = fetch_tool(self._url)

        if status != 200:
            return result

        try:
            threads = data['threads']
        except KeyError as e:
            utils.inform('{} {} {}'.format(e, threads.keys(), self._url), level=utils.WARNING)
            return result

        for thread in threads:
            number = int(thread['num'])
            if number not in self._threads:
                result.append(Thread(self._section, number))
                self._threads.add(number)

            alive_threads.add(number)

        utils.inform('Found {} threads'.format(len(result)), level=utils.INFO)
        self._threads = alive_threads
        return result
예제 #2
0
    def get_threads(self, fetch_tool):
        alive_threads = set()
        result = []
        status, data = fetch_tool(self._url)

        if status != 200:
            return result

        try:
            threads = data['threads']
        except KeyError as e:
            utils.inform('{} {} {}'.format(e, threads.keys(), self._url),
                         level=utils.WARNING)
            return result

        for thread in threads:
            number = int(thread['num'])
            if number not in self._threads:
                result.append(Thread(self._section, number))
                self._threads.add(number)

            alive_threads.add(number)

        utils.inform('Found {} threads'.format(len(result)), level=utils.INFO)
        self._threads = alive_threads
        return result
예제 #3
0
 def _get_response(self, request):
     utils.inform('Requesting {}'.format(request), level=utils.INFO)
     self._conn.request('GET', request, headers=self._headers)
     resp = self._conn.getresponse()
     utils.inform('Response is {}: {}'.format(resp.status, resp.reason),
                  level=utils.INFO)
     return resp
예제 #4
0
파일: connection.py 프로젝트: aq1/WillBeams
 def _get_response(self, request):
     utils.inform('Requesting {}'.format(request), level=utils.INFO)
     self._conn.request('GET', request, headers=self._headers)
     resp = self._conn.getresponse()
     utils.inform('Response is {}: {}'.format(
         resp.status, resp.reason), level=utils.INFO)
     return resp
예제 #5
0
파일: connection.py 프로젝트: aq1/WillBeams
 def _f(self, *args, **kwargs):
     while True:
         try:
             return function(self, *args, **kwargs)
         except exceptions as e:
             utils.inform(e, level=utils.WARNING)
             self._connect()
             time.sleep(1)
예제 #6
0
 def _f(self, *args, **kwargs):
     while True:
         try:
             return function(self, *args, **kwargs)
         except exceptions as e:
             utils.inform(e, level=utils.WARNING)
             self._connect()
             time.sleep(1)
예제 #7
0
def callback(ch, method, properties, body):

    try:
        url, thumb, md5, size = body.decode('utf8').split()
    except (ValueError, UnicodeDecodeError):
        utils.inform('unexpected format: "{}"'.format(body), level=utils.ERROR)

    utils.inform(body, level=utils.IMPORTANT_INFO)
    ch.basic_ack(delivery_tag=method.delivery_tag)
예제 #8
0
파일: qhandler.py 프로젝트: aq1/WillBeams
def check_task_q(task_q):
    try:
        task = task_q.get(timeout=0.1)
    except queue.Empty:
        return None
    else:
        if task == utils.STOP_SIGNAL:
            utils.inform("I'm done", level=utils.IMPORTANT_INFO)
            exit()
예제 #9
0
파일: downloader.py 프로젝트: aq1/WillBeams
def callback(ch, method, properties, body):

    try:
        url, thumb, md5, size = body.decode('utf8').split()
    except (ValueError, UnicodeDecodeError):
        utils.inform('unexpected format: "{}"'.format(body), level=utils.ERROR)

    utils.inform(body, level=utils.IMPORTANT_INFO)
    ch.basic_ack(delivery_tag=method.delivery_tag)
예제 #10
0
def check_task_q(task_q):
    try:
        task = task_q.get(timeout=0.1)
    except queue.Empty:
        return None
    else:
        if task == utils.STOP_SIGNAL:
            utils.inform("I'm done", level=utils.IMPORTANT_INFO)
            exit()
예제 #11
0
    def get_json(self, request):
        json_data = {}

        resp = self._get_response(request)
        data = resp.read()

        try:
            data = data.decode('utf8')
        except (AttributeError, UnicodeDecodeError):
            pass

        if resp.status == 200:
            try:
                json_data = json.loads(data)
            except ValueError as e:
                utils.inform(e, level=utils.WARNING)
            except TypeError as e:
                utils.inform(e, level=utils.WARNING)

        return resp.status, json_data
예제 #12
0
파일: connection.py 프로젝트: aq1/WillBeams
    def get_json(self, request):
        json_data = {}

        resp = self._get_response(request)
        data = resp.read()

        try:
            data = data.decode('utf8')
        except (AttributeError, UnicodeDecodeError):
            pass

        if resp.status == 200:
            try:
                json_data = json.loads(data)
            except ValueError as e:
                utils.inform(e, level=utils.WARNING)
            except TypeError as e:
                utils.inform(e, level=utils.WARNING)

        return resp.status, json_data
예제 #13
0
 def check(self):
     try:
         self.room_info = self.get_room_info()
         if self.room_info['status']:
             utils.print_log(self.room_id, self.room_info['roomname'])
             if self.recording_lock.locked():
                 return True
             else:
                 # Calls inform only when it is enabled and no inform was sent in the last 100 secs
                 # This prevents unexpectedly spamming the infrom service in edge cases, for example, when the streamer had a bad network connnection or the streamer didn't turn off live after disconnection.
                 if self.enable_inform and time.time(
                 ) - self.last_inform > 100:
                     utils.inform(self.room_id, self.room_info['roomname'],
                                  self.inform_url)
                     self.last_inform = time.time()
                 return self.get_live_urls()
         else:
             utils.print_log(self.room_id, '等待开播')
             return False
     except Exception as e:
         utils.print_log(self.room_id, 'Error:' + str(e))
     return False
예제 #14
0
    def get_webms(self, fetch_tool):
        utils.inform('Searching for webms. Last post: {}'.format(
            self._last_post),
                     level=utils.INFO)
        webms = []
        status, data = fetch_tool(self._url)

        if status == 404:
            return None
        elif status != 200:
            return webms

        found_webms_count = 0
        try:
            posts = data['threads'][0]['posts']
        except KeyError as e:
            utils.inform('Key error: {}'.format(e), level=utils.WARNING)
            return webms

        for post in posts:

            if post['num'] <= self._last_post:
                continue

            try:
                files = post['files']
            except KeyError:
                continue

            for f in files:
                if f.get('type', None) == utils.WEBM:
                    found_webms_count += 1
                    url = utils.RESOURCE_URL.format(self._section, f['path'])
                    thumb = utils.RESOURCE_URL.format(self._section,
                                                      f['thumbnail'])
                    md5 = f['md5']
                    size = f['size']
                    webms.append(Webm(url, thumb, md5, size))

        utils.inform('Found {} webms'.format(found_webms_count),
                     level=utils.INFO)
        self._last_post = data['threads'][0]['posts'][-1]['num']
        return webms
예제 #15
0
파일: parser.py 프로젝트: aq1/WillBeams
    def get_webms(self, fetch_tool):
        utils.inform('Searching for webms. Last post: {}'.format(
            self._last_post), level=utils.INFO)
        webms = []
        status, data = fetch_tool(self._url)

        if status == 404:
            return None
        elif status != 200:
            return webms

        found_webms_count = 0
        try:
            posts = data['threads'][0]['posts']
        except KeyError as e:
            utils.inform('Key error: {}'.format(e), level=utils.WARNING)
            return webms

        for post in posts:

            if post['num'] <= self._last_post:
                continue

            try:
                files = post['files']
            except KeyError:
                continue

            for f in files:
                if f.get('type', None) == utils.WEBM:
                    found_webms_count += 1
                    url = utils.RESOURCE_URL.format(self._section, f['path'])
                    thumb = utils.RESOURCE_URL.format(self._section, f['thumbnail'])
                    md5 = f['md5']
                    size = f['size']
                    webms.append(Webm(url, thumb, md5, size))

        utils.inform('Found {} webms'.format(found_webms_count), level=utils.INFO)
        self._last_post = data['threads'][0]['posts'][-1]['num']
        return webms
예제 #16
0
파일: parser.py 프로젝트: aq1/WillBeams
def start_thread(task_q, webm_q, url, sections):
    threading.Thread(target=work, args=(task_q, webm_q, url, sections)).start()
    utils.inform('Parser thread started', level=utils.IMPORTANT_INFO)
예제 #17
0
파일: downloader.py 프로젝트: aq1/WillBeams
def start_thread(channel, q_name):
    threading.Thread(target=work,
                     args=[channel, q_name]).start()

    utils.inform('Download thread started', level=utils.IMPORTANT_INFO)
예제 #18
0
def start_thread(task_q, webm_q, url, sections):
    threading.Thread(target=work, args=(task_q, webm_q, url, sections)).start()
    utils.inform('Parser thread started', level=utils.IMPORTANT_INFO)
예제 #19
0
파일: connection.py 프로젝트: aq1/WillBeams
 def _connect(self):
     self._conn = http.HTTPSConnection(self._host)
     self._set_headers()
     utils.inform('Connected to {}'.format(self._host), level=utils.INFO)
예제 #20
0
파일: parser.py 프로젝트: aq1/WillBeams
            if webms is None:
                del threads[i]
                length -= 1
                continue

            i += 1
            for webm in webms:
                qhandler.put(webm_q, webm)


def start_thread(task_q, webm_q, url, sections):
    threading.Thread(target=work, args=(task_q, webm_q, url, sections)).start()
    utils.inform('Parser thread started', level=utils.IMPORTANT_INFO)


def stop_threads(task_q, workers):
    for _ in range(workers):
        task_q.put(utils.STOP_SIGNAL)


if __name__ == '__main__':
    task_q = qhandler.get_task_q()
    webm_q = qhandler.create_channel()
    url = utils.BOARD_URL
    sections = ['b']
    try:
        work(task_q, webm_q, url, sections)
    except (KeyboardInterrupt, SystemExit):
        utils.inform("I'm done", level=utils.IMPORTANT_INFO)
        exit()
예제 #21
0
파일: parser.py 프로젝트: aq1/WillBeams
def print_output(webm):
    utils.inform(webm, level=utils.INFO)
예제 #22
0
def print_output(webm):
    utils.inform(webm, level=utils.INFO)
예제 #23
0
 def _connect(self):
     self._conn = http.HTTPSConnection(self._host)
     self._set_headers()
     utils.inform('Connected to {}'.format(self._host), level=utils.INFO)
예제 #24
0
def start_thread(channel, q_name):
    threading.Thread(target=work, args=[channel, q_name]).start()

    utils.inform('Download thread started', level=utils.IMPORTANT_INFO)
예제 #25
0
            if webms is None:
                del threads[i]
                length -= 1
                continue

            i += 1
            for webm in webms:
                qhandler.put(webm_q, webm)


def start_thread(task_q, webm_q, url, sections):
    threading.Thread(target=work, args=(task_q, webm_q, url, sections)).start()
    utils.inform('Parser thread started', level=utils.IMPORTANT_INFO)


def stop_threads(task_q, workers):
    for _ in range(workers):
        task_q.put(utils.STOP_SIGNAL)


if __name__ == '__main__':
    task_q = qhandler.get_task_q()
    webm_q = qhandler.create_channel()
    url = utils.BOARD_URL
    sections = ['b']
    try:
        work(task_q, webm_q, url, sections)
    except (KeyboardInterrupt, SystemExit):
        utils.inform("I'm done", level=utils.IMPORTANT_INFO)
        exit()