Ejemplo n.º 1
0
class User():
    def __init__(self):
        self.http = HttpClient()        
        
    def login(self,loginData):
        signup_url = "http://localhost/signup/"
        xsrf = self.http.getXsrf(signup_url)
        print xsrf
        loginData['_xsrf']      = xsrf
        
        regQuery = urllib.urlencode(loginData)
        #print regQuery
        url = "http://localhost/login/"
        print url
        req = urllib2.Request(url,regQuery)
        res = urllib2.urlopen(req)
        print res.read(20)
        self.http.saveCookie()
    
    def reg(self,regData):
        signup_url = "http://localhost/signup/"
        xsrf = self.http.getXsrf(signup_url)
        print xsrf
        regData['_xsrf']      = xsrf
        regQuery = urllib.urlencode(regData)
        url = "http://localhost/signup/"
        req = urllib2.Request(url,regQuery)
        res = urllib2.urlopen(req)
        print res.read(100)
    
    def delete(self,user_key):
                
        url = "http://localhost/manager/user/delete/%s" % user_key
        res = urllib2.urlopen(url)
        print res.code
Ejemplo n.º 2
0
 def make_request(self, url, data='', headers={}, cb=lambda x: None):
     file_name = base64.b64encode(url)
     
     if LOCAL_WORK:
         logging.info('Local request: {0}'.format(url))
         with open(LOCAL_DIR + file_name, 'r') as f:
             cb(ResponseStub(pickle.loads(f.read())))
         return
             
     def dumped_cb(response):
         if LOCAL_DUMP:
             with open(LOCAL_DIR + file_name, 'w+') as f:
                 f.write(pickle.dumps(response.body))
         cb(response)
     
     body = urlencode(data) if isinstance(data, dict) else data
     if body:
         url = url + '?' + body
     
     req = httpclient.HTTPRequest(
                 url=url,
                 method='GET',
                 headers=headers,
                 connect_timeout=5,
                 request_timeout=10)
     
     logging.info('External request: {0}'.format(url))
     HttpClient.instance().fetch(req, dumped_cb)
Ejemplo n.º 3
0
 def __init__(self,
              top_level_classifier_host="ml-services.production.indix.tv",
              threshold=0.75):
     self.threshold = threshold
     self.classifier_query = ClassifierQuery("http",
                                             top_level_classifier_host,
                                             "/api/classify")
     self.client = HttpClient()
 def __init__(self,
              driver=chrome_driver(),
              wait_interval=10,
              page_load_interval=5):
     HttpClient.__init__(self, wait_interval)
     self.driver = driver
     # wait up to wait_interval seconds for the elements to become available
     self.driver.implicitly_wait(page_load_interval)
Ejemplo n.º 5
0
def get_keys(host, port, bucket):
    bucket_type, bucket_name = bucket
    context = "/types/{}/buckets/{}/keys?keys=true".format(
        HttpClient.escape_slash(bucket_type),
        HttpClient.escape_slash(bucket_name))
    client = HttpClient(host, port)
    body = client.get(context)['body']
    log("Context: {} BODY: {}".format(context, body))
    data = json.loads(body.decode())
    return data['keys']
Ejemplo n.º 6
0
def main(argv):
    parser = create_option_parser()
    (options, args) = parser.parse_args()
    try:
        if not options.bucket_name:
            parser.print_help()
            return 1
        if not options.key:
            parser.print_help()
            return 1
        params = get_params(options)
        client = HttpClient(options.host, options.port)
        if options.bucket_type:
            context = "/types/{}/buckets/{}/keys/{}/".format(
                client.escape_slash(options.bucket_type),
                client.escape_slash(options.bucket_name),
                client.escape_slash(options.key))
        else:
            context = "/buckets/{}/keys/{}/".format(
                client.escape_slash(options.bucket_name),
                client.escape_slash(options.key))
        response = client.get(context, params=params)
        client.pretty_print_response(response, options.verbose, options.b64)
        return 0
    except Exception as e:
        print("An error occurred creating {{{{{}, {}}}, {}}}: {}".format(
            options.bucket_type, options.bucket_name, options.key, e))
        #import traceback
        #traceback.print_exc()
        return -1
Ejemplo n.º 7
0
def get_bucket_names(host, port, bucket_type):
    context = "/types/{}/buckets?buckets=true".format(bucket_type)
    client = HttpClient(host, port)
    try:
        data = json.loads(client.get(context)['body'].decode())
    except Exception as e:
        print(
            "An error ocurred decoding JSON payload from request for bucket names.  Error: {}; body: {}"
            .format(e,
                    client.get(context)['body']))
    #print("data: {}".format(data))
    return data['buckets']
Ejemplo n.º 8
0
def main(argv):
    parser = create_option_parser()
    (options, args) = parser.parse_args()
    try:
        (options, args) = parser.parse_args()
        client = HttpClient(options.host, options.port)
        response = client.post("/reboot", "")
        client.pretty_print_response(response, False)
        return 0
    except Exception as e:
        import traceback
        traceback.print_exc()
        return -1
Ejemplo n.º 9
0
def load_srv_json(file, srv_url=None):
    global HTTP_CLIENT
    if not srv_url:
        srv_url = updates_url()
    file_url = srv_url + file + '.json'
    try:
        if not HTTP_CLIENT:
            HTTP_CLIENT = HttpClient()
        return HTTP_CLIENT.get_json(file_url)
    except Exception as exc:
        LOG.exc(exc, 'Error loading server data: %s' % file)
        return None
    finally:
        manage_memory()
Ejemplo n.º 10
0
class CategoryTagger(object):
    def __init__(self,
                 top_level_classifier_host="ml-services.production.indix.tv",
                 threshold=0.75):
        self.threshold = threshold
        self.classifier_query = ClassifierQuery("http",
                                                top_level_classifier_host,
                                                "/api/classify")
        self.client = HttpClient()

    def tag(self, title):
        title = title.lower().strip()
        q = self.classifier_query.getSearchQuery(title)
        json_response = json.loads(self.client.query(q))
        conf_scores = [
            float(score) for score in json_response['confidence'].split(",")
        ]
        result_dict = {}
        result_dict['brands'] = []
        result_dict['stores'] = []
        result_dict['categories'] = []
        if conf_scores[0] > self.threshold:
            matches = {}
            matches['matches'] = [json_response['category_id']]
            matches['token'] = title
            result_dict['categories'] = [matches]
        return (result_dict, title)
Ejemplo n.º 11
0
class Babelfy:
    def __init__(self):
        self.client = HttpClient()

    endpoint = "https://babelfy.io/v1/disambiguate"
    api_key = "24857588-e014-40b6-92e9-f993306255a9"
    lang = "uk"
    client = None

    def send_text(self, text):
        parameters = {"text": text, "lang": self.lang, "key": self.api_key}
        groups = []

        try:
            response = self.client.get(self.endpoint, parameters)

            if len(response) > 0:
                for entity_group in response:
                    token_fragment = entity_group['tokenFragment']
                    if token_fragment['end'] - token_fragment['start'] > 0:
                        groups.append(
                            range(token_fragment['start'],
                                  token_fragment['end'] + 1))
        except Exception:
            return []

        return groups

    def set_client(self, _client):
        self.client = _client
Ejemplo n.º 12
0
def get(host, path):
    """Sends HTTP GET using HttpClient and returns the request response body.
    :param host: host ip addr
    :param path: resource endpoint path
    :return: response body
    """
    print('Sending GET ' + path + ' HTTP/1.1 request to ' + host)
    window.refresh()
    c = HttpClient(49512)
    req = c.create_get_request(host, path)
    logger.debug('GET request created.')
    logger.debug(req)
    res = c.request(bytes(req), host)
    print('Response: ' + res.split(b'\r\n')[0].decode())
    window.refresh()

    return res
Ejemplo n.º 13
0
def main(argv):
    parser = create_option_parser()
    (options, args) = parser.parse_args()
    try:
        (options, args) = parser.parse_args()
        if not options.bucket_name:
            parser.print_help()
            return 1
        if not options.key:
            parser.print_help()
            return 1
        if not options.value and not options.file:
            parser.print_help()
            return 1
        params = get_params(options)
        client = HttpClient(options.host, options.port)
        if options.bucket_type:
            context = "/types/{}/buckets/{}/keys/{}/".format(
                client.escape_slash(options.bucket_type),
                client.escape_slash(options.bucket_name),
                client.escape_slash(options.key))
        else:
            context = "/buckets/{}/keys/{}/".format(
                client.escape_slash(options.bucket_name),
                client.escape_slash(options.key))
        vclock = None
        if not options.force:
            result = client.get(context)
            if result['status'] != 404:
                if 'X-Riak-Vclock' in result['headers']:
                    vclock = result['headers']['X-Riak-Vclock']
        body = get_data(options)
        headers = {'X-Riak-Vclock': vclock} if vclock else {}
        response = client.put(context,
                              body=body,
                              content_type=options.content_type,
                              headers=headers,
                              params=params)
        client.pretty_print_response(response, options.verbose)
        return 0
    except Exception as e:
        print("An error occurred creating {{{{{}, {}}}, {}}}: {}".format(
            options.bucket_type, options.bucket_name, options.key, e))
        import traceback
        traceback.print_exc()
        return -1
Ejemplo n.º 14
0
def main(argv):
    parser = create_option_parser()
    (options, args) = parser.parse_args()
    try:
        (options, args) = parser.parse_args()
        client = HttpClient(options.host, options.port)
        response = client.get("/config/camera?param=frame_size")
        print("frame_size: %s" % response['body'])
        response = client.get("/config/camera?param=jpeg_quality")
        print("jpeg_quality: %s" % response['body'])
        response = client.get("/config/camera?param=flash")
        print("flash: %s" % response['body'])
        #client.pretty_print_response(response, False)
        return 0
    except Exception as e:
        import traceback
        traceback.print_exc()
        return -1
Ejemplo n.º 15
0
def main(argv):
    parser = create_option_parser()
    (options, args) = parser.parse_args()
    try:
        (options, args) = parser.parse_args()
        client = HttpClient(options.host, options.port)
        response = client.get("/config/wifi?param=mode")
        print("mode: %s" % response['body'])
        response = client.get("/config/wifi?param=ssid")
        print("ssid: %s" % response['body'])
        response = client.get("/config/wifi?param=device_name")
        print("device_name: %s" % response['body'])
        #client.pretty_print_response(response, False)
        return 0
    except Exception as e:
        import traceback
        traceback.print_exc()
        return -1
Ejemplo n.º 16
0
 def on_parent(self, widget, parent):
     print("on_parent event -> full layout available")
     pizzas_json = StorageManager().get_data(
         self.STORAGE_FILENAME)  # get pizzas from file
     self.recycleView.data = Pizza.build_list_data_for_rw(pizzas_json)
     HttpClient().get_pizzas(self.on_server_data, self.on_error_data,
                             self.on_failure_data, self.on_progress_data)
     print(
         "HttpClient().get_pizzas is called to get pizzas from web service")
Ejemplo n.º 17
0
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        """self.pizzas = [
            Pizza("4 fromages", "chèvre, emmental, brie, comté", 9.5, True),
            Pizza("Chorizo", "tomates, chorizo, parmesan", 11.2, False),
            Pizza("Calzone", "fromage, jambon, champignons", 10, False)
        ]"""

        HttpClient().get_pizzas(self.on_server_data, self.on_server_error)
Ejemplo n.º 18
0
def perform_software_update():
    global HTTP_CLIENT
    if not HTTP_CLIENT:
        HTTP_CLIENT = HttpClient()
    machine.WDT()
    manage_memory()
    version_data = load_version()
    device_type = get_device_type()
    srv_url = updates_url()
    srv_index = load_srv_json('index', srv_url=srv_url)
    machine.resetWDT()
    srv_versions = load_srv_json('devices', srv_url=srv_url)
    machine.resetWDT()
    if srv_index:
        for path, entry in srv_index.items():
            if 'devices_types' in entry and 'base' not in entry[
                    'devices_types'] and device_type not in entry[
                        'devices_types']:
                continue
            if path in version_data['files'] and version_data['files'][
                    path] == entry['hash']:
                continue
            local_path = entry['path'] if 'path' in entry else path
            print(local_path)
            ensure_file_path(local_path)
            file_url = srv_url + 'software/' + path
            print(file_url)
            while version_data['files'].get(path) != entry['hash']:
                if HTTP_CLIENT.get_to_file(file_url, 'file_buf'):
                    try:
                        uos.remove(local_path)
                    except OSError:
                        pass
                    uos.rename('file_buf', local_path)
                    if version_data['hash']:
                        version_data['hash'] = None
                    version_data['files'][path] = entry['hash']
                    save_version(version_data)
                    print('complete')
        machine.resetWDT()
        version_data['hash'] = srv_versions[device_type]
        version_data['update'] = False
        save_version(version_data)
    machine.reset()
Ejemplo n.º 19
0
def post(host, path, data):
    """Sends POST request using HttpClient and the data from GUI form.
    :param host: host ip addr
    :param path: resource endpoint path
    :param data: data to be sent as body of the POST request
    :return: HTTP response body
    """
    cover = open(data, 'rb').read()
    print('Sending POST ' + path + ' HTTP/1.1 request to ' + host)
    window.refresh()
    print('RSTEG not activated.')
    window.refresh()
    c = HttpClient(49512)
    req = c.create_post_request(host, path, cover, 'image/jpg')
    res = c.request(bytes(req), host)
    print('Response: ' + res.split(b'\r\n')[0].decode())
    window.refresh()

    return res
Ejemplo n.º 20
0
 def test_link_checker(self):
     valid_urls = ['http://domain.com',
                   'http://example.com/path/to/page?name=ferret&color=purple',
                   'https://docs.python.org/2/library/unittest.html',
                   'http://127.0.0.1:8080',
                   'http://localhost:8080',
                   'http://web-nginx.example.com:80'
                   ]
     invalid_urls = ['domain.com',
                     'http://domain',
                     'http://.com',
                     'file:///etc/fstab,'
                     ]
     # test valid urls
     for url in valid_urls:
         self.assertTrue(HttpClient.is_link_ok(url))
     # test invalid urls
     for url in invalid_urls:
         self.assertFalse(HttpClient.is_link_ok(url))
Ejemplo n.º 21
0
    def get_content(self, s):
        
        # Instancia el cliente para el peer, y hace la peticion.

        request_headers = self.headers[s.getpeername()]['str']
        #url = self.headers[s.getpeername()]['dic']['Host']
        method, resource, version = self.headers[s.getpeername()]['dic']['Request-Line'].split()
        if self.en_cache(resource):
            filename, header = self.get_from_cache(resource)
        else:
            client = HttpClient(download_dir=self.cache_dir)
            filename, header = client.retrieve_con_headers(request_headers, resource, method)
            header = self.adaptar_header(header)
            self.set_to_cache(resource,filename,header)

        self.requests[s.getpeername()] = {}
        self.requests[s.getpeername()]['filename'] = filename
        self.requests[s.getpeername()]['header'] = header

        return True
Ejemplo n.º 22
0
def rpost(host, path, data, secret_data, rprob):
    """Sends rPOST request using HttpClient and the data from GUI form.
    :param host: host ip addr
    :param path: resource endpoint path
    :param data: data to be sent as body of the POST request
    :param secret_data: secret to be sent using the RSTEG mechanism
    :param rprob: retransmission probability
    :return: HTTP response body
    """
    cover = open(data, 'rb').read()
    secret = open(secret_data, 'rb').read()
    print('Sending POST ' + path + ' HTTP/1.1 request to ' + host)
    print('Using RSTEG to send secret.')
    window.refresh()
    c = HttpClient(float(rprob), 49512)
    req = c.create_post_request(host, path, cover, 'image/jpg')
    res = c.rsteg_request(bytes(req), secret, host)
    print('Response: ' + res.split(b'\r\n')[0].decode())

    return res
Ejemplo n.º 23
0
class Board():
    def __init__(self):
        self.http = HttpClient()        
        
    def login(self,loginData):
        signup_url = "http://localhost/signup/"
        xsrf = self.http.getXsrf(signup_url)
        print xsrf
        loginData['_xsrf']      = xsrf
        
        regQuery = urllib.urlencode(loginData)
        #print regQuery
        url = "http://localhost/login/"
        print url
        req = urllib2.Request(url,regQuery)
        res = urllib2.urlopen(req)
        print res.read(20)
        self.http.saveCookie()
        
    def new(self,bData):        
        signup_url = "http://localhost/"
        xsrf = self.http.getXsrf(signup_url)
        print xsrf               
        bData['_xsrf']       = xsrf
        
        bQuery = urllib.urlencode(bData)
        url = "http://localhost/ajax/addboard/"
        req = urllib2.Request(url,bQuery)
        res = urllib2.urlopen(req)
        print res.read()
    def newBoard(self,bData):        
        signup_url = "http://localhost/"
        xsrf = self.http.getXsrf(signup_url)
        print xsrf               
        bData['_xsrf']       = xsrf
        
        bQuery = urllib.urlencode(bData)
        url = "http://localhost/manager/board/new/"
        req = urllib2.Request(url,bQuery)
        res = urllib2.urlopen(req)
        print res.read()
Ejemplo n.º 24
0
def niceview_svr(num=10):
    '''
        同步实现 图片获取
    :param num: 图片数量
    :return: 状态码和图片信息,
    '''
    header = {
        "apikey": "af8a128ea2f8f494cf7ab9456208a762",
        "Content-Type": "application/json"
    }
    host = 'apis.baidu.com'
    url = '/txapi/mvtp/meinv?num=%d' % num
    http_cli = HttpClient(host, 80, 30)
    http_res = http_cli.send_request(url, 'GET', header=header)

    ret = {}
    ret["status"] = http_res["status"]
    if ret["status"] != 200:
        return ret
    ret["picList"] = http_res["body"]["newslist"]
    return ret
    def test_get(self):
        args = {
            'params' : {'query': 'test'},
            'headers': {'X-Requested-With': 'XMLHttpRequest'}
        }

        response = HttpClient.get('http://curtaincall.weblike.jp/prog/php/test-api/get.php', **args)

        data = json.loads(response.text)

        self.assertTrue(data['result'])
        self.assertEqual(data['messages'], ['Success'])
Ejemplo n.º 26
0
def niceview_svr(num=10):
    '''
        同步实现 图片获取
    :param num: 图片数量
    :return: 状态码和图片信息,
    '''
    header = {
        "apikey": "af8a128ea2f8f494cf7ab9456208a762",
        "Content-Type": "application/json"
    }
    host = 'apis.baidu.com'
    url = '/txapi/mvtp/meinv?num=%d' % num
    http_cli = HttpClient(host, 80, 30)
    http_res = http_cli.send_request(url, 'GET', header=header)

    ret = {}
    ret["status"] = http_res["status"]
    if ret["status"] != 200:
        return ret
    ret["picList"] = http_res["body"]["newslist"]
    return ret
Ejemplo n.º 27
0
    def __init__(self,
                 server_host,
                 server_port=None,
                 username="******",
                 password="******",
                 use_tls=False,
                 version=API_CURRENT_VERSION):
        """
    Creates a Resource object that provides API endpoints.

    @param server_host: The hostname of the Cloudera Manager server.
    @param server_port: The port of the server. Defaults to 7180 (http) or
      7183 (https).
    @param username: Login name.
    @param password: Login password.
    @param use_tls: Whether to use tls (https).
    @param version: API version.
    @return: Resource object referring to the root.
    """
        self._version = version
        protocol = use_tls and "https" or "http"
        if server_port is None:
            server_port = use_tls and 7183 or 7180
        base_url = "%s://%s:%s/api/v%s" % \
            (protocol, server_host, server_port, version)

        client = HttpClient(base_url, exc_class=ApiException)
        client.set_basic_auth(username, password, API_AUTH_REALM)
        client.set_headers({"Content-Type": "application/json"})
        Resource.__init__(self, client)
def main(argv):
    parser = create_option_parser()
    (options, args) = parser.parse_args()
    try:
        (options, args) = parser.parse_args()
        client = HttpClient(options.host,
                            options.port,
                            verbose=options.verbose)
        params = {}
        if options.frame_size:
            params['frame_size'] = options.frame_size
        if options.jpeg_quality:
            params['jpeg_quality'] = options.jpeg_quality
        if options.flash:
            params['flash'] = options.flash
        response = client.post("/config/camera", "", params=params)
        client.pretty_print_response(response, False)
        return 0
    except Exception as e:
        import traceback
        traceback.print_exc()
        return -1
Ejemplo n.º 29
0
def main(argv):
    parser = create_option_parser()
    (options, args) = parser.parse_args()
    try:
        (options, args) = parser.parse_args()
        client = HttpClient(options.host, options.port)
        params = {}
        if options.mode:
            params['mode'] = options.mode
        if options.ssid:
            params['ssid'] = options.ssid
        if options.password:
            params['password'] = options.password
        if options.device_name:
            params['device_name'] = options.device_name
        response = client.post("/config/wifi", "", params=params)
        client.pretty_print_response(response, False)
        return 0
    except Exception as e:
        import traceback
        traceback.print_exc()
        return -1
Ejemplo n.º 30
0
def get_value(host, port, bucket_type, bucket_name, key):
    #quoted_key = urllib.quote(key.encode('utf8'), safe="")
    context = "/types/{}/buckets/{}/keys/{}".format(
        HttpClient.escape_slash(bucket_type), HttpClient.escape_slash(bucket_name), HttpClient.escape_slash(key)
    )
    riak_util.log("Context: {}".format(context))
    client = HttpClient(host, port)
    return client.get(context)['body']
Ejemplo n.º 31
0
def main(argv):
    try:
        path = argv[0]
    except:
        raise TypeError(HELP)

    try:
        opts, args = getopt.getopt(argv[1:], "hs:p:t")
    except getopt.GetoptError:
        raise TypeError(HELP)

    server = None
    port = None
    transport = None
    for opt, arg in opts:
        if opt == '-h':
            raise TypeError(HELP)
        elif opt == '-s':
            server = arg
        elif opt == '-p':
            port = arg
        elif opt == '-t':
            transport = arg

    if not server:
        server = "127.0.0.1"

    # Open connection with scribe
    zipkin = None
    if (transport == 'scribe'):
        zipkin = ZipkinScribeClient(port, server)
        if not port:
            port = 9410
    else:
        transport = "http"
        zipkin = HttpClient(port, server)
        if not port:
            port = 9411

    # Create TraceCollection and add trace:
    traces = TraceCollection()
    trace_handle = traces.add_trace(path, "ctf")
    if trace_handle is None:
        raise IOError("Error adding trace")

    print("Sending traces to " + server + ":" + port + " using " + transport +
          "\n")
    zipkin.send_annotations(traces.events)
    print("Done sending\n")
    return
Ejemplo n.º 32
0
class Q2Category(object):
    def __init__(self, qas_host="qas01.staging.indix.tv:8080"):
        self.qas_query = QASQuery("http", qas_host, "/api/annotate")
        self.client = HttpClient()

    def getCategories(self, search_term, threshold=0.025):
        query = self.qas_query.getSearchQuery(search_term)
        qas_response = self.client.query(query)
        cat_conf_dict = qas_response["ids"][0]["classes"]
        categories = []
        for cat, conf in cat_conf_dict.items():
            if conf >= threshold:
                categories.append(int(cat))
        return categories
    def test_file(self):
        args = {
            'fieldname': 'file',
            'filename' : 'test.json',
            'data'     : {'body': 'test'},
            'headers'  : {'X-Requested-With': 'XMLHttpRequest'}
        }

        response = HttpClient.file('http://curtaincall.weblike.jp/prog/php/test-api/file.php', **args)

        data = json.loads(response.text)

        self.assertTrue(data['result'])
        self.assertEqual(data['messages'], ['Success'])
Ejemplo n.º 34
0
class Cat():
    def __init__(self):
        self.http = HttpClient()        
        
    def add(self,catData):
        cat_url = "http://localhost/manager/cat/new/"
        xsrf = self.http.getXsrf(cat_url)
        print xsrf
        catData['_xsrf']      = xsrf
        
        catQuery = urllib.urlencode(catData)
        #print regQuery
        req = urllib2.Request(cat_url,catQuery)
        res = urllib2.urlopen(req)
        
        print res.read(20)
        self.http.saveCookie()      
        
    def delete(self,user_key):
                
        url = "http://localhost/manager/user/delete/%s" % user_key
        res = urllib2.urlopen(url)
        print res.code
 def page_links(self, link):
     """
     Downloads defined url from link with content like images, css, JavaScript, then parse all urls in a href tags.
      and return it as list.
     :param link: web page to be processed
     :return: list of parsed urls.
     """
     self.driver.get(link)
     links = self.driver.find_elements_by_tag_name('a')
     hrefs = []
     for link in links:
         url = link.get_attribute("href")
         if HttpClient.is_link_ok(url):
             hrefs.append(url)
     self.wait()
     return hrefs
Ejemplo n.º 36
0
def scrape():
    logging.info('Scraping calcalist')

    url = CALCALIST_INIT_URL
    logging.info('Initial URL: %s' % url)
    i = 0
    while url is not None:
        logging.info('iteration #%d with url: %s' % (i, url))
        page_source = HttpClient.download_page_source(url)
        nominations_page = NominationsPage(url, BASE_URL, page_source)

        for nomination in nominations_page.nominations():
            nomination['source'] = 'calcalist'
            yield nomination

        url = nominations_page.next_url()
        logging.info('next URL: %s' % url)
        i += 1
Ejemplo n.º 37
0
class API:
    def __init__(self):
        uri = 'http://localhost:5001/'

        self.client = HttpClient(uri=uri)

    @db_breaker
    def remote_post(self, wait, status_code):
        path = 'handle_post'
        params = {
            'wait': wait,
            'status_code': status_code,
        }
        try:
            response_text = self.client.post(path=path, params=params)
        except HTTPError as e:
            if e.response.status_code >= 500:
                raise

        return response_text
Ejemplo n.º 38
0
    def fetch_all_posts(self):
        """
        Fetches all the posts.

        Parameters
        ----------

        Returns
        -------
        posts: list
            -- Returns a list of posts available on the server
        """

        posts = []
        retry = 0
        current_page_no = 1
        # Infinite loop to give retries a chance.
        while True:
            try:
                sl_token = self.get_app_token()
                # 1-10 pages of data available on server
                for page_no in range(current_page_no, 11):

                    fetch_posts_data = {"sl_token": sl_token, "page": page_no}
                    posts_per_page = HttpClient.get_posts(fetch_posts_data)
                    # save current page_no to ensure we dont start from Page 1 on failure-retry sequence
                    current_page_no = page_no
                    # Reset Retry
                    retry = 0
                    posts.extend(posts_per_page)
                # everything went well break out of the loop
                break
            except InvalidSLTokenException as e:
                # Try to get latest token with a retry (3 times max)
                if retry < 3:
                    retry = retry + 1
                else:
                    raise Exception(
                        "Unable to fetch posts -- Check 'sltoken' &/or `url` are valid"
                    )
        return posts
Ejemplo n.º 39
0
    def __init__(self, QR_CODE_PATH, smart_qq_refer, cookie_file):

        self.smart_qq_refer = smart_qq_refer
        self.client = HttpClient(smart_qq_refer = smart_qq_refer, cookie_file=cookie_file)

        # cache
        self.friend_uin_list = {}
        self.group_code_list = {}
        self.group_id_list = {}
        self.group_member_info = {}

        self._group_sig_list = {}
        self._self_info = {}

        self.client_id = 53999199
        self.ptwebqq = ''
        self.psessionid = ''
        self.appid = 0
        self.vfwebqq = ''
        self.qrcode_path = QR_CODE_PATH
        self.username = ''
        self.account = 0
Ejemplo n.º 40
0
    def __init__(self, token):

        self.api_token = token
        self.client = HttpClient(token)
Ejemplo n.º 41
0
class QQBot(object):
    def __init__(self, QR_CODE_PATH, smart_qq_refer, cookie_file):

        self.smart_qq_refer = smart_qq_refer
        self.client = HttpClient(smart_qq_refer = smart_qq_refer, cookie_file=cookie_file)

        # cache
        self.friend_uin_list = {}
        self.group_code_list = {}
        self.group_id_list = {}
        self.group_member_info = {}

        self._group_sig_list = {}
        self._self_info = {}

        self.client_id = 53999199
        self.ptwebqq = ''
        self.psessionid = ''
        self.appid = 0
        self.vfwebqq = ''
        self.qrcode_path = QR_CODE_PATH
        self.username = ''
        self.account = 0

    @property
    def bkn(self):
        skey = self.client.get_cookie('skey')
        hash_str = 5381
        for i in skey:
            hash_str += (hash_str << 5) + ord(i)
        hash_str = int(hash_str & 2147483647)
        return hash_str

    def _get_group_sig(self, guin, tuin, service_type=0):
        key = '%s --> %s' % (guin, tuin)
        if key not in self._group_sig_list:
            url = "http://d1.web2.qq.com/channel/get_c2cmsg_sig2?id=%s&to_uin=%s&service_type=%s&clientid=%s&psessionid=%s&t=%d" % (
                guin, tuin, service_type, self.client_id, self.psessionid, int(time.time() * 100))
            response = self.client.get(url)
            rsp_json = json.loads(response)
            if rsp_json["retcode"] != 0:
                return ""
            sig = rsp_json["result"]["value"]
            self._group_sig_list[key] = sig
        if key in self._group_sig_list:
            return self._group_sig_list[key]
        return ""

    def _login_by_cookie(self):
        logger.info("Try cookie login...")

        self.client.load_cookie()
        self.ptwebqq = self.client.get_cookie('ptwebqq')

        response = self.client.post(
            'http://d1.web2.qq.com/channel/login2',
            {
                'r': '{{"ptwebqq":"{0}","clientid":{1},"psessionid":"{2}","status":"online"}}'.format(
                    self.ptwebqq,
                    self.client_id,
                    self.psessionid
                )
            },
            self.smart_qq_refer
        )
        try:
            ret = json.loads(response)
        except ValueError:
            logger.warning("Cookies login fail, response decode error.")
            return
        if ret['retcode'] != 0:
            raise CookieLoginFailed("Login step 1 failed with response:\n %s " % ret)

        response2 = self.client.get(
                "http://s.web2.qq.com/api/getvfwebqq?ptwebqq={0}&clientid={1}&psessionid={2}&t={3}".format(
                        self.ptwebqq,
                        self.client_id,
                        self.psessionid,
                        self.client.get_timestamp()
                ))
        ret2 = json.loads(response2)

        if ret2['retcode'] != 0:
            raise CookieLoginFailed(
                "Login step 2 failed with response:\n %s " % ret
            )

        self.psessionid = ret['result']['psessionid']
        self.account = ret['result']['uin']
        self.vfwebqq = ret2['result']['vfwebqq']

        logger.info("Login by cookie succeed. account: %s" % self.account)
        return True

    def _login_by_qrcode(self, no_gui):
        logger.info("RUNTIMELOG Trying to login by qrcode.")
        logger.info("RUNTIMELOG Requesting the qrcode login pages...")
        qr_validation_url = 'https://ssl.ptlogin2.qq.com/ptqrlogin?' \
                            'webqq_type=10&remember_uin=1&login2qq=1&aid={0}' \
                            '&u1=http%3A%2F%2Fw.qq.com%2Fproxy.html%3Flogin2qq%3D1%26webqq_type%3D10' \
                            '&ptredirect=0&ptlang=2052&daid=164&from_ui=1&pttype=1&dumy=' \
                            '&fp=loginerroralert&action=0-0-{1}&mibao_css={2}' \
                            '&t=undefined&g=1&js_type=0&js_ver={3}&login_sig={4}'

        init_url = "https://ui.ptlogin2.qq.com/cgi-bin/login?" \
                   "daid=164&target=self&style=16&mibao_css=m_webqq" \
                   "&appid=501004106&enable_qlogin=0&no_verifyimg=1" \
                   "&s_url=http%3A%2F%2Fw.qq.com%2Fproxy.html" \
                   "&f_url=loginerroralert&strong_login=1" \
                   "&login_state=10&t=20131024001"
        html = self.client.get(
            init_url,
        )
        appid = find_first_result(
            html,
            r'<input type="hidden" name="aid" value="(\d+)" />', 'Get AppId Error',
            True
        )
        sign = find_first_result(
            html,
            r'g_login_sig=encodeURIComponent\("(.*?)"\)', 'Get Login Sign Error',
        )
        js_ver = find_first_result(
            html,
            r'g_pt_version=encodeURIComponent\("(\d+)"\)',
            'Get g_pt_version Error',
            True,
        )
        mibao_css = find_first_result(
            html,
            r'g_mibao_css=encodeURIComponent\("(.+?)"\)',
            'Get g_mibao_css Error',
            True
        )

        star_time = date_to_millis(datetime.datetime.utcnow())

        error_times = 0
        ret_code = None
        login_result = None
        redirect_url = None

        while True:
            error_times += 1
            logger.info("Downloading QRCode file...")
            self.client.download(
                'https://ssl.ptlogin2.qq.com/ptqrshow?appid={0}&e=0&l=L&s=8&d=72&v=4'.format(appid),
                self.qrcode_path
            )
            if not no_gui:
                thread = Thread(target=show_qr, args=(self.qrcode_path, ))
                thread.setDaemon(True)
                thread.start()

            while True:
                ret_code, redirect_url = self._get_qr_login_status(
                    qr_validation_url, appid, star_time, mibao_css, js_ver,
                    sign, init_url
                )

                if ret_code in (
                        QR_CODE_STATUS['succeed'], QR_CODE_STATUS["qr_code_expired"]
                ):
                    break
                time.sleep(1)

            if ret_code == QR_CODE_STATUS['succeed'] or error_times > 10:
                break

        if os.path.exists(self.qrcode_path):
            os.remove(self.qrcode_path)

        login_failed_tips = "QRCode validation response is:\n%s" % login_result

        if ret_code is not None and (ret_code != 0):
            raise QRLoginFailed(login_failed_tips)
        elif redirect_url is None:
            raise QRLoginFailed(login_failed_tips)
        else:
            html = self.client.get(redirect_url)
            logger.debug("QR Login redirect_url response: %s" % html)
            return True

    def _get_qr_login_status(
            self, qr_validation_url, appid, star_time,
            mibao_css, js_ver, sign, init_url
    ):
        redirect_url = None
        login_result = self.client.get(
            qr_validation_url.format(
                appid,
                date_to_millis(datetime.datetime.utcnow()) - star_time,
                mibao_css,
                js_ver,
                sign
            ),
            init_url
        )
        ret_code = int(find_first_result(login_result, r"\d+?", None))
        redirect_info = re.findall(r"(http.*?)\'", login_result)
        if redirect_info:
            logger.debug("redirect_info match is: %s" % redirect_info)
            redirect_url = redirect_info[0]
        return ret_code, redirect_url

    def login(self, no_gui=False):
        try:
            self._login_by_cookie()
        except CookieLoginFailed:
            logger.info("Cookie login failed.")
            while True:
                if self._login_by_qrcode(no_gui):
                    if self._login_by_cookie():
                        break
                time.sleep(4)
        user_info = self.get_self_info()
        self.get_online_friends_list()
        self.get_group_list_with_group_id()
        self.get_group_list_with_group_code()
        try:
            self.username = user_info['nick']
            logger.info(
                "User information got: user name is [%s]" % self.username
            )
        except KeyError:
            logger.exception(
                "User info access failed, check your login and response:\n%s"
                % user_info
            )
            exit(1)
        logger.info("RUNTIMELOG QQ:{0} login successfully, Username:{1}".format(self.account, self.username))

    def check_msg(self):

        # Pooling the message
        response = self.client.post(
            'http://d1.web2.qq.com/channel/poll2',
            {
                'r': json.dumps(
                    {
                        "ptwebqq": self.ptwebqq,
                        "clientid": self.client_id,
                        "psessionid": self.psessionid,
                        "key": ""
                    }
                )
            },
            self.smart_qq_refer
        )
        logger.debug("Pooling returns response: %s" % response)
        if response == "":
            return
        try:
            ret = json.loads(response)
        except ValueError:
            logger.warning("RUNTIMELOG decode poll response error.")
            logger.debug("RESPONSE {}".format(response))
            return

        ret_code = ret['retcode']

        if ret_code in (103, ):
            logger.warning(
                "Pooling received retcode: " + str(ret_code) + ": Check error. 请前往http://w.qq.com/ 手动登陆SmartQQ一次."
            )
        elif ret_code in (121,):
            logger.warning("Pooling error with retcode %s" % ret_code)
        elif ret_code == 0:
            if 'result' not in ret or len(ret['result']) == 0:
                logger.info("Pooling ends, no new message received.")
            else:
                return ret['result']
        elif ret_code == 100006:
            logger.error("Pooling request error, response is: %s" % ret)
        elif ret_code == 116:
            self.ptwebqq = ret['p']
            logger.debug("ptwebqq updated in this pooling")
        else:
            logger.warning("Pooling returns unknown retcode %s" % ret_code)
        return None

    def uin_to_account(self, tuin):
        """
        将uin转换成用户QQ号
        :param tuin:
        :return:str 用户QQ号
        """
        uin_str = str(tuin)
        try:
            logger.info("RUNTIMELOG Requesting the account by uin:    " + str(tuin))
            info = json.loads(
                self.client.get(
                    'http://s.web2.qq.com/api/get_friend_uin2?tuin={0}&type=1&vfwebqq={1}&t={2}'.format(
                        uin_str,
                        self.vfwebqq,
                        self.client.get_timestamp()
                    ),
                    self.smart_qq_refer
                )
            )
            logger.debug("RESPONSE uin_to_account html:    " + str(info))
            if info['retcode'] != 0:
                raise TypeError('uin_to_account retcode error')
            info = info['result']['account']
            return info

        except Exception:
            logger.exception("RUNTIMELOG uin_to_account fail")
            return None

    def get_self_info(self):
        """
        获取自己的信息, 并存入self._self_info
        get_self_info2
        {"retcode":0,"result":{"birthday":{"month":1,"year":1989,"day":30},"face":555,"phone":"","occupation":"","allow":1,"college":"","uin":2609717081,"blood":0,"constel":1,"lnick":"","vfwebqq":"68b5ff5e862ac589de4fc69ee58f3a5a9709180367cba3122a7d5194cfd43781ada3ac814868b474","homepage":"","vip_info":0,"city":"青岛","country":"中国","personal":"","shengxiao":5,"nick":"要有光","email":"","province":"山东","account":2609717081,"gender":"male","mobile":""}}
        :return:dict
        """
        try_times = 0

        while len(self._self_info) is 0:
            url = "http://s.web2.qq.com/api/get_self_info2?t={}".format(time.time())
            response = self.client.get(url)
            logger.debug("get_self_info2 response:{}".format(response))
            rsp_json = json.loads(response)
            if rsp_json["retcode"] != 0:
                try_times += 1
                logger.warning("get_self_info2 fail. {}".format(try_times))
                if try_times >= 5:
                    return {}
            try:
                self._self_info = rsp_json["result"]
            except KeyError:
                logger.warning("get_self_info2 failed. Retrying.")
                continue
        return self._self_info

    def get_online_friends_list(self):
        """
        获取在线好友列表
        get_online_buddies2
        :return:list
        """
        logger.info("RUNTIMELOG Requesting the online buddies.")
        response = self.client.get(
            'http://d1.web2.qq.com/channel/get_online_buddies2?vfwebqq={0}&clientid={1}&psessionid={2}&t={3}'.format(
                self.vfwebqq,
                self.client_id,
                self.psessionid,
                self.client.get_timestamp(),
            )
        )  # {"result":[],"retcode":0}
        logger.debug("RESPONSE get_online_buddies2 html:{}".format(response))
        try:
            online_buddies = json.loads(response)
        except ValueError:
            logger.warning("get_online_buddies2 response decode as json fail.")
            return None

        if online_buddies['retcode'] != 0:
            logger.warning('get_online_buddies2 retcode is not 0. returning.')
            return None

        online_buddies = online_buddies['result']
        return online_buddies

    def get_friend_info(self, tuin):
        """
        获取好友详情信息
        get_friend_info
        {"retcode":0,"result":{"face":0,"birthday":{"month":1,"year":1989,"day":30},"occupation":"","phone":"","allow":1,"college":"","uin":3964575484,"constel":1,"blood":3,"homepage":"http://blog.lovewinne.com","stat":20,"vip_info":0,"country":"中国","city":"","personal":"","nick":" 信","shengxiao":5,"email":"*****@*****.**","province":"山东","gender":"male","mobile":"158********"}}
        :return:dict
        """

        uin = str(tuin)
        if uin not in self.friend_uin_list:
            logger.info("RUNTIMELOG Requesting the account info by uin: {}".format(uin))
            info = json.loads(self.client.get(
                'http://s.web2.qq.com/api/get_friend_info2?tuin={0}&vfwebqq={1}&clientid={2}&psessionid={3}&t={4}'.format(
                    uin,
                    self.vfwebqq,
                    self.client_id,
                    self.psessionid,
                    self.client.get_timestamp()
                )
            ))
            logger.debug("get_friend_info2 html: {}".format(str(info)))
            if info['retcode'] != 0:
                logger.warning('get_friend_info2 retcode unknown: {}'.format(info))
                return None
            info = info['result']
            info['account'] = self.uin_to_account(uin)
            info['longnick'] = self.get_friend_longnick(uin)
            self.friend_uin_list[uin] = info

        try:
            return self.friend_uin_list[uin]
        except:
            logger.warning("RUNTIMELOG get_friend_info return fail.")
            logger.debug("RUNTIMELOG now uin list:    " + str(self.friend_uin_list[uin]))

    def get_friend_longnick(self, tuin):
        """
        获取好友的签名信息
        {"retcode":0,"result":[{"uin":3964575484,"lnick":"幸福,知道自己在哪里,知道下一个目标在哪里,心不累~"}]}
        :return:dict
        """
        url = "http://s.web2.qq.com/api/get_single_long_nick2?tuin=%s&vfwebqq=%s&t=%s" % (
            tuin, self.vfwebqq, int(time.time() * 100))
        response = self.client.get(url)
        rsp_json = json.loads(response)
        if rsp_json["retcode"] != 0:
            return {}
        return rsp_json["result"]

    def get_group_list_with_group_code(self):
        """
        获取包含群名和group_code的列表, 并存入cache, 其中code为group_code
        :type group_code: str
        :return:list
        [
            {
                u 'code': 1131597161, # 这是真实group_code
                u 'flag': 184550417,
                u 'gid': 1802239929,  # 这是msg.group_code, 即假group_code
                u 'name': u '测试'
            },
            {
                u 'code': 1131597161,
                u 'flag': 184550417,
                u 'gid': 1802239929,
                u 'name': u '测试'
            }
        ]
        """

        def _hash_digest(uin, ptwebqq):
            """
            计算hash,貌似TX的这个算法会经常变化,暂时不使用
            get_group_code_list 会依赖此数据
            提取自http://pub.idqqimg.com/smartqq/js/mq.js
            :param uin:
            :param ptwebqq:
            :return:
            """
            N = [0, 0, 0, 0]
            # print(N[0])
            for t in range(len(ptwebqq)):
                N[t % 4] ^= ord(ptwebqq[t])
            U = ["EC", "OK"]
            V = [0, 0, 0, 0]
            V[0] = int(uin) >> 24 & 255 ^ ord(U[0][0])
            V[1] = int(uin) >> 16 & 255 ^ ord(U[0][1])
            V[2] = int(uin) >> 8 & 255 ^ ord(U[1][0])
            V[3] = int(uin) & 255 ^ ord(U[1][1])
            U = [0, 0, 0, 0, 0, 0, 0, 0]
            for T in range(8):
                if T % 2 == 0:
                    U[T] = N[T >> 1]
                else:
                    U[T] = V[T >> 1]
            N = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F"]
            V = ""
            for T in range(len(U)):
                V += N[U[T] >> 4 & 15]
                V += N[U[T] & 15]
            return V


        logger.info("RUNTIMELOG Requesting the group list.")
        response = self.client.post(
            'http://s.web2.qq.com/api/get_group_name_list_mask2',
            {
                'r': json.dumps(
                    {
                        "vfwebqq": self.vfwebqq,
                        "hash": _hash_digest(self._self_info['uin'], self.ptwebqq),
                    }
                )
            },
        )
        try:
            response = json.loads(response)
        except ValueError:
            logger.warning("RUNTIMELOG The response of group list request can't be load as json")
            return
        logger.debug("RESPONSE get_group_name_list_mask2 html:    " + str(response))
        if response['retcode'] != 0:
            raise TypeError('get_online_buddies2 result error')
        for group in response['result']['gnamelist']:
            self.group_code_list[str(group['gid'])] = group

        return response['result']['gnamelist']

    def get_group_list_with_group_id(self):
        """
        获取包含群名和群号的列表, 并存入cache, 其中gc为群号
        :type group_id: str
        :return:list

        return list sample
        [
            {
                "gc": 114302207,
                "gn": "测试群1",
                "owner": 484216451
            },
            {
                "gc": 125299202,
                "gn": "测试群2",
                "owner": 242917661
            }
        ]
        """
        url = "http://qun.qq.com/cgi-bin/qun_mgr/get_group_list"
        data = {'bkn': self.bkn}
        response = self.client.post(url, data=data, refer='http://qun.qq.com/member.html')
        logger.debug("get_group_list response: {}".format(response))
        rsp_json = json.loads(response)
        if rsp_json['ec'] == 0:
            group_id_list = list()
            group_id_list.extend(rsp_json.get('join') or [])
            group_id_list.extend(rsp_json.get('manage') or [])
            if group_id_list:
                for group in group_id_list:
                    self.group_id_list[str(group['gc'])] = group
                return group_id_list
            else:
                logger.debug("seems this account didn't join any group: {}".format(response))
        else:
            logger.warning("get_group_list code unknown: {}".format(response))
            return None

    def get_true_group_code(self, fake_group_code):
        """
        通过假group_code获取真group_code
        :type fake_group_code: str
        :return str
        """
        fake_group_code = str(fake_group_code)
        logger.debug("正在查询group_code:{}对应的真实group_code".format(fake_group_code))
        if fake_group_code not in self.group_code_list:
            logger.info("尝试更新群列表信息")
            self.get_group_list_with_group_code()  # 先尝试更新群列表
            if fake_group_code not in self.group_code_list:
                logger.warning("没有所查询的group_code, 请检查group_code是否错误")
                return 0
        return str(self.group_code_list[fake_group_code]['code'])

    def get_group_info(self, group_code=None, group_id=None):
        """
        通过group_code或者group_id(群号)获取对应群信息
        :type group_code: str
        :type group_id: str
        :return dict
        {
            'name':         "群名",
            'id':            12345678,
            'group_code':    87654321
        }
        """
        if group_code or group_id:
            if group_code:
                assert isinstance(group_code, str), "group_code类型错误, 应该为str"
                t_group_code = self.get_true_group_code(group_code)
                if t_group_code not in self.group_code_list:
                    self.get_group_list_with_group_code()
                group_code_info = self.group_code_list.get(t_group_code)

                group_id_list = self.get_group_list_with_group_id()
                result = {
                    'name':         group_code_info['name'],
                    'id':           "",
                    'group_code':   group_code_info['code']
                }
                group_id_list = filter(lambda x:x['gn'] == group_code_info['name'], group_id_list)
                if len(group_id_list) == 1:
                    result['id'] = group_id_list[0].get('gc')
                    return result
                else:
                    raise KeyError('QQ{qq}的群列表中含有{count}个同名群:"{group_name}"'.format(
                        qq=self.account,
                        count=len(group_id_list),
                        group_name=group_code_info['name']
                    ))
            elif group_id:
                assert isinstance(group_id, str), "group_id类型错误, 应该为str"
                if group_id not in self.group_id_list:
                    self.get_group_list_with_group_id()
                group_id_info = self.group_id_list.get(group_id)
                group_code_list = self.get_group_list_with_group_code()
                result = {
                    'name': group_id_info['gn'],
                    'id': group_id_info['gc'],
                    'group_code': 0
                }
                group_code_list = filter(lambda x:x['name'] == group_id_info['gn'], group_code_list)
                if len(group_code_list) == 1:
                    result['group_code'] = group_code_list[0].get('code')
                    return result
                else:
                    raise KeyError('QQ{qq}的群列表中含有{count}个同名群:"{group_name}"'.format(
                        qq=self.account,
                        count=len(group_code_list),
                        group_name=group_id_info['gn']
                    ))
        else:
            raise KeyError("请输入group_code或group_id之一")

    def get_group_member_info_list(self, group_code):
        """
        获取指定群的成员信息
        :group_code: int, can be "ture" of "fake" group_code
        {"retcode":0,"result":{"stats":[],"minfo":[{"nick":" 信","province":"山东","gender":"male","uin":3964575484,"country":"中国","city":""},{"nick":"崔震","province":"","gender":"unknown","uin":2081397472,"country":"","city":""},{"nick":"云端的猫","province":"山东","gender":"male","uin":3123065696,"country":"中国","city":"青岛"},{"nick":"要有光","province":"山东","gender":"male","uin":2609717081,"country":"中国","city":"青岛"},{"nick":"小莎机器人","province":"广东","gender":"female","uin":495456232,"country":"中国","city":"深圳"}],"ginfo":{"face":0,"memo":"http://hujj009.ys168.com\r\n0086+区(没0)+电话\r\n0086+手机\r\nhttp://john123951.xinwen365.net/qq/index.htm","class":395,"fingermemo":"","code":3943922314,"createtime":1079268574,"flag":16778241,"level":0,"name":"ぁQQぁ","gid":3931577475,"owner":3964575484,"members":[{"muin":3964575484,"mflag":192},{"muin":2081397472,"mflag":65},{"muin":3123065696,"mflag":128},{"muin":2609717081,"mflag":0},{"muin":495456232,"mflag":0}],"option":2},"cards":[{"muin":3964575484,"card":"●s.Εx2(22222)□"},{"muin":495456232,"card":"小莎机器人"}],"vipinfo":[{"vip_level":0,"u":3964575484,"is_vip":0},{"vip_level":0,"u":2081397472,"is_vip":0},{"vip_level":0,"u":3123065696,"is_vip":0},{"vip_level":0,"u":2609717081,"is_vip":0},{"vip_level":0,"u":495456232,"is_vip":0}]}}
        :return:dict
        """
        if group_code == 0:
            return
        try:
            url = "http://s.web2.qq.com/api/get_group_member_info_ext2?gcode=%s&vfwebqq=%s&t=%s" % (
                group_code, self.vfwebqq, int(time.time() * 100))
            response = self.client.get(url)
            rsp_json = json.loads(response)
            logger.debug("get_group_member_info_ext2 info response: {}".format(rsp_json))
            retcode = rsp_json["retcode"]
            if retcode == 0:
                result = rsp_json["result"]
            elif retcode == 6:
                logger.debug("get_group_member_info_ext2 retcode is 6, trying to get true code.")
                result = self.get_group_member_info_list(self.get_true_group_code(group_code))
            else:
                logger.warning("group_code error.")
                return
            self.group_member_info[str(group_code)] = result    # 缓存群成员信息, 此处会把真假group_code都加入cache
            return result
        except Exception as ex:
            logger.warning("RUNTIMELOG get_group_member_info_ext2. Error: " + str(ex))
            return

    def get_group_member_info(self, group_code, uin):
        """
        获取群中某一指定成员的信息
        :type group_code:   int, can be "ture" of "fake" group_code
        :type uin:  int
        :return:    dict
        """
        group_code = str(group_code)
        if group_code not in self.group_member_info:
            logger.info("group_code not in cache, try to request info")
            result = self.get_group_member_info_list(group_code)
            if result is False:
                logger.warning("没有所查询的group_code信息")
                return

        for member in self.group_member_info[group_code]['minfo']:
            if member['uin'] == uin:
                return member

    # 发送群消息
    def send_group_msg(self, reply_content, group_code, msg_id, fail_times=0):
        fix_content = str(reply_content.replace("\\", "\\\\\\\\").replace("\n", "\\\\n").replace("\t", "\\\\t"))
        rsp = ""
        try:
            logger.info("Starting send group message: %s" % reply_content)
            req_url = "http://d1.web2.qq.com/channel/send_qun_msg2"
            data = (
                ('r',
                 '{{"group_uin":{0}, "face":564,"content":"[\\"{4}\\",[\\"font\\",{{\\"name\\":\\"Arial\\",\\"size\\":\\"10\\",\\"style\\":[0,0,0],\\"color\\":\\"000000\\"}}]]","clientid":{1},"msg_id":{2},"psessionid":"{3}"}}'.format(
                         group_code, self.client_id, msg_id, self.psessionid, fix_content)),
                ('clientid', self.client_id),
                ('psessionid', self.psessionid)
            )
            rsp = self.client.post(req_url, data, self.smart_qq_refer)
            rsp_json = json.loads(rsp)
            if 'retcode' in rsp_json and rsp_json['retcode'] not in MESSAGE_SENT:
                raise ValueError("RUNTIMELOG reply group chat error" + str(rsp_json['retcode']))
            logger.info("RUNTIMELOG send_qun_msg: Reply '{}' successfully.".format(reply_content))
            logger.debug("RESPONSE send_qun_msg: Reply response: " + str(rsp))
            return rsp_json
        except:
            logger.warning("RUNTIMELOG send_qun_msg fail")
            if fail_times < 5:
                logger.warning("RUNTIMELOG send_qun_msg: Response Error.Wait for 2s and Retrying." + str(fail_times))
                logger.debug("RESPONSE send_qun_msg rsp:" + str(rsp))
                time.sleep(2)
                self.send_group_msg(reply_content, group_code, msg_id, fail_times + 1)
            else:
                logger.warning("RUNTIMELOG send_qun_msg: Response Error over 5 times.Exit.reply content:" + str(reply_content))
                return False


    # 发送私密消息
    def send_friend_msg(self, reply_content, uin, msg_id, fail_times=0):
        fix_content = str(reply_content.replace("\\", "\\\\\\\\").replace("\n", "\\\\n").replace("\t", "\\\\t"))
        rsp = ""
        try:
            req_url = "http://d1.web2.qq.com/channel/send_buddy_msg2"
            data = (
                ('r',
                 '{{"to":{0}, "face":594, "content":"[\\"{4}\\", [\\"font\\", {{\\"name\\":\\"Arial\\", \\"size\\":\\"10\\", \\"style\\":[0, 0, 0], \\"color\\":\\"000000\\"}}]]", "clientid":{1}, "msg_id":{2}, "psessionid":"{3}"}}'.format(
                         uin, self.client_id, msg_id, self.psessionid, fix_content)),
                ('clientid', self.client_id),
                ('psessionid', self.psessionid)
            )
            rsp = self.client.post(req_url, data, self.smart_qq_refer)
            rsp_json = json.loads(rsp)
            if 'errCode' in rsp_json and rsp_json['errCode'] != 0:
                raise ValueError("reply pmchat error" + str(rsp_json['retcode']))
            logger.info("RUNTIMELOG Reply successfully.")
            logger.debug("RESPONSE Reply response: " + str(rsp))
            return rsp_json
        except:
            if fail_times < 5:
                logger.warning("RUNTIMELOG Response Error.Wait for 2s and Retrying." + str(fail_times))
                logger.debug("RESPONSE " + str(rsp))
                time.sleep(2)
                self.send_friend_msg(reply_content, uin, msg_id, fail_times + 1)
            else:
                logger.warning("RUNTIMELOG Response Error over 5 times.Exit.reply content:" + str(reply_content))
                return False


    def reply_msg(self, msg, reply_content=None, return_function=False):
        """
        :type msg: QMessage类, 例如 GroupMsg, PrivateMsg, SessMsg
        :type reply_content: string, 回复的内容.
        :return: 服务器的响应内容. 如果 return_function 为 True, 则返回的是一个仅有 reply_content 参数的便捷回复函数.
        """
        msg_id = randint(1, 100000)
        import functools
        assert isinstance(msg, QMessage)
        if isinstance(msg, GroupMsg):
            if return_function:
                return functools.partial(self.send_group_msg, group_code=msg.group_code, msg_id=msg_id)
            return self.send_group_msg(reply_content=reply_content, group_code=msg.group_code, msg_id=msg_id)
        if isinstance(msg, PrivateMsg):
            if return_function:
                return functools.partial(self.send_friend_msg, uin=msg.from_uin, msg_id=msg_id)
            return self.send_friend_msg(reply_content=reply_content, uin=msg.from_uin, msg_id=msg_id)
        if isinstance(msg, SessMsg):
            # 官方已废弃临时消息接口, 等官方重启后再完善此函数
            pass
Ejemplo n.º 42
0
Archivo: pin.py Proyecto: sousajmr/ptpy
class Pin:
    def __init__(self):
        self.http = HttpClient()

    def login(self, loginData):
        signup_url = "http://localhost/signup/"
        xsrf = self.http.getXsrf(signup_url)
        print xsrf
        loginData["_xsrf"] = xsrf

        regQuery = urllib.urlencode(loginData)
        # print regQuery
        url = "http://localhost/login/"
        print url
        req = urllib2.Request(url, regQuery)
        res = urllib2.urlopen(req)
        print res.read(20)
        self.http.saveCookie()

    def new(self, bData):
        signup_url = "http://localhost/uploader/file/"
        xsrf = self.http.getXsrf(signup_url)
        print xsrf

        bData["_xsrf"] = xsrf

        bQuery = urllib.urlencode(bData)
        url = "http://localhost/ajax/addboard/"
        req = urllib2.Request(url, bQuery)
        res = urllib2.urlopen(req)
        print res.read()

    def uploadFile(self):
        signup_url = "http://localhost/uploader/file/"
        xsrf = self.http.getXsrf(signup_url)
        print xsrf

        fields = [("_xsrf", xsrf)]

        ifile = "/home/sites/lihuashu.com/test/1.gif"

        imgdata = file(ifile, "rb")
        files = [("ifile", imgdata.name, imgdata.read())]

        content_type, upload_data = uploadfile(fields, files)

        uploadheader = {"Content-Type": content_type, "Content-Length": str(len(upload_data))}
        request = urllib2.Request("http://localhost/upload/", upload_data, uploadheader)
        res = urllib2.urlopen(request)
        print res.read(100)
        print self.http.cookieJar
        for c in self.http.cookieJar:
            if c.name == "_xsrf":
                self._xsrf = c.value
                print c.value
            if c.name == "pic_url":
                self.pic_url = c.value
                print c.value
            if c.name == "thumb_url":
                self.thumb_url = c.value
                print c.value
            if c.name == "board":
                self.board = c.value
                print c.value

    def addPin(self):
        print self._xsrf

        url = "http://localhost/service/form/"
        pData = {}
        pData["board"] = self.board
        pData["thumb_url"] = self.thumb_url
        pData["pic_url"] = self.pic_url
        pData["content"] = "content"
        pData["_xsrf"] = self._xsrf

        req = urllib2.Request(url, urllib.urlencode(pData))
        r = urllib2.urlopen(req)
 def __init__(self, analyzer, stream_id, admin_if_addr, _map):
     self.response = None
     self.stream_id = stream_id
     self.analyzer = analyzer
     self.analyze_result = None
     HttpClient.__init__(self, admin_if_addr, _map)
Ejemplo n.º 44
0
 def __init__(self):
     self.http = HttpClient()        
Ejemplo n.º 45
0
import sys
import json

sys.path.append('../src')

from http_client import HttpClient

args = {
    'fieldname': 'file',
    'filename' : 'test.json',
    'data'     : {'body': 'test'},
    'headers'  : {'X-Requested-With': 'XMLHttpRequest'}
}

response = HttpClient.file('http://curtaincall.weblike.jp/prog/php/test-api/file.php', **args)

data = json.loads(response.text)

print(data['result'])
print(data['messages'])
Ejemplo n.º 46
0
import sys
import json

sys.path.append('../src')

from http_client import HttpClient

http_client = HttpClient()

args = {
    'data'   : {'body': 'test'},
    'headers': {'X-Requested-With': 'XMLHttpRequest'}
}

response = HttpClient.post('http://curtaincall.weblike.jp/prog/php/test-api/post.php', **args)

data = json.loads(response.text)

print(data['result'])
print(data['messages'])
Ejemplo n.º 47
0
from http_client import HttpClient

httpmethod = "GET";
url = "http://api.openapi.io/ppurio/"
parameters = "?parameter=parameterValue&";
clientKey = "CLIENT_KEY";
contentType = "application/x-www-form-urlencoded";

# Create HttpClient
client = HttpClient()
     
# Http Method
response = client.sendRequest(httpmethod, url, parameters, clientKey, contentType)

# now you can do something with the response.
print vars(response)
Ejemplo n.º 48
0
class Bot():
    """
    @brief Facebook messenger bot
    """

    def __init__(self, token):

        self.api_token = token
        self.client = HttpClient(token)

    def send_message(self, message, recipient):

        payload = {
            'recipient': {
                'id': recipient,
            },
            'message': message.to_json()
        }

        payload = json.dumps(payload)

        response, error = self.client.submit_request(
                            '/me/messages',
                            'POST',
                            payload)

        if error is not None:
            print 'Error Encountered! Could not send message\n'
            print 'Message: %s' % error

        return response, error

    def set_welcome(self, message):

        greeting = {
            'setting_type': 'greeting',
            'greeting': {
                'text': message
            }
        }

        data = json.dumps(greeting)

        response, error = self.client.submit_request(
            '/me/thread_settings',
            'POST',
            data)

        return response, error
   
    def get_profile(self, profile):
        response, error = self.client.submit_request(
	    '/' +str(profile) +'fields=first_name,last_name,profile_pic,locale,timezone,gender',
	    'GET')
	
	return response, error
    
    def messages_for_request(self, request):
        """
        Handle incoming requests from the webhook
        """

        entries = request.json['entry']
        messages = []
        for entry in entries:
            for msg in entry['messaging']:
                if msg.get('message') and msg['message'].get('text'):
                    messages.append(ReceivedMessage(msg))

        return messages

    def message_from_reciept(receipt):
        """
        Handle receipts
        """

        raise NotImplementedError

    def set_persistent_menu(self, buttons):
        """
        Sets the persistent menu. 
        @params: buttons -> list of button objects
        @moreinfo: https://developers.facebook.com/docs/messenger-platform/thread-settings/persistent-menu
        """
        MAX_BUTTONS = 5
        if len(buttons) > MAX_BUTTONS:
            raise TypeError(
                "Persistent menus have maximum %d buttons. " \
                "Found: %d" % (MAX_BUTTONS, len(buttons))
            )

        payload = {
            "setting_type": "call_to_actions",
            "thread_state": "existing_thread",
            "call_to_actions": [    
                button.to_json() for button in buttons
            ]
        }
        
        data = json.dumps(payload)
        response, error = self.client.submit_request(
            '/me/thread_settings',
            'POST',
             data)

        return response, error