コード例 #1
0
def get_info(url, cw=None):
    print_ = get_print(cw)
    info = {}

    html = downloader.read_html(url)
    if '"cafe_cautionpage"' in html:
        raise errors.LoginRequired()
    url_article = re.find(r'''//cafe\.naver\.com/ArticleRead\.nhn\?articleid=[0-9]+&clubid=[0-9]+''', html, err='no iframe')
    url_article = urljoin(url, url_article)

    print_(url_article)

    articleid = re.find(r'articleid=([0-9]+)', url_article)
    clubid = re.find(r'clubid=([0-9]+)', url_article)
    url_api = f'https://apis.naver.com/cafe-web/cafe-articleapi/v2/cafes/{clubid}/articles/{articleid}?query=&useCafeId=true&requestFrom=A'

    j = downloader.read_json(url_api, url)

    info['title'] = j['result']['article']['subject']
    info['cafename'] = j['result']['cafe']['url']
    info['cafeid'] = clubid
    info['id'] = articleid
    
    html_content = j['result']['article']['contentHtml']
    soup = Soup(html_content)

    imgs = []

    pairs = []

    for video in soup.findAll('span', class_='_naverVideo'):
        vid = video.attrs['vid']
        key = video.attrs['key']
        pairs.append((vid, key))

    for script in soup.findAll('script', class_='__se_module_data'):
        data_raw = script['data-module']
        data = json.loads(data_raw)['data']
        vid = data.get('vid')
        if not vid:
            continue
        key = data['inkey']
        pairs.append((vid, key))

    for vid, key in pairs:
        url_api = 'https://apis.naver.com/rmcnmv/rmcnmv/vod/play/v2.0/{}?key={}'.format(vid, key)
        data_raw = downloader.read_html(url_api)
        data = json.loads(data_raw)
        fs = data['videos']['list']
        fs = sorted(fs, key=lambda f: f['size'], reverse=True)
        video = Image(fs[0]['source'], url_article, len(imgs))
        imgs.append(video)
        
    for img in soup.findAll('img'):
        img = Image(urljoin(url_article, img['src']), url, len(imgs))
        imgs.append(img)

    info['imgs'] = imgs

    return info
コード例 #2
0
ファイル: connection.py プロジェクト: rboulton/xaprun
 def _handle_message(self, buf):
     i = buf.find(' ')
     if i == -1:
         self._failed("Invalid response message - no message id")
     msgid = buf[:i]
     buf = buf[i + 1:]
     cb = self.pending.get(msgid, None)
     if cb is None:
         self._failed("Response for unknown message id (%r)" % msgid)
     del self.pending[msgid]
     response = ''
     try:
         if len(buf) > 0:
             if buf[0] == 'S':
                 response = {'ok': 1, 'msg': buf[1:]}
             elif buf[0] == 'J':
                 response = json.loads(buf[1:])
             elif buf[0] == 'E':
                 response = json.loads(buf[1:])
             elif buf[0] == 'F':
                 response = json.loads(buf[1:])
             else:
                 response = {'ok': 0,
                     'msg': "Unknown response type code (%r)" % buf[0]}
     except:
         cb({'ok': 0, 'msg': 'Unable to parse response (%r)' % buf})
     else:
         cb(response)
コード例 #3
0
 def get_json(jsonmethod, sort=None, filters=None, fields=None, limits=None,
              returntype=None, optparam=None, filtertype=None):
     """method to get details from the kodi json api"""
     kodi_json = {}
     kodi_json["jsonrpc"] = "2.0"
     kodi_json["method"] = jsonmethod
     kodi_json["params"] = {}
     if optparam:
         if isinstance(optparam, list):
             for param in optparam:
                 kodi_json["params"][param[0]] = param[1]
         else:
             kodi_json["params"][optparam[0]] = optparam[1]
     kodi_json["id"] = 1
     if sort:
         kodi_json["params"]["sort"] = sort
     if filters:
         if not filtertype:
             filtertype = "and"
         if len(filters) > 1:
             kodi_json["params"]["filter"] = {filtertype: filters}
         else:
             kodi_json["params"]["filter"] = filters[0]
     if fields:
         kodi_json["params"]["properties"] = fields
     if limits:
         kodi_json["params"]["limits"] = {"start": limits[0], "end": limits[1]}
     json_response = xbmc.executeJSONRPC(try_encode(json.dumps(kodi_json)))
     if sys.version_info.major == 3:
         json_object = json.loads(json_response)
     else:
         json_object = json.loads(json_response.decode('utf-8', 'replace'))
     # set the default returntype to prevent errors
     if "details" in jsonmethod.lower():
         result = {}
     else:
         result = []
     if 'result' in json_object:
         if returntype and returntype in json_object['result']:
             # returntype specified, return immediately
             result = json_object['result'][returntype]
         else:
             # no returntype specified, we'll have to look for it
             if sys.version_info.major == 3:
                 for key, value in json_object['result'].items():
                     if not key == "limits" and (isinstance(value, list) or isinstance(value, dict)):
                         result = value
             else:
                 for key, value in json_object['result'].iteritems():
                     if not key == "limits" and (isinstance(value, list) or isinstance(value, dict)):
                         result = value
     else:
         log_msg(json_response)
         log_msg(kodi_json)
     return result
コード例 #4
0
 def set_json(jsonmethod, params):
     """method to set info in the kodi json api"""
     kodi_json = {}
     kodi_json["jsonrpc"] = "2.0"
     kodi_json["method"] = jsonmethod
     kodi_json["params"] = params
     kodi_json["id"] = 1
     json_response = xbmc.executeJSONRPC(try_encode(json.dumps(kodi_json)))
     if sys.version_info.major == 3:
         return json.loads(json_response)
     else:
         return json.loads(json_response.decode('utf-8', 'replace'))
コード例 #5
0
ファイル: fields.py プロジェクト: hfeeki/bhwsg
    def clean(self, value):

        if not value and not self.required:
            return None

        value = super(JSONFormField, self).clean(value)

        if isinstance(value, basestring):
            try:
                json.loads(value)
            except ValueError:
                raise FormValidationError(_("Enter valid JSON"))
        return value
コード例 #6
0
def read(project_id: str, model_id: str) -> List[Dict[str, Any]]:
    """
    Get a list with every story name
    :return: list with the names
    """
    pipeline = [
        {
            "$lookup": {
                "from": "interactions",
                "localField": "_id",
                "foreignField": "story_id",
                "as": "interactions",
            }
        },
        {
            "$match": {
                "project_id": ObjectId(project_id),
            }
        },
        {
            "$project": {
                "name": 1,
                "interactions_length": {
                    "$size": "$interactions"
                }
            }
        },
    ]

    return json.objectid_to_id(
        json.loads(json_mongo.dumps(STORIES_COLL.aggregate(pipeline))))
コード例 #7
0
ファイル: cache.py プロジェクト: adityamangla/metaStudio
    def get(self, url, data={}, headers=DEFAULT_HEADERS, timeout=-1, value="data"):
        r = None
        if timeout == 0:
            return r
        if data:
            url_hash = hashlib.sha1(url + '?' + data).hexdigest()
        else:
            url_hash = hashlib.sha1(url).hexdigest()

        conn = self.connect()
        c = conn.cursor()
        sql = 'SELECT %s, compressed FROM cache WHERE url_hash=?' % value
        if timeout > 0:
            now = time.mktime(time.localtime())
            t = (url_hash, now-timeout)
            sql += ' AND created > ?'
        else:
            t = (url_hash, )
        if value != "headers":
            sql += ' AND only_headers != 1 '
        c.execute(sql, t)
        for row in c:
            r = row[0]
            if value == 'headers':
                r = json.loads(r)
            elif value == 'data':
                if row[1] == 1:
                    r = zlib.decompress(r)
                else:
                    r = str(r)
            break

        c.close()
        conn.close()
        return r
コード例 #8
0
ファイル: twfy.py プロジェクト: yuhonghong7035/cvn.org
def getConstituencies(**kw):
    """
    A list of constituencies

    args - either:
      date - the list of constituencies as it was on this date

    or:
      latitude, longitude, distance - list of constituencies
                                 within distance of lat, lng
    """
    geoargs = ("latitude", "longitude", "distance")
    validargs = ("date", ) + geoargs

    invalid_args = list(k for k in kw.keys() if k not in validargs)
    if len(invalid_args) > 0:
        raise ValueError("Invalid args %r" % ",".join(invalid_args))

    if any(kw.has_key(k) for k in geoargs):
        if not all(kw.has_key(k) for k in geoargs):
            raise ValueError("Need all geoargs")

    params = dict((k, v) for k, v in kw.items() if v != None)
    headers, result = fetch(svcurl("getConstituencies", params))
    return [x['name'] for x in json.loads(result, encoding=charset(headers))]
コード例 #9
0
ファイル: snsbase.py プロジェクト: qiyeboy/snsapi
 def _http_post(self, baseurl, params):
     for p in params:
         params[p] = self._unicode_encode(params[p])
     data = urllib.urlencode(params)
     resp = urllib.urlopen(baseurl,data)
     json_objs = json.loads(resp.read())
     return json_objs
コード例 #10
0
ファイル: twfy.py プロジェクト: tfgg/cvn.org
def getConstituencies(**kw):
    """
    A list of constituencies

    args - either:
      date - the list of constituencies as it was on this date

    or:
      latitude, longitude, distance - list of constituencies
                                 within distance of lat, lng
    """
    geoargs = ("latitude", "longitude", "distance")
    validargs = ("date", ) + geoargs

    invalid_args = list(k for k in kw.keys() if k not in validargs)
    if len(invalid_args) > 0:
        raise ValueError("Invalid args %r" % ",".join(invalid_args))

    if any(kw.has_key(k) for k in geoargs):
        if not all(kw.has_key(k) for k in geoargs):
            raise ValueError("Need all geoargs")

    params = dict((k, v) for k,v in kw.items() if v != None)
    headers, result = fetch(svcurl("getConstituencies", params))
    return [x['name'] for x in json.loads(result, encoding=charset(headers))]
コード例 #11
0
def getConstituencies(**kw):
    """
    A list of constituencies

    args - either:
      date - the list of constituencies as it was on this date

    or:
      latitude, longitude, distance - list of constituencies
                                 within distance of lat, lng
    """
    geoargs = ("latitude", "longitude", "distance")
    validargs = ("date", ) + geoargs

    invalid_args = list(k for k in kw.keys() if k not in validargs)
    if len(invalid_args) > 0:
        raise ValueError("Invalid args %r" % ",".join(invalid_args))

    if any(kw.has_key(k) for k in geoargs):
        if not all(kw.has_key(k) for k in geoargs):
            raise ValueError("Need all geoargs")

    params = dict((k, v) for k,v in kw.items() if v != None)
    new_constituency_date = "2010"
    date = kw.get('date', '2009')
    if date < new_constituency_date:
        headers, result = fetch(svcurl("getConstituencies", params))
        return json.loads(result, encoding=charset(headers))
    else:
        # XXX hopefully there will be a proper API for this in the future
        tsv_data = csv.DictReader(urllib.urlopen(URL_OF_2010_TSV),
                                  dialect='excel-tab')
        return tsv_data
コード例 #12
0
def read() -> List[Dict[str, Any]]:
    """
    Read project data from database
    :return: project data
    """
    pipeline = [
        {
            "$lookup": {
                "from": "models",
                "localField": "_id",
                "foreignField": "project_id",
                "as": "models",
            }
        },
        {
            "$project": {
                "name": 1,
                "models_length": {
                    "$size": "$models"
                }
            }
        },
    ]

    return json.objectid_to_id(
        json.loads(json_mongo.dumps(PROJECTS_COLL.aggregate(pipeline))))
コード例 #13
0
 def _http_post(self, baseurl, params):
     for p in params:
         params[p] = self._unicode_encode(params[p])
     data = urllib.urlencode(params)
     resp = urllib.urlopen(baseurl, data)
     json_objs = json.loads(resp.read())
     return json_objs
コード例 #14
0
def read(project_id: str, model_id: str) -> List[Dict[str, Any]]:
    """
    Get the information of every stored intent
    :return: a dictionary with every intent and its related data
    """
    pipeline = [
        {
            "$lookup": {
                "from": "templates",
                "localField": "_id",
                "foreignField": "intent_id",
                "as": "templates",
            }
        },
        {
            "$match": {
                "project_id": ObjectId(project_id),
                "model_id": ObjectId(model_id),
            }
        },
        {
            "$project": {
                "name": 1,
                "templates_length": {
                    "$size": "$templates"
                }
            }
        },
    ]

    return json.objectid_to_id(
        json.loads(json_mongo.dumps(INTENTS_COLL.aggregate(pipeline))))
コード例 #15
0
ファイル: test_pos.py プロジェクト: ProudDuck/Java-Jieba
def test_json():
    pt.initialize()
    res = [pt.lcut(s) for s in sentences]
    j = json.dumps(res, cls=PairJSONEncoder)
    print(j)
    k = json.loads(j, cls=PairJSONDecoder)
    print(k)
コード例 #16
0
    def onNotification(self, sender, method, data):
        """builtin function for the xbmc.Monitor class"""
        try:
            log_msg("Kodi_Monitor: sender %s - method: %s  - data: %s" %
                    (sender, method, data))
            data = json.loads(data.decode('utf-8'))
            mediatype = ""
            dbid = 0
            transaction = False
            if data and isinstance(data, dict):
                if data.get("item"):
                    mediatype = data["item"].get("type", "")
                    dbid = data["item"].get("id", 0)
                elif data.get("type"):
                    mediatype = data["type"]
                    dbid = data.get("id", 0)
                if data.get("transaction"):
                    transaction = True

            if method == "System.OnQuit":
                self.win.setProperty("SkinHelperShutdownRequested", "shutdown")

            if method == "VideoLibrary.OnUpdate":
                self.process_db_update(mediatype, dbid, transaction)

            if method == "AudioLibrary.OnUpdate":
                self.process_db_update(mediatype, dbid, transaction)

            if method == "Player.OnStop":
                self.monitoring_stream = False
                self.infopanelshown = False
                self.win.clearProperty("Skinhelper.PlayerPlaying")
                self.win.clearProperty("TrailerPlaying")
                self.reset_win_props()

            if method == "Player.OnPlay":
                if not self.monitoring_stream and not getCondVisibility(
                        "Player.DisplayAfterSeek"):
                    self.reset_win_props()
                if self.wait_for_player():
                    if getCondVisibility("Player.HasAudio"):
                        if getCondVisibility("Player.IsInternetStream"):
                            self.monitor_radiostream()
                        else:
                            self.set_music_properties()
                    if getCondVisibility("Pvr.IsPlayingRadio"):
                        if getCondVisibility("!Player.IsInternetStream"):
                            self.monitor_radiostream()
                        else:
                            self.set_music_properties()
                    elif getCondVisibility(
                            "VideoPlayer.Content(livetv) | String.StartsWith(Player.FileNameAndPath,pvr://)"
                    ):
                        self.monitor_livetv()
                    else:
                        self.set_video_properties(mediatype, dbid)
                        self.show_info_panel()
        except Exception as exc:
            log_exception(__name__, exc)
コード例 #17
0
ファイル: fields.py プロジェクト: hfeeki/bhwsg
 def to_python(self, value):
     """Convert string value to JSON"""
     if isinstance(value, basestring):
         try:
             return json.loads(value, **self.load_kwargs)
         except ValueError:
             pass
     return value
コード例 #18
0
ファイル: twfy.py プロジェクト: tfgg/cvn.org
def getConstituency(postcode):
    "Constituency postcode is in"
    _, response = fetch(svcurl("getConstituency", {"postcode": postcode}))
    result = json.loads(response)
    if not result.has_key('error'):
        return result['name']
    else:
        return None
コード例 #19
0
ファイル: twfy.py プロジェクト: yuhonghong7035/cvn.org
def getConstituency(postcode):
    "Constituency postcode is in"
    _, response = fetch(svcurl("getConstituency", {"postcode": postcode}))
    result = json.loads(response)
    if not result.has_key('error'):
        return result['name']
    else:
        return None
コード例 #20
0
ファイル: jsonc.py プロジェクト: adityamangla/metaStudio
def loads(source):
    try:
        minified = minify(source)
        return json.loads(minified)
    except json.JSONDecodeError, e:
        s = minified.split('\n')
        context = s[e.lineno-1][max(0, e.colno-1):e.colno+30]
        msg = e.msg + ' at ' + context
        raise json.JSONDecodeError(msg, minified, e.pos)
コード例 #21
0
def cookie_decode(str):
    if str is None:
        return None, None
    try:
        ds = _decode(str)
        dd = json.loads(ds)
        return dd['email'], dd['pwd']
    except Exception:
        return None, None
コード例 #22
0
ファイル: kodidb.py プロジェクト: camster1/RTOTV
 def set_json(jsonmethod, params):
     '''method to set info in the kodi json api'''
     kodi_json = {}
     kodi_json["jsonrpc"] = "2.0"
     kodi_json["method"] = jsonmethod
     kodi_json["params"] = params
     kodi_json["id"] = 1
     json_response = xbmc.executeJSONRPC(try_encode(json.dumps(kodi_json)))
     return json.loads(json_response.decode('utf-8', 'replace'))
コード例 #23
0
 def __set_std(self, data, output, key):
     if data:
         try:
             data = data.replace('Size of the terminal is too small, output could be missaligned.', '') \
                        .replace(' id', ' n') \
                        .replace('Default Policy', 'Default-Policy')  # FIXME avoid code for specific program combability
             output[key] = loads(data)
         except Exception:
             output[key] = table_to_dict(data.splitlines())
コード例 #24
0
ファイル: snsbase.py プロジェクト: qiyeboy/snsapi
 def _http_get(self, baseurl, params):
     # Support unicode parameters. 
     # We should encode them as exchanging stream (e.g. utf-8)
     # before URL encoding and issue HTTP requests. 
     for p in params:
         params[p] = self._unicode_encode(params[p])
     uri = urllib.urlencode(params)
     url = baseurl + "?" + uri
     resp = urllib.urlopen(url)
     json_objs = json.loads(resp.read())
     return json_objs
コード例 #25
0
 def _http_get(self, baseurl, params):
     # Support unicode parameters.
     # We should encode them as exchanging stream (e.g. utf-8)
     # before URL encoding and issue HTTP requests.
     for p in params:
         params[p] = self._unicode_encode(params[p])
     uri = urllib.urlencode(params)
     url = baseurl + "?" + uri
     resp = urllib.urlopen(url)
     json_objs = json.loads(resp.read())
     return json_objs
コード例 #26
0
ファイル: snsbase.py プロジェクト: hzhua/snsapi
 def _http_post(self, baseurl, params):
     try:
         for p in params:
             params[p] = self._unicode_encode(params[p])
         data = urllib.urlencode(params)
         resp = urllib.urlopen(baseurl,data)
         json_objs = json.loads(resp.read())
         return json_objs
     except Exception, e:
         logger.warning("_http_post fail: %s", e)
         return {}
コード例 #27
0
def getGeometry(name=None):
    """
    Centre and bounding box of constituencies

    Don't provide any argument to get all constituencies
    """
    if name:
        params = dict(name=name)
    else:
        params = dict()
    if settings.CONSTITUENCY_YEAR.strftime("%Y") >= "2010":
        # XXX this is temporary hack until elegant TWFY API solution
        # happens 
        params['future'] = 1
    headers, result = fetch(svcurl("getGeometry", params))
    if name:
        data = json.loads(result, encoding=charset(headers))
    else:
        data = json.loads(result, encoding=charset(headers))["data"]
    return data
コード例 #28
0
def getCurrentMP(name):
    """Return the current MP data structure for the named constituency
    
    """
    params = dict(constituency=name)
    headers, result = fetch(svcurl("getMP", params))
    try:
        data = json.loads(result, encoding=charset(headers))
    except ValueError:
        data = "Unknown (server error)"
    return data
コード例 #29
0
 def body_dict(self):
     '''
     Parses the request body as JSON, and returns the dict representation
     '''
     #memoize
     if hasattr(self, '_body_dict'):
         return self._body_dict
         
     try:
         self._body_dict = json.loads(self.request.body)
         return self._body_dict
     except Exception, e:
         raise HTTPError(400, e.args[0])
コード例 #30
0
def getConstituency(postcode):
    "Constituency postcode is in"
    params = {"postcode": postcode}
    if settings.CONSTITUENCY_YEAR.strftime("%Y") >= "2010":
        # XXX this is temporary hack until elegant TWFY API solution
        # happens 
        params['future'] = 1
    _, response = fetch(svcurl("getConstituency", params))
    result = json.loads(response)
    if not result.has_key('error'):
        return result['name']
    else:
        return None
コード例 #31
0
    def onNotification(self, sender, method, data):
        '''builtin function for the xbmc.Monitor class'''
        try:
            log_msg("Kodi_Monitor: sender %s - method: %s  - data: %s" % (sender, method, data))
            data = json.loads(data.decode('utf-8'))
            mediatype = ""
            dbid = 0
            transaction = False
            if data and isinstance(data, dict):
                if data.get("item"):
                    mediatype = data["item"].get("type", "")
                    dbid = data["item"].get("id", 0)
                elif data.get("type"):
                    mediatype = data["type"]
                    id = data.get("id", 0)
                if data.get("transaction"):
                    transaction = True

            if method == "System.OnQuit":
                self.win.setProperty("SkinHelperShutdownRequested", "shutdown")

            if method == "VideoLibrary.OnUpdate":
                self.process_db_update(mediatype, dbid, transaction)

            if method == "AudioLibrary.OnUpdate":
                self.process_db_update(mediatype, dbid, transaction)

            if method == "Player.OnStop":
                self.monitoring_stream = False
                self.infopanelshown = False
                self.win.clearProperty("Skinhelper.PlayerPlaying")
                self.win.clearProperty("TrailerPlaying")
                self.reset_win_props()
                if not dbid:
                    self.process_db_update(mediatype, "", transaction)

            if method == "Player.OnPlay":
                self.reset_win_props()
                if self.wait_for_player():
                    if xbmc.getCondVisibility("Player.HasAudio"):
                        if xbmc.getCondVisibility("Player.IsInternetStream"):
                            self.monitor_radiostream()
                        else:
                            self.set_music_properties()
                    elif xbmc.getCondVisibility("VideoPlayer.Content(livetv)"):
                        self.monitor_livetv()
                    else:
                        self.set_video_properties(mediatype, dbid)
                        self.show_info_panel()
        except Exception as exc:
            log_exception(__name__, exc)
コード例 #32
0
ファイル: twfy.py プロジェクト: yuhonghong7035/cvn.org
def getGeometry(name=None):
    """
    Centre and bounding box of constituencies

    Don't provide any argument to get all constituencies
    """
    if name:
        params = dict(name=name)
    else:
        params = dict()

    headers, result = fetch(svcurl("getGeometry", params))
    data = json.loads(result, encoding=charset(headers))["data"]
    return data
コード例 #33
0
ファイル: twfy.py プロジェクト: tfgg/cvn.org
def getGeometry(name=None):
    """
    Centre and bounding box of constituencies

    Don't provide any argument to get all constituencies
    """
    if name:
        params = dict(name=name)
    else:
        params = dict()

    headers, result = fetch(svcurl("getGeometry", params))
    data = json.loads(result, encoding=charset(headers))["data"]
    return data
コード例 #34
0
ファイル: fb.py プロジェクト: osiloke/FamilyFeed
def parse_signed_request(signed_request, secret):
    encoded_sig, payload = signed_request.split('.', 2)

    sig = base64_url_decode(encoded_sig)
    data = json.loads(base64_url_decode(payload))

    if data.get('algorithm').upper() != 'HMAC-SHA256':
        raise Exception('Unknown algorithm')

    expected_sig = hmac.new(secret, msg=payload, digestmod=hashlib.sha256)\
        .digest()
    if sig != expected_sig:
        raise Exception('Signature error')

    return data
コード例 #35
0
 def __set_std(self, data, output, key, output_format):
     if data:
         data = data.strip()
         if output_format == 'plain':
             output[key] = data
         elif output_format == 'lines':
             output[key] = data.splitlines()
         else:
             try:
                 output[key] = loads(data)
             except Exception as e:
                 msg = f'Not valid JSON for {key}'
                 self.log.exception(msg, e)
                 output.update(description=msg, exception=extract_info(e))
                 output[key] = data
コード例 #36
0
ファイル: snsbase.py プロジェクト: hzhua/snsapi
 def _http_get(self, baseurl, params):
     # Support unicode parameters. 
     # We should encode them as exchanging stream (e.g. utf-8)
     # before URL encoding and issue HTTP requests. 
     try:
         for p in params:
             params[p] = self._unicode_encode(params[p])
         uri = urllib.urlencode(params)
         url = baseurl + "?" + uri
         resp = urllib.urlopen(url)
         json_objs = json.loads(resp.read())
         return json_objs
     except Exception, e:
         # Tolerate communication fault, like network failure. 
         logger.warning("_http_get fail: %s", e)
         return {}
コード例 #37
0
 def _api_request(self, endpoint, params=None):
     '''get info from json api'''
     result = {}
     url = "http://localhost:4000/api/%s" % endpoint
     params = params if params else {}
     try:
         response = requests.get(url, params=params, timeout=10)
         if response and response.content and response.status_code == 200:
             if "{" in response.content:
                 result = json.loads(response.content.decode('utf-8', 'replace'))
             else:
                 result = response.content.decode('utf-8')
         else:
             self.logger.error("Invalid or empty reponse from server - endpoint: %s - server response: %s - %s" %
                     (endpoint, response.status_code, response.content))
     except Exception as exc:
         self.logger.error(exc)
         result = None
     return result
コード例 #38
0
def read(project_id: str, model_id: str) -> List[Dict[str, Any]]:
    """
    Return information from entities
    :return: data of the entities
    """
    pipeline = [
        {
            "$lookup": {
                "from": "values",
                "localField": "_id",
                "foreignField": "entity_id",
                "as": "values",
            }
        },
        {
            "$match": {
                "project_id": ObjectId(project_id),
            }
        },
        {
            "$project": {
                "name": 1,
                "color": 1,
                "intents": 1,
                "text_color": 1,
                "intents_length": {"$size": "$intents"},
                "values_length": {"$size": "$values"},
            }
        },
    ]

    data = list(ENTITIES_COLL.aggregate(pipeline))

    for doc in data:
        intent_list = doc["intents"]
        entity = doc["name"]
        regx = re.compile(r"\${" + entity + "}", re.IGNORECASE)
        for intent in intent_list:
            intent["templates_length"] = TEMPLATES_COLL.count_documents(
                {"intent_id": intent["id"], "name": {"$regex": regx}}
            )

    return json.objectid_to_id(json.loads(json_mongo.dumps(data)))
コード例 #39
0
def read_values(project_id: str, entity_id: str) -> List[Dict[str, Any]]:
    """
    Return data from an entity with id [entity_id]
    :return: entity data
    """

    return json.objectid_to_id(
        json.loads(
            json_mongo.dumps(
                VALUES_COLL.find(
                    {
                        "project_id": ObjectId(project_id),
                        "entity_id": ObjectId(entity_id),
                    },
                    {"project_id": 0, "model_id": 0, "entity_id": 0},
                )
            )
        )
    )
コード例 #40
0
def read_models(project_id: str) -> List[Dict[str, Any]]:
    """
    Read data of every model stored related to a project with id [project_id]
    :param project_id: id of the project
    :return: a list with the data of the models
    """
    pipeline = [
        {
            "$lookup": {
                "from": "stories",
                "localField": "_id",
                "foreignField": "model_id",
                "as": "stories",
            }
        },
        {
            "$match": {
                "project_id": ObjectId(project_id)
            }
        },
        {
            "$project": {
                "name": 1,
                "stories_length": {
                    "$size": "$stories"
                }
            }
        },
    ]

    data = list(MODELS_COLL.aggregate(pipeline))
    project_name = read_project_name_from_id(project_id)
    for doc in data:
        model_name = read_model_name_from_id(doc["_id"])
        if os.path.exists(os.path.join(MODEL_PATH, project_name, model_name)):
            doc["last_trained_timestamp"] = datetime.fromtimestamp(
                os.path.getctime(
                    get_latest_model(
                        os.path.join(MODEL_PATH, project_name,
                                     model_name)))).isoformat()

    return json.objectid_to_id(json.loads(json_mongo.dumps(data)))
コード例 #41
0
def read_templates(project_id: str,
                   model_id: str,
                   intent_id: str = None) -> List[Dict[str, Any]]:
    """
    Get templates of every intent or an specific one with id [intent_id]
    :param intent_id: id of the intent
    :return:
    """
    data = {"project_id": ObjectId(project_id), "model_id": ObjectId(model_id)}

    if intent_id:
        data["intent_id"] = ObjectId(intent_id)

    return json.objectid_to_id(
        json.loads(
            json_mongo.dumps(
                TEMPLATES_COLL.find(data, {
                    "project_id": 0,
                    "model_id": 0
                }))))
コード例 #42
0
def read_interactions(project_id: str, story_id: str) -> List[Dict[str, Any]]:
    """
    Gets every interaction associated to the story with id [story_id]
    :param project_id:
    :param story_id:
    :return:
    """

    return json.objectid_to_id(
        json.loads(
            json_mongo.dumps(
                INTERACTIONS_COLL.find(
                    {
                        "project_id": ObjectId(project_id),
                        "story_id": ObjectId(story_id),
                    },
                    {
                        "project_id": 0,
                        "model_id": 0,
                        "story_id": 0
                    },
                ))))
コード例 #43
0
ファイル: oembed.py プロジェクト: adityamangla/metaStudio
def get_embed_code(url, maxwidth=None, maxheight=None):
    embed = {}
    header = cache.get_headers(url)
    if header.get('content-type', '').startswith('text/html'):
        html = cache.read_url(url)
        json_oembed = filter(lambda l: 'json+oembed' in l, re.compile('<link.*?>').findall(html))
        xml_oembed = filter(lambda l: 'xml+oembed' in l, re.compile('<link.*?>').findall(html))
        if json_oembed:
            oembed_url = find_re(json_oembed[0], 'href="(.*?)"')
            if maxwidth:
                oembed_url += '&maxwidth=%d' % maxwidth
            if maxheight:
                oembed_url += '&maxheight=%d' % maxheight
            embed = json.loads(cache.read_url(oembed_url))
        elif xml_oembed:
            oembed_url = find_re(json_oembed[0], 'href="(.*?)"')
            if maxwidth:
                oembed_url += '&maxwidth=%d' % maxwidth
            if maxheight:
                oembed_url += '&maxheight=%d' % maxheight
            data = cache.read_url(oembed_url)
            for e in ET.fromstring(data):
                embed[e.tag] = e.text
    return embed