Beispiel #1
0
    def do_source(self, arg):
        """Add source to given artifact or most recently added artifact if not specified

        Usage: source                            # adds to last created artifact
               source <artifact name|session id> # adds to specific artifact
        """
        if arg == '':
            last = self.session.receive('artifacts')
            _type = detect_type(last)
        else:
            _type = detect_type(arg)
            is_key, value = lookup_key(self.session, arg)

            if is_key and value is None:
                error('Unable to find artifact key in session (%s)' % arg)
                return
            elif is_key and value is not None:
                arg = value
            else:
                pass

        if self.db.exists(_type, {'name': last}):
            self.db.update_one(_type, {'name': last}, {'source': arg})
            success('Added source to artifact entry (%s: %s)' % (last, arg))
        else:
            warning('Failed to find last artifact in MongoDB. Run "new <artifact name>" before using the source command')
Beispiel #2
0
    def do_report(self, arg):
        """Save artifact report as JSON file

        Usage: report <artifact name>
               report <session id>"""
        is_key, value = lookup_key(self.session, arg)

        if is_key and value is None:
            error('Unable to find artifact key in session (%s)' % arg)
            return
        elif is_key and value is not None:
            arg = value
        else:
            pass

        _type = detect_type(arg)

        result = self.db.find(_type, {'name': arg}, one=True)
        if len(result) == 0:
            warning('No entry found for artifact (%s)' % arg)
        else:
            report = storage.JSON(data=result, file_path=output_dir)
            report.save()
            if os.path.exists(report.file_path):
                success('Saved artifact report (%s)' % report.file_path)
            else:
                error('Failed to properly save report')
Beispiel #3
0
    def default(self, arg):
        """Override default function for custom error message"""
        if arg.startswith('#'):
            return

        error('Unknown command')
        return
Beispiel #4
0
 def do_session(self, arg):
     """Open a new session"""
     self.session = RedisCache(config)
     if self.session.db is None:
         error('Failed to connect to Redis back-end. Please ensure the Redis service is running')
     else:
         success('Opened new session')
Beispiel #5
0
 def do_session(self, arg):
     """Open a new session"""
     self.session = RedisCache(config)
     if self.session.db is None:
         error('Failed to connect to Redis back-end. Please ensure the Redis service is running')
     else:
         success('Opened new session')
Beispiel #6
0
    def default(self, arg):
        """Override default function for custom error message"""
        if arg.startswith('#'):
            return

        error('Unknown command')
        return
Beispiel #7
0
    def do_cat(self, arg):
        """View artifact details or list API keys

        Usage: cat apikeys
               cat <artifact name>"""
        if arg == 'apikeys':
            data = json.load(open(common.API_CONF, 'rb'))
            print json.dumps(data, indent=2)
        else:
            is_key, value = lookup_key(self.session, arg)

            if is_key and value is None:
                error('Unable to find artifact key in session (%s)' % arg)
                return
            elif is_key and value is not None:
                arg = value
            else:
                pass

            artifact_type = detect_type(arg)
            result = self.db.find(artifact_type, {'name': arg}, one=True)
            if len(result) == 0:
                info('No entry found for artifact (%s)' % arg)
            else:
                print json.dumps(result, indent=2, separators=(',', ':'))
Beispiel #8
0
    def do_report(self, arg):
        """Save artifact report as JSON file

        Usage: report <artifact name>
               report <session id>"""
        is_key, value = lookup_key(self.session, arg)

        if is_key and value is None:
            error('Unable to find artifact key in session (%s)' % arg)
            return
        elif is_key and value is not None:
            arg = value
        else:
            pass

        _type = detect_type(arg)

        result = self.db.find(_type, {'name': arg}, one=True)
        if len(result) == 0:
            warning('No entry found for artifact (%s)' % arg)
        else:
            report = storage.JSON(data=result, file_path=output_dir)
            report.save()
            if os.path.exists(report.file_path):
                success('Saved artifact report (%s)' % report.file_path)
            else:
                error('Failed to properly save report')
Beispiel #9
0
    def do_source(self, arg):
        """Add source to given artifact or most recently added artifact if not specified

        Usage: source                            # adds to last created artifact
               source <artifact name|session id> # adds to specific artifact
        """
        if arg == '':
            last = self.session.receive('artifacts')
            _type = detect_type(last)
        else:
            _type = detect_type(arg)
            is_key, value = lookup_key(self.session, arg)

            if is_key and value is None:
                error('Unable to find artifact key in session (%s)' % arg)
                return
            elif is_key and value is not None:
                arg = value
            else:
                pass

        if self.db.exists(_type, {'name': last}):
            self.db.update_one(_type, {'name': last}, {'source': arg})
            success('Added source to artifact entry (%s: %s)' % (last, arg))
        else:
            warning('Failed to find last artifact in MongoDB. Run "new <artifact name>" before using the source command')
Beispiel #10
0
    def do_cat(self, arg):
        """View artifact details or list API keys

        Usage: cat apikeys
               cat <artifact name>"""
        if arg == 'apikeys':
            data = json.load(open(common.API_CONF, 'rb'))
            print json.dumps(data, indent=2)
        else:
            is_key, value = lookup_key(self.session, arg)

            if is_key and value is None:
                error('Unable to find artifact key in session (%s)' % arg)
                return
            elif is_key and value is not None:
                arg = value
            else:
                pass

            artifact_type = detect_type(arg)
            result = self.db.find(artifact_type, {'name': arg}, one=True)
            if len(result) == 0:
                info('No entry found for artifact (%s)' % arg)
            else:
                print json.dumps(result, indent=2, separators=(',', ':'))
Beispiel #11
0
def __decode(js):
    docJs = """
        function unbox() { 
            var evalCode = ""; 
            var document = { 
                write: function(str) { evalCode = str; } 
            }; 
            var bdecodeb = function(str, key) { return {str: str, key: key}; }; """ + js + """ return evalCode; }"""
    strAndKey = js2py.eval_js(docJs)()
    decodedStr = __bdecodeb(strAndKey.str, strAndKey.key)
    decodedStr = BeautifulSoup(decodedStr, 'html5lib').get_text()
    if "token" in decodedStr:
        test = js2py.eval_js('function add() { ' + decodedStr +
                             '; return { token: token, hken: hken}; }')
        tokenObj = test()
        return tokenObj
    else:
        presetCode = """
            function secondUnbox() {
                var plus = { video: { VideoPlayer: function(str, code) { return code; } } };
                var document = { addEventListener: function(event, func) {} };
                """ + __bdecode_str() + ";" + decodedStr + """
                vstPlay();
                return video.src;
            }
        """
        common.error(presetCode)
        src = js2py.eval_js(presetCode)()
        return src
Beispiel #12
0
def play_mirror(url):
    with common.busy_indicator():
        common.error("What is the url: " + url)
        (title, vidurl) = scrapers.episodeVideo(url.split('?')[0]) # common.resolve(url)
        if vidurl:
            li = xbmcgui.ListItem(title)
            li.setThumbnailImage('')
            if 'User-Agent=' not in vidurl:
                vidurl = vidurl + '|User-Agent=' + urllib.quote(get_ua())
            xbmc.Player().play(vidurl, li)
Beispiel #13
0
    def get(self, pagename='index'):
        ''' 静的なページの出力 '''
        template_values = {}
        path = os.path.join(os.path.dirname(__file__), 'templates/%s.html' % (pagename))

        if not os.path.exists(path):
            common.error(self, 404, "Page not found.")
            return

        self.response.out.write(template.render(path, template_values))
Beispiel #14
0
    def get(self):
        user = common.currentuser()
        if not user:
            common.error(self, 404, 'ユーザが見つかりません')
            return

        myfeeds = models.CustomFeed.all().ancestor(user).fetch(1000)
        template_values = {'username': user.user, 'userid': user.key().id(), 'feeds': myfeeds}
        path = os.path.join(os.path.dirname(__file__), 'templates/mypage.html')

        self.response.out.write(template.render(path, template_values))
Beispiel #15
0
    def get(self):
        user = common.currentuser()
        if not user:
            common.error(self, 404, "User not found.")
            return

        logs = models.Log.all().order('-time').ancestor(user).fetch(1000)

        template_values = {'logs': logs}
        path = os.path.join(os.path.dirname(__file__), 'templates/log.html')

        self.response.out.write(template.render(path, template_values))
Beispiel #16
0
def play_mirror(url):
    with common.busy_indicator():
        common.error("What is the url: " + url)
        (title, vidurl) = scrapers.episodeVideo(
            url.split('?')[0])  # common.resolve(url)
        if vidurl:
            li = xbmcgui.ListItem(title)
            for pxyDomain in config.domain_using_proxy:
                if (re.search(pxyDomain, vidurl)):
                    vidurl = config.local_proxy + '/' + re.sub(
                        '\://', '/', vidurl, 1)
                    break
            if 'User-Agent=' not in vidurl:
                vidurl = vidurl + '|User-Agent=' + quote(get_ua())
            xbmc.Player().play(vidurl, li)
Beispiel #17
0
    def get(self):
        user = common.currentuser()
        if not user:
            common.error(self, 404, "ユーザを確認出来ませんでした。")
            return

        ct = models.CustomTest.all().ancestor(user).get()
        if not ct:
            ct = models.CustomTest(name='custom', parent=user)
            ct.put()

        template_values = {'ct': ct}
        path = os.path.join(os.path.dirname(__file__), 'templates/customtest.html')
        
        self.response.out.write(template.render(path, template_values))
Beispiel #18
0
    def get(self, feedname):
        u = common.currentuser()
        if not u:
            common.error(self, 404, 'not accept user')
            return

        cf = models.CustomFeed.get_by_key_name(feedname, parent=u)
        if not cf:
            common.error(self, 404, "フィードが存在しません。");
            return

        template_values = {'feedname': feedname, 'cf': cf}
        path = os.path.join(os.path.dirname(__file__), 'templates/edit.html')

        self.response.out.write(template.render(path, template_values))
Beispiel #19
0
    def do_delete(self, arg):
        """Remove artifact from database by name or ID

        Usage: delete <name>
               delete <session id>"""
        is_key, value = lookup_key(self.session, arg)

        if is_key and value is None:
            error('Unable to find artifact key in session (%s)' % arg)
            return
        elif is_key and value is not None:
            arg = value
        else:
            pass

        artifact_type = detect_type(arg)
        self.db.delete_one(artifact_type, {'name': arg})
Beispiel #20
0
    def do_delete(self, arg):
        """Remove artifact from database by name or ID

        Usage: delete <name>
               delete <session id>"""
        is_key, value = lookup_key(self.session, arg)

        if is_key and value is None:
            error('Unable to find artifact key in session (%s)' % arg)
            return
        elif is_key and value is not None:
            arg = value
        else:
            pass

        artifact_type = detect_type(arg)
        self.db.delete_one(artifact_type, {'name': arg})
Beispiel #21
0
    def setselectors(self):
        '''POSTデータをカスタムテストエンティティに保存します'''
        user = common.currentuser()
        if not user:
            common.error(self, 404, "User not found")
            return False

        ct = models.CustomTest.all().ancestor(user).get()
        if not ct:
            ct = models.CustomTest(parent=user)

        ct.setbypost(self.request.POST)

        if not ct.put():
            common.error(self, 200, "fail to save.")
            return False

        return True
Beispiel #22
0
    def do_rm(self, arg):
        """Remove artifact from session by ID

        Usage: rm <session id>"""
        try:
            arg = int(arg)
        except:
            error('Artifact ID must be an integer')
            return

        if self.session is not None:
            if self.session.exists(arg):
                self.session.delete(arg)
                success('Removed artifact from cache (%s)' % arg)
            else:
                warning('Unable to find artifact by ID (%s)' % arg)
        else:
            warning('No active session; start a new session by running the "session" command')
Beispiel #23
0
    def do_rm(self, arg):
        """Remove artifact from session by ID

        Usage: rm <session id>"""
        try:
            arg = int(arg)
        except:
            error('Artifact ID must be an integer')
            return

        if self.session is not None:
            if self.session.exists(arg):
                self.session.delete(arg)
                success('Removed artifact from cache (%s)' % arg)
            else:
                warning('Unable to find artifact by ID (%s)' % arg)
        else:
            warning('No active session; start a new session by running the "session" command')
Beispiel #24
0
    def seturl(self):
        '''URLとURLからフェッチして保存します'''
        user = common.currentuser()
        if not user:
            common.error(self, 404, "User not found.")
            return

        ct = models.CustomTest.all().ancestor(user).get()
        if not ct:
            ct = models.CustomTest(parent=user)

        ct.setbypost(self.request.POST)

        if not ct.rss_link:
            soup = Soup(defines.defaulttesthtml)
        else:
            result = urlfetch.fetch(ct.rss_link)
            if result.status_code != 200:
                common.error(self, 200, "Url Fetch Error")
                return
            soup = Soup(result.content)

        try: 
            ct.data = soup.prettify().decode('UTF-8')
        except ValueError, message:
            common.error(self, 200, message)
            return
Beispiel #25
0
    def get(self, userid, feedname, feedtype=None):

        user = models.User.get_by_id(int(userid))
        if not user:
            common.error(self, 404, 'User ID is not found.')
            return

        cf = models.CustomFeed.get_by_key_name(feedname, parent=user)
        if not cf:
            common.error(self, 404, 'CutomFeed is not found.')
            return
        
        feed = models.FeedData.get_by_key_name(feedname, parent=cf)
        if not feed:
            common.error(self, 404, 'Feed data is not found.')
            return

        if feedtype == None:
            template_values = {'userid': userid, 'cf': cf, 'nickname': ''}
            path = os.path.join(os.path.dirname(__file__), 'templates/feed.html')

            self.response.out.write(template.render(path, template_values))

        else: 
            self.response.out.write(getattr(feed, feedtype))
Beispiel #26
0
    def getselectorstexts(self):
        '''カスタムテストエンティティのセレクタ属性セットから各テキストを取得します'''
        user = common.currentuser()
        if not user:
            common.error(self, 404, "User not found")
            return False

        ct = models.CustomTest.all().ancestor(user).get()
        if not ct:
            return False

        soup = Soup(ct.data)

        texts = {}
        if ct.item_title_enable and ct.item_title_selector:
            texts['titles'] = common.selectortext(soup, ct.item_title_selector, ct.item_title_attr)
        if ct.item_link_enable and ct.item_link_selector:
            texts['links'] = common.selectortext(soup, ct.item_link_selector, ct.item_link_attr)
        if ct.item_description_enable and ct.item_description_selector:
            texts['descriptions'] = common.selectortext(soup, ct.item_description_selector, ct.item_description_attr)
        if ct.item_date_enable and ct.item_date_selector:
            texts['dates'] = common.selectortext(soup, ct.item_date_selector, ct.item_date_attr)

        return texts
Beispiel #27
0
    def post(self, feedname):
        u = common.currentuser()
        if not u:
            common.error(self, 404, 'not accept user')
            return

        feed = models.CustomFeed.get_by_key_name(feedname, parent=u)
        if not feed:
            common.error(self, 404, 'Feed not found')
            return

        try:
            feed.setbypost(self.request.POST)
        except ValueError:
            common.error(self, 200, "Invalid input data")
            return

        if not feed.put():
            common.error(self, 200, "fail to save.")
            return

        self.redirect('/edit/' + feedname)
Beispiel #28
0
    def post(self):
        user = common.currentuser()
        if not user:
            common.error(self, 404, 'ユーザが見つかりません')

        newfeedname = self.request.POST['name']
        if models.CustomFeed.get_by_key_name(newfeedname, parent=user):
            common.error(self, 409, "既にページが存在します。")
            return

        cf = models.CustomFeed(parent=user, key_name=newfeedname, name=newfeedname)
        if not cf.put():
            common.error(self, 400, "フィードの作成に失敗しました。")
            return

        self.redirect("/mypage")
Beispiel #29
0
    def post(self):
        user = common.currentuser()
        if not user:
            common.error(self, 404, 'ユーザが見つかりません')
            return

        feedname = self.request.POST['name']
        cf = models.CustomFeed.get_by_key_name(feedname, parent=user)
        if not cf:
            common.error(self, 400, "ページは存在しません。")
            return

        cf.delete()
        if cf.is_saved():
            common.error(self, 400, "削除に失敗しました。")
            return

        self.redirect('/mypage')
Beispiel #30
0
def categories(url):
    soup = _soup(url)
    token = None
    for tag in soup.find_all('script'):
        if 'bdecodeb' in tag.getText():
            token = __decode(tag.getText())
            break

    show_list = []
    common.error(token)
    if hasattr(token, 'token'):
        options = soup.select('#playURL > option')
        for t in options:
            all_title = t.getText()
            show_url = t['value']
            common.error(show_url)
            show_url = __decodeUrl(show_url, token.hken, token.token)
            common.error(show_url)
            show_list.append((all_title, show_url, ''))
    else:
        titleTile = soup.find('title')
        show_list.append((titleTile.getText(), token, ''))

    return show_list
Beispiel #31
0
    def get(self):
        user = nextusermodel()
        if not user:
            common.error(self, 404, "user not found")
            return

        models.db.run_in_transaction(logdeletion, user.key())

        customfeeds = models.CustomFeed.all().ancestor(user).order("time")
        for cf in customfeeds:
            if not cf.rss_link:
                continue

            ref = fetch(cf.rss_link).content
            soup = Soup(ref)

            dict_compilelist = []

            message = u""
            if cf.item_title_enable:
                titles = common.selectortext(soup, cf.item_title_selector, cf.item_title_attr)
                dlist_title = [("title", t) for t in titles]
                dict_compilelist.append(dlist_title)
                message += u"(title %d 個) " % (len(titles))

            if cf.item_link_enable:
                links = common.selectortext(soup, cf.item_link_selector, cf.item_link_attr)
                dlist_link = [("link", l) for l in links]
                dict_compilelist.append(dlist_link)
                message += u"(links %d 個) " % (len(links))

            if cf.item_description_enable:
                descriptions = common.selectortext(soup, cf.item_description_selector, cf.item_description_attr)
                dlist_description = [("description", d) for d in descriptions]
                dict_compilelist.append(dlist_description)
                message += u"(descriptions %d 個) " % (len(descriptions))

            if cf.item_date_enable:
                dates = common.selectortext(soup, cf.item_date_selector, cf.item_date_attr)
                dlist_date = [("pubdate", dateparser(d)) for d in dates]
                dict_compilelist.append(dlist_date)
                message += u"(dates %d 個) " % (len(dates))

            message += u"見つかりました。"
            models.Log(feedname=cf.name, type=models.Log._types["info"], message=message, parent=user).put()

            items = []
            for dl in zip(*dict_compilelist):
                d = {"title": "", "link": "", "description": ""}
                d.update(dict(dl))
                items.append(d)

            feeddata = models.FeedData.get_by_key_name(cf.name, parent=cf)
            if not feeddata:
                feeddata = models.FeedData(parent=cf, key_name=cf.name)

            try:
                rss_title = cf.rss_title.encode("UTF-8")
                rss_link = cf.rss_link.encode("UTF-8")
                rss_description = cf.rss_description.encode("UTF-8")

                feeddata.atom = common.buildatom("Anon", rss_title, rss_link, rss_description, items).decode("UTF-8")
                feeddata.rss = common.buildrss("Anon", rss_title, rss_link, rss_description, items).decode("UTF-8")
                feeddata.rdf = common.buildrdf("Anon", rss_title, rss_link, rss_description, items).decode("UTF-8")

                if not feeddata.put():
                    raise ValueError  # TODO: save Error
            except:
                message = u"何かエラーが発生しました。"
                models.Log(feedname=cf.name, type=models.Log._types["error"], message=message, parent=user).put()
                raise

            else:
                message = u"カスタムフィードの作成に成功しました。"
                models.Log(feedname=cf.name, type=models.Log._types["success"], message=message, parent=user).put()
Beispiel #32
0
        action='store',
        default='%s/reports' % os.path.dirname(os.path.realpath(__file__)),
        required=False)

    ob_group.add_argument('-d', '--debug',
        help='enable full traceback on exceptions',
        action='store_true',
        default=False,
        required=False)

    args = parser.parse_args()

    config = '%s/etc/omnibus.conf' % os.path.dirname(os.path.realpath(__file__))

    output_dir = args.output
    DEBUG = args.debug

    info('Using configuration file (%s) ...' % config)
    info('Debug: %s' % DEBUG)

    if os.path.exists(output_dir):
        if not os.path.isdir(output_dir):
            error('Specified report output location is not a directory; exiting ...')
            sys.exit(1)
    else:
        info('Creating report output directory (%s) ...' % output_dir)
        mkdir(output_dir)

    console = Console()
    console.cmdloop()
Beispiel #33
0
        action='store',
        default='%s/reports' % os.path.dirname(os.path.realpath(__file__)),
        required=False)

    ob_group.add_argument('-d', '--debug',
        help='enable full traceback on exceptions',
        action='store_true',
        default=False,
        required=False)

    args = parser.parse_args()

    config = '%s/etc/omnibus.conf' % os.path.dirname(os.path.realpath(__file__))

    output_dir = args.output
    DEBUG = args.debug

    info('Using configuration file (%s) ...' % config)
    info('Debug: %s' % DEBUG)

    if os.path.exists(output_dir):
        if not os.path.isdir(output_dir):
            error('Specified report output location is not a directory; exiting ...')
            sys.exit(1)
    else:
        info('Creating report output directory (%s) ...' % output_dir)
        mkdir(output_dir)

    console = Console()
    console.cmdloop()