Пример #1
0
def do_cabpost():
    print('==============================')
    print('For Post ... ')
    from model_ent.post_model import MPost as MPostOld
    mpost_old = MPostOld()

    from torcms.model.post_model import MPost

    mpost = MPost()
    post_recs = mpost_old.query_all()
    for post_rec in post_recs:
        # print(post_rec.uid)

        if post_rec.uid.startswith('m') or post_rec.uid.startswith('g'):
            pass
            # continue
        post_data = {
            'title': post_rec.title,
            'user_name': post_rec.user_name,
            'logo': post_rec.logo,
            'cnt_md': unescape(unescape(post_rec.cnt_md)),
            'keywords': post_rec.keywords,
            'time_create': post_rec.time_create,
            'time_update': post_rec.time_update,
            'kind': '1',
            'valid': 1,
        }
        print(post_rec.uid)
        mpost.add_or_update(buqi_postid(post_rec.uid), post_data)
Пример #2
0
def do_wiki():
    print('==============================')
    print('For Wiki ... ')
    from torcms.model.wiki_model import MWiki
    mwiki = MWiki()
    #
    from torcms.model.page_model import MPage
    mpage = MPage()

    from model_ent.wiki_model import MWiki as OldWiki
    from model_ent.page_model import MPage as OldPage
    oldwiki = OldWiki()
    oldpage = OldPage()

    print('got it')

    abc = oldwiki.query_all(limit_num=2000)

    ded = oldpage.query_all()
    for rec in oldwiki.query_all(limit_num=20000):
        post_data = {
            "uid": '_' + ''.join(rec.uid.split('-')),
            "title": rec.title,
            "date": rec.date,
            "time_create": rec.time_create,
            "user_name": rec.user_name,
            "time_update": rec.time_update,
            "view_count": rec.view_count,
            "cnt_md": unescape(rec.cnt_md),
            "cnt_html": rec.cnt_html,
            "kind": '1',
        }
        # print(rec.title)
        mwiki.insert_data(post_data)

    for rec in ded:
        print(rec.slug)
        post_data = {
            "uid": rec.slug,
            "title": rec.title,
            "date": rec.date,
            "time_create": rec.time_create,
            "user_name": '',
            "time_update": rec.time_update,
            "view_count": rec.view_count,
            "cnt_md": unescape(rec.cnt_md),
            "cnt_html": rec.cnt_html,
            "kind": '2',
        }
        # print(rec.title)
        mwiki.insert_data(post_data)

    print('QED')
Пример #3
0
 def get(self, actionType, encFilePath):
     filePath = unescape(encFilePath)
     logging.debug("Action (%s) File(%s)" % (actionType, filePath))
     if (actionType == "play"):
         try:
             logging.debug("Playing: %s" % filePath)
             pB_audio.playSong(filePath)
         except Exception, e:
             logging.error("Exception raised from playListModify")
             logging.error(traceback.format_exc())
Пример #4
0
 def get(self, actionType, encFilePath):
     filePath = unescape(encFilePath)
     logging.debug("Action (%s) File(%s)" % (actionType, filePath))
     if (actionType == "play"):
         try:
             logging.debug("Playing: %s" % filePath)
             pB_audio.playSong(filePath)
         except Exception, e:
             logging.error("Exception raised from playListModify")
             logging.error(traceback.format_exc())
Пример #5
0
 def get(self, search):
     loader = tornado.template.Loader('templates/')
     
     # parse the query
     try:
         parsed = ply.yacc.parse(unescape(search))
         ((query, args), typeofdata) = convert(parsed)
         
         # execute the query
         conn = sqlite3.connect('db.sqlite')
         cur = conn.cursor()
         cur.execute(query, *(args,))
         results = cur.fetchall()
         
         cur.close()
         
         self.write(loader.load('search_results.html').generate(results=results,
                                                                typeofdata=typeofdata))
     except QueryError as e:
         self.write('Your query was incorrect: %s' % e.message)
Пример #6
0
def do_tabapp():
    print('==============================')
    print('For Infor ... ')
    from torcms.model.post_model import MPost
    from torcms.model.infor2catalog_model import MInfor2Catalog
    mpost = MPost()
    mpost2tag = MInfor2Catalog()

    from model_ent.infor_model import MInfor
    minfor = MInfor()
    info_recs = minfor.query_all(20000)
    for info_rec in info_recs:
        # print(info_rec.uid)
        # info_tag = mpost2tag.get_entry_catalog(info_rec.uid).uid
        extinfo = info_rec.extinfo
        if 'def_cat_uid' in extinfo:
            old_cat_id = extinfo['def_cat_uid']
            extinfo['def_cat_uid'] = retag(old_cat_id)
            extinfo['def_cat_pid'] = retag(old_cat_id)[:2] + '00'
        # print(extinfo['def_cat_uid'])

        post_data = {
            'title': info_rec.title,
            'user_name': info_rec.user_name,
            'logo': info_rec.logo,
            'cnt_md': unescape(info_rec.cnt_md),
            'keywords': info_rec.keywords,
            'kind': '2',
            'extinfo': extinfo,
            # 'time_create': info_rec.time_create,
            'time_update': info_rec.time_update,
        }
        # mpost.insert_data('m' + info_rec.uid, post_data)
        # print(info_rec.uid)
        mpost.add_or_update(info_rec.uid, post_data)
        if 'def_cat_uid' in extinfo:
            try:
                mpost2tag.add_record(info_rec.uid, extinfo['def_cat_uid'], 1)
            except:
                print(info_rec.title)
Пример #7
0
def _index():
    # Gather, convert and process assets
    DOCTYPE = '<!DOCTYPE html>'
    style = StringIO()
    head = E.HEAD()
    body = E.BODY()
    templates = []
    template_names = []

    def visit(node, f):
        for c in node.getchildren():
            visit(c, f)
            if c.tag != 'template':
                continue
            names = [n[1:] for n in c.keys() if n and n[0] == ':']
            if not names:
                _log.error('Unbound template found (%s)', f)
                continue

            for name in names:
                if name in template_names:
                    _log.error('Duplicate template "%s" found (%s)', name, f)
                    continue

                template = E.SCRIPT(
                    id='template-{0}'.format(name),
                    type='text/x-angulate-template'
                )
                template.text = c.text
                template.extend(c.getchildren())
                templates.append(template)

            template_names.extend(names)
            node.remove(c)
        return

    for dirpath, dirnames, filenames in os.walk(_view_path):
        for filename in filenames:
            ext = os.path.splitext(filename)[-1]
            filename = os.path.join(dirpath, filename)

            handler = build.style_handler.get(ext)
            if handler:
                style.write(handler(filename))
                continue

            handler = build.template_handler.get(ext)
            if not handler:
                continue
            contents = handler(filename)

            if not contents:
                _log.warning('View is empty (%s)', filename)
                continue

            try:
                dom = html.fromstring('<head></head>' + contents)
            except Exception as e:
                _log.error('Parse error (%s) %s', filename, e)
                continue

            for e in dom.getchildren():
                if e.tag == 'head':
                    head.extend(e.getchildren())
                elif e.tag == 'body':
                    visit(e, filename)
                    body.text = (body.text or '') + (e.text or '')
                    body.extend(e.getchildren())
                elif e.tag == 'template':
                    visit(E.BODY(e), filename)
                else:
                    _log.error('View is invalid (%s)', filename)
                    continue

            s = 'angulate.registerTemplate("{0}", "{1}");'
            templates.append(
                E.SCRIPT(
                    '\n'.join([
                        s.format(name, 'template-{0}'.format(name))
                        for name in template_names
                    ]),
                    type='text/javascript'))

    # Append styles
    head.append(E.STYLE(style.getvalue()))

    # Append compiled runtime and Javascript functions
    body.extend([
        E.SCRIPT(
            compiler.runtime(),
            type='text/javascript'),
        E.SCRIPT(
            '\n'.join(f for f in client.compiled()),
            type='text/javascript')
    ])

    # Append bundle
    for b in _bundle_files:
        assert len(b) in [2, 3], 'Invalid bundle file config'
        if len(b) == 2:
            body.append(E.SCRIPT(
                src='bundle/{0}'.format(b[1]),
                type='text/javascript'))
        elif _cdn:
            link = html.tostring(E.SCRIPT(
                src='bundle/{0}'.format(b[2]),
                type='text/javascript'
            ), encoding='utf-8')

            link = link.decode('utf-8').replace('</script>', '<\/script>')
            body.extend([
                E.SCRIPT(
                    src=b[1],
                    type='text/javascript'),
                E.SCRIPT(
                    "window.{0} || document.write('{1}')".format(b[0], link),
                    type='text/javascript')
            ])
        else:
            body.append(E.SCRIPT(
                src='bundle/{0}'.format(b[2]),
                type='text/javascript'))

    # Append templates
    body.extend(templates)

    # Bootstrap angular
    body.append(E.SCRIPT(
        '\n'.join([
            'window.app = angular.module("app", ["ngAnimate", "angulate"]);',
            'window.app.run(["$rootScope", function($rootScope) {',
            '  $rootScope._session = avalon.session;',
            '  avalon.scope = $rootScope;'
            '}])',
            'angular.bootstrap(document, ["app"]);'
        ]),
        type='text/javascript'))

    return unescape(html.tostring(E.HTML(head, body), doctype=DOCTYPE,
                                  encoding='utf-8'))
Пример #8
0
    def post(self):
        '''Create, auto-generate, modify, reassign and search URL redirects.
    
        Form submission based operations are performed, based on the
        'action' parameter. All operations (except search) require auth.
        User is redirected to the login endpoint if request doesn't carry auth.
        '''
        # Update in-memory stats counters
        stats = Stats()
        stats.update_stats('total_request_count')
        if self.request.protocol == 'http':
            self.redirect('https://' + self.request.host + self.request.uri)
            return

        self.check_xsrf_cookie()

        irs = self.application.settings['irs'] # Database
        shorturls = self.application.settings['shorturls'] # Collection
        shortname = self.get_body_argument('shortname', None)
        # Replace underscores in shortnames with hyphen.
        url = self.get_body_argument('url', None)
        action = self.get_body_argument('action', None)
        host_header_based_redirect = self.get_body_argument('hostbasedredirect', 'False')

        if host_header_based_redirect == 'True':
            host_header_based_redirect = True
        else:
            host_header_based_redirect = False

        username = self.get_current_user()

        # All operations except search require user login
        if not username and action != 'search':
            login_url = self.application.settings.get('login_url') 
            redirect_url = login_url
            self.redirect(redirect_url)
            return

        log.info('Request from user %s.' % (username))
        log.info('Request Body: %s' % self.request.body)

        if action == 'create':
            # We need a URL and shortname to work with!
            if not url or not shortname:
               log.info('Both URL and shortname are required! Got URL: %s and shortname: %s' % (url, shortname))
               self.send_error(status_code=400) 
               return

            # This standardizes short urls (to a degree) and enhances findability.
            shortname = shortname.replace('_', '-')

            # Check if the short name entry already exists
            redirect_entry = shorturls.find_one({'shortname': shortname, 'host_header_based_redirect': host_header_based_redirect})
            if redirect_entry:
                self.write('Shortcut <b>%s</b> already exists! Please edit the existing entry.' % (escape(shortname)))
            else:
                if shortname in RESERVED_URL_SHORTNAMES:
                    log.info('Shortcut %s is on the blacklist %s. Refusing to create shortcut!' % (shortname, RESERVED_URL_SHORTNAMES))
                    self.send_error(status_code=400)
                    return
               
                if host_header_based_redirect:
                    log.info('Creating Host: header based redirect. Host: %s URL: %s' %(shortname, url))

                parsed_url = urlparse.urlparse(url)
                if not parsed_url.scheme in ['http', 'https']:
                    log.info('URL target is not a supported protocol.')
                    self.send_error(status_code=400)
                    return

                if parsed_url.netloc in self.application.settings['self_hostnames']:
                    log.info('Target URL seems self referential. Could potentially generate a redirect loop.')
                    self.send_error(status_code=400)
                    return
  
                # Strip the target url of a trailing '/' (if any) so we don't
                # end up with redirects to urls ending in '//'
                #url = url.rstrip('/')
                url = url[:-1] if url.endswith('//') else url

                current_time = datetime.datetime.utcnow()
                shorturls.insert({
                    'shortname': shortname,
                    'url': url,
                    'owner': username,
                    'group': None,
                    'host_header_based_redirect': host_header_based_redirect,
                    'created': current_time,
                    'modified': current_time,
                    'redirect_count': 0
                })
                # Update in-memory stats counters
                stats = Stats()
                stats.update_stats('shortname_count')
                stats.update_stats('distinct_owner_count', len(shorturls.distinct('owner')), reset=True)
                collstats = irs.command('collstats', 'shorturls')
                stats.update_stats('collection_size_bytes', collstats['size'], reset=True)
                log.info('Shortcut %s for %s created.' % (shortname, url))
                self.write('Shortcut <b>%s</b> for URL <b>%s</b> created' % (escape(shortname), escape(url)))
                self.redirect('/_app/')
                return
        elif action == 'generate':
            # We need a URL to work with
            if not url:
               log.info('URL required! Got URL: %s' % (url))
               self.send_error(status_code=400) 
               return

            # Gotta love the ascii art!
            gibberish = ['_']

            # Generate a random 7 character string sequence lower and upper case
            # chars and digits
            # We should be good for (62!)/(55!) short URLs.
            for _ in xrange(7):
                gibberish.append(random.choice(string.ascii_letters + string.digits)) 
            shortname = ''.join(gibberish)

            if shorturls.find_one({'shortname': shortname}):
                self.write('Shortcut <b>%s</b> already exists! Please try again.' % shortname)
            else:
                if shortname in RESERVED_URL_SHORTNAMES:
                    log.info('Shortcut %s is reserved: %s. Refusing to create shortcut!' % (shortname, RESERVED_URL_SHORTNAMES))
                    self.send_error(status_code=400)
                    return

                parsed_url = urlparse.urlparse(url)
                if not parsed_url.scheme in ['http', 'https']:
                    log.info('URL target is not a supported protocol.')
                    self.send_error(status_code=400)
                    return

                if parsed_url.netloc in self.application.settings['self_hostnames']:
                    log.info('Target URL seems self referential. Could potentially generate a redirect loop.')
                    self.send_error(status_code=400)
                    return
  
                # Strip the target url of a trailing '/' (if any) so we don't
                # end up with redirects to urls ending in '//'
                #url = url.rstrip('/')
                url = url[:-1] if url.endswith('//') else url

                current_time = datetime.datetime.utcnow()
                shorturls.insert({
                    'shortname': shortname,
                    'url': url,
                    'owner': username,
                    'group': None,
                    'host_header_based_redirect': False,
                    'created': current_time,
                    'modified': current_time,
                    'redirect_count': 0
                })
                log.info('Shortcut %s for %s created.' % (shortname, url))
                # Update in-memory stats counters
                stats = Stats()
                stats.update_stats('shortname_count')
                stats.update_stats('distinct_owner_count', len(shorturls.distinct('owner')), reset=True)
                collstats = irs.command('collstats', 'shorturls')
                stats.update_stats('collection_size_bytes', collstats['size'], reset=True)
                self.write('Shortcut <b>%s</b> for URL <b>%s</b> created' % (escape(shortname), escape(url)))
                self.redirect('/_app/')
                return
        elif action == 'edit':
            # The shortname hidden form field and the url field is rendered escaped.
            # Need to unescape it now so database lookups with shortname as the key
            # works correctly.
            shortname = unescape(self.get_body_argument('shortname'))
            url = unescape(self.get_body_argument('url'))
            # We need a URL and shortname to work with
            if not url or not shortname:
               log.info('Both URL and shortname are required! Got URL: %s and shortname: %s' % (url, shortname))
               self.send_error(status_code=400) 
               return

            # Find the short name entry to edit
            redirect_entry = shorturls.find_one({'shortname': shortname, 'host_header_based_redirect': host_header_based_redirect})
            if not redirect_entry:
                self.write('Shortcut <b>%s</b> does not exist! Please create it first.' % (escape(shortname)))
            else:
                saved_url = redirect_entry.get('url', None)
                # Check authorization to modify the short name entry
                if username != redirect_entry['owner'] and username != self.application.settings['adminuser']:
                    self.set_status(403)
                    self.write('Shortcut <b>%s</b> is not owned by you! Please contact the owner %s.' % (escape(shortname), escape(redirect_entry['owner'])))
                    return
                else:
                    parsed_url = urlparse.urlparse(url)
                    if not parsed_url.scheme in ['http', 'https']:
                        log.info('URL target is not a supported protocol.')
                        self.send_error(status_code=400)
                        return

                    if parsed_url.netloc in self.application.settings['self_hostnames']:
                        log.info('Target URL seems self referential. Could potentially generate a redirect loop.')
                        self.send_error(status_code=400)
                        return
  
                    # Strip the target url of a trailing '/' (if any) so we don't
                    # end up with redirects to urls ending in '//'
                    #url = url.rstrip('/')
                    url = url[:-1] if url.endswith('//') else url

                    current_time = datetime.datetime.utcnow()
                    shorturls.update(
                        {
                            'shortname': shortname, 'host_header_based_redirect': host_header_based_redirect
                        },
                        {
                            '$set': {'url': url, 'modified': current_time}
                        }
                    )
                    # Update stats
                    collstats = irs.command('collstats', 'shorturls')
                    stats.update_stats('collection_size_bytes', collstats['size'], reset=True)
                    log.info('Shortcut %s edited. Old URL: %s New URL: %s' % (shortname, saved_url, url))
                    self.write('Shortcut <b>%s</b> edited successfully.' % (escape(shortname)))
                    self.redirect('/_app/')
                    return
        elif action == 'search':
            regex = self.get_body_argument('regex', None) 
            self.render('search.html', regex=regex, shorturls=shorturls, username=self.get_current_user())
        elif action == 'delete':
            # The shortname hidden form field is rendered escaped. Unescape it.
            shortname = unescape(shortname)
            redirect_entry = shorturls.find_one({'shortname': shortname, 'host_header_based_redirect': host_header_based_redirect})
            if redirect_entry:
                saved_url = redirect_entry.get('url', None)
                # Check authorization to delete the short name entry
                if username != redirect_entry['owner'] and username != self.application.settings['adminuser']:
                    self.set_status(403)
                    self.write('Shortcut <b>%s</b> is not owned by you! Please contact the owner %s.' % (escape(shortname), escape(redirect_entry['owner'])))
                    return
                shorturls.remove({'shortname': shortname, 'host_header_based_redirect': host_header_based_redirect})
                log.info('Shortcut %s for URL %s deleted.' % (shortname, saved_url))
                # Update in-memory stats counters. Decrement here.
                stats = Stats()
                stats.update_stats('shortname_count', -1)
                stats.update_stats('distinct_owner_count', len(shorturls.distinct('owner')), reset=True)
                collstats = irs.command('collstats', 'shorturls')
                stats.update_stats('collection_size_bytes', collstats['size'], reset=True)
                self.write('Shortcut <b>%s</b> for URL <b>%s</b> deleted' % (escape(shortname), escape(saved_url)))
                self.redirect('/_app/')
                return
            else:
                self.write('Shortcut <b>%s</b> does not exist!' % (escape(shortname)))
        elif action == 'reassign':
            # The shortname hidden form field is rendered escaped. Unescape it.
            shortname = unescape(shortname)
            redirect_entry = shorturls.find_one({'shortname': shortname, 'host_header_based_redirect': host_header_based_redirect})
            newowner = self.get_body_argument('newowner', None)
            if username != redirect_entry['owner'] and username != self.application.settings['adminuser']:
                self.set_status(403)
                self.write('You cannot reassign shortcut <b>%s</b> not owned by you! Please contact the owner %s.' % (escape(shortname), escape(redirect_entry['owner'])))
                return
            else:
                # Should really check if the user exists in LDAP/AD. However
                # AD (by default) does not allow anonymous searches. For now just 
                # make do with a basic regex check for valid userPrincipalName.
                if re.match(r'^[^@][email protected]$', newowner):
                    current_time = datetime.datetime.utcnow()
                    shorturls.update(
                        {
                            'shortname': shortname, 'host_header_based_redirect': host_header_based_redirect
                        },
                        {
                            '$set': {'owner': newowner, 'modified': current_time}
                        }
                    )
                    stats.update_stats('distinct_owner_count', len(shorturls.distinct('owner')), reset=True)
                    log.info('Ownership of short name %s re-assigned. From: %s -> %s' % (shortname, redirect_entry['owner'], newowner))
                    self.write('Ownership of short name %s re-assigned. From: %s -> %s' % (escape(shortname), escape(redirect_entry['owner']), escape(newowner)))
                    self.redirect('/')
                    return
                else:
                    self.set_status(400)
                    self.write('%s is not a valid user name. Should be [email protected]' % (escape(newowner)))
                    return
        else:
            log.debug('Unrecognized action specified: shortname: %s url: %s action: %s' % (shortname, url, action))
            self.send_error(status_code=400)
            return