def do_cabpost():
print('==============================')
print('For Post ... ')
from model_ent.post_model import MPost as MPostOld
mpost_old = MPostOld()
from torcms.model.post_model import MPost
mpost = MPost()
post_recs = mpost_old.query_all()
for post_rec in post_recs:
# print(post_rec.uid)
if post_rec.uid.startswith('m') or post_rec.uid.startswith('g'):
pass
# continue
post_data = {
'title': post_rec.title,
'user_name': post_rec.user_name,
'logo': post_rec.logo,
'cnt_md': unescape(unescape(post_rec.cnt_md)),
'keywords': post_rec.keywords,
'time_create': post_rec.time_create,
'time_update': post_rec.time_update,
'kind': '1',
'valid': 1,
}
print(post_rec.uid)
mpost.add_or_update(buqi_postid(post_rec.uid), post_data)
def do_wiki():
print('==============================')
print('For Wiki ... ')
from torcms.model.wiki_model import MWiki
mwiki = MWiki()
#
from torcms.model.page_model import MPage
mpage = MPage()
from model_ent.wiki_model import MWiki as OldWiki
from model_ent.page_model import MPage as OldPage
oldwiki = OldWiki()
oldpage = OldPage()
print('got it')
abc = oldwiki.query_all(limit_num=2000)
ded = oldpage.query_all()
for rec in oldwiki.query_all(limit_num=20000):
post_data = {
"uid": '_' + ''.join(rec.uid.split('-')),
"title": rec.title,
"date": rec.date,
"time_create": rec.time_create,
"user_name": rec.user_name,
"time_update": rec.time_update,
"view_count": rec.view_count,
"cnt_md": unescape(rec.cnt_md),
"cnt_html": rec.cnt_html,
"kind": '1',
}
# print(rec.title)
mwiki.insert_data(post_data)
for rec in ded:
print(rec.slug)
post_data = {
"uid": rec.slug,
"title": rec.title,
"date": rec.date,
"time_create": rec.time_create,
"user_name": '',
"time_update": rec.time_update,
"view_count": rec.view_count,
"cnt_md": unescape(rec.cnt_md),
"cnt_html": rec.cnt_html,
"kind": '2',
}
# print(rec.title)
mwiki.insert_data(post_data)
print('QED')
def get(self, actionType, encFilePath):
filePath = unescape(encFilePath)
logging.debug("Action (%s) File(%s)" % (actionType, filePath))
if (actionType == "play"):
try:
logging.debug("Playing: %s" % filePath)
pB_audio.playSong(filePath)
except Exception, e:
logging.error("Exception raised from playListModify")
logging.error(traceback.format_exc())
def get(self, actionType, encFilePath):
filePath = unescape(encFilePath)
logging.debug("Action (%s) File(%s)" % (actionType, filePath))
if (actionType == "play"):
try:
logging.debug("Playing: %s" % filePath)
pB_audio.playSong(filePath)
except Exception, e:
logging.error("Exception raised from playListModify")
logging.error(traceback.format_exc())
def get(self, search):
loader = tornado.template.Loader('templates/')
# parse the query
try:
parsed = ply.yacc.parse(unescape(search))
((query, args), typeofdata) = convert(parsed)
# execute the query
conn = sqlite3.connect('db.sqlite')
cur = conn.cursor()
cur.execute(query, *(args,))
results = cur.fetchall()
cur.close()
self.write(loader.load('search_results.html').generate(results=results,
typeofdata=typeofdata))
except QueryError as e:
self.write('Your query was incorrect: %s' % e.message)
def do_tabapp():
print('==============================')
print('For Infor ... ')
from torcms.model.post_model import MPost
from torcms.model.infor2catalog_model import MInfor2Catalog
mpost = MPost()
mpost2tag = MInfor2Catalog()
from model_ent.infor_model import MInfor
minfor = MInfor()
info_recs = minfor.query_all(20000)
for info_rec in info_recs:
# print(info_rec.uid)
# info_tag = mpost2tag.get_entry_catalog(info_rec.uid).uid
extinfo = info_rec.extinfo
if 'def_cat_uid' in extinfo:
old_cat_id = extinfo['def_cat_uid']
extinfo['def_cat_uid'] = retag(old_cat_id)
extinfo['def_cat_pid'] = retag(old_cat_id)[:2] + '00'
# print(extinfo['def_cat_uid'])
post_data = {
'title': info_rec.title,
'user_name': info_rec.user_name,
'logo': info_rec.logo,
'cnt_md': unescape(info_rec.cnt_md),
'keywords': info_rec.keywords,
'kind': '2',
'extinfo': extinfo,
# 'time_create': info_rec.time_create,
'time_update': info_rec.time_update,
}
# mpost.insert_data('m' + info_rec.uid, post_data)
# print(info_rec.uid)
mpost.add_or_update(info_rec.uid, post_data)
if 'def_cat_uid' in extinfo:
try:
mpost2tag.add_record(info_rec.uid, extinfo['def_cat_uid'], 1)
except:
print(info_rec.title)
def _index():
# Gather, convert and process assets
DOCTYPE = '<!DOCTYPE html>'
style = StringIO()
head = E.HEAD()
body = E.BODY()
templates = []
template_names = []
def visit(node, f):
for c in node.getchildren():
visit(c, f)
if c.tag != 'template':
continue
names = [n[1:] for n in c.keys() if n and n[0] == ':']
if not names:
_log.error('Unbound template found (%s)', f)
continue
for name in names:
if name in template_names:
_log.error('Duplicate template "%s" found (%s)', name, f)
continue
template = E.SCRIPT(
id='template-{0}'.format(name),
type='text/x-angulate-template'
)
template.text = c.text
template.extend(c.getchildren())
templates.append(template)
template_names.extend(names)
node.remove(c)
return
for dirpath, dirnames, filenames in os.walk(_view_path):
for filename in filenames:
ext = os.path.splitext(filename)[-1]
filename = os.path.join(dirpath, filename)
handler = build.style_handler.get(ext)
if handler:
style.write(handler(filename))
continue
handler = build.template_handler.get(ext)
if not handler:
continue
contents = handler(filename)
if not contents:
_log.warning('View is empty (%s)', filename)
continue
try:
dom = html.fromstring('<head></head>' + contents)
except Exception as e:
_log.error('Parse error (%s) %s', filename, e)
continue
for e in dom.getchildren():
if e.tag == 'head':
head.extend(e.getchildren())
elif e.tag == 'body':
visit(e, filename)
body.text = (body.text or '') + (e.text or '')
body.extend(e.getchildren())
elif e.tag == 'template':
visit(E.BODY(e), filename)
else:
_log.error('View is invalid (%s)', filename)
continue
s = 'angulate.registerTemplate("{0}", "{1}");'
templates.append(
E.SCRIPT(
'\n'.join([
s.format(name, 'template-{0}'.format(name))
for name in template_names
]),
type='text/javascript'))
# Append styles
head.append(E.STYLE(style.getvalue()))
# Append compiled runtime and Javascript functions
body.extend([
E.SCRIPT(
compiler.runtime(),
type='text/javascript'),
E.SCRIPT(
'\n'.join(f for f in client.compiled()),
type='text/javascript')
])
# Append bundle
for b in _bundle_files:
assert len(b) in [2, 3], 'Invalid bundle file config'
if len(b) == 2:
body.append(E.SCRIPT(
src='bundle/{0}'.format(b[1]),
type='text/javascript'))
elif _cdn:
link = html.tostring(E.SCRIPT(
src='bundle/{0}'.format(b[2]),
type='text/javascript'
), encoding='utf-8')
link = link.decode('utf-8').replace('</script>', '<\/script>')
body.extend([
E.SCRIPT(
src=b[1],
type='text/javascript'),
E.SCRIPT(
"window.{0} || document.write('{1}')".format(b[0], link),
type='text/javascript')
])
else:
body.append(E.SCRIPT(
src='bundle/{0}'.format(b[2]),
type='text/javascript'))
# Append templates
body.extend(templates)
# Bootstrap angular
body.append(E.SCRIPT(
'\n'.join([
'window.app = angular.module("app", ["ngAnimate", "angulate"]);',
'window.app.run(["$rootScope", function($rootScope) {',
' $rootScope._session = avalon.session;',
' avalon.scope = $rootScope;'
'}])',
'angular.bootstrap(document, ["app"]);'
]),
type='text/javascript'))
return unescape(html.tostring(E.HTML(head, body), doctype=DOCTYPE,
encoding='utf-8'))
def post(self):
'''Create, auto-generate, modify, reassign and search URL redirects.
Form submission based operations are performed, based on the
'action' parameter. All operations (except search) require auth.
User is redirected to the login endpoint if request doesn't carry auth.
'''
# Update in-memory stats counters
stats = Stats()
stats.update_stats('total_request_count')
if self.request.protocol == 'http':
self.redirect('https://' + self.request.host + self.request.uri)
return
self.check_xsrf_cookie()
irs = self.application.settings['irs'] # Database
shorturls = self.application.settings['shorturls'] # Collection
shortname = self.get_body_argument('shortname', None)
# Replace underscores in shortnames with hyphen.
url = self.get_body_argument('url', None)
action = self.get_body_argument('action', None)
host_header_based_redirect = self.get_body_argument('hostbasedredirect', 'False')
if host_header_based_redirect == 'True':
host_header_based_redirect = True
else:
host_header_based_redirect = False
username = self.get_current_user()
# All operations except search require user login
if not username and action != 'search':
login_url = self.application.settings.get('login_url')
redirect_url = login_url
self.redirect(redirect_url)
return
log.info('Request from user %s.' % (username))
log.info('Request Body: %s' % self.request.body)
if action == 'create':
# We need a URL and shortname to work with!
if not url or not shortname:
log.info('Both URL and shortname are required! Got URL: %s and shortname: %s' % (url, shortname))
self.send_error(status_code=400)
return
# This standardizes short urls (to a degree) and enhances findability.
shortname = shortname.replace('_', '-')
# Check if the short name entry already exists
redirect_entry = shorturls.find_one({'shortname': shortname, 'host_header_based_redirect': host_header_based_redirect})
if redirect_entry:
self.write('Shortcut <b>%s</b> already exists! Please edit the existing entry.' % (escape(shortname)))
else:
if shortname in RESERVED_URL_SHORTNAMES:
log.info('Shortcut %s is on the blacklist %s. Refusing to create shortcut!' % (shortname, RESERVED_URL_SHORTNAMES))
self.send_error(status_code=400)
return
if host_header_based_redirect:
log.info('Creating Host: header based redirect. Host: %s URL: %s' %(shortname, url))
parsed_url = urlparse.urlparse(url)
if not parsed_url.scheme in ['http', 'https']:
log.info('URL target is not a supported protocol.')
self.send_error(status_code=400)
return
if parsed_url.netloc in self.application.settings['self_hostnames']:
log.info('Target URL seems self referential. Could potentially generate a redirect loop.')
self.send_error(status_code=400)
return
# Strip the target url of a trailing '/' (if any) so we don't
# end up with redirects to urls ending in '//'
#url = url.rstrip('/')
url = url[:-1] if url.endswith('//') else url
current_time = datetime.datetime.utcnow()
shorturls.insert({
'shortname': shortname,
'url': url,
'owner': username,
'group': None,
'host_header_based_redirect': host_header_based_redirect,
'created': current_time,
'modified': current_time,
'redirect_count': 0
})
# Update in-memory stats counters
stats = Stats()
stats.update_stats('shortname_count')
stats.update_stats('distinct_owner_count', len(shorturls.distinct('owner')), reset=True)
collstats = irs.command('collstats', 'shorturls')
stats.update_stats('collection_size_bytes', collstats['size'], reset=True)
log.info('Shortcut %s for %s created.' % (shortname, url))
self.write('Shortcut <b>%s</b> for URL <b>%s</b> created' % (escape(shortname), escape(url)))
self.redirect('/_app/')
return
elif action == 'generate':
# We need a URL to work with
if not url:
log.info('URL required! Got URL: %s' % (url))
self.send_error(status_code=400)
return
# Gotta love the ascii art!
gibberish = ['_']
# Generate a random 7 character string sequence lower and upper case
# chars and digits
# We should be good for (62!)/(55!) short URLs.
for _ in xrange(7):
gibberish.append(random.choice(string.ascii_letters + string.digits))
shortname = ''.join(gibberish)
if shorturls.find_one({'shortname': shortname}):
self.write('Shortcut <b>%s</b> already exists! Please try again.' % shortname)
else:
if shortname in RESERVED_URL_SHORTNAMES:
log.info('Shortcut %s is reserved: %s. Refusing to create shortcut!' % (shortname, RESERVED_URL_SHORTNAMES))
self.send_error(status_code=400)
return
parsed_url = urlparse.urlparse(url)
if not parsed_url.scheme in ['http', 'https']:
log.info('URL target is not a supported protocol.')
self.send_error(status_code=400)
return
if parsed_url.netloc in self.application.settings['self_hostnames']:
log.info('Target URL seems self referential. Could potentially generate a redirect loop.')
self.send_error(status_code=400)
return
# Strip the target url of a trailing '/' (if any) so we don't
# end up with redirects to urls ending in '//'
#url = url.rstrip('/')
url = url[:-1] if url.endswith('//') else url
current_time = datetime.datetime.utcnow()
shorturls.insert({
'shortname': shortname,
'url': url,
'owner': username,
'group': None,
'host_header_based_redirect': False,
'created': current_time,
'modified': current_time,
'redirect_count': 0
})
log.info('Shortcut %s for %s created.' % (shortname, url))
# Update in-memory stats counters
stats = Stats()
stats.update_stats('shortname_count')
stats.update_stats('distinct_owner_count', len(shorturls.distinct('owner')), reset=True)
collstats = irs.command('collstats', 'shorturls')
stats.update_stats('collection_size_bytes', collstats['size'], reset=True)
self.write('Shortcut <b>%s</b> for URL <b>%s</b> created' % (escape(shortname), escape(url)))
self.redirect('/_app/')
return
elif action == 'edit':
# The shortname hidden form field and the url field is rendered escaped.
# Need to unescape it now so database lookups with shortname as the key
# works correctly.
shortname = unescape(self.get_body_argument('shortname'))
url = unescape(self.get_body_argument('url'))
# We need a URL and shortname to work with
if not url or not shortname:
log.info('Both URL and shortname are required! Got URL: %s and shortname: %s' % (url, shortname))
self.send_error(status_code=400)
return
# Find the short name entry to edit
redirect_entry = shorturls.find_one({'shortname': shortname, 'host_header_based_redirect': host_header_based_redirect})
if not redirect_entry:
self.write('Shortcut <b>%s</b> does not exist! Please create it first.' % (escape(shortname)))
else:
saved_url = redirect_entry.get('url', None)
# Check authorization to modify the short name entry
if username != redirect_entry['owner'] and username != self.application.settings['adminuser']:
self.set_status(403)
self.write('Shortcut <b>%s</b> is not owned by you! Please contact the owner %s.' % (escape(shortname), escape(redirect_entry['owner'])))
return
else:
parsed_url = urlparse.urlparse(url)
if not parsed_url.scheme in ['http', 'https']:
log.info('URL target is not a supported protocol.')
self.send_error(status_code=400)
return
if parsed_url.netloc in self.application.settings['self_hostnames']:
log.info('Target URL seems self referential. Could potentially generate a redirect loop.')
self.send_error(status_code=400)
return
# Strip the target url of a trailing '/' (if any) so we don't
# end up with redirects to urls ending in '//'
#url = url.rstrip('/')
url = url[:-1] if url.endswith('//') else url
current_time = datetime.datetime.utcnow()
shorturls.update(
{
'shortname': shortname, 'host_header_based_redirect': host_header_based_redirect
},
{
'$set': {'url': url, 'modified': current_time}
}
)
# Update stats
collstats = irs.command('collstats', 'shorturls')
stats.update_stats('collection_size_bytes', collstats['size'], reset=True)
log.info('Shortcut %s edited. Old URL: %s New URL: %s' % (shortname, saved_url, url))
self.write('Shortcut <b>%s</b> edited successfully.' % (escape(shortname)))
self.redirect('/_app/')
return
elif action == 'search':
regex = self.get_body_argument('regex', None)
self.render('search.html', regex=regex, shorturls=shorturls, username=self.get_current_user())
elif action == 'delete':
# The shortname hidden form field is rendered escaped. Unescape it.
shortname = unescape(shortname)
redirect_entry = shorturls.find_one({'shortname': shortname, 'host_header_based_redirect': host_header_based_redirect})
if redirect_entry:
saved_url = redirect_entry.get('url', None)
# Check authorization to delete the short name entry
if username != redirect_entry['owner'] and username != self.application.settings['adminuser']:
self.set_status(403)
self.write('Shortcut <b>%s</b> is not owned by you! Please contact the owner %s.' % (escape(shortname), escape(redirect_entry['owner'])))
return
shorturls.remove({'shortname': shortname, 'host_header_based_redirect': host_header_based_redirect})
log.info('Shortcut %s for URL %s deleted.' % (shortname, saved_url))
# Update in-memory stats counters. Decrement here.
stats = Stats()
stats.update_stats('shortname_count', -1)
stats.update_stats('distinct_owner_count', len(shorturls.distinct('owner')), reset=True)
collstats = irs.command('collstats', 'shorturls')
stats.update_stats('collection_size_bytes', collstats['size'], reset=True)
self.write('Shortcut <b>%s</b> for URL <b>%s</b> deleted' % (escape(shortname), escape(saved_url)))
self.redirect('/_app/')
return
else:
self.write('Shortcut <b>%s</b> does not exist!' % (escape(shortname)))
elif action == 'reassign':
# The shortname hidden form field is rendered escaped. Unescape it.
shortname = unescape(shortname)
redirect_entry = shorturls.find_one({'shortname': shortname, 'host_header_based_redirect': host_header_based_redirect})
newowner = self.get_body_argument('newowner', None)
if username != redirect_entry['owner'] and username != self.application.settings['adminuser']:
self.set_status(403)
self.write('You cannot reassign shortcut <b>%s</b> not owned by you! Please contact the owner %s.' % (escape(shortname), escape(redirect_entry['owner'])))
return
else:
# Should really check if the user exists in LDAP/AD. However
# AD (by default) does not allow anonymous searches. For now just
# make do with a basic regex check for valid userPrincipalName.
if re.match(r'^[^@][email protected]$', newowner):
current_time = datetime.datetime.utcnow()
shorturls.update(
{
'shortname': shortname, 'host_header_based_redirect': host_header_based_redirect
},
{
'$set': {'owner': newowner, 'modified': current_time}
}
)
stats.update_stats('distinct_owner_count', len(shorturls.distinct('owner')), reset=True)
log.info('Ownership of short name %s re-assigned. From: %s -> %s' % (shortname, redirect_entry['owner'], newowner))
self.write('Ownership of short name %s re-assigned. From: %s -> %s' % (escape(shortname), escape(redirect_entry['owner']), escape(newowner)))
self.redirect('/')
return
else:
self.set_status(400)
self.write('%s is not a valid user name. Should be [email protected]' % (escape(newowner)))
return
else:
log.debug('Unrecognized action specified: shortname: %s url: %s action: %s' % (shortname, url, action))
self.send_error(status_code=400)
return