def do_source(self, arg):
"""Add source to given artifact or most recently added artifact if not specified
Usage: source # adds to last created artifact
source <artifact name|session id> # adds to specific artifact
"""
if arg == '':
last = self.session.receive('artifacts')
_type = detect_type(last)
else:
_type = detect_type(arg)
is_key, value = lookup_key(self.session, arg)
if is_key and value is None:
error('Unable to find artifact key in session (%s)' % arg)
return
elif is_key and value is not None:
arg = value
else:
pass
if self.db.exists(_type, {'name': last}):
self.db.update_one(_type, {'name': last}, {'source': arg})
success('Added source to artifact entry (%s: %s)' % (last, arg))
else:
warning('Failed to find last artifact in MongoDB. Run "new <artifact name>" before using the source command')
def do_report(self, arg):
"""Save artifact report as JSON file
Usage: report <artifact name>
report <session id>"""
is_key, value = lookup_key(self.session, arg)
if is_key and value is None:
error('Unable to find artifact key in session (%s)' % arg)
return
elif is_key and value is not None:
arg = value
else:
pass
_type = detect_type(arg)
result = self.db.find(_type, {'name': arg}, one=True)
if len(result) == 0:
warning('No entry found for artifact (%s)' % arg)
else:
report = storage.JSON(data=result, file_path=output_dir)
report.save()
if os.path.exists(report.file_path):
success('Saved artifact report (%s)' % report.file_path)
else:
error('Failed to properly save report')
def default(self, arg):
"""Override default function for custom error message"""
if arg.startswith('#'):
return
error('Unknown command')
return
def do_session(self, arg):
"""Open a new session"""
self.session = RedisCache(config)
if self.session.db is None:
error('Failed to connect to Redis back-end. Please ensure the Redis service is running')
else:
success('Opened new session')
def do_session(self, arg):
"""Open a new session"""
self.session = RedisCache(config)
if self.session.db is None:
error('Failed to connect to Redis back-end. Please ensure the Redis service is running')
else:
success('Opened new session')
def default(self, arg):
"""Override default function for custom error message"""
if arg.startswith('#'):
return
error('Unknown command')
return
def do_cat(self, arg):
"""View artifact details or list API keys
Usage: cat apikeys
cat <artifact name>"""
if arg == 'apikeys':
data = json.load(open(common.API_CONF, 'rb'))
print json.dumps(data, indent=2)
else:
is_key, value = lookup_key(self.session, arg)
if is_key and value is None:
error('Unable to find artifact key in session (%s)' % arg)
return
elif is_key and value is not None:
arg = value
else:
pass
artifact_type = detect_type(arg)
result = self.db.find(artifact_type, {'name': arg}, one=True)
if len(result) == 0:
info('No entry found for artifact (%s)' % arg)
else:
print json.dumps(result, indent=2, separators=(',', ':'))
def do_report(self, arg):
"""Save artifact report as JSON file
Usage: report <artifact name>
report <session id>"""
is_key, value = lookup_key(self.session, arg)
if is_key and value is None:
error('Unable to find artifact key in session (%s)' % arg)
return
elif is_key and value is not None:
arg = value
else:
pass
_type = detect_type(arg)
result = self.db.find(_type, {'name': arg}, one=True)
if len(result) == 0:
warning('No entry found for artifact (%s)' % arg)
else:
report = storage.JSON(data=result, file_path=output_dir)
report.save()
if os.path.exists(report.file_path):
success('Saved artifact report (%s)' % report.file_path)
else:
error('Failed to properly save report')
def do_source(self, arg):
"""Add source to given artifact or most recently added artifact if not specified
Usage: source # adds to last created artifact
source <artifact name|session id> # adds to specific artifact
"""
if arg == '':
last = self.session.receive('artifacts')
_type = detect_type(last)
else:
_type = detect_type(arg)
is_key, value = lookup_key(self.session, arg)
if is_key and value is None:
error('Unable to find artifact key in session (%s)' % arg)
return
elif is_key and value is not None:
arg = value
else:
pass
if self.db.exists(_type, {'name': last}):
self.db.update_one(_type, {'name': last}, {'source': arg})
success('Added source to artifact entry (%s: %s)' % (last, arg))
else:
warning('Failed to find last artifact in MongoDB. Run "new <artifact name>" before using the source command')
def do_cat(self, arg):
"""View artifact details or list API keys
Usage: cat apikeys
cat <artifact name>"""
if arg == 'apikeys':
data = json.load(open(common.API_CONF, 'rb'))
print json.dumps(data, indent=2)
else:
is_key, value = lookup_key(self.session, arg)
if is_key and value is None:
error('Unable to find artifact key in session (%s)' % arg)
return
elif is_key and value is not None:
arg = value
else:
pass
artifact_type = detect_type(arg)
result = self.db.find(artifact_type, {'name': arg}, one=True)
if len(result) == 0:
info('No entry found for artifact (%s)' % arg)
else:
print json.dumps(result, indent=2, separators=(',', ':'))
def __decode(js):
docJs = """
function unbox() {
var evalCode = "";
var document = {
write: function(str) { evalCode = str; }
};
var bdecodeb = function(str, key) { return {str: str, key: key}; }; """ + js + """ return evalCode; }"""
strAndKey = js2py.eval_js(docJs)()
decodedStr = __bdecodeb(strAndKey.str, strAndKey.key)
decodedStr = BeautifulSoup(decodedStr, 'html5lib').get_text()
if "token" in decodedStr:
test = js2py.eval_js('function add() { ' + decodedStr +
'; return { token: token, hken: hken}; }')
tokenObj = test()
return tokenObj
else:
presetCode = """
function secondUnbox() {
var plus = { video: { VideoPlayer: function(str, code) { return code; } } };
var document = { addEventListener: function(event, func) {} };
""" + __bdecode_str() + ";" + decodedStr + """
vstPlay();
return video.src;
}
"""
common.error(presetCode)
src = js2py.eval_js(presetCode)()
return src
def play_mirror(url):
with common.busy_indicator():
common.error("What is the url: " + url)
(title, vidurl) = scrapers.episodeVideo(url.split('?')[0]) # common.resolve(url)
if vidurl:
li = xbmcgui.ListItem(title)
li.setThumbnailImage('')
if 'User-Agent=' not in vidurl:
vidurl = vidurl + '|User-Agent=' + urllib.quote(get_ua())
xbmc.Player().play(vidurl, li)
def get(self, pagename='index'):
''' 静的なページの出力 '''
template_values = {}
path = os.path.join(os.path.dirname(__file__), 'templates/%s.html' % (pagename))
if not os.path.exists(path):
common.error(self, 404, "Page not found.")
return
self.response.out.write(template.render(path, template_values))
def get(self):
user = common.currentuser()
if not user:
common.error(self, 404, 'ユーザが見つかりません')
return
myfeeds = models.CustomFeed.all().ancestor(user).fetch(1000)
template_values = {'username': user.user, 'userid': user.key().id(), 'feeds': myfeeds}
path = os.path.join(os.path.dirname(__file__), 'templates/mypage.html')
self.response.out.write(template.render(path, template_values))
def get(self):
user = common.currentuser()
if not user:
common.error(self, 404, "User not found.")
return
logs = models.Log.all().order('-time').ancestor(user).fetch(1000)
template_values = {'logs': logs}
path = os.path.join(os.path.dirname(__file__), 'templates/log.html')
self.response.out.write(template.render(path, template_values))
def play_mirror(url):
with common.busy_indicator():
common.error("What is the url: " + url)
(title, vidurl) = scrapers.episodeVideo(
url.split('?')[0]) # common.resolve(url)
if vidurl:
li = xbmcgui.ListItem(title)
for pxyDomain in config.domain_using_proxy:
if (re.search(pxyDomain, vidurl)):
vidurl = config.local_proxy + '/' + re.sub(
'\://', '/', vidurl, 1)
break
if 'User-Agent=' not in vidurl:
vidurl = vidurl + '|User-Agent=' + quote(get_ua())
xbmc.Player().play(vidurl, li)
def get(self):
user = common.currentuser()
if not user:
common.error(self, 404, "ユーザを確認出来ませんでした。")
return
ct = models.CustomTest.all().ancestor(user).get()
if not ct:
ct = models.CustomTest(name='custom', parent=user)
ct.put()
template_values = {'ct': ct}
path = os.path.join(os.path.dirname(__file__), 'templates/customtest.html')
self.response.out.write(template.render(path, template_values))
def get(self, feedname):
u = common.currentuser()
if not u:
common.error(self, 404, 'not accept user')
return
cf = models.CustomFeed.get_by_key_name(feedname, parent=u)
if not cf:
common.error(self, 404, "フィードが存在しません。");
return
template_values = {'feedname': feedname, 'cf': cf}
path = os.path.join(os.path.dirname(__file__), 'templates/edit.html')
self.response.out.write(template.render(path, template_values))
def do_delete(self, arg):
"""Remove artifact from database by name or ID
Usage: delete <name>
delete <session id>"""
is_key, value = lookup_key(self.session, arg)
if is_key and value is None:
error('Unable to find artifact key in session (%s)' % arg)
return
elif is_key and value is not None:
arg = value
else:
pass
artifact_type = detect_type(arg)
self.db.delete_one(artifact_type, {'name': arg})
def do_delete(self, arg):
"""Remove artifact from database by name or ID
Usage: delete <name>
delete <session id>"""
is_key, value = lookup_key(self.session, arg)
if is_key and value is None:
error('Unable to find artifact key in session (%s)' % arg)
return
elif is_key and value is not None:
arg = value
else:
pass
artifact_type = detect_type(arg)
self.db.delete_one(artifact_type, {'name': arg})
def setselectors(self):
'''POSTデータをカスタムテストエンティティに保存します'''
user = common.currentuser()
if not user:
common.error(self, 404, "User not found")
return False
ct = models.CustomTest.all().ancestor(user).get()
if not ct:
ct = models.CustomTest(parent=user)
ct.setbypost(self.request.POST)
if not ct.put():
common.error(self, 200, "fail to save.")
return False
return True
def do_rm(self, arg):
"""Remove artifact from session by ID
Usage: rm <session id>"""
try:
arg = int(arg)
except:
error('Artifact ID must be an integer')
return
if self.session is not None:
if self.session.exists(arg):
self.session.delete(arg)
success('Removed artifact from cache (%s)' % arg)
else:
warning('Unable to find artifact by ID (%s)' % arg)
else:
warning('No active session; start a new session by running the "session" command')
def do_rm(self, arg):
"""Remove artifact from session by ID
Usage: rm <session id>"""
try:
arg = int(arg)
except:
error('Artifact ID must be an integer')
return
if self.session is not None:
if self.session.exists(arg):
self.session.delete(arg)
success('Removed artifact from cache (%s)' % arg)
else:
warning('Unable to find artifact by ID (%s)' % arg)
else:
warning('No active session; start a new session by running the "session" command')
def seturl(self):
'''URLとURLからフェッチして保存します'''
user = common.currentuser()
if not user:
common.error(self, 404, "User not found.")
return
ct = models.CustomTest.all().ancestor(user).get()
if not ct:
ct = models.CustomTest(parent=user)
ct.setbypost(self.request.POST)
if not ct.rss_link:
soup = Soup(defines.defaulttesthtml)
else:
result = urlfetch.fetch(ct.rss_link)
if result.status_code != 200:
common.error(self, 200, "Url Fetch Error")
return
soup = Soup(result.content)
try:
ct.data = soup.prettify().decode('UTF-8')
except ValueError, message:
common.error(self, 200, message)
return
def get(self, userid, feedname, feedtype=None):
user = models.User.get_by_id(int(userid))
if not user:
common.error(self, 404, 'User ID is not found.')
return
cf = models.CustomFeed.get_by_key_name(feedname, parent=user)
if not cf:
common.error(self, 404, 'CutomFeed is not found.')
return
feed = models.FeedData.get_by_key_name(feedname, parent=cf)
if not feed:
common.error(self, 404, 'Feed data is not found.')
return
if feedtype == None:
template_values = {'userid': userid, 'cf': cf, 'nickname': ''}
path = os.path.join(os.path.dirname(__file__), 'templates/feed.html')
self.response.out.write(template.render(path, template_values))
else:
self.response.out.write(getattr(feed, feedtype))
def getselectorstexts(self):
'''カスタムテストエンティティのセレクタ属性セットから各テキストを取得します'''
user = common.currentuser()
if not user:
common.error(self, 404, "User not found")
return False
ct = models.CustomTest.all().ancestor(user).get()
if not ct:
return False
soup = Soup(ct.data)
texts = {}
if ct.item_title_enable and ct.item_title_selector:
texts['titles'] = common.selectortext(soup, ct.item_title_selector, ct.item_title_attr)
if ct.item_link_enable and ct.item_link_selector:
texts['links'] = common.selectortext(soup, ct.item_link_selector, ct.item_link_attr)
if ct.item_description_enable and ct.item_description_selector:
texts['descriptions'] = common.selectortext(soup, ct.item_description_selector, ct.item_description_attr)
if ct.item_date_enable and ct.item_date_selector:
texts['dates'] = common.selectortext(soup, ct.item_date_selector, ct.item_date_attr)
return texts
def post(self, feedname):
u = common.currentuser()
if not u:
common.error(self, 404, 'not accept user')
return
feed = models.CustomFeed.get_by_key_name(feedname, parent=u)
if not feed:
common.error(self, 404, 'Feed not found')
return
try:
feed.setbypost(self.request.POST)
except ValueError:
common.error(self, 200, "Invalid input data")
return
if not feed.put():
common.error(self, 200, "fail to save.")
return
self.redirect('/edit/' + feedname)
def post(self):
user = common.currentuser()
if not user:
common.error(self, 404, 'ユーザが見つかりません')
newfeedname = self.request.POST['name']
if models.CustomFeed.get_by_key_name(newfeedname, parent=user):
common.error(self, 409, "既にページが存在します。")
return
cf = models.CustomFeed(parent=user, key_name=newfeedname, name=newfeedname)
if not cf.put():
common.error(self, 400, "フィードの作成に失敗しました。")
return
self.redirect("/mypage")
def post(self):
user = common.currentuser()
if not user:
common.error(self, 404, 'ユーザが見つかりません')
return
feedname = self.request.POST['name']
cf = models.CustomFeed.get_by_key_name(feedname, parent=user)
if not cf:
common.error(self, 400, "ページは存在しません。")
return
cf.delete()
if cf.is_saved():
common.error(self, 400, "削除に失敗しました。")
return
self.redirect('/mypage')
def categories(url):
soup = _soup(url)
token = None
for tag in soup.find_all('script'):
if 'bdecodeb' in tag.getText():
token = __decode(tag.getText())
break
show_list = []
common.error(token)
if hasattr(token, 'token'):
options = soup.select('#playURL > option')
for t in options:
all_title = t.getText()
show_url = t['value']
common.error(show_url)
show_url = __decodeUrl(show_url, token.hken, token.token)
common.error(show_url)
show_list.append((all_title, show_url, ''))
else:
titleTile = soup.find('title')
show_list.append((titleTile.getText(), token, ''))
return show_list
def get(self):
user = nextusermodel()
if not user:
common.error(self, 404, "user not found")
return
models.db.run_in_transaction(logdeletion, user.key())
customfeeds = models.CustomFeed.all().ancestor(user).order("time")
for cf in customfeeds:
if not cf.rss_link:
continue
ref = fetch(cf.rss_link).content
soup = Soup(ref)
dict_compilelist = []
message = u""
if cf.item_title_enable:
titles = common.selectortext(soup, cf.item_title_selector, cf.item_title_attr)
dlist_title = [("title", t) for t in titles]
dict_compilelist.append(dlist_title)
message += u"(title %d 個) " % (len(titles))
if cf.item_link_enable:
links = common.selectortext(soup, cf.item_link_selector, cf.item_link_attr)
dlist_link = [("link", l) for l in links]
dict_compilelist.append(dlist_link)
message += u"(links %d 個) " % (len(links))
if cf.item_description_enable:
descriptions = common.selectortext(soup, cf.item_description_selector, cf.item_description_attr)
dlist_description = [("description", d) for d in descriptions]
dict_compilelist.append(dlist_description)
message += u"(descriptions %d 個) " % (len(descriptions))
if cf.item_date_enable:
dates = common.selectortext(soup, cf.item_date_selector, cf.item_date_attr)
dlist_date = [("pubdate", dateparser(d)) for d in dates]
dict_compilelist.append(dlist_date)
message += u"(dates %d 個) " % (len(dates))
message += u"見つかりました。"
models.Log(feedname=cf.name, type=models.Log._types["info"], message=message, parent=user).put()
items = []
for dl in zip(*dict_compilelist):
d = {"title": "", "link": "", "description": ""}
d.update(dict(dl))
items.append(d)
feeddata = models.FeedData.get_by_key_name(cf.name, parent=cf)
if not feeddata:
feeddata = models.FeedData(parent=cf, key_name=cf.name)
try:
rss_title = cf.rss_title.encode("UTF-8")
rss_link = cf.rss_link.encode("UTF-8")
rss_description = cf.rss_description.encode("UTF-8")
feeddata.atom = common.buildatom("Anon", rss_title, rss_link, rss_description, items).decode("UTF-8")
feeddata.rss = common.buildrss("Anon", rss_title, rss_link, rss_description, items).decode("UTF-8")
feeddata.rdf = common.buildrdf("Anon", rss_title, rss_link, rss_description, items).decode("UTF-8")
if not feeddata.put():
raise ValueError # TODO: save Error
except:
message = u"何かエラーが発生しました。"
models.Log(feedname=cf.name, type=models.Log._types["error"], message=message, parent=user).put()
raise
else:
message = u"カスタムフィードの作成に成功しました。"
models.Log(feedname=cf.name, type=models.Log._types["success"], message=message, parent=user).put()
action='store',
default='%s/reports' % os.path.dirname(os.path.realpath(__file__)),
required=False)
ob_group.add_argument('-d', '--debug',
help='enable full traceback on exceptions',
action='store_true',
default=False,
required=False)
args = parser.parse_args()
config = '%s/etc/omnibus.conf' % os.path.dirname(os.path.realpath(__file__))
output_dir = args.output
DEBUG = args.debug
info('Using configuration file (%s) ...' % config)
info('Debug: %s' % DEBUG)
if os.path.exists(output_dir):
if not os.path.isdir(output_dir):
error('Specified report output location is not a directory; exiting ...')
sys.exit(1)
else:
info('Creating report output directory (%s) ...' % output_dir)
mkdir(output_dir)
console = Console()
console.cmdloop()
action='store',
default='%s/reports' % os.path.dirname(os.path.realpath(__file__)),
required=False)
ob_group.add_argument('-d', '--debug',
help='enable full traceback on exceptions',
action='store_true',
default=False,
required=False)
args = parser.parse_args()
config = '%s/etc/omnibus.conf' % os.path.dirname(os.path.realpath(__file__))
output_dir = args.output
DEBUG = args.debug
info('Using configuration file (%s) ...' % config)
info('Debug: %s' % DEBUG)
if os.path.exists(output_dir):
if not os.path.isdir(output_dir):
error('Specified report output location is not a directory; exiting ...')
sys.exit(1)
else:
info('Creating report output directory (%s) ...' % output_dir)
mkdir(output_dir)
console = Console()
console.cmdloop()