def list(self):
list_xml = urlopen(self.list_xml_url + self.region + '.xml')
soup = BeautifulSoup(list_xml)
names = soup.find_all('name')
ids = soup.find_all('id')
stations = {name.string: id.string for name, id in zip(names, ids)}
print 'Your region code is "' + self.region + '", and stations in your region are:'
print pp_str(stations)
def dump(self, an_obj):
"""Prints a debug value without misconversion.
Prints a debug value without a misconversion for UTF-8 strings
in a list or a dict class, because it's not supported by Python.
Args:
an_obj: An instance of a list or a dict class.
"""
sys.stderr.write(prettyprint.pp_str(an_obj) + '\n')
def testSet(self):
set1 = set(['John', 'Jane', 'Jack', 'Janice'])
pp(set1)
expected='''
[
"Jane",
"Janice",
"John",
"Jack"
]
'''.strip()
self.assertEqual(pp_str(set1), expected)
def WriteData(data, sDataPyFilePath):
'''
将python数据写入data.py
:param data: Python数据对象
:param sDataPyFilePath: 导出py文件绝对路径
:return: NULL
'''
import prettyprint
sData = prettyprint.pp_str(data)
f = open(sDataPyFilePath, "w")
f.write(OUT_TEMPLATE % sData)
f.close()
def run(self):
# [(Exception, func), ]
failed_funcs = []
for func in self.functions:
try:
self._run_job(*func)
except Exception as e:
logger.exception('Jobs failed: func={}'.format(func))
failed_funcs.append((e, func))
if failed_funcs:
raise Exception('Jobs failed {}'
.format(prettyprint.pp_str(failed_funcs)))
def testObject(self):
class MyClass(object):
def __str__(self):
return "<MyClass>"
ls = list([1, MyClass()])
pp(ls)
expected='''
[
1,
"<MyClass>"
]
'''.strip()
self.assertEqual(pp_str(ls), expected)
def nico_video_post(self, search_keyword, prev_datetime):
logger.debug("Call nico_video_post({}, {})".format(search_keyword, prev_datetime))
if prev_datetime:
now_date = datetime.datetime.now()
if now_date - prev_datetime < datetime.timedelta(1):
old_prev_datetime = prev_datetime
prev_datetime = prev_datetime - datetime.timedelta(hours=2)
logger.debug("Change prev_datetime: {} -> {}".format(old_prev_datetime, prev_datetime))
with DbManager() as db_manager:
nico = NicoSearch(db_manager, self.nico_user_id, self.nico_pass_word)
nico.login()
# Search latest videos by NicoNico.
videos = nico.search_videos(search_keyword, prev_datetime)
tweet_count = 0
failed_list = []
for video in reversed(videos):
# Check if the video is already posted.
old_post_video = (
db_manager.db_session.query(PostVideo)
.filter(sqlalchemy.and_(PostVideo.video_id == video.id))
.first()
)
if old_post_video:
logger.debug("Skip posted video: video={}".format(video))
continue
# Add new post_video to database when not registerd
post_video = PostVideo(video.id)
logger.info("Add new post_video to database : post_video={}".format(post_video))
db_manager.db_session.add(post_video)
# Make message for twitter.
str_first_retrieve = video.first_retrieve.strftime("%y/%m/%d %H:%M")
msg = utils.make_tweet_msg(
self.TW_NICO_VIDEO_TWEET_FORMAT, str_first_retrieve, title=video.title, url=video.get_url()
)
try:
self.tweet_msg(msg, is_sleep=True)
tweet_count += 1
db_manager.db_session.commit()
except Exception as e:
db_manager.db_session.rollback()
logger.exception("Tweet failed msg={}".format(msg))
failed_list.append((e, msg))
logger.info("nico_latest_commenting_video(): {} tweet".format(tweet_count))
if failed_list:
raise Exception("Tweet faild {}".format(prettyprint.pp_str(failed_list)))
def tweet_msgs(self, msgs):
if not msgs:
logger.debug("No tweet messages")
return
# [(Exception, tweet), ]
tweet_count = 0
failed_list = []
for msg in msgs:
try:
self.tweet_msg(msg, is_sleep=True)
tweet_count += 1
except Exception as e:
logger.exception("Tweet failed msg={}".format(msg))
failed_list.append((e, msg))
logger.info("nico_latest_commenting_video(): {} tweet".format(tweet_count))
if failed_list:
raise Exception("Tweet faild {}".format(prettyprint.pp_str(failed_list)))
def nico_latest_commenting_video_post(
self, search_keyword, prev_datetime, number_of_results=3, expire_days=30, max_post_count=1
):
logger.debug(
"Call nico_latest_commenting_video_post({}, {}, {}, {}, {})".format(
search_keyword, prev_datetime, number_of_results, expire_days, max_post_count
)
)
with DbManager() as db_manager:
nico = NicoSearch(db_manager, self.nico_user_id, self.nico_pass_word)
nico.login()
# Search latest commenting videos by NicoNico.
it = nico.search_latest_commenting_videos(
search_keyword, prev_datetime, number_of_results, expire_days, max_post_count
)
tweet_count = 0
failed_list = []
for video in it:
# Make message to tweet.
str_first_retrieve = video.first_retrieve.strftime("%y/%m/%d %H:%M")
msg = utils.make_tweet_msg(
self.TW_NICO_DETAIL_VIDEO_TWEET_FORMAT,
str_first_retrieve,
video.view_counter,
video.num_res,
video.mylist_counter,
title=video.title,
url=video.get_url(),
)
try:
self.tweet_msg(msg, is_sleep=True)
tweet_count += 1
db_manager.db_session.commit()
except Exception as e:
db_manager.db_session.rollback()
logger.exception("Tweet failed msg={}".format(msg))
failed_list.append((e, msg))
logger.info("nico_latest_commenting_video(): {} tweet".format(tweet_count))
if failed_list:
raise Exception("Tweet faild {}".format(prettyprint.pp_str(failed_list)))
def testNestedSet(self):
set1 = list([6, set([2,1,3]), 5,[3,1,2], None])
pp(set1)
expected='''
[
6,
[
1,
2,
3
],
5,
[
3,
1,
2
],
null
]
'''.strip()
self.assertEqual(pp_str(set1), expected)
def nico_comment_post(
self, search_keyword, prev_datetime, max_comment_num=1500, max_tweet_num_per_video=3, filter_func=None
):
logger.debug(
"Call nico_comment_post({}, {}, {}, {}, {})".format(
search_keyword, prev_datetime, max_comment_num, max_tweet_num_per_video, filter_func
)
)
with DbManager() as db_manager:
nico = NicoSearch(db_manager, self.nico_user_id, self.nico_pass_word)
nico.login()
# Search latest comments by NicoNico.
videos = nico.search_videos_with_comments(search_keyword, prev_datetime, max_comment_num)
tweet_count = 0
failed_list = []
for video in videos:
if filter_func and filter_func(video):
continue
for nico_comment in video.get_latest_comments(max_tweet_num_per_video):
# Make message for twitter.
msg = utils.make_tweet_msg(
self.TW_NICO_COMMENT_TWEET_FORMAT,
nico_comment.vpos,
nico_comment.post_datetime,
comment=nico_comment.comment,
title=video.title,
url=video.get_url(),
)
try:
self.tweet_msg(msg, is_sleep=True)
tweet_count += 1
except Exception as e:
logger.exception("Tweet failed msg={}".format(msg))
failed_list.append((e, msg))
logger.info("nico_latest_commenting_video(): {} tweet".format(tweet_count))
if failed_list:
raise Exception("Tweet faild {}".format(prettyprint.pp_str(failed_list)))
def testTuple(self): orig = (u'output prettily', u'綺麗に出力せよ') res = pp_str(orig) self.assertEqual(orig[0], json.loads(res)[0]) self.assertEqual(orig[1], json.loads(res)[1])
type=int,
help='Timeout decay (seconds)')
parser.add_argument('--sleep',
default=sleep,
type=float,
help='Loop sleep time (seconds)')
parser.add_argument('--debug',
default=debug,
type=int,
help='Debug Level (0-{})'.format(D_FULL))
args = parser.parse_args()
if args.debug >= D_DEBUG: print('ARGS=', pp_str(args))
timeoutDelta = timedelta(seconds=args.timeout)
scanner = Scanner()
if not scanner.isOpen(): scanner.discover()
if not scanner.isOpen():
print('Scanner did not initialize!')
raise RuntimeError
if args.scanmode != scanmode:
cmd = 'JPM,SVC_MODE,' + Scanner.ServiceModes[args.scanmode]
if args.debug >= D_DEBUG: print('Sending:', cmd)
resp = scanner.cmd(cmd, cooked=Scanner.DECODED)
if cmd.startswith('`'): # A '`' requests a RAW output
cmd = cmd[1:]
cookit = Scanner.RAW
print('Asking for RAW')
elif cmd.startswith('~'): # A '~' requests a DECODED output
cmd = cmd[1:]
cookit = Scanner.DECODED
print('Asking for DECODED')
else:
cookit = Scanner.COOKED # Default is COOKED
print('Asking for COOKED')
# Convert lower->UPPER, Mixed->Mixed
cmd = ','.join([a.upper() if a.islower() else a for a in cmd.split(',')])
ans = S.cmd(cmd, cookit)
if isinstance(ans, dict):
print('The {cmd} command returned:'.format(cmd = cmd))
print(pp_str(ans))
else: print('The {cmd} command returned: {ans}'.
format(cmd = cmd, ans = ans))
# Check for exit programming mode and return to scanning
if (cookit == Scanner.COOKED and ans == 'EPG,OK') or \
(cookit == Scanner.RAW and ans == b'EPG,OK'): # We just exited Program Mode, restart scanning
print('Restarting scanning...')
ans = S.cmd('JPM,SCN_MODE,0')
if ans == 'JPM,OK': print('Scanning restarted.')
else: print('Error ({ans}) restarting scanning.'.format(ans = ans))
if cmd.startswith('`'): # A '`' requests a RAW output
cmd = cmd[1:]
cookit = Scanner.RAW
print('Asking for RAW')
elif cmd.startswith('~'): # A '~' requests a DECODED output
cmd = cmd[1:]
cookit = Scanner.DECODED
print('Asking for DECODED')
else:
cookit = Scanner.COOKED # Default is COOKED
print('Asking for COOKED')
# Convert lower->UPPER, Mixed->Mixed
cmd = ','.join([a.upper() if a.islower() else a for a in cmd.split(',')])
ans = S.cmd(cmd, cookit)
if isinstance(ans, dict):
print('The {cmd} command returned:'.format(cmd=cmd))
print(pp_str(ans))
else:
print('The {cmd} command returned: {ans}'.format(cmd=cmd, ans=ans))
# Check for exit programming mode and return to scanning
if (cookit == Scanner.COOKED and ans == 'EPG,OK') or \
(cookit == Scanner.RAW and ans == b'EPG,OK'): # We just exited Program Mode, restart scanning
print('Restarting scanning...')
ans = S.cmd('JPM,SCN_MODE,0')
if ans == 'JPM,OK': print('Scanning restarted.')
else: print('Error ({ans}) restarting scanning.'.format(ans=ans))
all_words |= set(words)
file_words_index[os.path.basename(file)] = words
# Select 1000 top words
all_words = sorted(list(all_words))
all_words = all_words[:1000]
word_files_index = {}
# Invert words and files
for idx, word in enumerate(all_words):
print('{} of {}. Processing word {}'.format(idx + 1, len(all_words), word.encode('utf-8')))
print('=' * 30)
files = []
for file, words in file_words_index.items():
if word in words:
files.append(file)
word_files_index[word] = sorted(set(files))
filename = '1_2_index.txt'
with open(filename, 'wb') as f:
print('Writing to file {}'.format(filename))
str_word_files_index = prettyprint.pp_str(word_files_index)
f.write(str_word_files_index)
print('Done!')
def testList(self): orig = [u'output prettily', u'綺麗に出力せよ'] res = pp_str(orig) self.assertEqual(orig, json.loads(res))
def testDict(self):
orig = {'en':u'output prettily', 'ja':u'綺麗に出力せよ'}
res = pp_str(orig)
self.assertEqual(orig, json.loads(res))
type = int,
help = 'Timeout decay (seconds)')
parser.add_argument('--sleep',
default = sleep,
type = float,
help = 'Loop sleep time (seconds)')
parser.add_argument('--debug',
default = debug,
type = int,
help = 'Debug Level (0-{})'.format(D_FULL))
args = parser.parse_args()
if args.debug >= D_DEBUG: print('ARGS=', pp_str(args))
timeoutDelta = timedelta(seconds = args.timeout)
scanner = Scanner()
if not scanner.isOpen(): scanner.discover()
if not scanner.isOpen():
print('Scanner did not initialize!')
raise RuntimeError
if args.scanmode != scanmode:
cmd = 'JPM,SVC_MODE,' + Scanner.ServiceModes[args.scanmode]
if args.debug >= D_DEBUG: print('Sending:', cmd)
resp = scanner.cmd(cmd, cooked = Scanner.DECODED)
