def setup_win_unicode_console(enable):
""""Enables or disables unicode display on windows."""
enable = to_bool(enable)
if ON_WINDOWS and win_unicode_console:
if enable:
win_unicode_console.enable()
else:
win_unicode_console.disable()
return enable
def setup_win_unicode_console(enable):
""""Enables or disables unicode display on windows."""
enable = to_bool(enable)
if ON_WINDOWS and win_unicode_console:
if enable:
win_unicode_console.enable()
else:
win_unicode_console.disable()
return enable
def main(argv):
# parse config file
configfile = './config.cfg'
if len(argv) == 2:
configfile = argv[1]
config = parse_config(configfile)
win_unicode_console.enable()
result = check_update(config)
win_unicode_console.disable()
return result
def setup_win_unicode_console(enable):
""""Enables or disables unicode display on windows."""
try:
import win_unicode_console
except ImportError:
win_unicode_console = False
enable = to_bool(enable)
if ON_WINDOWS and win_unicode_console:
if enable:
win_unicode_console.enable()
else:
win_unicode_console.disable()
return enable
def setup_win_unicode_console(enable):
""""Enables or disables unicode display on windows."""
try:
import win_unicode_console
except ImportError:
win_unicode_console = False
enable = to_bool(enable)
if ON_WINDOWS and win_unicode_console:
if enable:
win_unicode_console.enable()
else:
win_unicode_console.disable()
return enable
def deinit(self):
if self.stream is None:
raise RuntimeError('Console has not been initialized.')
self.cursor.show_cursor()
global orig_stdout
global orig_stderr
sys.stdout = orig_stdout
sys.stderr = orig_stderr
orig_stdout = None
orig_stderr = None
self.stream = None
self.wrapper = None
self.cursor = None
colorama.deinit()
win_unicode_console.disable()
# print get_linear_model_status('999999', filter='y', dl=30, ptype='low')
# print powerCompute_df(['300134','002171'], dtype='d',end=None, dl=10, filter='y')
# # print powerCompute_df(['601198', '002791', '000503'], dtype='d', end=None, dl=30, filter='y')
# print get_linear_model_status('999999', filter='y', dl=34, ptype='low', days=1)
# print pct.get_linear_model_status('601519', filter='y', dl=34, ptype='low', days=1)
# sys.exit()
import re
if cct.isMac():
cct.set_console(80, 19)
else:
cct.set_console(80, 19)
parser = parseArgmain()
parser.print_help()
if cct.get_os_system().find('win') >= 0:
import win_unicode_console
win_unicode_console.disable()
while 1:
try:
# log.setLevel(LoggerFactory.INFO)
# log.setLevel(LoggerFactory.DEBUG)
code = raw_input("code:")
args = parser.parse_args(code.split())
# if not code.lower() == 'q' and not code.lower() == 'quit' and not code.lower() == 'exit' and not code == 'q' and not code == 'e' and not str(args.code) == 'None' and (args.wencai == 'y' or re.match('[a-zA-Z]+',code) is not None or re.match('[ \u4e00 -\u9fa5]+',code) == None ):
# if not cct.get_os_system() == 'mac':
# import ipdb;ipdb.set_trace()
if not code.lower() == 'q' and not code.lower() == 'quit' and not code.lower() == 'exit' and not code == 'q' and not code == 'e' \
and not str(args.code) == 'None' and (args.wencai == 'y' and (re.match('[a-zA-Z]+',code) is None and re.match(ur"[\u4e00-\u9fa5]+",code) is not None ) ):
df = wcd.get_wencai_Market_url(code,200,pct=False)
print df.shape,df[:8]
if len(df) == 1:
def main():
"""
The primary CLI function for the Topic Explorer.
"""
if platform.system() == 'Windows':
win_unicode_console.enable()
# Create the master argparse object.
parser = ThrowingArgumentParser()
# Adding the benchmarks flags.
benchmark_group = parser.add_mutually_exclusive_group()
benchmark_group.add_argument('-t', '--time', help="Print execution time",
action='store_true')
benchmark_group.add_argument('-p', '--profile', help="""Profile the command.
Optional filename saves results for use with snakeviz, pstats, or
cprofilev. Automatically launches snakeviz, if installed.""",
nargs='?', metavar='STATS_FILE')
# Using add_subparsers(metavar) until argparse.SUPPRESS support is fixed.
# See issue http://bugs.python.org/issue22848
parsers = parser.add_subparsers(help="select a command",
parser_class=ArgumentParser,
metavar='{version,demo,update,init,prep,train,launch,notebook,metadata}')
version_parser = parsers.add_parser('version', help="Print the version and exit")
version_parser.set_defaults(func='version')
# Init Parser
parser_init = parsers.add_parser('init', help="Initialize the topic explorer")
init.populate_parser(parser_init)
parser_init.set_defaults(func="init")
# Prep Parser
parser_prep = parsers.add_parser('prep', help="Prep the corpus",
formatter_class=argparse.RawDescriptionHelpFormatter)
prep.populate_parser(parser_prep)
parser_prep.set_defaults(func="prep")
# Train Parser
parser_train = parsers.add_parser('train', help="Train the LDA models")
train.populate_parser(parser_train)
parser_train.set_defaults(func="train")
# Launch Parser
parser_launch = parsers.add_parser('launch', help="Serve the trained LDA models")
server.populate_parser(parser_launch)
parser_launch.set_defaults(func="launch")
# Serve Parser
parser_serve = parsers.add_parser('serve',
help="Serve a single LDA model, helper for `topicexplorer launch`," +
"rarely called directly")
server.populate_parser(parser_serve)
parser_serve.set_defaults(func="serve")
# Notebook Parser
parser_nb = parsers.add_parser('notebook',
help="Create a set of IPython Notebooks")
notebook.populate_parser(parser_nb)
parser_nb.set_defaults(func="notebook")
# Demo Parser
parser_demo = parsers.add_parser('demo',
help="Download and run the AP demo")
parser_demo.set_defaults(func="demo")
# Update Parser
parser_update = parsers.add_parser('update',
help="Update the Topic Explorer")
parser_update.set_defaults(func="update")
# Metadata Parser
parser_metadata = parsers.add_parser('metadata',
help="Add spaces before unicode chars")
metadata.populate_parser(parser_metadata)
parser_metadata.set_defaults(func="metadata")
# Export Parser
parser_export = parsers.add_parser('export', help="Export a tez archive")
export.populate_parser(parser_export)
parser_export.set_defaults(func="export")
# Export HTML Parser
parser_export_html = parsers.add_parser('export-html', help="Export the topic cluster visualization")
export_html.populate_parser(parser_export_html)
parser_export_html.set_defaults(func="export-html")
# Import Parser
parser_import = parsers.add_parser('import', help="Import the tez archive")
tezimport.populate_parser(parser_import)
parser_import.set_defaults(func="import")
# fancy arg validation for manually injecting tempfile to profile arg
try:
try:
args = parser.parse_args()
except ArgumentParserError as e:
import sys
new_args = sys.argv[1:]
try:
# If the error was thrown by the '-p' argument not having a
# valid file, fix by manually injecting a nargs break
profile = new_args.index('-p')
if (len(new_args) > (profile + 1) and
new_args[profile + 1] in parsers.choices.keys()):
new_args.insert(profile + 1, '-')
args = parser.parse_args(new_args)
else:
raise e
except ValueError:
raise e
except ArgumentParserError as e:
import sys
# Check to see if error occurs with a subparser and cause the exception
# to arise from the subparser instead
for p in parsers.choices.keys():
if p in sys.argv[1:]:
subargs_idx = sys.argv.index(p) + 1
subargs = sys.argv[subargs_idx:]
subparser = locals()['parser_' + p]
# this might cause an error in the subparser, in which case
# we actually want to show that error first
args = subparser.parse_args(subargs)
# Use the default error mechanism for the master parser.
# If the code gets here, it means the error was not in a subparser
ArgumentParser.error(parser, e)
if args.profile:
if args.profile == '-':
import tempfile
temphandle, args.profile = tempfile.mkstemp(suffix='.prof', prefix='vsm.')
print("Saving benchmark data to", args.profile)
from profilehooks import profile
def benchmark(fn):
return profile(fn, immediate=True, filename=args.profile, stdout=None)
elif args.time:
from profilehooks import timecall
def benchmark(fn):
return timecall(fn, immediate=False)
else:
def benchmark(fn):
return fn
if args.func == 'version':
from topicexplorer.version import __pretty_version__
print(__pretty_version__, end='')
elif args.func == 'init':
args.config_file = benchmark(init.main)(args)
print("\nTIP: Only initalizing corpus object and config file.")
print(" Next prepare the corpus using:")
print(" topicexplorer prep", args.config_file)
print(" Or skip directly to training LDA models using:")
print(" topicexplorer train", args.config_file)
elif args.func == 'prep':
benchmark(prep.main)(args)
print("\nTIP: Train the LDA models with:")
print(" topicexplorer train", args.config_file)
elif args.func == 'train':
benchmark(train.main)(args)
if not args.dry_run:
print("\nTIP: launch the topic explorer with:")
print(" topicexplorer launch", args.config_file)
print(" or the notebook server with:")
print(" topicexplorer notebook", args.config_file)
elif args.func == 'launch' or args.func == 'serve':
# Note that we are only benchmarking the creation process - obviously
# benches of the serve process will take longer
app = benchmark(server.create_app)(args)
server.main(args, app)
elif args.func == 'notebook':
benchmark(notebook.main)(args)
elif args.func == 'demo':
benchmark(demo.main)(args)
elif args.func == 'update':
benchmark(update.main)(args)
elif args.func == 'metadata':
benchmark(metadata.main)(args)
elif args.func == 'export':
benchmark(export.main)(args)
elif args.func == 'export-html':
benchmark(export_html.main)(args)
elif args.func == 'import':
benchmark(tezimport.main)(args)
if args.profile:
try:
import snakeviz.cli
print("\n\n")
snakeviz.cli.main([args.profile])
except ImportError:
print("""\nSnakeviz is not installed. Install with `pip install snakeviz`,
then run `snakeviz {}`.""".format(args.profile))
if platform.system() == 'Windows':
win_unicode_console.disable()
def main():
# Fix console for windows users
if platform.system() == 'Windows':
import win_unicode_console
win_unicode_console.enable()
args = docopt(__doc__, version=('lyrico ' + __version__))
# The check_config flag instructs the "Config.load_config" to skip the 'BadConfigError's.
# So only when user is running downloads, the config must be valid.
# When user is running cmds to update config, it will be always loaded
# regardless of values of the settings.
check_config = not (args['--settings'] or args['disable'] or args['enable']
or args['set'])
Config.load_config(check_config)
if not Config.is_loaded:
# Config not loaded due to exceptions. Error logged by exception handlers.
return
if args['--settings']:
# show current settings
Config.show_settings()
return
if args['disable'] or args['enable'] or args['set']:
# User is updating config
if args['set']:
# setting 'lyrics_dir' or 'source_dir'
# This general try catch block is intended for os.makedirs call if
# it raises OSError which is not due to directory already existing or
# some other error than OSError
try:
Config.set_dir(args['<dir_type>'], args['<full_path_to_dir>'])
except Exception as e:
print(e)
if args['enable'] or args['disable']:
# setting 'save_to_file', 'save_to_tag' or 'overwrite'.
# detect wether user wants to enable or disable a lyrico action
update_type = 'enable' if args['enable'] else 'disable'
Config.update_lyrico_actions(args['<lyrico_action>'], update_type)
else:
# User wants to download lyrics.
if args['<source_dir>']:
# if lyrico <source_dir> invocation is used:
# update user's "source_dir" in config
# update Config class' 'source_dir' class variable
# This general try catch block is intended for os.makedirs call if
# it raises OSError which is not due to directory already existing or
# some other error than OSError
try:
set_dir_success = Config.set_dir('source_dir',
args['<source_dir>'])
except Exception as e:
print(e)
# Don't go ahead with excution since user gave bad path or might have
# correct system settings?
return
# For this usage if user provides non existing dir, return by using boolean
# return value of Config.set_dir
if not set_dir_success:
return
# update class variable so that new setting is reflected across modules.
Config.source_dir = args['<source_dir>']
song_list = [
Song(song_path) for song_path in get_song_list(Config.source_dir)
]
print(len(song_list), 'songs detected.')
print('Metadata extracted for',
(str(Song.valid_metadata_count) + '/' + str(len(song_list))),
'songs.')
for song in song_list:
# Only download lyrics if 'title' and 'artist' is present
# Error str is already present in song.error
if song.artist and song.title:
song.download_lyrics()
# Show immidiate log in console
else:
# If title was present, use that
if song.title:
print(song.title, 'was ignored.', song.error)
# else use audio file path
else:
print(song.path, 'was ignored.', song.error)
print('\nBuilding log...')
Song.log_results(song_list)
print('FINISHED')
# Disable windows unicode console anyways
if platform.system() == 'Windows':
win_unicode_console.disable()
def main():
# Fix console for windows users
if platform.system() == 'Windows':
import win_unicode_console
win_unicode_console.enable()
args = docopt(__doc__, version = ('lyrico ' + __version__))
# The check_config flag instructs the "Config.load_config" to skip the 'BadConfigError's.
# So only when user is running downloads, the config must be valid.
# When user is running cmds to update config, it will be always loaded
# regardless of values of the settings.
check_config = not(args['--settings'] or args['disable'] or args['enable'] or args['set'])
Config.load_config(check_config)
if not Config.is_loaded:
# Config not loaded due to exceptions. Error logged by exception handlers.
return
if args['--settings']:
# show current settings
Config.show_settings()
return
if args['disable'] or args['enable'] or args['set']:
# User is updating config
if args['set']:
# setting 'lyrics_dir' or 'source_dir'
# This general try catch block is intended for os.makedirs call if
# it raises OSError which is not due to directory already existing or
# some other error than OSError
try:
Config.set_dir(args['<dir_type>'], args['<full_path_to_dir>'])
except Exception as e:
print(e)
if args['enable'] or args['disable']:
# setting 'save_to_file', 'save_to_tag' or 'overwrite'.
# detect wether user wants to enable or disable a lyrico action
update_type = 'enable' if args['enable'] else 'disable'
Config.update_lyrico_actions(args['<lyrico_action>'], update_type)
else:
# User wants to download lyrics.
if args['<source_dir>']:
# if lyrico <source_dir> invocation is used:
# update user's "source_dir" in config
# update Config class' 'source_dir' class variable
# This general try catch block is intended for os.makedirs call if
# it raises OSError which is not due to directory already existing or
# some other error than OSError
try:
set_dir_success = Config.set_dir('source_dir', args['<source_dir>'])
except Exception as e:
print(e)
# Don't go ahead with excution since user gave bad path or might have
# correct system settings?
return
# For this usage if user provides non existing dir, return by using boolean
# return value of Config.set_dir
if not set_dir_success:
return
# update class variable so that new setting is reflected across modules.
Config.source_dir = args['<source_dir>']
song_list = [Song(song_path) for song_path in get_song_list(Config.source_dir)]
print(len(song_list), 'songs detected.')
print('Metadata extracted for', (str(Song.valid_metadata_count) + '/' + str(len(song_list))), 'songs.')
for song in song_list:
# Only download lyrics if 'title' and 'artist' is present
# Error str is already present in song.error
if song.artist and song.title:
song.download_lyrics()
# Show immidiate log in console
else:
# If title was present, use that
if song.title:
print(song.title, 'was ignored.', song.error)
# else use audio file path
else:
print(song.path, 'was ignored.', song.error)
print('\nBuilding log...')
Song.log_results(song_list)
print('FINISHED')
# Disable windows unicode console anyways
if platform.system() == 'Windows':
win_unicode_console.disable()
def main():
# Fix console for windows users
if platform.system() == 'Windows':
import win_unicode_console
win_unicode_console.enable()
args = docopt(__doc__, version=('lyrico ' + __version__))
Config.load_config()
if args['--settings']:
# show current settings
Config.show_settings()
return
if args['set']:
# setting 'lyrics_dir' or 'source_dir'
# This general try catch block is intended for os.makedirs call if
# it raises OSError which is not due to directory already existing or
# some other error than OSError
try:
Config.set_dir(args['<dir_type>'], args['<full_path_to_dir>'])
Config.save()
except Exception as e:
print(e)
return
if args['enable'] or args['disable']:
# setting 'save_to_file', 'save_to_tag' or 'overwrite'.
# detect wether user wants to enable or disable a lyrico action
update_type = 'enable' if args['enable'] else 'disable'
Config.update_lyrico_actions(args['<lyrico_action>'], update_type)
Config.save()
return
# User wants to download lyrics.
if args['<source_dir>']:
# if lyrico <source_dir> invocation is used:
# update user's "source_dir" in config
# update Config class' 'source_dir' class variable
# This general try catch block is intended for os.makedirs call if
# it raises OSError which is not due to directory already existing or
# some other error than OSError
try:
set_dir_success = Config.set_dir('source_dir',
args['<source_dir>'])
except Exception as e:
print(e)
# Don't go ahead with excution since user gave bad path or might have
# correct system settings?
return
# For this usage if user provides non existing dir, return by using boolean
# return value of Config.set_dir
if not set_dir_success:
return
#settings changes are done, we need a valid config now
if not Config.check():
return
song_list = [
Song(song_path) for song_path in get_song_list(Config.source_dir)
]
print(len(song_list), 'songs detected.')
print('Metadata extracted for',
(str(Song.valid_metadata_count) + '/' + str(len(song_list))),
'songs.')
for song in song_list:
# Only download lyrics if 'title' and 'artist' is present
# Error str is already present in song.error
if song.artist and song.title:
song.download_lyrics()
# Show immidiate log in console
else:
# If title was present, use that
if song.title:
print(song.title, 'was ignored.', song.error)
# else use audio file path
else:
print(song.path, 'was ignored.', song.error)
print('\nBuilding log...')
Song.log_results(song_list)
print(
'{songs} songs, {tagged} tagged, {files} lyric files, {existing} existing, {errors} errors'
.format(songs=len(song_list),
tagged=Song.lyrics_saved_to_tag_count,
files=Song.lyrics_saved_to_file_count,
existing=Song.lyrics_existing_count,
errors=Song.lyrics_errors_count))
print('FINISHED')
# Disable windows unicode console anyways
if platform.system() == 'Windows':
win_unicode_console.disable()