def run():
cli = CLI()
try:
(options, args) = cli.parse()
if options.simple_db_migrate_version:
msg = 'simple-db-migrate v%s' % SIMPLE_DB_MIGRATE_VERSION
cli.info_and_exit(msg)
if options.show_colors:
CLI.show_colors()
# Create config
config = FileConfig(options.config_file, options.environment)
config.put('schema_version', options.schema_version)
config.put('show_sql', options.show_sql)
config.put('show_sql_only', options.show_sql_only)
config.put('new_migration', options.new_migration)
config.put('drop_db_first', options.drop_db_first)
config.put('paused_mode', options.paused_mode)
config.put('log_dir', options.log_dir)
config.put('label_version', options.label_version)
config.put('force_use_files_on_down', options.force_use_files_on_down)
config.put('force_execute_old_migrations_versions',
options.force_execute_old_migrations_versions)
# paused mode forces log_level to 2
log_level = int(options.log_level)
if options.paused_mode:
log_level = 2
config.put('log_level', log_level)
# Ask the password for user if configured
if config.get('db_password') == '<<ask_me>>':
if options.password:
passwd = options.password
else:
cli.msg(
'\nPlease inform password to connect to database "%s@%s:%s"'
% (config.get('db_user'), config.get('db_host'),
config.get('db_name')))
passwd = getpass()
config.remove('db_password')
config.put('db_password', passwd)
# If CLI was correctly parsed, execute db-migrate.
Main(config).execute()
except KeyboardInterrupt:
cli.info_and_exit("\nExecution interrupted by user...")
except Exception, e:
cli.error_and_exit(str(e))
def __init__(self, section_name: str = 'DB'):
self.db = None
config = FileConfig()
try:
self.db = connect(host=config.get(f'{section_name}.host'),
user=config.get(f'{section_name}.user'),
passwd=config.get(f'{section_name}.pass'),
database=config.get(f'{section_name}.name'))
except ProgrammingError as e:
print(f'[DB] [ERROR] {e.msg}')
raise e
def run():
cli = CLI()
try:
(options, args) = cli.parse()
if options.simple_db_migrate_version:
msg = 'simple-db-migrate v%s' % SIMPLE_DB_MIGRATE_VERSION
cli.info_and_exit(msg)
if options.show_colors:
CLI.show_colors()
# Create config
config = FileConfig(options.config_file)
config.put('schema_version', options.schema_version)
config.put('show_sql', options.show_sql)
config.put('show_sql_only', options.show_sql_only)
config.put('new_migration', options.new_migration)
config.put('drop_db_first', options.drop_db_first)
config.put('paused_mode', options.paused_mode)
config.put('log_dir', options.log_dir)
# paused mode forces log_level to 2
log_level = int(options.log_level)
if options.paused_mode:
log_level = 2
config.put('log_level', log_level)
# Ask the password for user if configured
if config.get('db_password') == '<<ask_me>>':
cli.msg('\nPlease inform password to connect to database "%s@%s:%s"' % (config.get('db_user'), config.get('db_host'), config.get('db_name')))
passwd = getpass()
config.remove('db_password')
config.put('db_password', passwd)
# If CLI was correctly parsed, execute db-migrate.
Main(config).execute()
except KeyboardInterrupt:
cli.info_and_exit("\nExecution interrupted by user...")
except Exception, e:
cli.error_and_exit(str(e))
def __init__(self, fast: int, slow: int, signal: int, period: int):
self.db = None
config = FileConfig()
try:
self.db = connect(host=config.get('DB.host'),
user=config.get('DB.user'),
passwd=config.get('DB.pass'),
database=config.get('DB.name'))
except ProgrammingError as e:
print(f'[DB] [ERROR] {e.msg}')
raise e
self.fast = fast
self.slow = slow
self.signal = signal
self.period = period
self.__table = '_'.join(
map(str, ['a', self.fast, self.signal, self.period]))
self.__key = '_'.join(
map(str, [self.fast, self.slow, self.signal, self.period]))
def run(options):
try:
if options.get('simple_db_migrate_version'):
msg = 'simple-db-migrate v%s' % SIMPLE_DB_MIGRATE_VERSION
CLI.info_and_exit(msg)
if options.get('show_colors'):
CLI.show_colors()
# Create config
if options.get('config_file'):
config = FileConfig(options.get('config_file'), options.get('environment'))
else:
config = Config()
config.update('schema_version', options.get('schema_version'))
config.update('show_sql', options.get('show_sql'))
config.update('show_sql_only', options.get('show_sql_only'))
config.update('new_migration', options.get('new_migration'))
config.update('drop_db_first', options.get('drop_db_first'))
config.update('paused_mode', options.get('paused_mode'))
config.update('log_dir', options.get('log_dir'))
config.update('label_version', options.get('label_version'))
config.update('force_use_files_on_down', options.get('force_use_files_on_down'))
config.update('force_execute_old_migrations_versions', options.get('force_execute_old_migrations_versions'))
config.update('utc_timestamp', options.get('utc_timestamp'))
config.update('database_user', options.get('database_user'))
config.update('database_password', options.get('database_password'))
config.update('database_host', options.get('database_host'))
config.update('database_name', options.get('database_name'))
if options.get('database_migrations_dir'):
config.update("database_migrations_dir", Config._parse_migrations_dir(options.get('database_migrations_dir')))
config.update('database_engine', options.get('database_engine'))
if not config.get('database_engine', None):
config.update('database_engine', "mysql")
config.update('database_version_table', options.get('database_version_table'))
if not config.get('database_version_table', None):
config.update('database_version_table', "__db_version__")
# paused mode forces log_level to 2
log_level = int(options.get('log_level'))
if options.get('paused_mode'):
log_level = 2
config.update('log_level', log_level)
# Ask the password for user if configured
if config.get('database_password') == '<<ask_me>>':
if options.get('password'):
passwd = options.get('password')
else:
CLI.msg('\nPlease inform password to connect to database "%s@%s:%s"' % (config.get('database_user'), config.get('database_host'), config.get('database_name')))
passwd = getpass()
config.update('database_password', passwd)
# If CLI was correctly parsed, execute db-migrate.
Main(config).execute()
except KeyboardInterrupt:
CLI.info_and_exit("\nExecution interrupted by user...")
except Exception, e:
CLI.error_and_exit(str(e))
session.query(Backtest).filter(Backtest.id == bid).update({
'ts_start': start,
'ts_end': end,
'data': f'{dumps(dict(statistics=statistics))}',
'total_month6': float(statistics['4']['total'] - statistics['4']['fees']),
'total_month3': float(statistics['3']['total'] - statistics['3']['fees']),
'total_month1': float(statistics['2']['total'] - statistics['2']['fees']),
'total_week': float(statistics['1']['total'] - statistics['1']['fees'])
})
session.commit()
session.close()
@staticmethod
def is_calculated(p_in, p_out, start, end):
SQL = f'SELECT COUNT(`id`) FROM `main_backtest` ' \
f'WHERE `buy_fast` = {p_in[0]} AND `buy_slow` = {p_in[1]} ' \
f'AND `buy_signal` = {p_in[2]} AND `buy_period` = {p_in[3]} ' \
f'AND `sell_fast` = {p_out[0]} AND `sell_slow` = {p_out[1]} ' \
f'AND `sell_signal` = {p_out[2]} AND `sell_period` = {p_out[3]} ' \
f'AND `ts_end` = {end} ' \
f'AND `active` > 0'
session = Backtest.__get_session()
c, = session.execute(SQL).fetchone()
session.close()
return c >= 2
Backtest.CON_STRING = config.get('APP.BT_CONNECTION_STRING', '')
else:
ModelBacktest.do_update(last.id, start, end, statistics[_type])
else:
print(f'Can not calculate statistics for {[p_in, p_out]}')
del b
print(f'{[p_in, p_out]}: {time() - _s}')
if __name__ == '__main__':
config = FileConfig()
end = 1537514011#time()
end -= end%ONE_DAY
start = end - 180*ONE_DAY
process_count = config.get('APP.POOL_PROCESSES', 4, int)
max_tasks = config.get('APP.POOL_TASK_PER_CHILD', 10, int)
use_pool = config.get('APP.USE_POOL', True, bool)
start_at = time()
if use_pool:
pool = Pool(processes=process_count, maxtasksperchild=max_tasks)
pool.starmap(processor2, generator(start, end))
pool.close()
pool.join()
else:
for item in generator(start, end):
processor2(*item)
print(f'Done in {time() - start_at}s')
def run(options):
""" Initial Module. Treat Parameters and call Main Module for execution """
try:
if options.get('simple_virtuoso_migrate_version'):
msg = ('simple-virtuoso-migrate v%s' %
SIMPLE_VIRTUOSO_MIGRATE_VERSION)
CLI.info_and_exit(msg)
if options.get('show_colors'):
CLI.show_colors()
# Create config
if options.get('config_file'):
config = FileConfig(options.get('config_file'),
options.get('environment'))
else:
config = Config()
config.update('schema_version', options.get('schema_version'))
config.update('show_sparql', options.get('show_sparql'))
config.update('show_sparql_only', options.get('show_sparql_only'))
config.update('file_migration', options.get('file_migration'))
#config.update('add_ttl', options.get('add_ttl'))
config.update('load_ttl', options.get('load_ttl'))
config.update('log_dir', options.get('log_dir'))
config.update('database_user', options.get('database_user'))
config.update('database_password', options.get('database_password'))
config.update('host_user', options.get('host_user'))
config.update('host_password', options.get('host_password'))
config.update('virtuoso_dirs_allowed',
options.get('virtuoso_dirs_allowed'))
config.update('database_host', options.get('database_host'))
config.update('database_port', options.get('database_port'))
config.update('database_endpoint', options.get('database_endpoint'))
config.update('database_graph', options.get('database_graph'))
config.update('database_ontology', options.get('database_ontology'))
if options.get('database_migrations_dir'):
config.update("database_migrations_dir",
Config._parse_migrations_dir(
options.get(
'database_migrations_dir')))
config.update("database_migrations_dir",
config.get("database_migrations_dir")[0])
config.update('log_level', int(options.get('log_level')))
# Ask the password for user if configured
if config.get('database_password') == '<<ask_me>>':
CLI.msg('\nPlease inform password to connect to virtuoso (DATABASE) "%s@%s"' % (config.get('database_user'),
config.get('database_host')))
passwd = getpass()
config.update('database_password', passwd)
is_local = config.get('database_host', '').lower() in ["localhost",
"127.0.0.1"]
if config.get('load_ttl') and\
config.get('virtuoso_dirs_allowed') is None and\
not is_local:
if config.get('host_password') == '<<ask_me>>':
msg = '\nPlease inform password to connect to virtuoso (HOST) "%s@%s"'
CLI.msg(msg % (config.get('host_user'),
config.get('database_host')))
passwd = getpass()
config.update('host_password', passwd)
# If CLI was correctly parsed, execute db-virtuoso.
Main(config).execute()
except KeyboardInterrupt:
CLI.info_and_exit("\nExecution interrupted by user...")
except Exception, e:
CLI.error_and_exit(unicode(e))
def run(options):
try:
if options.get('simple_db_migrate_version'):
msg = 'simple-db-migrate v%s' % SIMPLE_DB_MIGRATE_VERSION
CLI.info_and_exit(msg)
if options.get('show_colors'):
CLI.show_colors()
# Create config
if options.get('config_file') or os.path.exists('simple-db-migrate.conf'):
config = FileConfig(options.get('config_file') or 'simple-db-migrate.conf', options.get('environment'))
else:
config = Config()
config.update('schema_version', options.get('schema_version'))
config.update('show_sql', options.get('show_sql'))
config.update('show_sql_only', options.get('show_sql_only'))
config.update('new_migration', options.get('new_migration'))
config.update('drop_db_first', options.get('drop_db_first'))
config.update('paused_mode', options.get('paused_mode'))
config.update('log_dir', options.get('log_dir'))
config.update('label_version', options.get('label_version'))
config.update('force_use_files_on_down', options.get('force_use_files_on_down'))
config.update('force_execute_old_migrations_versions', options.get('force_execute_old_migrations_versions'))
config.update('utc_timestamp', options.get('utc_timestamp'))
config.update('database_user', options.get('database_user'))
config.update('database_password', options.get('database_password'))
config.update('database_host', options.get('database_host'))
config.update('database_port', options.get('database_port'))
config.update('database_name', options.get('database_name'))
if config.get('database_port', None):
config.update('database_port', int(config.get('database_port')))
if options.get('database_migrations_dir'):
config.update("database_migrations_dir", Config._parse_migrations_dir(options.get('database_migrations_dir')))
config.update('database_engine', options.get('database_engine'))
if not config.get('database_engine', None):
config.update('database_engine', "mysql")
config.update('database_version_table', options.get('database_version_table'))
if not config.get('database_version_table', None):
config.update('database_version_table', "__db_version__")
# paused mode forces log_level to 2
log_level = int(options.get('log_level'))
if options.get('paused_mode'):
log_level = 2
config.update('log_level', log_level)
# Ask the password for user if configured
if config.get('database_password') == '<<ask_me>>':
if options.get('password'):
passwd = options.get('password')
else:
CLI.msg('\nPlease inform password to connect to database "%s@%s:%s"' % (config.get('database_user'), config.get('database_host'), config.get('database_name')))
passwd = getpass()
config.update('database_password', passwd)
if options.get('info_database'):
if options.get('info_database').lower() == 'last_label':
CLI.info_and_exit(Main(config).last_label() or "NONE")
elif options.get('info_database').lower() == 'labels':
labels = Main(config).labels()
CLI.info_and_exit(labels and "\n".join(labels) or "NONE")
else:
CLI.error_and_exit("The '%s' is a wrong parameter for info" % options.get('info_database').lower())
# If CLI was correctly parsed, execute db-migrate.
Main(config).execute()
except KeyboardInterrupt:
CLI.info_and_exit("\nExecution interrupted by user...")
except Exception, e:
CLI.error_and_exit(str(e))
def run(options):
""" Initial Module. Treat Parameters and call Main Module for execution """
try:
if options.get("simple_virtuoso_migrate_version"):
msg = "simple-virtuoso-migrate v%s" % SIMPLE_VIRTUOSO_MIGRATE_VERSION
CLI.info_and_exit(msg)
if options.get("show_colors"):
CLI.show_colors()
# Create config
if options.get("config_file"):
config = FileConfig(options.get("config_file"), options.get("environment"))
else:
config = Config()
config.update("schema_version", options.get("schema_version"))
config.update("show_sparql", options.get("show_sparql"))
config.update("show_sparql_only", options.get("show_sparql_only"))
config.update("file_migration", options.get("file_migration"))
# config.update('add_ttl', options.get('add_ttl'))
config.update("load_ttl", options.get("load_ttl"))
config.update("log_dir", options.get("log_dir"))
config.update("database_user", options.get("database_user"))
config.update("database_password", options.get("database_password"))
config.update("host_user", options.get("host_user"))
config.update("host_password", options.get("host_password"))
config.update("virtuoso_dirs_allowed", options.get("virtuoso_dirs_allowed"))
config.update("database_host", options.get("database_host"))
config.update("database_port", options.get("database_port"))
config.update("database_endpoint", options.get("database_endpoint"))
config.update("database_graph", options.get("database_graph"))
config.update("database_ontology", options.get("database_ontology"))
if options.get("database_migrations_dir"):
config.update(
"database_migrations_dir", Config._parse_migrations_dir(options.get("database_migrations_dir"))
)
config.update("database_migrations_dir", config.get("database_migrations_dir")[0])
config.update("log_level", int(options.get("log_level")))
# Ask the password for user if configured
if config.get("database_password") == "<<ask_me>>":
CLI.msg(
'\nPlease inform password to connect to\
virtuoso (DATABASE) "%s@%s"'
% (config.get("database_user"), config.get("database_host"))
)
passwd = getpass()
config.update("database_password", passwd)
is_local = config.get("database_host", "").lower() in ["localhost", "127.0.0.1"]
# import pdb; pdb.set_trace()
if config.get("load_ttl") and config.get("virtuoso_dirs_allowed") is None and not is_local:
if config.get("host_password") == "<<ask_me>>":
CLI.msg(
'\nPlease inform password to connect to\
virtuoso (HOST) "%s@%s"'
% (config.get("host_user"), config.get("database_host"))
)
passwd = getpass()
config.update("host_password", passwd)
# If CLI was correctly parsed, execute db-virtuoso.
Main(config).execute()
except KeyboardInterrupt:
CLI.info_and_exit("\nExecution interrupted by user...")
except Exception, e:
CLI.error_and_exit(unicode(e))
