class CacheDatabase():
_index_key = 'date'
def __init__(self, path):
self.db = Database(path)
if not self.db.exists():
index = CacheIndex(self.db.path, self._index_key)
self.db.create()
self.db.add_index(index)
else:
self.db.open()
def insert(self, date, record):
self.db.insert(dict(date=date, record=record))
def insert_all(self, records):
now = mktime(gmtime())
for record in records:
self.insert(now, record)
def since(self, date):
return self.db.get_many(self._index_key,
limit=-1,
inclusive_start=False,
with_doc=True,
start=date)
class CacheDatabase():
_index_key = 'date'
def __init__(self, path):
self.db = Database(path)
if not self.db.exists():
index = CacheIndex(self.db.path, self._index_key)
self.db.create()
self.db.add_index(index)
else:
self.db.open()
def insert(self, date, record):
self.db.insert(dict(date=date, record=record))
def insert_all(self, records):
now = mktime(gmtime())
for record in records:
self.insert(now, record)
def since(self, date):
return self.db.get_many(
self._index_key, limit=-1, inclusive_start=False, with_doc=True,
start=date
)
def __init__(self, name=''):
self.name = name
self.old_db_path = ''
self.db_path = os.path.join(sickrage.app.data_dir, 'database',
self.name)
self.db = SuperThreadSafeDatabase(self.db_path)
def __init__(self, name=''):
self.name = name
self.old_db_path = ''
self.db_path = os.path.join(sickrage.DATA_DIR, 'database', self.name)
self.db = SuperThreadSafeDatabase(self.db_path)
if self.db.exists():
self.db.open()
def __init__(self, path=None):
super(CodernityDbStorage, self).__init__()
self.path = path
self.database = SuperThreadSafeDatabase(path)
if os.path.exists(path):
self.database.open()
def __init__(self, path):
self.db = Database(path)
if not self.db.exists():
index = CacheIndex(self.db.path, self._index_key)
self.db.create()
self.db.add_index(index)
else:
self.db.open()
def __init__(self, path):
self.db = Database(path)
if not self.db.exists():
index = CacheIndex(self.db.path, self._index_key)
self.db.create()
self.db.add_index(index)
else:
self.db.open()
def __init__(self, path=None):
super(CodernityDbStorage, self).__init__()
self.path = path
self.database = SuperThreadSafeDatabase(path)
if os.path.exists(path):
self.database.open()
class srDatabase(object):
_indexes = {}
_migrate_list = {}
def __init__(self, name=''):
self.name = name
self.old_db_path = ''
self.db_path = os.path.join(sickrage.app.data_dir, 'database', self.name)
self.db = SuperThreadSafeDatabase(self.db_path)
def initialize(self):
# Remove database folder if both exists
if self.db.exists() and os.path.isfile(self.old_db_path):
self.db.open()
self.db.destroy()
if self.db.exists():
self.backup()
self.db.open()
else:
self.db.create()
# setup database indexes
self.setup_indexes()
def backup(self):
# Backup before start and cleanup old backups
backup_path = os.path.join(sickrage.app.data_dir, 'db_backup', self.name)
backup_count = 5
existing_backups = []
if not os.path.isdir(backup_path):
os.makedirs(backup_path)
for root, dirs, files in os.walk(backup_path):
# Only consider files being a direct child of the backup_path
if root == backup_path:
for backup_file in sorted(files):
ints = re.findall('\d+', backup_file)
# Delete non zip files
if len(ints) != 1:
try:
os.remove(os.path.join(root, backup_file))
except:
pass
else:
existing_backups.append((int(ints[0]), backup_file))
else:
# Delete stray directories.
shutil.rmtree(root)
# Remove all but the last 5
for eb in existing_backups[:-backup_count]:
os.remove(os.path.join(backup_path, eb[1]))
# Create new backup
new_backup = os.path.join(backup_path, '%s.tar.gz' % int(time.time()))
with tarfile.open(new_backup, 'w:gz') as zipf:
for root, dirs, files in os.walk(self.db_path):
for zfilename in files:
zipf.add(os.path.join(root, zfilename),
arcname='database/%s/%s' % (
self.name,
os.path.join(root[len(self.db_path) + 1:], zfilename))
)
def compact(self, try_repair=True, **kwargs):
# Removing left over compact files
for f in os.listdir(self.db.path):
for x in ['_compact_buck', '_compact_stor']:
if f[-len(x):] == x:
os.unlink(os.path.join(self.db.path, f))
try:
start = time.time()
size = float(self.db.get_db_details().get('size', 0))
sickrage.app.log.info(
'Compacting {} database, current size: {}MB'.format(self.name, round(size / 1048576, 2)))
self.db.compact()
new_size = float(self.db.get_db_details().get('size', 0))
sickrage.app.log.info(
'Done compacting {} database in {}s, new size: {}MB, saved: {}MB'.format(
self.name, round(time.time() - start, 2),
round(new_size / 1048576, 2), round((size - new_size) / 1048576, 2))
)
except (IndexException, AttributeError, TypeError) as e:
if try_repair:
sickrage.app.log.debug('Something wrong with indexes, trying repair')
# Remove all indexes
old_indexes = self._indexes.keys()
for index_name in old_indexes:
try:
self.db.destroy_index(index_name)
except IndexNotFoundException:
pass
except:
sickrage.app.log.debug('Failed removing old index %s', index_name)
# Add them again
for index_name in self._indexes:
try:
self.db.add_index(self._indexes[index_name](self.db.path, index_name))
self.db.reindex_index(index_name)
except IndexConflict:
pass
except:
sickrage.app.log.debug('Failed adding index %s', index_name)
raise
self.compact(try_repair=False)
else:
sickrage.app.log.debug('Failed compact: {}'.format(traceback.format_exc()))
except:
sickrage.app.log.debug('Failed compact: {}'.format(traceback.format_exc()))
def setup_indexes(self):
# setup database indexes
for index_name in self._indexes:
try:
# Make sure store and bucket don't exist
exists = []
for x in ['buck', 'stor']:
full_path = os.path.join(self.db.path, '%s_%s' % (index_name, x))
if os.path.exists(full_path):
exists.append(full_path)
if index_name not in self.db.indexes_names:
# Remove existing buckets if index isn't there
for x in exists:
os.unlink(x)
self.db.add_index(self._indexes[index_name](self.db.path, index_name))
self.db.reindex_index(index_name)
else:
# Previous info
previous_version = self.db.indexes_names[index_name]._version
current_version = self._indexes[index_name]._version
self.check_versions(index_name, current_version, previous_version)
except:
sickrage.app.log.debug('Failed adding index {}'.format(index_name))
def check_versions(self, index_name, current_version, previous_version):
# Only edit index if versions are different
if previous_version < current_version:
self.db.destroy_index(self.db.indexes_names[index_name])
self.db.add_index(self._indexes[index_name](self.db.path, index_name))
self.db.reindex_index(index_name)
def close(self):
self.db.close()
def upgrade(self):
pass
def cleanup(self):
pass
@property
def version(self):
try:
dbData = list(self.all('version'))[-1]
except IndexError:
dbData = {
'_t': 'version',
'database_version': 1
}
dbData.update(self.insert(dbData))
return dbData['database_version']
@property
def opened(self):
return self.db.opened
def check_integrity(self):
for index_name in self._indexes:
sickrage.app.log.debug('Checking data integrity for index {}'.format(index_name))
data = []
failed = False
# check integrity of index data
for x in self.db.all(index_name):
try:
data += [self.get('id', x.get('_id'))]
except Exception:
failed = True
# check if we failed integrity check, if so then destroy index
if failed and index_name in self.db.indexes_names:
self.db.destroy_index(self.db.indexes_names[index_name])
# check if index exists, if not then add it
if index_name not in self.db.indexes_names:
self.db.add_index(self._indexes[index_name](self.db.path, index_name))
# rebuild index if failed
if failed:
sickrage.app.log.debug('Failed data integrity check, rebuilding index {}'.format(index_name))
for x in data:
del x['_id'], x['_rev']
self.insert(x)
# cleanup
del data
def migrate(self):
if os.path.isfile(self.old_db_path):
sickrage.app.log.info('=' * 30)
sickrage.app.log.info('Migrating %s database, please wait...', self.name)
migrate_start = time.time()
import sqlite3
conn = sqlite3.connect(self.old_db_path)
conn.text_factory = lambda x: (x.decode('utf-8', 'ignore'))
migrate_data = {}
rename_old = False
try:
c = conn.cursor()
for ml in self._migrate_list:
migrate_data[ml] = {}
rows = self._migrate_list[ml]
try:
c.execute('SELECT {} FROM `{}`'.format('`' + '`,`'.join(rows) + '`', ml))
except:
# ignore faulty destination_id database
rename_old = True
raise
for p in c.fetchall():
columns = {}
for row in self._migrate_list[ml]:
columns[row] = p[rows.index(row)]
if not migrate_data[ml].get(p[0]):
migrate_data[ml][p[0]] = columns
else:
if not isinstance(migrate_data[ml][p[0]], list):
migrate_data[ml][p[0]] = [migrate_data[ml][p[0]]]
migrate_data[ml][p[0]].append(columns)
sickrage.app.log.info('Getting data took %s', (time.time() - migrate_start))
if not self.db.opened:
return
for t_name in migrate_data:
t_data = migrate_data.get(t_name, {})
sickrage.app.log.info('Importing %s %s' % (len(t_data), t_name))
for k, v in t_data.items():
if isinstance(v, list):
for d in v:
d.update({'_t': t_name})
self.insert(d)
else:
v.update({'_t': t_name})
self.insert(v)
sickrage.app.log.info('Total migration took %s', (time.time() - migrate_start))
sickrage.app.log.info('=' * 30)
rename_old = True
except OperationalError:
sickrage.app.log.debug('Migrating from unsupported/corrupt %s database version', self.name)
rename_old = True
except:
sickrage.app.log.debug('Migration of %s database failed', self.name)
finally:
conn.close()
# rename old database
if rename_old:
random = randomString()
sickrage.app.log.info('Renaming old database to %s.%s_old' % (self.old_db_path, random))
os.rename(self.old_db_path, '{}.{}_old'.format(self.old_db_path, random))
if os.path.isfile(self.old_db_path + '-wal'):
os.rename(self.old_db_path + '-wal', '{}-wal.{}_old'.format(self.old_db_path, random))
if os.path.isfile(self.old_db_path + '-shm'):
os.rename(self.old_db_path + '-shm', '{}-shm.{}_old'.format(self.old_db_path, random))
def delete_corrupted(self, _id, traceback_error=''):
try:
sickrage.app.log.debug('Deleted corrupted document "{}": {}'.format(_id, traceback_error))
corrupted = self.db.get('id', _id, with_storage=False)
self.db._delete_id_index(corrupted.get('_id'), corrupted.get('_rev'), None)
except:
log.debug('Failed deleting corrupted: {}'.format(traceback.format_exc()))
def all(self, *args, **kwargs):
with_doc = kwargs.pop('with_doc', True)
for data in self.db.all(*args, **kwargs):
if with_doc :
try:
doc = self.db.get('id', data['_id'])
yield doc
except (RecordDeleted, RecordNotFound):
sickrage.app.log.debug('Record not found, skipping: {}'.format(data['_id']))
except (ValueError, EOFError):
self.delete_corrupted(data.get('_id'), traceback_error=traceback.format_exc(0))
else:
yield data
def get_many(self, *args, **kwargs):
with_doc = kwargs.pop('with_doc', True)
for data in self.db.get_many(*args, **kwargs):
if with_doc:
try:
doc = self.db.get('id', data['_id'])
yield doc
except (RecordDeleted, RecordNotFound):
sickrage.app.log.debug('Record not found, skipping: {}'.format(data['_id']))
except (ValueError, EOFError):
self.delete_corrupted(data.get('_id'), traceback_error=traceback.format_exc(0))
else:
yield data
def get(self, *args, **kwargs):
try:
x = self.db.get(with_doc=kwargs.get('with_doc', True), *args, **kwargs)
return x.get('doc', x)
except (RecordDeleted, RecordNotFound):
pass
def delete(self, *args):
return self.db.delete(*args)
def update(self, *args):
return self.db.update(*args)
def insert(self, *args):
return self.db.insert(*args)
def runCouchPotato(options,
base_path,
args,
data_dir=None,
log_dir=None,
Env=None,
desktop=None):
try:
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
encoding = None
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
encoding = 'UTF-8'
Env.set('encoding', encoding)
# Do db stuff
db_path = sp(os.path.join(data_dir, 'database'))
old_db_path = os.path.join(data_dir, 'couchpotato.db')
# Remove database folder if both exists
if os.path.isdir(db_path) and os.path.isfile(old_db_path):
db = SuperThreadSafeDatabase(db_path)
db.open()
db.destroy()
# Check if database exists
db = SuperThreadSafeDatabase(db_path)
db_exists = db.exists()
if db_exists:
# Backup before start and cleanup old backups
backup_path = sp(os.path.join(data_dir, 'db_backup'))
backup_count = 5
existing_backups = []
if not os.path.isdir(backup_path): os.makedirs(backup_path)
for root, dirs, files in os.walk(backup_path):
# Only consider files being a direct child of the backup_path
if root == backup_path:
for backup_file in sorted(files):
ints = re.findall('\d+', backup_file)
# Delete non zip files
if len(ints) != 1:
os.remove(os.path.join(root, backup_file))
else:
existing_backups.append((int(ints[0]), backup_file))
else:
# Delete stray directories.
shutil.rmtree(root)
# Remove all but the last 5
for eb in existing_backups[:-backup_count]:
os.remove(os.path.join(backup_path, eb[1]))
# Create new backup
new_backup = sp(
os.path.join(backup_path, '%s.tar.gz' % int(time.time())))
zipf = tarfile.open(new_backup, 'w:gz')
for root, dirs, files in os.walk(db_path):
for zfilename in files:
zipf.add(os.path.join(root, zfilename),
arcname='database/%s' %
os.path.join(root[len(db_path) + 1:], zfilename))
zipf.close()
# Open last
db.open()
else:
db.create()
# Force creation of cachedir
log_dir = sp(log_dir)
cache_dir = sp(os.path.join(data_dir, 'cache'))
python_cache = sp(os.path.join(cache_dir, 'python'))
if not os.path.exists(cache_dir):
os.mkdir(cache_dir)
if not os.path.exists(python_cache):
os.mkdir(python_cache)
# Register environment settings
Env.set('app_dir', sp(base_path))
Env.set('data_dir', sp(data_dir))
Env.set('log_path', sp(os.path.join(log_dir, 'CouchPotato.log')))
Env.set('db', db)
Env.set('http_opener', requests.Session())
Env.set('cache_dir', cache_dir)
Env.set('cache', FileSystemCache(python_cache))
Env.set('console_log', options.console_log)
Env.set('quiet', options.quiet)
Env.set('desktop', desktop)
Env.set('daemonized', options.daemon)
Env.set('args', args)
Env.set('options', options)
# Determine debug
debug = options.debug or Env.setting('debug', default=False, type='bool')
Env.set('debug', debug)
# Development
development = Env.setting('development', default=False, type='bool')
Env.set('dev', development)
# Disable logging for some modules
for logger_name in [
'enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado',
'requests'
]:
logging.getLogger(logger_name).setLevel(logging.ERROR)
for logger_name in ['gntp']:
logging.getLogger(logger_name).setLevel(logging.WARNING)
# Disable SSL warning
disable_warnings()
# Use reloader
reloader = debug is True and development and not Env.get(
'desktop') and not options.daemon
# Logger
logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s',
'%m-%d %H:%M:%S')
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
logging.addLevelName(19, 'INFO')
# To screen
if (debug or
options.console_log) and not options.quiet and not options.daemon:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'),
'a',
500000,
10,
encoding=Env.get('encoding'))
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Start logging & enable colors
# noinspection PyUnresolvedReferences
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug('Started with options %s', options)
# Check available space
try:
total_space, available_space = getFreeSpace(data_dir)
if available_space < 100:
log.error(
'Shutting down as CP needs some space to work. You\'ll get corrupted data otherwise. Only %sMB left',
available_space)
return
except:
log.error('Failed getting diskspace: %s', traceback.format_exc())
def customwarn(message, category, filename, lineno, file=None, line=None):
log.warning('%s %s %s line:%s', (category, message, filename, lineno))
warnings.showwarning = customwarn
# Create app
from couchpotato import WebHandler
web_base = ('/' + Env.setting('url_base').lstrip('/') +
'/') if Env.setting('url_base') else '/'
Env.set('web_base', web_base)
api_key = Env.setting('api_key')
if not api_key:
api_key = uuid4().hex
Env.setting('api_key', value=api_key)
api_base = r'%sapi/%s/' % (web_base, api_key)
Env.set('api_base', api_base)
# Basic config
host = Env.setting('host', default='0.0.0.0')
# app.debug = development
config = {
'use_reloader': reloader,
'port': tryInt(Env.setting('port', default=5050)),
'host': host if host and len(host) > 0 else '0.0.0.0',
'ssl_cert': Env.setting('ssl_cert', default=None),
'ssl_key': Env.setting('ssl_key', default=None),
}
# Load the app
application = Application(
[],
log_function=lambda x: None,
debug=config['use_reloader'],
gzip=True,
cookie_secret=api_key,
login_url='%slogin/' % web_base,
)
Env.set('app', application)
# Request handlers
application.add_handlers(
".*$",
[
(r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler),
# API handlers
(r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler
(r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key
(r'%s' % api_base, RedirectHandler, {
"url": web_base + 'docs/'
}), # API docs
# Login handlers
(r'%slogin(/?)' % web_base, LoginHandler),
(r'%slogout(/?)' % web_base, LogoutHandler),
# Catch all webhandlers
(r'%s(.*)(/?)' % web_base, WebHandler),
(r'(.*)', WebHandler),
])
# Static paths
static_path = '%sstatic/' % web_base
for dir_name in ['fonts', 'images', 'scripts', 'style']:
application.add_handlers(
".*$",
[('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {
'path':
sp(os.path.join(base_path, 'couchpotato', 'static', dir_name))
})])
Env.set('static_path', static_path)
# Load configs & plugins
loader = Env.get('loader')
loader.preload(root=sp(base_path))
loader.run()
# Fill database with needed stuff
fireEvent('database.setup')
if not db_exists:
fireEvent('app.initialize', in_order=True)
fireEvent('app.migrate')
# Go go go!
from tornado.ioloop import IOLoop
from tornado.autoreload import add_reload_hook
loop = IOLoop.current()
# Reload hook
def reload_hook():
fireEvent('app.shutdown')
add_reload_hook(reload_hook)
# Some logging and fire load event
try:
log.info('Starting server on port %(port)s', config)
except:
pass
fireEventAsync('app.load')
ssl_options = None
if config['ssl_cert'] and config['ssl_key']:
ssl_options = {
'certfile': config['ssl_cert'],
'keyfile': config['ssl_key'],
}
server = HTTPServer(application,
no_keep_alive=True,
ssl_options=ssl_options)
try_restart = True
restart_tries = 5
while try_restart:
try:
server.listen(config['port'], config['host'])
loop.start()
server.close_all_connections()
server.stop()
loop.close(all_fds=True)
except Exception as e:
log.error('Failed starting: %s', traceback.format_exc())
try:
nr, msg = e
if nr == 48:
log.info(
'Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds',
(config.get('port'), restart_tries))
time.sleep(1)
restart_tries -= 1
if restart_tries > 0:
continue
else:
return
except ValueError:
return
except:
pass
raise
try_restart = False
class CodernityDbStorage(ProviderStorage, Plugin):
__key__ = 'codernitydb'
def __init__(self, path=None):
super(CodernityDbStorage, self).__init__()
self.path = path
self.database = SuperThreadSafeDatabase(path)
if os.path.exists(path):
self.database.open()
#
# Provider methods
#
def create(self, source, target):
if os.path.exists(self.path):
return True
# Create database
self.database.create()
# Add indices
self.database.add_index(
CollectionKeyIndex(self.database.path, 'collection_key'))
self.database.add_index(
MetadataKeyIndex(self.database.path, 'metadata_key'))
self.database.add_index(
MetadataCollectionIndex(self.database.path, 'metadata_collection'))
self.database.add_index(ItemKeyIndex(self.database.path, 'item_key'))
return True
def open_database(self, source, target, database=None):
return ModelRegistry['Database'].load(
DatabaseCodernityDbStorage.open(self, source, target, database),
source, target)
#
# Collection methods
#
def has_collection(self, source, target):
try:
self.database.get('collection_key', (source, target))
return True
except RecordNotFound:
pass
return False
def get_collection_version(self, source, target):
try:
item = self.database.get('collection_key', (source, target),
with_doc=True)
if not item or 'doc' not in item:
return None
return Version(item['doc'].get('version'))
except RecordNotFound:
pass
return None
@Elapsed.track
def update_collection(self, source, target, version):
# Retrieve current item
try:
current = self.database.get('collection_key', (source, target),
with_doc=True)
if 'doc' in current:
current = current['doc']
else:
current = None
except RecordNotFound:
current = None
# Build collection metadata
item = {
'_': {
't': 'collection',
'c': {
's': source,
't': target
}
},
# Collection attributes
'version': str(version)
}
# Add current item identifier
if current and '_id' in current and '_rev' in current:
item['_id'] = current['_id']
item['_rev'] = current['_rev']
# Update database
if current:
# Update existing item
try:
self.database.update(item)
except Exception, ex:
log.warn('Unable to update collection: %s', ex, exc_info=True)
return False
log.debug('[%s -> %s] Updated collection', source, target)
return True
# Insert new item
try:
self.database.insert(item)
except Exception, ex:
log.warn('Unable to insert collection: %s', ex, exc_info=True)
return False
def __init__(self, name=''):
self.name = name
self.old_db_path = ''
self.db_path = os.path.join(sickrage.app.data_dir, 'database', self.name)
self.db = SuperThreadSafeDatabase(self.db_path)
class srDatabase(object):
_indexes = {}
_migrate_list = {}
def __init__(self, name=''):
self.name = name
self.old_db_path = ''
self.db_path = os.path.join(sickrage.app.data_dir, 'database', self.name)
self.db = SuperThreadSafeDatabase(self.db_path)
def initialize(self):
# Remove database folder if both exists
if self.db.exists() and os.path.isfile(self.old_db_path):
self.db.open()
self.db.destroy()
if self.db.exists():
# Backup before start and cleanup old backups
backup_path = os.path.join(sickrage.app.data_dir, 'db_backup', self.name)
backup_count = 5
existing_backups = []
if not os.path.isdir(backup_path): os.makedirs(backup_path)
for root, dirs, files in os.walk(backup_path):
# Only consider files being a direct child of the backup_path
if root == backup_path:
for backup_file in sorted(files):
ints = re.findall('\d+', backup_file)
# Delete non zip files
if len(ints) != 1:
try:
os.remove(os.path.join(root, backup_file))
except:
pass
else:
existing_backups.append((int(ints[0]), backup_file))
else:
# Delete stray directories.
shutil.rmtree(root)
# Remove all but the last 5
for eb in existing_backups[:-backup_count]:
os.remove(os.path.join(backup_path, eb[1]))
# Create new backup
new_backup = os.path.join(backup_path, '%s.tar.gz' % int(time.time()))
with tarfile.open(new_backup, 'w:gz') as zipf:
for root, dirs, files in os.walk(self.db_path):
for zfilename in files:
zipf.add(os.path.join(root, zfilename),
arcname='database/%s/%s' % (
self.name,
os.path.join(root[len(self.db_path) + 1:], zfilename))
)
self.db.open()
else:
self.db.create()
# setup database indexes
self.setupIndexes()
def compact(self, try_repair=True, **kwargs):
# Removing left over compact files
for f in os.listdir(self.db.path):
for x in ['_compact_buck', '_compact_stor']:
if f[-len(x):] == x:
os.unlink(os.path.join(self.db.path, f))
try:
start = time.time()
size = float(self.db.get_db_details().get('size', 0))
sickrage.app.log.info(
'Compacting {} database, current size: {}MB'.format(self.name, round(size / 1048576, 2)))
self.db.compact()
new_size = float(self.db.get_db_details().get('size', 0))
sickrage.app.log.info(
'Done compacting {} database in {}s, new size: {}MB, saved: {}MB'.format(
self.name, round(time.time() - start, 2),
round(new_size / 1048576, 2), round((size - new_size) / 1048576, 2))
)
except (IndexException, AttributeError, TypeError) as e:
if try_repair:
sickrage.app.log.debug('Something wrong with indexes, trying repair')
# Remove all indexes
old_indexes = self._indexes.keys()
for index_name in old_indexes:
try:
self.db.destroy_index(index_name)
except IndexNotFoundException:
pass
except:
sickrage.app.log.debug('Failed removing old index %s', index_name)
# Add them again
for index_name in self._indexes:
try:
self.db.add_index(self._indexes[index_name](self.db.path, index_name))
self.db.reindex_index(index_name)
except IndexConflict:
pass
except:
sickrage.app.log.debug('Failed adding index %s', index_name)
raise
self.compact(try_repair=False)
else:
sickrage.app.log.debug('Failed compact: {}'.format(traceback.format_exc()))
except:
sickrage.app.log.debug('Failed compact: {}'.format(traceback.format_exc()))
def setupIndexes(self):
# setup database indexes
for index_name in self._indexes:
try:
# Make sure store and bucket don't exist
exists = []
for x in ['buck', 'stor']:
full_path = os.path.join(self.db.path, '%s_%s' % (index_name, x))
if os.path.exists(full_path):
exists.append(full_path)
if index_name not in self.db.indexes_names:
# Remove existing buckets if index isn't there
for x in exists:
os.unlink(x)
self.db.add_index(self._indexes[index_name](self.db.path, index_name))
self.db.reindex_index(index_name)
else:
# Previous info
previous_version = self.db.indexes_names[index_name]._version
current_version = self._indexes[index_name]._version
self.check_versions(index_name, current_version, previous_version)
except:
sickrage.app.log.debug('Failed adding index {}'.format(index_name))
def check_versions(self, index_name, current_version, previous_version):
# Only edit index if versions are different
if previous_version < current_version:
self.db.destroy_index(self.db.indexes_names[index_name])
self.db.add_index(self._indexes[index_name](self.db.path, index_name))
self.db.reindex_index(index_name)
def close(self):
self.db.close()
def cleanup(self):
pass
@property
def opened(self):
return self.db.opened
def check_integrity(self):
for index_name in self._indexes:
try:
for x in self.db.all(index_name):
try:
self.db.get('id', x.get('_id'), with_doc=True)
except (ValueError, TypeError) as e:
self.db.delete(self.db.get(index_name, x.get('key'), with_doc=True)['doc'])
except Exception as e:
if index_name in self.db.indexes_names:
self.db.destroy_index(self.db.indexes_names[index_name])
def migrate(self):
if os.path.isfile(self.old_db_path):
sickrage.app.log.info('=' * 30)
sickrage.app.log.info('Migrating %s database, please wait...', self.name)
migrate_start = time.time()
import sqlite3
conn = sqlite3.connect(self.old_db_path)
conn.text_factory = lambda x: (x.decode('utf-8', 'ignore'))
migrate_data = {}
rename_old = False
try:
c = conn.cursor()
for ml in self._migrate_list:
migrate_data[ml] = {}
rows = self._migrate_list[ml]
try:
c.execute('SELECT {} FROM `{}`'.format('`' + '`,`'.join(rows) + '`', ml))
except:
# ignore faulty destination_id database
rename_old = True
raise
for p in c.fetchall():
columns = {}
for row in self._migrate_list[ml]:
columns[row] = p[rows.index(row)]
if not migrate_data[ml].get(p[0]):
migrate_data[ml][p[0]] = columns
else:
if not isinstance(migrate_data[ml][p[0]], list):
migrate_data[ml][p[0]] = [migrate_data[ml][p[0]]]
migrate_data[ml][p[0]].append(columns)
sickrage.app.log.info('Getting data took %s', (time.time() - migrate_start))
if not self.db.opened:
return
for t_name in migrate_data:
t_data = migrate_data.get(t_name, {})
sickrage.app.log.info('Importing %s %s' % (len(t_data), t_name))
for k, v in t_data.items():
if isinstance(v, list):
for d in v:
d.update({'_t': t_name})
self.db.insert(d)
else:
v.update({'_t': t_name})
self.db.insert(v)
sickrage.app.log.info('Total migration took %s', (time.time() - migrate_start))
sickrage.app.log.info('=' * 30)
rename_old = True
except OperationalError:
sickrage.app.log.debug('Migrating from unsupported/corrupt %s database version', self.name)
rename_old = True
except:
sickrage.app.log.debug('Migration of %s database failed', self.name)
finally:
conn.close()
# rename old database
if rename_old:
random = randomString()
sickrage.app.log.info('Renaming old database to %s.%s_old' % (self.old_db_path, random))
os.rename(self.old_db_path, '{}.{}_old'.format(self.old_db_path, random))
if os.path.isfile(self.old_db_path + '-wal'):
os.rename(self.old_db_path + '-wal', '{}-wal.{}_old'.format(self.old_db_path, random))
if os.path.isfile(self.old_db_path + '-shm'):
os.rename(self.old_db_path + '-shm', '{}-shm.{}_old'.format(self.old_db_path, random))
def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, Env = None, desktop = None):
try:
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
encoding = None
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
encoding = 'UTF-8'
Env.set('encoding', encoding)
# Do db stuff
db_path = sp(os.path.join(data_dir, 'database'))
# Check if database exists
db = SuperThreadSafeDatabase(db_path)
db_exists = db.exists()
if db_exists:
# Backup before start and cleanup old backups
backup_path = sp(os.path.join(data_dir, 'db_backup'))
backup_count = 5
existing_backups = []
if not os.path.isdir(backup_path): os.makedirs(backup_path)
for root, dirs, files in os.walk(backup_path):
for backup_file in sorted(files):
ints = re.findall('\d+', backup_file)
# Delete non zip files
if len(ints) != 1:
os.remove(os.path.join(backup_path, backup_file))
else:
existing_backups.append((int(ints[0]), backup_file))
# Remove all but the last 5
for eb in existing_backups[:-backup_count]:
os.remove(os.path.join(backup_path, eb[1]))
# Create new backup
new_backup = sp(os.path.join(backup_path, '%s.tar.gz' % int(time.time())))
zipf = tarfile.open(new_backup, 'w:gz')
for root, dirs, files in os.walk(db_path):
for zfilename in files:
zipf.add(os.path.join(root, zfilename), arcname = 'database/%s' % os.path.join(root[len(db_path) + 1:], zfilename))
zipf.close()
# Open last
db.open()
else:
db.create()
# Force creation of cachedir
log_dir = sp(log_dir)
cache_dir = sp(os.path.join(data_dir, 'cache'))
python_cache = sp(os.path.join(cache_dir, 'python'))
if not os.path.exists(cache_dir):
os.mkdir(cache_dir)
if not os.path.exists(python_cache):
os.mkdir(python_cache)
# Register environment settings
Env.set('app_dir', sp(base_path))
Env.set('data_dir', sp(data_dir))
Env.set('log_path', sp(os.path.join(log_dir, 'CouchPotato.log')))
Env.set('db', db)
Env.set('cache_dir', cache_dir)
Env.set('cache', FileSystemCache(python_cache))
Env.set('console_log', options.console_log)
Env.set('quiet', options.quiet)
Env.set('desktop', desktop)
Env.set('daemonized', options.daemon)
Env.set('args', args)
Env.set('options', options)
# Determine debug
debug = options.debug or Env.setting('debug', default = False, type = 'bool')
Env.set('debug', debug)
# Development
development = Env.setting('development', default = False, type = 'bool')
Env.set('dev', development)
# Disable logging for some modules
for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado', 'requests']:
logging.getLogger(logger_name).setLevel(logging.ERROR)
for logger_name in ['gntp']:
logging.getLogger(logger_name).setLevel(logging.WARNING)
# Use reloader
reloader = debug is True and development and not Env.get('desktop') and not options.daemon
# Logger
logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%m-%d %H:%M:%S')
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
logging.addLevelName(19, 'INFO')
# To screen
if (debug or options.console_log) and not options.quiet and not options.daemon:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get('log_path'), 'a', 500000, 10, encoding = Env.get('encoding'))
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Start logging & enable colors
# noinspection PyUnresolvedReferences
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug('Started with options %s', options)
def customwarn(message, category, filename, lineno, file = None, line = None):
log.warning('%s %s %s line:%s', (category, message, filename, lineno))
warnings.showwarning = customwarn
# Create app
from couchpotato import WebHandler
web_base = ('/' + Env.setting('url_base').lstrip('/') + '/') if Env.setting('url_base') else '/'
Env.set('web_base', web_base)
api_key = Env.setting('api_key')
if not api_key:
api_key = uuid4().hex
Env.setting('api_key', value = api_key)
api_base = r'%sapi/%s/' % (web_base, api_key)
Env.set('api_base', api_base)
# Basic config
host = Env.setting('host', default = '0.0.0.0')
# app.debug = development
config = {
'use_reloader': reloader,
'port': tryInt(Env.setting('port', default = 5050)),
'host': host if host and len(host) > 0 else '0.0.0.0',
'ssl_cert': Env.setting('ssl_cert', default = None),
'ssl_key': Env.setting('ssl_key', default = None),
}
# Load the app
application = Application(
[],
log_function = lambda x: None,
debug = config['use_reloader'],
gzip = True,
cookie_secret = api_key,
login_url = '%slogin/' % web_base,
)
Env.set('app', application)
# Request handlers
application.add_handlers(".*$", [
(r'%snonblock/(.*)(/?)' % api_base, NonBlockHandler),
# API handlers
(r'%s(.*)(/?)' % api_base, ApiHandler), # Main API handler
(r'%sgetkey(/?)' % web_base, KeyHandler), # Get API key
(r'%s' % api_base, RedirectHandler, {"url": web_base + 'docs/'}), # API docs
# Login handlers
(r'%slogin(/?)' % web_base, LoginHandler),
(r'%slogout(/?)' % web_base, LogoutHandler),
# Catch all webhandlers
(r'%s(.*)(/?)' % web_base, WebHandler),
(r'(.*)', WebHandler),
])
# Static paths
static_path = '%sstatic/' % web_base
for dir_name in ['fonts', 'images', 'scripts', 'style']:
application.add_handlers(".*$", [
('%s%s/(.*)' % (static_path, dir_name), StaticFileHandler, {'path': sp(os.path.join(base_path, 'couchpotato', 'static', dir_name))})
])
Env.set('static_path', static_path)
# Load configs & plugins
loader = Env.get('loader')
loader.preload(root = sp(base_path))
loader.run()
# Fill database with needed stuff
fireEvent('database.setup')
if not db_exists:
fireEvent('app.initialize', in_order = True)
fireEvent('app.migrate')
# Go go go!
from tornado.ioloop import IOLoop
from tornado.autoreload import add_reload_hook
loop = IOLoop.current()
# Reload hook
def test():
fireEvent('app.shutdown')
add_reload_hook(test)
# Some logging and fire load event
try: log.info('Starting server on port %(port)s', config)
except: pass
fireEventAsync('app.load')
if config['ssl_cert'] and config['ssl_key']:
server = HTTPServer(application, no_keep_alive = True, ssl_options = {
'certfile': config['ssl_cert'],
'keyfile': config['ssl_key'],
})
else:
server = HTTPServer(application, no_keep_alive = True)
try_restart = True
restart_tries = 5
while try_restart:
try:
server.listen(config['port'], config['host'])
loop.start()
except Exception as e:
log.error('Failed starting: %s', traceback.format_exc())
try:
nr, msg = e
if nr == 48:
log.info('Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds', (config.get('port'), restart_tries))
time.sleep(1)
restart_tries -= 1
if restart_tries > 0:
continue
else:
return
except:
pass
raise
try_restart = False
class srDatabase(object):
_indexes = {}
_migrate_list = {}
def __init__(self, name=''):
self.name = name
self.old_db_path = ''
self.db_path = os.path.join(sickrage.DATA_DIR, 'database', self.name)
self.db = SuperThreadSafeDatabase(self.db_path)
if self.db.exists():
self.db.open()
def initialize(self):
# Remove database folder if both exists
if self.db.exists() and os.path.isfile(self.old_db_path):
self.db.destroy()
if self.db.exists():
# Backup before start and cleanup old backups
backup_path = os.path.join(sickrage.DATA_DIR, 'db_backup',
self.name)
backup_count = 5
existing_backups = []
if not os.path.isdir(backup_path): os.makedirs(backup_path)
for root, dirs, files in os.walk(backup_path):
# Only consider files being a direct child of the backup_path
if root == backup_path:
for backup_file in sorted(files):
ints = re.findall('\d+', backup_file)
# Delete non zip files
if len(ints) != 1:
try:
os.remove(os.path.join(root, backup_file))
except:
pass
else:
existing_backups.append(
(int(ints[0]), backup_file))
else:
# Delete stray directories.
shutil.rmtree(root)
# Remove all but the last 5
for eb in existing_backups[:-backup_count]:
os.remove(os.path.join(backup_path, eb[1]))
# Create new backup
new_backup = os.path.join(backup_path,
'%s.tar.gz' % int(time.time()))
with tarfile.open(new_backup, 'w:gz') as zipf:
for root, dirs, files in os.walk(self.db_path):
for zfilename in files:
zipf.add(os.path.join(root, zfilename),
arcname='database/%s' % os.path.join(
root[len(self.db_path) + 1:], zfilename))
else:
self.db.create()
# setup database indexes
self.setupIndexes()
def compact(self, try_repair=True, **kwargs):
# Removing left over compact files
for f in os.listdir(self.db.path):
for x in ['_compact_buck', '_compact_stor']:
if f[-len(x):] == x:
os.unlink(os.path.join(self.db.path, f))
try:
start = time.time()
size = float(self.db.get_db_details().get('size', 0))
sickrage.srCore.srLogger.debug(
'Compacting {} database, current size: {}MB'.format(
self.name, round(size / 1048576, 2)))
self.db.compact()
new_size = float(self.db.get_db_details().get('size', 0))
sickrage.srCore.srLogger.debug(
'Done compacting {} database in {}s, new size: {}MB, saved: {}MB'
.format(self.name, round(time.time() - start, 2),
round(new_size / 1048576, 2),
round((size - new_size) / 1048576, 2)))
except Exception:
if try_repair:
sickrage.srCore.srLogger.error(
'Something wrong with indexes, trying repair')
# Remove all indexes
old_indexes = self._indexes.keys()
for index_name in old_indexes:
try:
self.db.destroy_index(index_name)
except IndexNotFoundException:
pass
except:
sickrage.srCore.srLogger.error(
'Failed removing old index %s', index_name)
# Add them again
for index_name in self._indexes:
try:
self.db.add_index(self._indexes[index_name](
self.db.path, index_name))
self.db.reindex_index(index_name)
except IndexConflict:
pass
except:
sickrage.srCore.srLogger.error(
'Failed adding index %s', index_name)
raise
self.compact(try_repair=False)
else:
sickrage.srCore.srLogger.error('Failed compact: {}'.format(
traceback.format_exc()))
except:
sickrage.srCore.srLogger.error('Failed compact: {}'.format(
traceback.format_exc()))
def setupIndexes(self):
# setup database indexes
for index_name in self._indexes:
try:
# Make sure store and bucket don't exist
exists = []
for x in ['buck', 'stor']:
full_path = os.path.join(self.db.path,
'%s_%s' % (index_name, x))
if os.path.exists(full_path):
exists.append(full_path)
if index_name not in self.db.indexes_names:
# Remove existing buckets if index isn't there
for x in exists:
os.unlink(x)
self.db.add_index(self._indexes[index_name](self.db.path,
index_name))
# self.db.reindex_index(self.db.indexes_names[index_name])
else:
# Previous info
previous_version = self.db.indexes_names[
index_name]._version
current_version = self._indexes[index_name]._version
# Only edit index if versions are different
if previous_version < current_version:
self.db.destroy_index(
self.db.indexes_names[index_name])
self.db.add_index(self._indexes[index_name](
self.db.path, index_name))
self.db.reindex_index(index_name)
except:
sickrage.srCore.srLogger.error(
'Failed adding index {}'.format(index_name))
def close(self):
self.db.close()
def check_integrity(self):
for index_name in self._indexes:
try:
for x in self.db.all(index_name):
try:
self.db.get('id', x.get('_id'), with_doc=True)
except (ValueError, TypeError) as e:
self.db.delete(
self.db.get(index_name,
x.get('key'),
with_doc=True)['doc'])
except Exception as e:
if index_name in self.db.indexes_names:
self.db.destroy_index(self.db.indexes_names[index_name])
def migrate(self):
if os.path.isfile(self.old_db_path):
sickrage.srCore.srLogger.info('=' * 30)
sickrage.srCore.srLogger.info(
'Migrating %s database, please wait...', self.name)
migrate_start = time.time()
import sqlite3
conn = sqlite3.connect(self.old_db_path)
migrate_data = {}
rename_old = False
try:
c = conn.cursor()
for ml in self._migrate_list:
migrate_data[ml] = {}
rows = self._migrate_list[ml]
try:
c.execute('SELECT {} FROM `{}`'.format(
'`' + '`,`'.join(rows) + '`', ml))
except:
# ignore faulty destination_id database
rename_old = True
raise
for p in c.fetchall():
columns = {}
for row in self._migrate_list[ml]:
columns[row] = p[rows.index(row)]
if not migrate_data[ml].get(p[0]):
migrate_data[ml][p[0]] = columns
else:
if not isinstance(migrate_data[ml][p[0]], list):
migrate_data[ml][p[0]] = [
migrate_data[ml][p[0]]
]
migrate_data[ml][p[0]].append(columns)
conn.close()
sickrage.srCore.srLogger.info('Getting data took %s',
(time.time() - migrate_start))
if not self.db.opened:
return
for t_name in migrate_data:
t_data = migrate_data.get(t_name, {})
sickrage.srCore.srLogger.info('Importing %s %s' %
(len(t_data), t_name))
for k, v in t_data.items():
if isinstance(v, list):
for d in v:
d.update({'_t': t_name})
self.db.insert(d)
else:
v.update({'_t': t_name})
self.db.insert(v)
sickrage.srCore.srLogger.info('Total migration took %s',
(time.time() - migrate_start))
sickrage.srCore.srLogger.info('=' * 30)
rename_old = True
except OperationalError:
sickrage.srCore.srLogger.error(
'Migrating from unsupported/corrupt %s database version',
self.name)
rename_old = True
except:
sickrage.srCore.srLogger.error(
'Migration of %s database failed', self.name)
# rename old database
if rename_old:
random = randomString()
sickrage.srCore.srLogger.info(
'Renaming old database to %s.%s_old' %
(self.old_db_path, random))
os.rename(self.old_db_path,
'{}.{}_old'.format(self.old_db_path, random))
if os.path.isfile(self.old_db_path + '-wal'):
os.rename(self.old_db_path + '-wal',
'{}-wal.{}_old'.format(self.old_db_path, random))
if os.path.isfile(self.old_db_path + '-shm'):
os.rename(self.old_db_path + '-shm',
'{}-shm.{}_old'.format(self.old_db_path, random))
def runCouchPotato(options, base_path, args, data_dir=None, log_dir=None, Env=None, desktop=None):
try:
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
encoding = None
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ("ANSI_X3.4-1968", "US-ASCII", "ASCII"):
encoding = "UTF-8"
Env.set("encoding", encoding)
# Do db stuff
db_path = sp(os.path.join(data_dir, "database"))
old_db_path = os.path.join(data_dir, "couchpotato.db")
# Remove database folder if both exists
if os.path.isdir(db_path) and os.path.isfile(old_db_path):
db = SuperThreadSafeDatabase(db_path)
db.open()
db.destroy()
# Check if database exists
db = SuperThreadSafeDatabase(db_path)
db_exists = db.exists()
if db_exists:
# Backup before start and cleanup old backups
backup_path = sp(os.path.join(data_dir, "db_backup"))
backup_count = 5
existing_backups = []
if not os.path.isdir(backup_path):
os.makedirs(backup_path)
for root, dirs, files in os.walk(backup_path):
# Only consider files being a direct child of the backup_path
if root == backup_path:
for backup_file in sorted(files):
ints = re.findall("\d+", backup_file)
# Delete non zip files
if len(ints) != 1:
try:
os.remove(os.path.join(root, backup_file))
except:
pass
else:
existing_backups.append((int(ints[0]), backup_file))
else:
# Delete stray directories.
shutil.rmtree(root)
# Remove all but the last 5
for eb in existing_backups[:-backup_count]:
os.remove(os.path.join(backup_path, eb[1]))
# Create new backup
new_backup = sp(os.path.join(backup_path, "%s.tar.gz" % int(time.time())))
zipf = tarfile.open(new_backup, "w:gz")
for root, dirs, files in os.walk(db_path):
for zfilename in files:
zipf.add(
os.path.join(root, zfilename),
arcname="database/%s" % os.path.join(root[len(db_path) + 1 :], zfilename),
)
zipf.close()
# Open last
db.open()
else:
db.create()
# Force creation of cachedir
log_dir = sp(log_dir)
cache_dir = sp(os.path.join(data_dir, "cache"))
python_cache = sp(os.path.join(cache_dir, "python"))
if not os.path.exists(cache_dir):
os.mkdir(cache_dir)
if not os.path.exists(python_cache):
os.mkdir(python_cache)
session = requests.Session()
session.max_redirects = 5
# Register environment settings
Env.set("app_dir", sp(base_path))
Env.set("data_dir", sp(data_dir))
Env.set("log_path", sp(os.path.join(log_dir, "CouchPotato.log")))
Env.set("db", db)
Env.set("http_opener", session)
Env.set("cache_dir", cache_dir)
Env.set("cache", FileSystemCache(python_cache))
Env.set("console_log", options.console_log)
Env.set("quiet", options.quiet)
Env.set("desktop", desktop)
Env.set("daemonized", options.daemon)
Env.set("args", args)
Env.set("options", options)
# Determine debug
debug = options.debug or Env.setting("debug", default=False, type="bool")
Env.set("debug", debug)
# Development
development = Env.setting("development", default=False, type="bool")
Env.set("dev", development)
# Disable logging for some modules
for logger_name in ["enzyme", "guessit", "subliminal", "apscheduler", "tornado", "requests"]:
logging.getLogger(logger_name).setLevel(logging.ERROR)
for logger_name in ["gntp"]:
logging.getLogger(logger_name).setLevel(logging.WARNING)
# Disable SSL warning
disable_warnings()
# Use reloader
reloader = debug is True and development and not Env.get("desktop") and not options.daemon
# Logger
logger = logging.getLogger()
formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s", "%m-%d %H:%M:%S")
level = logging.DEBUG if debug else logging.INFO
logger.setLevel(level)
logging.addLevelName(19, "INFO")
# To screen
if (debug or options.console_log) and not options.quiet and not options.daemon:
hdlr = logging.StreamHandler(sys.stderr)
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# To file
hdlr2 = handlers.RotatingFileHandler(Env.get("log_path"), "a", 500000, 10, encoding=Env.get("encoding"))
hdlr2.setFormatter(formatter)
logger.addHandler(hdlr2)
# Start logging & enable colors
# noinspection PyUnresolvedReferences
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug("Started with options %s", options)
# Check soft-chroot dir exists:
try:
# Load Soft-Chroot
soft_chroot = Env.get("softchroot")
soft_chroot_dir = Env.setting("soft_chroot", section="core", default=None, type="unicode")
soft_chroot.initialize(soft_chroot_dir)
except SoftChrootInitError as exc:
log.error(exc)
return
except:
log.error("Unable to check whether SOFT-CHROOT is defined")
return
# Check available space
try:
total_space, available_space = getFreeSpace(data_dir)
if available_space < 100:
log.error(
"Shutting down as CP needs some space to work. You'll get corrupted data otherwise. Only %sMB left",
available_space,
)
return
except:
log.error("Failed getting diskspace: %s", traceback.format_exc())
def customwarn(message, category, filename, lineno, file=None, line=None):
log.warning("%s %s %s line:%s", (category, message, filename, lineno))
warnings.showwarning = customwarn
# Create app
from couchpotato import WebHandler
web_base = ("/" + Env.setting("url_base").lstrip("/") + "/") if Env.setting("url_base") else "/"
Env.set("web_base", web_base)
api_key = Env.setting("api_key")
if not api_key:
api_key = uuid4().hex
Env.setting("api_key", value=api_key)
api_base = r"%sapi/%s/" % (web_base, api_key)
Env.set("api_base", api_base)
# Basic config
host = Env.setting("host", default="0.0.0.0")
host6 = Env.setting("host6", default="::")
config = {
"use_reloader": reloader,
"port": tryInt(Env.setting("port", default=5050)),
"host": host if host and len(host) > 0 else "0.0.0.0",
"host6": host6 if host6 and len(host6) > 0 else "::",
"ssl_cert": Env.setting("ssl_cert", default=None),
"ssl_key": Env.setting("ssl_key", default=None),
}
# Load the app
application = Application(
[],
log_function=lambda x: None,
debug=config["use_reloader"],
gzip=True,
cookie_secret=api_key,
login_url="%slogin/" % web_base,
)
Env.set("app", application)
# Request handlers
application.add_handlers(
".*$",
[
(r"%snonblock/(.*)(/?)" % api_base, NonBlockHandler),
# API handlers
(r"%s(.*)(/?)" % api_base, ApiHandler), # Main API handler
(r"%sgetkey(/?)" % web_base, KeyHandler), # Get API key
(r"%s" % api_base, RedirectHandler, {"url": web_base + "docs/"}), # API docs
# Login handlers
(r"%slogin(/?)" % web_base, LoginHandler),
(r"%slogout(/?)" % web_base, LogoutHandler),
# Catch all webhandlers
(r"%s(.*)(/?)" % web_base, WebHandler),
(r"(.*)", WebHandler),
],
)
# Static paths
static_path = "%sstatic/" % web_base
for dir_name in ["fonts", "images", "scripts", "style"]:
application.add_handlers(
".*$",
[
(
"%s%s/(.*)" % (static_path, dir_name),
StaticFileHandler,
{"path": sp(os.path.join(base_path, "couchpotato", "static", dir_name))},
)
],
)
Env.set("static_path", static_path)
# Load configs & plugins
loader = Env.get("loader")
loader.preload(root=sp(base_path))
loader.run()
# Fill database with needed stuff
fireEvent("database.setup")
if not db_exists:
fireEvent("app.initialize", in_order=True)
fireEvent("app.migrate")
# Go go go!
from tornado.ioloop import IOLoop
from tornado.autoreload import add_reload_hook
loop = IOLoop.current()
# Reload hook
def reload_hook():
fireEvent("app.shutdown")
add_reload_hook(reload_hook)
# Some logging and fire load event
try:
log.info("Starting server on port %(port)s", config)
except:
pass
fireEventAsync("app.load")
ssl_options = None
if config["ssl_cert"] and config["ssl_key"]:
ssl_options = {"certfile": config["ssl_cert"], "keyfile": config["ssl_key"]}
server = HTTPServer(application, no_keep_alive=True, ssl_options=ssl_options)
try_restart = True
restart_tries = 5
while try_restart:
try:
if config["host"].startswith("unix:"):
server.add_socket(bind_unix_socket(config["host"][5:]))
else:
server.listen(config["port"], config["host"])
if Env.setting("ipv6", default=False):
try:
server.listen(config["port"], config["host6"])
except:
log.info2("Tried to bind to IPV6 but failed")
loop.start()
server.close_all_connections()
server.stop()
loop.close(all_fds=True)
except Exception as e:
log.error("Failed starting: %s", traceback.format_exc())
try:
nr, msg = e
if nr == 48:
log.info(
"Port (%s) needed for CouchPotato is already in use, try %s more time after few seconds",
(config.get("port"), restart_tries),
)
time.sleep(1)
restart_tries -= 1
if restart_tries > 0:
continue
else:
return
except ValueError:
return
except:
pass
raise
try_restart = False
class CodernityDbStorage(ProviderStorage, Plugin):
__key__ = 'codernitydb'
def __init__(self, path=None):
super(CodernityDbStorage, self).__init__()
self.path = path
self.database = SuperThreadSafeDatabase(path)
if os.path.exists(path):
self.database.open()
#
# Provider methods
#
def create(self, source, target):
if os.path.exists(self.path):
return True
# Create database
self.database.create()
# Add indices
self.database.add_index(CollectionKeyIndex(self.database.path, 'collection_key'))
self.database.add_index(MetadataKeyIndex(self.database.path, 'metadata_key'))
self.database.add_index(MetadataCollectionIndex(self.database.path, 'metadata_collection'))
self.database.add_index(ItemKeyIndex(self.database.path, 'item_key'))
return True
def open_database(self, source, target, database=None):
return ModelRegistry['Database'].load(
DatabaseCodernityDbStorage.open(self, source, target, database),
source, target
)
#
# Collection methods
#
def has_collection(self, source, target):
try:
self.database.get('collection_key', (source, target))
return True
except RecordNotFound:
pass
return False
def get_collection_version(self, source, target):
try:
item = self.database.get('collection_key', (source, target), with_doc=True)
if not item or 'doc' not in item:
return None
return Version(item['doc'].get('version'))
except RecordNotFound:
pass
return None
@Elapsed.track
def update_collection(self, source, target, version):
# Retrieve current item
try:
current = self.database.get('collection_key', (source, target), with_doc=True)
if 'doc' in current:
current = current['doc']
else:
current = None
except RecordNotFound:
current = None
# Build collection metadata
item = {
'_': {
't': 'collection',
'c': {
's': source,
't': target
}
},
# Collection attributes
'version': str(version)
}
# Add current item identifier
if current and '_id' in current and '_rev' in current:
item['_id'] = current['_id']
item['_rev'] = current['_rev']
# Update database
if current:
# Update existing item
try:
self.database.update(item)
except Exception, ex:
log.warn('Unable to update collection: %s', ex, exc_info=True)
return False
log.debug('[%s -> %s] Updated collection', source, target)
return True
# Insert new item
try:
self.database.insert(item)
except Exception, ex:
log.warn('Unable to insert collection: %s', ex, exc_info=True)
return False
