def __init__(self, path, env=None, config=None):
self.path = os.path.normpath(os.path.abspath(path))
if not os.path.isdir(self.path):
raise Exception('No project can be found at %r' % self.path)
self.env = env
self.config = config
if not config and env:
self.config = env.config
if not self.config:
self.config = Config()
self.htdocs_path = join(path, 'htdocs')
self.daemon = None
self._db_backend = None
self.db_path = join(path, 'db')
if not os.path.isdir(self.db_path):
os.makedirs(self.db_path)
self._deployer = None
self.admin_log = logging.getLogger('synthesepy.admin')
self.admin_log.setLevel(logging.DEBUG)
self.admin_log_path = join(self.path, 'logs', 'admin.log')
class KeptClosedFileHandler(logging.FileHandler):
'''Extension of logging.FileHandler which tries to keep the log
file closed, so that multiple processes can write to it.
Concurrent access might have unpredictable results though'''
def emit(self, record):
logging.FileHandler.emit(self, record)
self.close()
# FIXME: older Pythons have an issue when overriding FileHandler
if sys.version_info < (2, 6, 6):
if not os.path.isfile(self.admin_log_path):
utils.maybe_makedirs(os.path.dirname(self.admin_log_path))
open(self.admin_log_path, 'wb').write('')
admin_handler = logging.FileHandler(self.admin_log_path)
else:
admin_handler = KeptClosedFileHandler(
self.admin_log_path, delay=True)
admin_formatter = logging.Formatter(
'%(asctime)s %(levelname)-12s %(message)s')
admin_handler.setFormatter(admin_formatter)
self.admin_log.addHandler(admin_handler)
self._read_config()
# env might not be available yet. set_env should be called once ready
# to complete the project initialization.
if self.env:
self.set_env(self.env)
class Project(object):
def __init__(self, path, env=None, config=None):
self.path = os.path.normpath(os.path.abspath(path))
if not os.path.isdir(self.path):
raise Exception('No project can be found at %r' % self.path)
self.env = env
self.config = config
if not config and env:
self.config = env.config
if not self.config:
self.config = Config()
self.htdocs_path = join(path, 'htdocs')
self.daemon = None
self._db_backend = None
self.db_path = join(path, 'db')
if not os.path.isdir(self.db_path):
os.makedirs(self.db_path)
self._deployer = None
self.admin_log = logging.getLogger('synthesepy.admin')
self.admin_log.setLevel(logging.DEBUG)
self.admin_log_path = join(self.path, 'logs', 'admin.log')
class KeptClosedFileHandler(logging.FileHandler):
'''Extension of logging.FileHandler which tries to keep the log
file closed, so that multiple processes can write to it.
Concurrent access might have unpredictable results though'''
def emit(self, record):
logging.FileHandler.emit(self, record)
self.close()
# FIXME: older Pythons have an issue when overriding FileHandler
if sys.version_info < (2, 6, 6):
if not os.path.isfile(self.admin_log_path):
utils.maybe_makedirs(os.path.dirname(self.admin_log_path))
open(self.admin_log_path, 'wb').write('')
admin_handler = logging.FileHandler(self.admin_log_path)
else:
admin_handler = KeptClosedFileHandler(
self.admin_log_path, delay=True)
admin_formatter = logging.Formatter(
'%(asctime)s %(levelname)-12s %(message)s')
admin_handler.setFormatter(admin_formatter)
self.admin_log.addHandler(admin_handler)
self._read_config()
# env might not be available yet. set_env should be called once ready
# to complete the project initialization.
if self.env:
self.set_env(self.env)
def set_env(self, env):
self.env = env
self.packages_loader = PackagesLoader(self)
self._load_sites()
self._load_packages()
self.daemon = daemon.Daemon(self.env, self)
self.imports_manager = imports.ImportsManager(self)
for env_config_name in self.config.env_config_names.split(','):
if not env_config_name:
continue
if env_config_name not in self.config.env_configs:
raise Exception('Invalid env_config_name %r. Possible ones are '
'%r' % (env_config_name, self.config.env_configs.keys()))
self.config.update_from_dict(
self.config.env_configs[env_config_name])
if not self.config.remote_db_path:
sep = '/' if self.config.remote_project_path.startswith('/') else '\\'
self.config.remote_db_path = sep.join([
self.config.remote_project_path, 'db', 'config.db3'])
manager_path = join(self.path, 'manager')
self.manager_module = None
if (not os.path.isdir(manager_path) or
not os.path.isfile(join(manager_path, 'main.py'))):
return
sys.path.append(manager_path)
self.manager_module = __import__('main')
log.debug('Manager module %r', self.manager_module)
def get_site(self, site_name):
for s in self.sites:
if s.name == site_name:
return s
return None
def _read_config(self):
for suffix in ['', '_local', '_local_' + socket.gethostname()]:
config_path = join(
self.path, 'config{suffix}.py'.format(suffix=suffix))
log.debug('Trying to read config file: %r', config_path)
if not os.path.isfile(config_path):
continue
log.debug('Reading config file: %r', config_path)
self.config.update_from_file(config_path)
# Set a path to the sqlite db if not set explicitly.
conn_info = db_backends.ConnectionInfo(self.config.conn_string)
if conn_info.backend == 'sqlite' and 'path' not in conn_info:
conn_info['path'] = join(self.path, 'db', 'config.db3')
self.config.conn_string = conn_info.conn_string
# Set defaults
self.config.project_path = self.path
if not self.config.project_name:
self.config.project_name = os.path.split(self.path)[1]
if not self.config.log_file:
self.config.log_file = join(self.path, 'logs', 'synthese.log')
log_dir = os.path.dirname(self.config.log_file)
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
log.debug('Config: %s', self.config)
def _load_sites(self):
self.sites = []
for site_path in sorted(glob.glob(join(self.path, 'sites', '*'))):
if not os.path.isdir(site_path):
continue
self.sites.append(Site(self, site_path))
log.debug('Found sites: %s', self.sites)
self.config.static_paths = []
admin_site = self.get_site('admin')
admin_package = None
if admin_site:
admin_package = admin_site.get_package('admin')
for site in self.sites:
for package in site.packages + [admin_package]:
if not package:
continue
self.config.static_paths.append(
(site.base_path, package.files_path))
if self.config.default_site:
site_ids = [s.id for s in self.sites if
s.name == self.config.default_site]
if len(site_ids) != 1:
raise Exception('Can\'t find site {0!r} (found: {1!r})'.format(
self.config.default_site, site_ids))
self.config.site_id = site_ids[0]
if self.config.site_id <= 0:
non_admin_sites = [s for s in self.sites if s.name != 'admin']
if len(non_admin_sites) > 0:
self.config.site_id = non_admin_sites[0].id
def _load_packages(self):
self.packages = []
packages_dir = join(self.path, 'packages')
if not os.path.isdir(packages_dir):
return
self.packages = self.packages_loader.load_packages([], packages_dir)
def _get_sites_and_packages(self):
for package in self.packages:
yield None, package
for site in self.sites:
for package in site.packages:
yield site, package
def _run_testdata_importer(self):
importer_path = self.env.testdata_importer_path
log.info('Runing testdata importer from %r', importer_path)
self.env.prepare_for_launch()
env = os.environ.copy()
env['SYNTHESE_TESTDATA_CONNSTRING'] = self.config.conn_string + \
',triggerCheck=0'
# FIXME: The testdata importer binary does not compile anymore we just skip its call.
#utils.call(importer_path, env=env)
@property
def db_backend(self):
if self._db_backend:
return self._db_backend
self._db_backend = db_backends.create_backend(
self.env, self.config.conn_string)
return self._db_backend
def clean(self):
self.db_backend.drop_db()
@command()
def load_data(self, local=False, overwrite=True):
"""Load data into the database."""
log.info('loading_data into project (local:%s)', local)
# TODO: don't import fixtures from a package more than once.
for site, package in self._get_sites_and_packages():
log.debug('Loading site:%s package:%s', site, package)
package.load_data(site, local, overwrite)
@command()
def load_local_data(self, overwrite):
"""
Load data into the database (data loaded only once meant to be edited)
"""
self.load_data(True, overwrite)
@command()
def reset(self):
"""
Delete database and load inital data.
"""
log.info('Resetting project')
self.clean()
self.db_backend.init_db()
self.load_data()
self.load_local_data(True)
def send_restart_mail(self, fail_kind, restart_count, last_start_s):
if not self.config.send_mail_on_restart:
return
log.info('Sending restart mail')
hostname = socket.gethostname()
LINE_COUNT = 500
try:
last_log = utils.tail(open(self.config.log_file, 'rb'), LINE_COUNT)
except IOError:
last_log = "[Not available]"
subject = ('Synthese {fail_kind} on {hostname} (project: {project}, '
'restarts: {restart_count})'.format(
fail_kind=fail_kind,
hostname=hostname,
project=self.config.project_name,
restart_count=restart_count))
body = '''
Detected Synthese {fail_kind} on {hostname}. It is going to restart.
Total restart count: {restart_count}. Seconds since last start: {uptime_s}.
Last {line_count} lines of log:
{last_log}
Have a nice day,
The synthese.py wrapper script.
'''.format(
fail_kind=fail_kind,
hostname=hostname,
restart_count=restart_count,
line_count=LINE_COUNT,
last_log=last_log,
uptime_s=int(time.time() - last_start_s))
utils.send_mail(self.env.config, self.config.mail_admins, subject, body)
@command()
def rundaemon(self, block=True):
"""Run Synthese daemon"""
migration.maybe_migrate_schema(self)
self.daemon.start()
if not block:
return
log.info('Daemon running, press ctrl-c to stop')
if (self.manager_module and
hasattr(self.manager_module, 'on_daemon_started')):
self.manager_module.on_daemon_started(self)
hang_detector_running = True
hang_detected_event = threading.Event()
def hang_detector():
HANG_CHECK_INTERVAL_S = 60
RESTART_HANG_COUNT = 2
hangs_count = 0
while hang_detector_running:
if not self.daemon.ready:
log.debug('Hang detector: daemon not ready')
time.sleep(HANG_CHECK_INTERVAL_S)
hangs_count = 0
continue
log.debug('Checking for hangs')
if not utils.can_connect(self.config.port, False, '/hang_check'):
hangs_count += 1
log.warn('Detected hang. Hangs count is %s', hangs_count)
else:
hangs_count = 0
if hangs_count >= RESTART_HANG_COUNT:
hangs_count = 0
log.warn('Hang detected, setting hang detected event.')
hang_detected_event.set()
time.sleep(HANG_CHECK_INTERVAL_S)
restart_if_crashed = (self.config.restart_if_crashed_or_hung or
self.config.restart_if_crashed)
restart_if_hung = (self.config.restart_if_crashed_or_hung or
self.config.restart_if_hung)
if restart_if_hung:
hang_detector_thread = threading.Thread(target=hang_detector)
hang_detector_thread.daemon = True
hang_detector_thread.start()
restart_count = 0
try:
while True:
restart = False
start_time = time.time()
while (self.daemon.is_running() and
not hang_detected_event.is_set()):
time.sleep(2)
if hang_detected_event.is_set():
log.info('Hang detected')
hang_detected_event.clear()
fail_kind = 'hang'
self.daemon.stop()
restart = True
else:
log.info('Daemon terminated')
crashed = not self.daemon.stopped
fail_kind = 'crash'
if crashed:
log.warn('Stop is unexpected, crash?')
restart_reason = 'Crash'
if restart_if_crashed:
restart = True
if not restart:
sys.exit(1)
break
log.info('Restarting daemon')
self.send_restart_mail(fail_kind, restart_count, start_time)
self.daemon.start(kill_proxy=False)
restart_count += 1
except:
raise
finally:
log.info('Stopping daemon')
hang_detector_running = False
self.daemon.stop()
@command()
def stopdaemon(self):
"""Stop Synthese daemon"""
self.daemon.stop()
# TODO: should use the HTTP method to stop the daemon once it works.
ports = [self.config.port]
if self.config.wsgi_proxy:
ports.append(self.config.wsgi_proxy_port)
for port in ports:
utils.kill_listening_processes(port)
@command()
def runproxy(self):
"""Run HTTP Proxy to serve static files"""
proxy.serve_forever(self.env, self)
# This is mostly for debugging. The webapp is also available with "runproxy"
# (however it doesn't auto-reload on change).
@command()
def runwebapp(self):
"""Run Web Frontend"""
# import here to avoid import cycles.
import synthesepy.web
web_app = synthesepy.web.get_application(project=self)
def root_app(environ, start_response):
status = '200 OK'
output = 'Dummy root app'
response_headers = [('Content-type', 'text/plain'),
('Content-Length', str(len(output)))]
start_response(status, response_headers)
return [output]
app = wsgi.DispatcherMiddleware(root_app, {
'/w': web_app
})
run_simple('0.0.0.0', 5000, app, use_reloader=True)
@command()
def project_command(self, args):
if not self.manager_module:
raise Exception('No manager module')
self.manager_module.project_command(self, args)
@command()
def db_view(self):
"""Open database in a GUI tool (if applicable)"""
if self.db_backend.name == 'sqlite':
utils.call(
[self.config.spatialite_gui_path,
self.db_backend.conn_info['path']],
bg=True)
else:
raise NotImplementedError("Not implemented for this backend")
@command()
def db_view_gis(self):
"""Open database in a GIS GUI tool (if applicable)"""
if self.db_backend.name == 'sqlite':
utils.call(
[self.config.spatialite_gis_path,
self.db_backend.conn_info['path']],
bg=True)
else:
raise NotImplementedError("Not implemented for this backend")
@command()
def db_shell(self, sql=None):
"""Open a SQL interpreter on the database or execute the given SQL"""
self.db_backend.shell(sql)
@command()
def db_dump(self, db_backend=None, prefix=''):
"""Dump database to text file"""
if not db_backend:
db_backend = self.db_backend
output = db_backend.dump()
max_id = 0
for d in os.listdir(self.db_path):
if 'sql' not in d:
continue
try:
max_id = max(max_id, int(d.split('-')[1]))
except:
pass
target = join(
self.db_path, '{prefix}config-{id:03}-{date}.sql.gz'.format(
prefix=prefix, id=max_id + 1,
date=datetime.datetime.now().strftime('%Y%m%d-%H%M')))
gzip.open(target, 'wb').write(output)
log.info('Db dumped to %r', target)
uncompressed_fname = 'config_{project_name}.sql'.format(
project_name=self.config.project_name)
uncompressed_target = join(self.db_path, uncompressed_fname)
open(uncompressed_target, 'wb').write(output)
return uncompressed_fname
@command()
def db_open_dump(self):
"""Open the latest database dump in a text editor"""
uncompressed_target = join(
self.db_path, 'config_{project_name}.sql'.format(
project_name=self.config.project_name))
if os.path.isfile(self.config.editor_path):
utils.call([self.config.editor_path, uncompressed_target], bg=True)
@command()
def db_restore(self, db_dump):
"""Restore a database from a text file dump"""
all_dumps = sorted(d for d in os.listdir(self.db_path) if 'sql' in d)
if not db_dump or db_dump == '-':
log.fatal('Name of dump (--dump) should be provided. '
'Possible dumps:')
for d in all_dumps:
print d
return
dumps = [d for d in all_dumps if db_dump in d]
if len(dumps) != 1:
raise Exception('Not only one dump matches %r (possible dumps: %r)' %
(db_dump, all_dumps))
sql_file = join(self.db_path, dumps[0])
log.info('Restoring %s', sql_file)
if sql_file.endswith('.gz'):
sql = gzip.open(sql_file, 'rb').read()
else:
sql = open(sql_file, 'rb').read()
log.info('Restoring %s bytes of sql', len(sql))
self.db_backend.restore(sql)
@command()
def db_sync_to_files(self):
db_sync.sync_to_files(self)
@command()
def db_sync_from_files(self, host, use_http):
db_sync.sync_from_files(self, host, use_http)
@command()
def db_sync(self, host=None):
db_sync.sync(self, host)
# Commands for syncing or managing a remote project.
@command()
def db_remote_dump(self):
"""Dump database from remote server"""
if not self.config.remote_server:
raise Exception('No remote server defined in configuration')
@contextlib.contextmanager
def remote_transaction_sqlite(conn_info):
remote_db_local_path = join(self.db_path, 'remote_config.db3')
log.info('Fetching db to %r', remote_db_local_path)
utils.rsync(self.config, '{remote_server}:' + self.config.remote_db_path,
utils.to_cygwin_path(remote_db_local_path))
remote_conn_info = self.db_backend.conn_info.copy()
remote_conn_info['path'] = remote_db_local_path
yield remote_conn_info
@contextlib.contextmanager
def remote_transaction_mysql(conn_info):
MYSQL_FORWARDED_PORT = 33000
utils.kill_listening_processes(MYSQL_FORWARDED_PORT)
if utils.can_connect(MYSQL_FORWARDED_PORT):
raise Exception('MySQL tunnel port (%s) is not '
'available' % MYSQL_FORWARDED_PORT)
ssh_command_line = utils.ssh_command_line(
self.config,
extra_opts='-N -L {forwarded_port}:localhost:3306'.format(
forwarded_port=MYSQL_FORWARDED_PORT))
log.debug('ssh command line for tunnel: %s', ssh_command_line)
p = subprocess.Popen(ssh_command_line, shell=True)
log.info('Waiting a bit for the tunnel to establish...')
time.sleep(3)
remote_conn_info = db_backends.ConnectionInfo(
self.config.remote_conn_string)
remote_conn_info.data['port'] = MYSQL_FORWARDED_PORT
remote_conn_info.data['host'] = '127.0.0.1'
yield remote_conn_info
p.kill()
if self.db_backend.name == 'sqlite':
remote_transaction = remote_transaction_sqlite
elif self.db_backend.name == 'mysql':
remote_transaction = remote_transaction_mysql
else:
raise Exception('Unsupported backend: %r', self.db_backend.name)
with remote_transaction(self.db_backend.conn_info.copy()) as remote_conn_info:
remote_conn_string = remote_conn_info.conn_string
log.info('Remote connection string: %r', remote_conn_string)
remote_backend = db_backends.create_backend(self.env, remote_conn_string)
return self.db_dump(remote_backend, self.config.remote_dump_prefix)
@command()
def db_remote_restore(self):
sql_fname = self.db_remote_dump()
self.db_restore(sql_fname)
@command()
def ssh(self):
"""Open a ssh shell on the remote server"""
utils.call(utils.ssh_command_line(self.config))
@command()
def imports(self, subcommand, template_id, import_id, dummy, no_mail, args):
"""Imports management"""
if subcommand == 'list_templates':
import_templates = self.imports_manager.get_import_templates()
for import_template in import_templates:
log.info('Import template: id=%s label=%r', import_template.id,
import_template.label)
for import_ in import_template.get_imports():
log.info('Import: %s, path=%s', import_.id, import_.path)
elif subcommand == 'list_imports':
import_template = self.imports_manager.get_import_template(
template_id)
for import_ in import_template.get_imports():
log.info('Import: %s, path=%s', import_.id, import_.path)
elif subcommand == 'create':
import_template = self.imports_manager.get_import_template(
template_id)
create_args = {}
if args:
create_args = json.loads(args)
import_ = import_template.create_import(create_args)
log.info('Created import with id: %s', import_.id)
elif subcommand == 'execute':
import_ = self.imports_manager.get_import(template_id, import_id)
import_.execute(None, dummy, no_mail)
else:
raise Exception('Unknown import subcommand: %s', subcommand)
# System install/uninstall
def _get_tools(self):
tools = [
external_tools.SyntheseSupervisor(self),
Apache(self),
external_tools.WSGI(self),
]
if self.config.use_udf_proxy:
tools.append(external_tools.UDFProxySupervisor(self))
if self.config.use_s3_proxy:
tools.append(external_tools.S3ProxySupervisor(self))
return tools
def _process_dot_in_file(self, content, vars, template_dir):
def get_template(name, from_template):
path = join(template_dir, name)
return from_template.__class__.from_filename(
path, namespace=from_template.namespace,
get_template=from_template.get_template)
tmpl = tempita.Template(content, get_template=get_template)
return tmpl.substitute(vars)
def _replace_dot_in_files(self):
for path, dirlist, filelist in os.walk(self.path):
for exclude in ['db', 'logs', 'imports', 'deploy', '.git', '.hg', '.svn']:
if exclude in dirlist:
dirlist.remove(exclude)
for name in filelist:
if not name.endswith('.in'):
continue
source = join(path, name)
target = source[:-len('.in')]
log.debug('Generating file: %r -> %r', source, target)
with open(target, 'wb') as f:
content = open(source).read()
vars = {
'generated_warning': 'WARNING: THIS FILE IS GENERATED, '
'DO NOT EDIT!!',
'project': self,
'p': self,
'config': self.config,
'c': self.config,
}
content = self._process_dot_in_file(content, vars, path)
f.write(content)
try:
shutil.copymode(source, target)
except OSError, e:
# This might happen if the user is not the owner.
log.warn('Error while updating mode: %s', e)