def dump_json(instance_dir, dump_file):
blocksize = 1000000
fd = open(dump_file, 'w')
fd.write('[')
first = True
for table in SDE_TABLES:
# Insert commas between tables of json data
if not first:
fd.write(',')
first = False
command = [sys.executable, 'manage.py', 'dumpdata', table]
proc = None
try:
log('... %s', table)
proc = subprocess.Popen(command, stdout=PIPE, stderr=PIPE, cwd=instance_dir)
data = proc.stdout.read(blocksize)[1:] # Strip out the first json character, a '['
while (len(data)):
if data[-1:] == ']': # Strip out the last json character, a ']'
fd.write(data[:-1])
else:
fd.write(data)
data = proc.stdout.read(blocksize)
(_, stderr) = proc.communicate()
if len(stderr):
stderr = 'Dump error, probable uninitialized database' + stderr
raise Exception(stderr)
except KeyboardInterrupt:
if proc is not None:
os.kill(proc.pid, signal.SIGTERM)
raise KeyboardInterrupt
fd.write(']')
fd.close()
def run(command, global_options, options, args):
if not args:
command.parser.error('Missing instance directory.')
instance_dir = args[0]
sqlite_db_dir = ''
config = SafeConfigParser()
if config.read([path.join(instance_dir, 'settings.ini')]):
sqlite_db_dir = config.get('database', 'sqlite_db_dir')
if not sqlite_db_dir:
sqlite_db_dir = path.join(instance_dir, 'db')
# upgrade files from template
upgrade_instance_files(instance_dir, config)
# run collectstatic
collect_static_files(instance_dir, options)
# migrate ecm db
if not options.no_syncdb:
migrate_ecm_db(instance_dir, options.upgrade_from_149)
log('')
log('ECM instance upgraded in "%s".' % instance_dir)
print_load_message(instance_dir, config.get('database', 'ecm_engine'))
print_usage_feedback_message()
def run(command, global_options, options, args):
if not args:
command.parser.error('Missing instance directory.')
instance_dir = args[0]
sqlite_db_dir = ''
settings_file = os.path.join(instance_dir, 'settings.ini')
config = SafeConfigParser()
if not config.read([settings_file]):
command.parser.error('Settings file "%s" not found.' % settings_file)
else:
sqlite_db_dir = config.get('database', 'sqlite_db_dir')
if not sqlite_db_dir:
sqlite_db_dir = os.path.join(instance_dir, 'db')
ecm_db_engine = config.get('database', 'ecm_engine')
# run collectstatic
collect_static_files(instance_dir, options)
# run syncdb
if 'sqlite' in ecm_db_engine and not os.path.exists(sqlite_db_dir):
os.makedirs(sqlite_db_dir)
init_ecm_db(instance_dir)
log('')
log('ECM instance initialized in "%s".' % instance_dir)
print_load_message(instance_dir, ecm_db_engine)
def _start(daemon):
log('Starting...')
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect( (daemon.address, daemon.port) )
log('ERROR: Address "%s:%s" already in use.' % (daemon.address, daemon.port))
sys.exit(1)
except socket.error:
pass
finally:
sock.close()
daemon.start()
try:
# wait to let the child process create the PID file
tries = 0
while tries < 10:
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect( (daemon.address, daemon.port) )
with open(daemon.pidfile, 'r') as pf:
pid = pf.read()
log('Server is listening on "%s:%s" (PID: %s)'
% (daemon.address, daemon.port, pid.strip()))
sys.exit(0)
except socket.error:
tries += 1
time.sleep(0.5)
finally:
sock.close()
raise socket.error()
except (IOError, socket.error):
log('ERROR: Failed to start instance.')
def print_load_message(instance_dir, db_engine):
if 'mysql' in db_engine:
engine = 'mysql'
elif 'sqlite' in db_engine:
engine = 'sqlite'
else:
engine = 'psql'
log('')
log('Now you need to load your database with EVE static data.')
log('Please execute `ecm-admin load %s <official_dump_file>` to do so.' % instance_dir)
log('You will find official dump conversions here http://releases.eve-corp-management.org/eve_sde/')
log('Be sure to take the latest one matching your db engine "%s".' % engine)
def print_load_message(instance_dir, db_engine):
if 'mysql' in db_engine:
engine = 'mysql'
elif 'sqlite' in db_engine:
engine = 'sqlite'
else:
engine = 'psql'
log('')
log('Now you need to load your database with EVE static data.')
log('Please execute `ecm-admin load %s <ecm_dump_file>` to do so.' % instance_dir)
log('You will find links to official dump conversions here https://github.com/evecm/ecm/wiki/Static-Data')
log('Be sure to take the latest one matching your db engine "%s".' % engine)
def run(command, global_options, options, args):
if not args:
command.parser.error('Missing instance directory.')
instance_dir = args[0]
config = SafeConfigParser()
settings_file = path.join(instance_dir, 'settings.ini')
if not config.read([settings_file]):
command.parser.error('Settings file "%s" not found.' % settings_file)
address = config.get('misc', 'server_bind_ip') or '127.0.0.1'
port = config.getint('misc', 'server_bind_port') or 8888
log('Starting...')
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect( (address, port) )
log('ERROR: Address "%s:%s" already in use.' % (address, port))
sys.exit(1)
except socket.error:
pass
finally:
sock.close()
log('GEvent WSGI server listening on %s:%s.' % (address, port))
log('Hit CTRL + C to stop.')
run_server(instance_dir, address, port, options.access_log)
def upgrade_instance_files(instance_dir, config):
log('Upgrading instance config files & examples...')
apache_mod_wsgi_vhost = path.join(instance_dir, 'examples/apache_mod_wsgi_vhost.example')
apache_proxy_vhost = path.join(instance_dir, 'examples/apache_reverse_proxy.example')
try:
with open(apache_proxy_vhost) as fd:
buff = fd.read()
match = SERVER_NAME_REGEXP.search(buff)
if match:
host_name = match.group(1)
else:
host_name = '???'
except IOError:
buff = ''
host_name = '???'
template_dir = path.abspath(path.dirname(instance_template.__file__))
shutil.copy(path.join(template_dir, 'settings.py'), instance_dir)
shutil.copy(path.join(template_dir, 'manage.py'), instance_dir)
dir_util.copy_tree(path.join(template_dir, 'wsgi'), path.join(instance_dir, 'wsgi'))
dir_util.copy_tree(path.join(template_dir, 'examples'), path.join(instance_dir, 'examples'))
if hasattr(os, 'chmod'):
os.chmod(path.join(instance_dir, 'manage.py'), 00755)
options = {
'host_name': host_name,
'instance_dir': path.abspath(instance_dir),
'bind_address': config.get('misc', 'server_bind_ip'),
'bind_port': config.get('misc', 'server_bind_port'),
}
try:
with open(apache_mod_wsgi_vhost, 'r') as fd:
buff = fd.read()
buff = buff % options
with open(apache_mod_wsgi_vhost, 'w') as fd:
buff = fd.write(buff)
except IOError, err:
log(err)
def migrate_ecm_db(instance_dir, upgrade_from_149=False):
instance_dir = path.abspath(instance_dir)
log("Migrating database...")
run_python_cmd('manage.py syncdb --noinput', instance_dir)
if upgrade_from_149:
log('Migrating from ECM 1.4.9...')
# we are upgrading from ECM 1.X.Y, we must perform the init migration
# on the 'hr' app (rename tables from 'roles_xxxxx' to 'hr_xxxxx')
pipe_to_django_shell('from south.models import MigrationHistory; '\
'MigrationHistory.objects.all().delete()' , instance_dir)
run_python_cmd('manage.py migrate hr 0001 --noinput', instance_dir)
# we MUST "fake" the first migration for 1.4.9 apps
# otherwise the migrate command will fail because DB tables already exist...
for app in ('common', 'scheduler', 'corp', 'assets', 'accounting'):
run_python_cmd('manage.py migrate %s 0001 --fake --noinput' % app, instance_dir)
run_python_cmd('manage.py migrate --all --noinput', instance_dir)
if upgrade_from_149:
pipe_to_django_shell('from ecm.apps.scheduler.models import ScheduledTask; '\
'ScheduledTask.objects.all().delete()' , instance_dir)
pipe_to_django_shell('from ecm.apps.common.models import UrlPermission; '\
'UrlPermission.objects.all().delete()' , instance_dir)
log('Database Migration successful.')
def run(command, global_options, options, args):
if not args:
command.parser.error('Missing instance directory.')
instance_dir = args.pop(0)
if not args:
command.parser.error('Missing dump file.')
dump_file = args.pop(0)
if not options.overwrite and os.path.exists(dump_file):
command.parser.error('Dump file already exists.')
config = SafeConfigParser()
if config.read([os.path.join(instance_dir, 'settings.ini')]):
db_engine = config.get('database', 'ecm_engine')
db_name = config.get('database', 'ecm_name')
db_user = config.get('database', 'ecm_user')
db_password = config.get('database', 'ecm_password')
else:
command.parser.error('Could not read "settings.ini" in instance dir.')
if not db_engine in SUPPORTED_ENGINES:
command.parser.error('Cannot dump patched EVE data with database engine %r. '
'Supported engines: %r' % (db_engine, SUPPORTED_ENGINES))
# remove existing file
if os.path.exists(dump_file):
os.remove(dump_file)
log('Dumping EVE data to %r...' % dump_file)
if options.json:
dump_json(instance_dir, dump_file)
elif 'postgresql' in db_engine:
dump_psql(instance_dir, dump_file, db_user, db_password, db_name)
elif 'mysql' in db_engine:
dump_mysql(instance_dir, dump_file, db_user, db_password, db_name)
elif 'sqlite' in db_engine:
dump_sqlite(instance_dir, dump_file)
log('EVE data successfully exported')
def run(command, global_options, options, args):
if not args:
command.parser.error('Missing instance directory.')
instance_dir = args.pop(0)
config = SafeConfigParser()
if config.read([os.path.join(instance_dir, 'settings.ini')]):
db_engine = config.get('database', 'ecm_engine')
db_password = config.get('database', 'ecm_password')
else:
command.parser.error('Could not read "settings.ini" in instance dir.')
if options.fuzzwork:
if not 'mysql' in db_engine:
command.parser.error('Fuzzwork download only supported with MySql database engine.')
if args:
dumppath = args.pop(0)
else:
dumppath = FUZZWORK_URL_PREFIX
elif not args:
command.parser.error('Missing dumppath or --fuzzwork option.')
else:
dumppath = args.pop(0)
try:
tempdir = tempfile.mkdtemp()
if options.save:
savedir = '.'
else:
savedir = tempdir
if options.fuzzwork:
for table in FUZZWORK_TABLES:
load_dump_file(instance_dir, savedir, tempdir, os.path.join(dumppath, table) + FUZZWORK_URL_SUFFIX, db_engine, db_password)
log('Patching CCP format SDE to match ours... (this also takes awhile)')
pipe_to_dbshell(os.path.join(SQL_ROOT, FUZZWORK_PATCH_SCRIPT), instance_dir, password=db_password)
else:
load_dump_file(instance_dir, savedir, tempdir, dumppath, db_engine, db_password)
log('EVE static data successfully imported.')
finally:
log('Removing temp files...')
shutil.rmtree(tempdir)
log('done')
def run(command, global_options, options, args):
"""
Create a new ECM instance.
"""
instance_dir = init_instance(command, args)
if options.quiet:
return
try:
prompt_missing_options(options)
write_settings(command, options, instance_dir)
log('')
log('New ECM instance created in "%s".' % instance_dir)
log('Please check the configuration in "%s" before running `ecm-admin init "%s"`.'
% (path.join(instance_dir, 'settings.ini'), instance_dir))
except:
# delete the created instance directory if something goes wrong
shutil.rmtree(instance_dir, ignore_errors=True)
raise
def run(command, global_options, optionsd, args):
if not args:
command.parser.error('Missing instance directory.')
instance_dir = args.pop(0)
if not args:
command.parser.error('Missing datadump.')
datadump = args.pop(0)
config = SafeConfigParser()
if config.read([os.path.join(instance_dir, 'settings.ini')]):
db_engine = config.get('database', 'ecm_engine')
db_password = config.get('database', 'ecm_password')
else:
command.parser.error('Could not read "settings.ini" in instance dir.')
try:
sql = CCP_DATA_DUMPS[db_engine]
except KeyError:
command.parser.error('Cannot load datadump with database engine %r. '
'Supported engines: %r' % (db_engine, CCP_DATA_DUMPS.keys()))
try:
tempdir = tempfile.mkdtemp()
if not os.path.exists(datadump):
# download from URL
dump_archive = os.path.join(tempdir, os.path.basename(datadump))
log('Downloading EVE original dump from %r to %r...', datadump, dump_archive)
req = urllib2.urlopen(datadump)
with open(dump_archive, 'wb') as fp:
shutil.copyfileobj(req, fp)
req.close()
log('Download complete.')
else:
dump_archive = datadump
extension = os.path.splitext(dump_archive)[-1]
if extension in ('.bz2', '.gz', '.zip'):
# decompress archive
log('Expanding %r to %r...', dump_archive, tempdir)
dump_file = expand(dump_archive, tempdir)
log('Expansion complete to %r.' % dump_file)
else:
dump_file = dump_archive
log('Patching and importing data (this can be long)...')
if 'sqlite' in db_engine:
import sqlite3
with open(os.path.join(SQL_ROOT, sql['PATCH'])) as f:
sql_script = f.read()
connection = sqlite3.connect(os.path.join(instance_dir, 'db/ecm.sqlite'))
cursor = connection.cursor()
cursor.execute('ATTACH DATABASE \'%s\' AS "eve";' % dump_file)
cursor.executescript(sql_script)
cursor.execute('DETACH DATABASE "eve";')
cursor.close()
connection.commit()
else:
pipe_to_dbshell(dump_file, instance_dir, password=db_password)
pipe_to_dbshell(os.path.join(SQL_ROOT, sql['PATCH']), instance_dir, password=db_password)
pipe_to_dbshell(os.path.join(SQL_ROOT, sql['DROP']), instance_dir, password=db_password)
log('EVE data successfully imported.')
finally:
log('Removing temp files...')
shutil.rmtree(tempdir)
log('done')
def collect_static_files(instance_dir, options):
log("Gathering static files...")
switches = '--noinput'
if os.name != 'nt' and options.symlink_files:
switches += ' --link'
run_python_cmd('manage.py collectstatic ' + switches, instance_dir)
def _stop(daemon):
log('Shutting down...')
daemon.stop()
log('Stopped')
def run(command, global_options, options, args):
if not args:
command.parser.error('Missing instance directory.')
instance_dir = args[0]
config = SafeConfigParser()
settings_file = path.join(instance_dir, 'settings.ini')
if not config.read([settings_file]):
command.parser.error('Settings file "%s" not found.' % settings_file)
pidfile = config.get('misc', 'pid_file') or 'ecm.pid'
address = config.get('misc', 'server_bind_ip') or '127.0.0.1'
port = config.getint('misc', 'server_bind_port') or 8888
run_as_user = config.get('misc', 'run_as_user') or None
if not path.isabs(pidfile):
pidfile = path.abspath(path.join(instance_dir, pidfile))
real_command = sys.argv[1]
if real_command == 'status':
if path.isfile(pidfile):
with open(pidfile, 'r') as pf:
pid = pf.read()
log('Instance is running with PID: %s' % pid.strip())
else:
log('Instance is stopped')
sys.exit(0)
else:
if run_as_user:
import pwd #@UnresolvedImport
try:
uid = pwd.getpwnam(run_as_user).pw_uid
except KeyError:
command.parser.error('User "%s" does not exist.' % run_as_user)
else:
uid = None
if options.logfile is not None:
logfile = path.abspath(options.logfile)
logdir = path.dirname(logfile)
if not path.exists(logdir):
os.makedirs(logdir)
else:
logfile = None
daemon = GEventWSGIDaemon(address=address,
port=port,
pidfile=pidfile,
working_dir=path.abspath(instance_dir),
uid=uid,
stdout=logfile,
stderr=logfile)
if real_command == 'start':
_start(daemon)
elif real_command == 'stop':
_stop(daemon)
elif real_command == 'restart':
_stop(daemon)
_start(daemon)
def init_ecm_db(instance_dir):
log("Initializing database...")
run_python_cmd('manage.py syncdb --noinput --migrate', instance_dir)
log('Database initialization successful.')
def print_usage_feedback_message():
log('')
log('*ATTENTION*')
log('')
log('ECM now has usage feedback system which is enabled by default.')
log('This feature is really helpfull for ECM developers because it allows')
log('us to know how much the project is used and helps our motivation ;-)')
log('')
log('It consists in a scheduled task that will run once each week and send')
log('*basic* and *anonymous* data to the official server %r.', ECM_USAGE_FEEDBACK_URL)
log('This is *NOT* a spying system nor a backdoor to enter your ECM instance.')
log('To know what data is sent in detail, please see the source code at http://eve-corp-management.org/projects/ecm/repository/entry/ecm/apps/common/tasks/usage_feedback.py')
log('')
log('If however you *really* want to disable the usage feedback on your')
log('instance, you need to go to the admin panel and change the scheduled')
log('task "ecm....send_feedback" to inactive.')
log('')
log('Thank you for using ECM. Fly safe.')
try:
with open(apache_mod_wsgi_vhost, 'r') as fd:
buff = fd.read()
buff = buff % options
with open(apache_mod_wsgi_vhost, 'w') as fd:
buff = fd.write(buff)
except IOError, err:
log(err)
try:
with open(apache_proxy_vhost, 'r') as fd:
buff = fd.read()
buff = buff % options
with open(apache_proxy_vhost, 'w') as fd:
buff = fd.write(buff)
except IOError, err:
log(err)
if not config.has_option('misc', 'secret_key'):
# Create a random SECRET_KEY hash to put it in the main settings.
chars = string.ascii_letters + string.digits + '.,;:!@#$^&*(-_+)[]{}'
config.set('misc', 'secret_key', get_random_string(50, chars))
template_dir = path.abspath(path.dirname(instance_template.__file__))
default_config = SafeConfigParser()
default_config.read([path.join(template_dir, 'settings.ini')])
for section in default_config.sections():
for option in default_config.options(section):
if not config.has_option(section, option):
config.set(section, option, default_config.get(section, option))
def load_dump_file(instance_dir, savedir, tempdir, datadump, db_engine, db_password):
if not os.path.exists(datadump):
# download from URL
dump_archive = os.path.join(savedir, os.path.basename(datadump))
log('Downloading %r to %r...', datadump, dump_archive)
req = urllib2.urlopen(datadump)
with open(dump_archive, 'wb') as fp:
shutil.copyfileobj(req, fp)
req.close()
else:
dump_archive = datadump
extension = os.path.splitext(dump_archive)[-1]
if extension in ('.bz2', '.gz', '.zip'):
# decompress archive
log('Expanding %r to %r...', dump_archive, tempdir)
dump_file = expand(dump_archive, tempdir)
extension = os.path.splitext(dump_file)[-1]
else:
dump_file = dump_archive
log('Importing %r... (this may take awhile!)', dump_file)
if extension in ('.json'):
load_json_dump(instance_dir, dump_file, tempdir)
elif 'sqlite' in db_engine:
import sqlite3
config = SafeConfigParser()
db_dir = ''
if config.read([os.path.join(instance_dir, 'settings.ini')]):
db_dir = config.get('database', 'sqlite_db_dir')
if not db_dir:
db_dir = os.path.join(instance_dir, 'db')
db_file = os.path.join(db_dir, 'ecm.sqlite')
# Connect to the instance DB and attach the SDE
connection = sqlite3.connect(db_file)
cursor = connection.cursor()
cursor.execute('ATTACH DATABASE \'%s\' AS "eve";' % dump_file)
# Get the tables from the SDE (import them all)
cursor.execute('SELECT "name","sql" FROM "eve"."sqlite_master" WHERE "type"="table" AND "sql" IS NOT NULL;')
tables = cursor.fetchall()
# Load the table data as brand new tables matching the dump file (to avoid unexplainable errors, maybe because Django/south doesn't set them up the same as the DB dump conversion scripts)
for table in tables:
tablename = table[0]
tablesql = table[1]
# Drop and recreate the table
cursor.execute('DROP TABLE "%s";' % tablename)
cursor.execute(tablesql)
# Insert the data
cursor.execute('INSERT INTO "%s" SELECT * FROM "eve"."%s";' % (tablename, tablename))
# Get the indicies of the attached DB and create them
cursor.execute('SELECT "sql" FROM "eve"."sqlite_master" WHERE "type"="index" AND "sql" IS NOT NULL;')
indicies = cursor.fetchall()
for index in indicies:
cursor.execute(index[0])
cursor.execute('DETACH DATABASE "eve";')
cursor.close()
connection.commit()
else:
pipe_to_dbshell(dump_file, instance_dir, password=db_password)