def load_dump_file(instance_dir, savedir, tempdir, datadump, db_engine, db_password):
if not os.path.exists(datadump):
# download from URL
dump_archive = os.path.join(savedir, os.path.basename(datadump))
log('Downloading %r to %r...', datadump, dump_archive)
req = urllib2.urlopen(datadump)
with open(dump_archive, 'wb') as fp:
shutil.copyfileobj(req, fp)
req.close()
else:
dump_archive = datadump
extension = os.path.splitext(dump_archive)[-1]
if extension in ('.bz2', '.gz', '.zip'):
# decompress archive
log('Expanding %r to %r...', dump_archive, tempdir)
dump_file = expand(dump_archive, tempdir)
extension = os.path.splitext(dump_file)[-1]
else:
dump_file = dump_archive
log('Importing %r... (this may take awhile!)', dump_file)
if extension in ('.json'):
load_json_dump(instance_dir, dump_file, tempdir)
elif 'sqlite' in db_engine:
import sqlite3
connection = sqlite3.connect(os.path.join(instance_dir, 'db/ecm.sqlite'))
cursor = connection.cursor()
cursor.execute('ATTACH DATABASE \'%s\' AS "eve";' % dump_file)
cursor.execute('DETACH DATABASE "eve";')
cursor.close()
connection.commit()
else:
pipe_to_dbshell(dump_file, instance_dir, password=db_password)
def run(command, global_options, options, args):
if not args:
command.parser.error('Missing instance directory.')
instance_dir = args.pop(0)
config = SafeConfigParser()
if config.read([os.path.join(instance_dir, 'settings.ini')]):
db_engine = config.get('database', 'ecm_engine')
db_password = config.get('database', 'ecm_password')
else:
command.parser.error('Could not read "settings.ini" in instance dir.')
if options.fuzzwork:
if not 'mysql' in db_engine:
command.parser.error(
'Fuzzwork download only supported with MySql database engine.')
if args:
dumppath = args.pop(0)
else:
dumppath = FUZZWORK_URL_PREFIX
elif not args:
command.parser.error('Missing dumppath or --fuzzwork option.')
else:
dumppath = args.pop(0)
try:
tempdir = tempfile.mkdtemp()
if options.save:
savedir = '.'
else:
savedir = tempdir
if options.fuzzwork:
for table in FUZZWORK_TABLES:
load_dump_file(
instance_dir, savedir, tempdir,
os.path.join(dumppath, table) + FUZZWORK_URL_SUFFIX,
db_engine, db_password)
log('Patching CCP format SDE to match ours... (this also takes awhile)'
)
pipe_to_dbshell(os.path.join(SQL_ROOT, FUZZWORK_PATCH_SCRIPT),
instance_dir,
password=db_password)
else:
load_dump_file(instance_dir, savedir, tempdir, dumppath, db_engine,
db_password)
log('EVE static data successfully imported.')
finally:
log('Removing temp files...')
shutil.rmtree(tempdir)
log('done')
def dump_mysql(instance_dir, dump_file, db_user, db_password, db_name):
tables = [ t for t in pipe_to_dbshell(r"SHOW TABLES LIKE 'eve\_%';", instance_dir).split() if t.startswith('eve_') ]
f = open(dump_file, 'w')
try:
try:
dump_cmd = MYSQL_DUMP_CMD % {
'user': db_user,
'password': db_password,
'database': db_name,
'tables': ' '.join(tables)
}
f.write('BEGIN;\n\n')
f.write('SET FOREIGN_KEY_CHECKS = 0;\n\n')
for t in tables:
f.write('TRUNCATE TABLE `%s`;\n' % t)
f.write('\n')
f.flush()
run_command(dump_cmd.split(), os.path.abspath(instance_dir), stdout=f)
f.flush()
f.write('SET FOREIGN_KEY_CHECKS = 1;\n\n')
f.write('COMMIT;\n\n')
except KeyboardInterrupt:
f.close()
sys.exit(1)
finally:
f.close()
def dump_psql(instance_dir, dump_file, db_user, db_password, db_name):
os.environ['PGDATABASE'] = db_name
os.environ['PGUSER'] = db_user
os.environ['PGPASSWORD'] = db_password
tables = [ t for t in pipe_to_dbshell(r"\dt eve_*", instance_dir).split() if t.startswith('eve_') ]
f = open(dump_file, 'w')
try:
try:
dump_cmd = 'pg_dump --format=p --encoding=utf-8 --no-owner --no-privileges '\
'--quote-all-identifiers --data-only --disable-triggers --table eve_*'
f.write('BEGIN;\n\n')
for t in tables:
f.write('TRUNCATE TABLE `%s`;\n' % t)
f.write('\n')
f.flush()
run_command(dump_cmd.split(), os.path.abspath(instance_dir), stdout=f)
f.flush()
f.write('COMMIT;\n\n')
except KeyboardInterrupt:
f.close()
sys.exit(1)
finally:
f.close()
def dump_mysql(instance_dir, dump_file, db_user, db_password, db_name):
tables = [
t for t in pipe_to_dbshell(r"SHOW TABLES LIKE 'eve\_%';",
instance_dir).split()
if t.startswith('eve_')
]
f = open(dump_file, 'w')
try:
try:
dump_cmd = MYSQL_DUMP_CMD % {
'user': db_user,
'password': db_password,
'database': db_name,
'tables': ' '.join(tables)
}
f.write('BEGIN;\n\n')
f.write('SET FOREIGN_KEY_CHECKS = 0;\n\n')
for t in tables:
f.write('TRUNCATE TABLE `%s`;\n' % t)
f.write('\n')
f.flush()
run_command(dump_cmd.split(),
os.path.abspath(instance_dir),
stdout=f)
f.flush()
f.write('SET FOREIGN_KEY_CHECKS = 1;\n\n')
f.write('COMMIT;\n\n')
except KeyboardInterrupt:
f.close()
sys.exit(1)
finally:
f.close()
def dump_psql(instance_dir, dump_file, db_user, db_password, db_name):
os.environ['PGDATABASE'] = db_name
os.environ['PGUSER'] = db_user
os.environ['PGPASSWORD'] = db_password
tables = [
t for t in pipe_to_dbshell(r"\dt eve_*", instance_dir).split()
if t.startswith('eve_')
]
f = open(dump_file, 'w')
try:
try:
dump_cmd = 'pg_dump --format=p --encoding=utf-8 --no-owner --no-privileges '\
'--quote-all-identifiers --data-only --disable-triggers --table eve_*'
f.write('BEGIN;\n\n')
for t in tables:
f.write('TRUNCATE TABLE `%s`;\n' % t)
f.write('\n')
f.flush()
run_command(dump_cmd.split(),
os.path.abspath(instance_dir),
stdout=f)
f.flush()
f.write('COMMIT;\n\n')
except KeyboardInterrupt:
f.close()
sys.exit(1)
finally:
f.close()
def run(command, global_options, options, args):
if not args:
command.parser.error('Missing instance directory.')
instance_dir = args.pop(0)
config = SafeConfigParser()
if config.read([os.path.join(instance_dir, 'settings.ini')]):
db_engine = config.get('database', 'ecm_engine')
db_password = config.get('database', 'ecm_password')
else:
command.parser.error('Could not read "settings.ini" in instance dir.')
if options.fuzzwork:
if not 'mysql' in db_engine:
command.parser.error('Fuzzwork download only supported with MySql database engine.')
if args:
dumppath = args.pop(0)
else:
dumppath = FUZZWORK_URL_PREFIX
elif not args:
command.parser.error('Missing dumppath or --fuzzwork option.')
else:
dumppath = args.pop(0)
try:
tempdir = tempfile.mkdtemp()
if options.save:
savedir = '.'
else:
savedir = tempdir
if options.fuzzwork:
for table in FUZZWORK_TABLES:
load_dump_file(instance_dir, savedir, tempdir, os.path.join(dumppath, table) + FUZZWORK_URL_SUFFIX, db_engine, db_password)
log('Patching CCP format SDE to match ours... (this also takes awhile)')
pipe_to_dbshell(os.path.join(SQL_ROOT, FUZZWORK_PATCH_SCRIPT), instance_dir, password=db_password)
else:
load_dump_file(instance_dir, savedir, tempdir, dumppath, db_engine, db_password)
log('EVE static data successfully imported.')
finally:
log('Removing temp files...')
shutil.rmtree(tempdir)
log('done')
def dump_sqlite(instance_dir, dump_file):
shutil.copy(os.path.join(instance_dir, 'db/ecm.sqlite'), dump_file)
tables = [ (t,) for t in pipe_to_dbshell('.tables', instance_dir).split() if not t.startswith('eve_') ]
import sqlite3
connection = sqlite3.connect(dump_file)
cursor = connection.cursor()
for table in tables:
cursor.execute('DROP TABLE "%s";' % table)
cursor.execute('VACUUM;')
cursor.close()
connection.commit()
connection.close()
def dump_sqlite(instance_dir, dump_file):
shutil.copy(os.path.join(instance_dir, 'db/ecm.sqlite'), dump_file)
tables = [(t, ) for t in pipe_to_dbshell('.tables', instance_dir).split()
if not t.startswith('eve_')]
import sqlite3
connection = sqlite3.connect(dump_file)
cursor = connection.cursor()
for table in tables:
cursor.execute('DROP TABLE "%s";' % table)
cursor.execute('VACUUM;')
cursor.close()
connection.commit()
connection.close()
def run(command, global_options, optionsd, args):
if not args:
command.parser.error('Missing instance directory.')
instance_dir = args.pop(0)
if not args:
command.parser.error('Missing datadump.')
datadump = args.pop(0)
config = SafeConfigParser()
if config.read([os.path.join(instance_dir, 'settings.ini')]):
db_engine = config.get('database', 'ecm_engine')
db_password = config.get('database', 'ecm_password')
else:
command.parser.error('Could not read "settings.ini" in instance dir.')
try:
sql = CCP_DATA_DUMPS[db_engine]
except KeyError:
command.parser.error('Cannot load datadump with database engine %r. '
'Supported engines: %r' % (db_engine, CCP_DATA_DUMPS.keys()))
try:
tempdir = tempfile.mkdtemp()
if not os.path.exists(datadump):
# download from URL
dump_archive = os.path.join(tempdir, os.path.basename(datadump))
log('Downloading EVE original dump from %r to %r...', datadump, dump_archive)
req = urllib2.urlopen(datadump)
with open(dump_archive, 'wb') as fp:
shutil.copyfileobj(req, fp)
req.close()
log('Download complete.')
else:
dump_archive = datadump
extension = os.path.splitext(dump_archive)[-1]
if extension in ('.bz2', '.gz', '.zip'):
# decompress archive
log('Expanding %r to %r...', dump_archive, tempdir)
dump_file = expand(dump_archive, tempdir)
log('Expansion complete to %r.' % dump_file)
else:
dump_file = dump_archive
log('Patching and importing data (this can be long)...')
if 'sqlite' in db_engine:
import sqlite3
with open(os.path.join(SQL_ROOT, sql['PATCH'])) as f:
sql_script = f.read()
connection = sqlite3.connect(os.path.join(instance_dir, 'db/ecm.sqlite'))
cursor = connection.cursor()
cursor.execute('ATTACH DATABASE \'%s\' AS "eve";' % dump_file)
cursor.executescript(sql_script)
cursor.execute('DETACH DATABASE "eve";')
cursor.close()
connection.commit()
else:
pipe_to_dbshell(dump_file, instance_dir, password=db_password)
pipe_to_dbshell(os.path.join(SQL_ROOT, sql['PATCH']), instance_dir, password=db_password)
pipe_to_dbshell(os.path.join(SQL_ROOT, sql['DROP']), instance_dir, password=db_password)
log('EVE data successfully imported.')
finally:
log('Removing temp files...')
shutil.rmtree(tempdir)
log('done')
def load_dump_file(instance_dir, savedir, tempdir, datadump, db_engine, db_password):
if not os.path.exists(datadump):
# download from URL
dump_archive = os.path.join(savedir, os.path.basename(datadump))
log('Downloading %r to %r...', datadump, dump_archive)
req = urllib2.urlopen(datadump)
with open(dump_archive, 'wb') as fp:
shutil.copyfileobj(req, fp)
req.close()
else:
dump_archive = datadump
extension = os.path.splitext(dump_archive)[-1]
if extension in ('.bz2', '.gz', '.zip'):
# decompress archive
log('Expanding %r to %r...', dump_archive, tempdir)
dump_file = expand(dump_archive, tempdir)
extension = os.path.splitext(dump_file)[-1]
else:
dump_file = dump_archive
log('Importing %r... (this may take awhile!)', dump_file)
if extension in ('.json'):
load_json_dump(instance_dir, dump_file, tempdir)
elif 'sqlite' in db_engine:
import sqlite3
config = SafeConfigParser()
db_dir = ''
if config.read([os.path.join(instance_dir, 'settings.ini')]):
db_dir = config.get('database', 'sqlite_db_dir')
if not db_dir:
db_dir = os.path.join(instance_dir, 'db')
db_file = os.path.join(db_dir, 'ecm.sqlite')
# Connect to the instance DB and attach the SDE
connection = sqlite3.connect(db_file)
cursor = connection.cursor()
cursor.execute('ATTACH DATABASE \'%s\' AS "eve";' % dump_file)
# Get the tables from the SDE (import them all)
cursor.execute('SELECT "name","sql" FROM "eve"."sqlite_master" WHERE "type"="table" AND "sql" IS NOT NULL;')
tables = cursor.fetchall()
# Load the table data as brand new tables matching the dump file (to avoid unexplainable errors, maybe because Django/south doesn't set them up the same as the DB dump conversion scripts)
for table in tables:
tablename = table[0]
tablesql = table[1]
# Drop and recreate the table
cursor.execute('DROP TABLE "%s";' % tablename)
cursor.execute(tablesql)
# Insert the data
cursor.execute('INSERT INTO "%s" SELECT * FROM "eve"."%s";' % (tablename, tablename))
# Get the indicies of the attached DB and create them
cursor.execute('SELECT "sql" FROM "eve"."sqlite_master" WHERE "type"="index" AND "sql" IS NOT NULL;')
indicies = cursor.fetchall()
for index in indicies:
cursor.execute(index[0])
cursor.execute('DETACH DATABASE "eve";')
cursor.close()
connection.commit()
else:
pipe_to_dbshell(dump_file, instance_dir, password=db_password)
def load_dump_file(instance_dir, savedir, tempdir, datadump, db_engine,
db_password):
if not os.path.exists(datadump):
# download from URL
dump_archive = os.path.join(savedir, os.path.basename(datadump))
log('Downloading %r to %r...', datadump, dump_archive)
req = urllib2.urlopen(datadump)
with open(dump_archive, 'wb') as fp:
shutil.copyfileobj(req, fp)
req.close()
else:
dump_archive = datadump
extension = os.path.splitext(dump_archive)[-1]
if extension in ('.bz2', '.gz', '.zip'):
# decompress archive
log('Expanding %r to %r...', dump_archive, tempdir)
dump_file = expand(dump_archive, tempdir)
extension = os.path.splitext(dump_file)[-1]
else:
dump_file = dump_archive
log('Importing %r... (this may take awhile!)', dump_file)
if extension in ('.json'):
load_json_dump(instance_dir, dump_file, tempdir)
elif 'sqlite' in db_engine:
import sqlite3
config = SafeConfigParser()
db_dir = ''
if config.read([os.path.join(instance_dir, 'settings.ini')]):
db_dir = config.get('database', 'sqlite_db_dir')
if not db_dir:
db_dir = os.path.join(instance_dir, 'db')
db_file = os.path.join(db_dir, 'ecm.sqlite')
# Connect to the instance DB and attach the SDE
connection = sqlite3.connect(db_file)
cursor = connection.cursor()
cursor.execute('ATTACH DATABASE \'%s\' AS "eve";' % dump_file)
# Get the tables from the SDE (import them all)
cursor.execute(
'SELECT "name","sql" FROM "eve"."sqlite_master" WHERE "type"="table" AND "sql" IS NOT NULL;'
)
tables = cursor.fetchall()
# Load the table data as brand new tables matching the dump file (to avoid unexplainable errors, maybe because Django/south doesn't set them up the same as the DB dump conversion scripts)
for table in tables:
tablename = table[0]
tablesql = table[1]
# Drop and recreate the table
cursor.execute('DROP TABLE "%s";' % tablename)
cursor.execute(tablesql)
# Insert the data
cursor.execute('INSERT INTO "%s" SELECT * FROM "eve"."%s";' %
(tablename, tablename))
# Get the indicies of the attached DB and create them
cursor.execute(
'SELECT "sql" FROM "eve"."sqlite_master" WHERE "type"="index" AND "sql" IS NOT NULL;'
)
indicies = cursor.fetchall()
for index in indicies:
cursor.execute(index[0])
cursor.execute('DETACH DATABASE "eve";')
cursor.close()
connection.commit()
else:
pipe_to_dbshell(dump_file, instance_dir, password=db_password)
