Пример #1
0
def dump_json(instance_dir, dump_file):
    blocksize = 1000000
    fd = open(dump_file, 'w')
    fd.write('[')
    first = True
    for table in SDE_TABLES:
        # Insert commas between tables of json data
        if not first:
            fd.write(',')
        first = False

        command = [sys.executable, 'manage.py', 'dumpdata', table]
        proc = None
        try:
            log('... %s', table)
            proc = subprocess.Popen(command, stdout=PIPE, stderr=PIPE, cwd=instance_dir)
            data = proc.stdout.read(blocksize)[1:] # Strip out the first json character, a '['
            while (len(data)):
                if data[-1:] == ']': # Strip out the last json character, a ']'
                    fd.write(data[:-1])
                else:
                    fd.write(data)
                data = proc.stdout.read(blocksize)
            (_, stderr) = proc.communicate()
            if len(stderr):
                stderr = 'Dump error, probable uninitialized database' + stderr
                raise Exception(stderr)
        except KeyboardInterrupt:
            if proc is not None:
                os.kill(proc.pid, signal.SIGTERM)
                raise KeyboardInterrupt

    fd.write(']')
    fd.close()
Пример #2
0
def run(command, global_options, options, args):
    if not args:
        command.parser.error('Missing instance directory.')
    instance_dir = args[0]
    sqlite_db_dir = ''
    config = SafeConfigParser()
    if config.read([path.join(instance_dir, 'settings.ini')]):
        sqlite_db_dir = config.get('database', 'sqlite_db_dir')
    if not sqlite_db_dir:
        sqlite_db_dir = path.join(instance_dir, 'db')

    # upgrade files from template
    upgrade_instance_files(instance_dir, config)

    # run collectstatic
    collect_static_files(instance_dir, options)

    # migrate ecm db
    if not options.no_syncdb:
        migrate_ecm_db(instance_dir, options.upgrade_from_149)

    log('')
    log('ECM instance upgraded in "%s".' % instance_dir)

    print_load_message(instance_dir, config.get('database', 'ecm_engine'))
    print_usage_feedback_message()
Пример #3
0
def run(command, global_options, options, args):
    if not args:
        command.parser.error('Missing instance directory.')
    instance_dir = args[0]
    sqlite_db_dir = ''
    settings_file = os.path.join(instance_dir, 'settings.ini')
    config = SafeConfigParser()
    if not config.read([settings_file]):
        command.parser.error('Settings file "%s" not found.' % settings_file)
    else:
        sqlite_db_dir = config.get('database', 'sqlite_db_dir')
    if not sqlite_db_dir:
        sqlite_db_dir = os.path.join(instance_dir, 'db')
    ecm_db_engine = config.get('database', 'ecm_engine')

    # run collectstatic
    collect_static_files(instance_dir, options)

    # run syncdb
    if 'sqlite' in ecm_db_engine and not os.path.exists(sqlite_db_dir):
        os.makedirs(sqlite_db_dir)
    init_ecm_db(instance_dir)

    log('')
    log('ECM instance initialized in "%s".' % instance_dir)

    print_load_message(instance_dir, ecm_db_engine)
Пример #4
0
def _start(daemon):
    log('Starting...')
    try:
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect( (daemon.address, daemon.port) )
        log('ERROR: Address "%s:%s" already in use.' % (daemon.address, daemon.port))
        sys.exit(1)
    except socket.error:
        pass
    finally:
        sock.close()

    daemon.start()
    try:
        # wait to let the child process create the PID file
        tries = 0
        while tries < 10:
            try:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                sock.connect( (daemon.address, daemon.port) )
                with open(daemon.pidfile, 'r') as pf:
                    pid = pf.read()
                log('Server is listening on "%s:%s" (PID: %s)'
                                     % (daemon.address, daemon.port, pid.strip()))
                sys.exit(0)
            except socket.error:
                tries += 1
                time.sleep(0.5)
            finally:
                sock.close()
        raise socket.error()
    except (IOError, socket.error):
        log('ERROR: Failed to start instance.')
Пример #5
0
def print_load_message(instance_dir, db_engine):
    
    if 'mysql' in db_engine:
        engine = 'mysql'
    elif 'sqlite' in db_engine:
        engine = 'sqlite'
    else:
        engine = 'psql'
    
    log('')
    log('Now you need to load your database with EVE static data.')
    log('Please execute `ecm-admin load %s <official_dump_file>` to do so.' % instance_dir)
    log('You will find official dump conversions here http://releases.eve-corp-management.org/eve_sde/')
    log('Be sure to take the latest one matching your db engine "%s".' % engine)
Пример #6
0
def print_load_message(instance_dir, db_engine):
    
    if 'mysql' in db_engine:
        engine = 'mysql'
    elif 'sqlite' in db_engine:
        engine = 'sqlite'
    else:
        engine = 'psql'
    
    log('')
    log('Now you need to load your database with EVE static data.')
    log('Please execute `ecm-admin load %s <ecm_dump_file>` to do so.' % instance_dir)
    log('You will find links to official dump conversions here https://github.com/evecm/ecm/wiki/Static-Data')
    log('Be sure to take the latest one matching your db engine "%s".' % engine)
Пример #7
0
def run(command, global_options, options, args):
    
    if not args:
        command.parser.error('Missing instance directory.')
    instance_dir = args[0]

    config = SafeConfigParser()
    settings_file = path.join(instance_dir, 'settings.ini')
    if not config.read([settings_file]):
        command.parser.error('Settings file "%s" not found.' % settings_file)

    address = config.get('misc', 'server_bind_ip') or '127.0.0.1'
    port = config.getint('misc', 'server_bind_port') or 8888

    log('Starting...')
    try:
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect( (address, port) )
        log('ERROR: Address "%s:%s" already in use.' % (address, port))
        sys.exit(1)
    except socket.error:
        pass
    finally:
        sock.close()
    log('GEvent WSGI server listening on %s:%s.' % (address, port))
    log('Hit CTRL + C to stop.')
    run_server(instance_dir, address, port, options.access_log)
Пример #8
0
def upgrade_instance_files(instance_dir, config):
    log('Upgrading instance config files & examples...')

    apache_mod_wsgi_vhost = path.join(instance_dir, 'examples/apache_mod_wsgi_vhost.example')
    apache_proxy_vhost = path.join(instance_dir, 'examples/apache_reverse_proxy.example')
    try:
        with open(apache_proxy_vhost) as fd:
            buff = fd.read()
        match = SERVER_NAME_REGEXP.search(buff)
        if match:
            host_name = match.group(1)
        else:
            host_name = '???'
    except IOError:
        buff = ''
        host_name = '???'

    template_dir = path.abspath(path.dirname(instance_template.__file__))
    shutil.copy(path.join(template_dir, 'settings.py'), instance_dir)
    shutil.copy(path.join(template_dir, 'manage.py'), instance_dir)
    dir_util.copy_tree(path.join(template_dir, 'wsgi'), path.join(instance_dir, 'wsgi'))
    dir_util.copy_tree(path.join(template_dir, 'examples'), path.join(instance_dir, 'examples'))
    if hasattr(os, 'chmod'):
        os.chmod(path.join(instance_dir, 'manage.py'), 00755)

    options = {
        'host_name': host_name,
        'instance_dir': path.abspath(instance_dir),
        'bind_address': config.get('misc', 'server_bind_ip'),
        'bind_port': config.get('misc', 'server_bind_port'),
    }

    try:
        with open(apache_mod_wsgi_vhost, 'r') as fd:
            buff = fd.read()
        buff = buff % options
        with open(apache_mod_wsgi_vhost, 'w') as fd:
            buff = fd.write(buff)
    except IOError, err:
        log(err)
Пример #9
0
def migrate_ecm_db(instance_dir, upgrade_from_149=False):
    instance_dir = path.abspath(instance_dir)

    log("Migrating database...")
    run_python_cmd('manage.py syncdb --noinput', instance_dir)

    if upgrade_from_149:
        log('Migrating from ECM 1.4.9...')
        # we are upgrading from ECM 1.X.Y, we must perform the init migration
        # on the 'hr' app (rename tables from 'roles_xxxxx' to 'hr_xxxxx')
        pipe_to_django_shell('from south.models import MigrationHistory; '\
                             'MigrationHistory.objects.all().delete()' , instance_dir)

        run_python_cmd('manage.py migrate hr 0001 --noinput', instance_dir)
        # we MUST "fake" the first migration for 1.4.9 apps
        # otherwise the migrate command will fail because DB tables already exist...
        for app in ('common', 'scheduler', 'corp', 'assets', 'accounting'):
            run_python_cmd('manage.py migrate %s 0001 --fake --noinput' % app, instance_dir)

    run_python_cmd('manage.py migrate --all --noinput', instance_dir)

    if upgrade_from_149:
        pipe_to_django_shell('from ecm.apps.scheduler.models import ScheduledTask; '\
                             'ScheduledTask.objects.all().delete()' , instance_dir)
        pipe_to_django_shell('from ecm.apps.common.models import UrlPermission; '\
                             'UrlPermission.objects.all().delete()' , instance_dir)

    log('Database Migration successful.')
Пример #10
0
def run(command, global_options, options, args):

    if not args:
        command.parser.error('Missing instance directory.')
    instance_dir = args.pop(0)
    if not args:
        command.parser.error('Missing dump file.')
    dump_file = args.pop(0)

    if not options.overwrite and os.path.exists(dump_file):
        command.parser.error('Dump file already exists.')

    config = SafeConfigParser()
    if config.read([os.path.join(instance_dir, 'settings.ini')]):
        db_engine = config.get('database', 'ecm_engine')
        db_name = config.get('database', 'ecm_name')
        db_user = config.get('database', 'ecm_user')
        db_password = config.get('database', 'ecm_password')
    else:
        command.parser.error('Could not read "settings.ini" in instance dir.')

    if not db_engine in SUPPORTED_ENGINES:
        command.parser.error('Cannot dump patched EVE data with database engine %r. '
                             'Supported engines: %r' % (db_engine, SUPPORTED_ENGINES))

    # remove existing file
    if os.path.exists(dump_file):
        os.remove(dump_file)

    log('Dumping EVE data to %r...' % dump_file)
    if options.json:
        dump_json(instance_dir, dump_file)
    elif 'postgresql' in db_engine:
        dump_psql(instance_dir, dump_file, db_user, db_password, db_name)
    elif 'mysql' in db_engine:
        dump_mysql(instance_dir, dump_file, db_user, db_password, db_name)
    elif 'sqlite' in db_engine:
        dump_sqlite(instance_dir, dump_file)
    log('EVE data successfully exported')
Пример #11
0
def run(command, global_options, options, args):
    
    if not args:
        command.parser.error('Missing instance directory.')
    instance_dir = args.pop(0)
    
    config = SafeConfigParser()
    if config.read([os.path.join(instance_dir, 'settings.ini')]):
        db_engine = config.get('database', 'ecm_engine')
        db_password = config.get('database', 'ecm_password')
    else:
        command.parser.error('Could not read "settings.ini" in instance dir.')
    
    if options.fuzzwork:
        if not 'mysql' in db_engine:
            command.parser.error('Fuzzwork download only supported with MySql database engine.')
        if args:
            dumppath = args.pop(0)
        else:
            dumppath = FUZZWORK_URL_PREFIX
    elif not args:
        command.parser.error('Missing dumppath or --fuzzwork option.')
    else:
        dumppath = args.pop(0)
    
    try:
        tempdir = tempfile.mkdtemp()
    
        if options.save:
            savedir = '.'
        else:
            savedir = tempdir
        
        if options.fuzzwork:
            for table in FUZZWORK_TABLES:
                load_dump_file(instance_dir, savedir, tempdir, os.path.join(dumppath, table) + FUZZWORK_URL_SUFFIX, db_engine, db_password)
            log('Patching CCP format SDE to match ours... (this also takes awhile)')
            pipe_to_dbshell(os.path.join(SQL_ROOT, FUZZWORK_PATCH_SCRIPT), instance_dir, password=db_password)
        else:
            load_dump_file(instance_dir, savedir, tempdir, dumppath, db_engine, db_password)        
        
        log('EVE static data successfully imported.')
    finally:
        log('Removing temp files...')
        shutil.rmtree(tempdir)
        log('done')
Пример #12
0
def run(command, global_options, options, args):
    """
    Create a new ECM instance.
    """
    instance_dir = init_instance(command, args)
    if options.quiet:
        return
    try:
        prompt_missing_options(options)
        write_settings(command, options, instance_dir)
        log('')
        log('New ECM instance created in "%s".' % instance_dir)
        log('Please check the configuration in "%s" before running `ecm-admin init "%s"`.'
                 % (path.join(instance_dir, 'settings.ini'), instance_dir))
    except:
        # delete the created instance directory if something goes wrong
        shutil.rmtree(instance_dir, ignore_errors=True)
        raise
Пример #13
0
def run(command, global_options, optionsd, args):
    
    if not args:
        command.parser.error('Missing instance directory.')
    instance_dir = args.pop(0)
    if not args:
        command.parser.error('Missing datadump.')
    datadump = args.pop(0)
    
    config = SafeConfigParser()
    if config.read([os.path.join(instance_dir, 'settings.ini')]):
        db_engine = config.get('database', 'ecm_engine')
        db_password = config.get('database', 'ecm_password')
    else:
        command.parser.error('Could not read "settings.ini" in instance dir.')
    try:
        sql = CCP_DATA_DUMPS[db_engine]
    except KeyError:
        command.parser.error('Cannot load datadump with database engine %r. '
                             'Supported engines: %r' % (db_engine, CCP_DATA_DUMPS.keys()))

    try:
        tempdir = tempfile.mkdtemp()
        
        if not os.path.exists(datadump):
            # download from URL
            dump_archive = os.path.join(tempdir, os.path.basename(datadump))
            log('Downloading EVE original dump from %r to %r...', datadump, dump_archive)
            req = urllib2.urlopen(datadump)
            with open(dump_archive, 'wb') as fp:
                shutil.copyfileobj(req, fp)
            req.close()
            log('Download complete.')
        else:
            dump_archive = datadump
        
        extension = os.path.splitext(dump_archive)[-1]
        if extension in ('.bz2', '.gz', '.zip'):
            # decompress archive
            log('Expanding %r to %r...', dump_archive, tempdir)
            dump_file = expand(dump_archive, tempdir)
            log('Expansion complete to %r.' % dump_file)
        else:
            dump_file = dump_archive
        
        log('Patching and importing data (this can be long)...')
        if 'sqlite' in db_engine:
            import sqlite3
            with open(os.path.join(SQL_ROOT, sql['PATCH'])) as f:
                sql_script = f.read() 
            connection = sqlite3.connect(os.path.join(instance_dir, 'db/ecm.sqlite'))
            cursor = connection.cursor()
            cursor.execute('ATTACH DATABASE \'%s\' AS "eve";' % dump_file)
            cursor.executescript(sql_script)
            cursor.execute('DETACH DATABASE "eve";')
            cursor.close()
            connection.commit()
        else:
            pipe_to_dbshell(dump_file, instance_dir, password=db_password)
            pipe_to_dbshell(os.path.join(SQL_ROOT, sql['PATCH']), instance_dir, password=db_password)
            pipe_to_dbshell(os.path.join(SQL_ROOT, sql['DROP']), instance_dir, password=db_password)
        
        log('EVE data successfully imported.')
    finally:
        log('Removing temp files...')
        shutil.rmtree(tempdir)
        log('done')
Пример #14
0
def collect_static_files(instance_dir, options):
    log("Gathering static files...")
    switches = '--noinput'
    if os.name != 'nt' and options.symlink_files:
        switches += ' --link'
    run_python_cmd('manage.py collectstatic ' + switches, instance_dir)
Пример #15
0
def _stop(daemon):
    log('Shutting down...')
    daemon.stop()
    log('Stopped')
Пример #16
0
def run(command, global_options, options, args):

    if not args:
        command.parser.error('Missing instance directory.')
    instance_dir = args[0]

    config = SafeConfigParser()
    settings_file = path.join(instance_dir, 'settings.ini')
    if not config.read([settings_file]):
        command.parser.error('Settings file "%s" not found.' % settings_file)

    pidfile = config.get('misc', 'pid_file') or 'ecm.pid'
    address = config.get('misc', 'server_bind_ip') or '127.0.0.1'
    port = config.getint('misc', 'server_bind_port') or 8888
    run_as_user = config.get('misc', 'run_as_user') or None

    if not path.isabs(pidfile):
        pidfile = path.abspath(path.join(instance_dir, pidfile))

    real_command = sys.argv[1]

    if real_command == 'status':
        if path.isfile(pidfile):
            with open(pidfile, 'r') as pf:
                pid = pf.read()
            log('Instance is running with PID: %s' % pid.strip())
        else:
            log('Instance is stopped')
        sys.exit(0)
    else:
        if run_as_user:
            import pwd #@UnresolvedImport
            try:
                uid = pwd.getpwnam(run_as_user).pw_uid
            except KeyError:
                command.parser.error('User "%s" does not exist.' % run_as_user)
        else:
            uid = None

        if options.logfile is not None:
            logfile = path.abspath(options.logfile)
            logdir = path.dirname(logfile)
            if not path.exists(logdir):
                os.makedirs(logdir)
        else:
            logfile = None

        daemon = GEventWSGIDaemon(address=address,
                                  port=port,
                                  pidfile=pidfile,
                                  working_dir=path.abspath(instance_dir),
                                  uid=uid,
                                  stdout=logfile,
                                  stderr=logfile)

        if real_command == 'start':
            _start(daemon)
        elif real_command == 'stop':
            _stop(daemon)
        elif real_command == 'restart':
            _stop(daemon)
            _start(daemon)
Пример #17
0
def init_ecm_db(instance_dir):
    log("Initializing database...")
    run_python_cmd('manage.py syncdb --noinput --migrate', instance_dir)
    log('Database initialization successful.')
Пример #18
0
def print_usage_feedback_message():
    log('')
    log('*ATTENTION*')
    log('')
    log('ECM now has usage feedback system which is enabled by default.')
    log('This feature is really helpfull for ECM developers because it allows')
    log('us to know how much the project is used and helps our motivation ;-)')
    log('')
    log('It consists in a scheduled task that will run once each week and send')
    log('*basic* and *anonymous* data to the official server %r.', ECM_USAGE_FEEDBACK_URL)
    log('This is *NOT* a spying system nor a backdoor to enter your ECM instance.')
    log('To know what data is sent in detail, please see the source code at http://eve-corp-management.org/projects/ecm/repository/entry/ecm/apps/common/tasks/usage_feedback.py')
    log('')
    log('If however you *really* want to disable the usage feedback on your')
    log('instance, you need to go to the admin panel and change the scheduled')
    log('task "ecm....send_feedback" to inactive.')
    log('')
    log('Thank you for using ECM. Fly safe.')
Пример #19
0
    try:
        with open(apache_mod_wsgi_vhost, 'r') as fd:
            buff = fd.read()
        buff = buff % options
        with open(apache_mod_wsgi_vhost, 'w') as fd:
            buff = fd.write(buff)
    except IOError, err:
        log(err)
    try:
        with open(apache_proxy_vhost, 'r') as fd:
            buff = fd.read()
        buff = buff % options
        with open(apache_proxy_vhost, 'w') as fd:
            buff = fd.write(buff)
    except IOError, err:
        log(err)

    if not config.has_option('misc', 'secret_key'):
        # Create a random SECRET_KEY hash to put it in the main settings.
        chars = string.ascii_letters + string.digits + '.,;:!@#$^&*(-_+)[]{}'
        config.set('misc', 'secret_key', get_random_string(50, chars))

    template_dir = path.abspath(path.dirname(instance_template.__file__))
    default_config = SafeConfigParser()
    default_config.read([path.join(template_dir, 'settings.ini')])

    for section in default_config.sections():
        for option in default_config.options(section):
            if not config.has_option(section, option):
                config.set(section, option, default_config.get(section, option))
Пример #20
0
def load_dump_file(instance_dir, savedir, tempdir, datadump, db_engine, db_password):
    if not os.path.exists(datadump):
        # download from URL
        dump_archive = os.path.join(savedir, os.path.basename(datadump))
        log('Downloading %r to %r...', datadump, dump_archive)
        req = urllib2.urlopen(datadump)
        with open(dump_archive, 'wb') as fp:
            shutil.copyfileobj(req, fp)
        req.close()
    else:
        dump_archive = datadump
    
    extension = os.path.splitext(dump_archive)[-1]
    if extension in ('.bz2', '.gz', '.zip'):
        # decompress archive
        log('Expanding %r to %r...', dump_archive, tempdir)
        dump_file = expand(dump_archive, tempdir)
        extension = os.path.splitext(dump_file)[-1]
    else:
        dump_file = dump_archive
        
    log('Importing %r... (this may take awhile!)', dump_file)
    
    if extension in ('.json'):
        load_json_dump(instance_dir, dump_file, tempdir)
    elif 'sqlite' in db_engine:
        import sqlite3
        config = SafeConfigParser()
        db_dir = ''
        if config.read([os.path.join(instance_dir, 'settings.ini')]):
            db_dir = config.get('database', 'sqlite_db_dir')
        if not db_dir:
            db_dir = os.path.join(instance_dir, 'db')
            
        db_file = os.path.join(db_dir, 'ecm.sqlite')
        
        # Connect to the instance DB and attach the SDE
        connection = sqlite3.connect(db_file)
        cursor = connection.cursor()
        cursor.execute('ATTACH DATABASE \'%s\' AS "eve";' % dump_file)
        
        # Get the tables from the SDE (import them all)
        cursor.execute('SELECT "name","sql" FROM "eve"."sqlite_master" WHERE "type"="table" AND "sql" IS NOT NULL;')
        tables = cursor.fetchall()
        
        # Load the table data as brand new tables matching the dump file (to avoid unexplainable errors, maybe because Django/south doesn't set them up the same as the DB dump conversion scripts)
        for table in tables:
            tablename = table[0]
            tablesql  = table[1]
            
            # Drop and recreate the table
            cursor.execute('DROP TABLE "%s";' % tablename)
            cursor.execute(tablesql)
            
            # Insert the data
            cursor.execute('INSERT INTO "%s" SELECT * FROM "eve"."%s";' % (tablename, tablename))

        # Get the indicies of the attached DB and create them
        cursor.execute('SELECT "sql" FROM "eve"."sqlite_master" WHERE "type"="index" AND "sql" IS NOT NULL;')
        indicies = cursor.fetchall()
        for index in indicies:
            cursor.execute(index[0])
            
        cursor.execute('DETACH DATABASE "eve";')
        cursor.close()
        connection.commit()
    else:
        pipe_to_dbshell(dump_file, instance_dir, password=db_password)