Example #1
0
def main(url_action=None, url_args=None):
    from helpers import Paths, Config
    import xcallback

    config = Config()

    wc = WorkingCopySync(Paths, config)
    xc = xcallback.Handler(Paths, Config.filename)

    if not url_action or url_action == editor.get_path():
        wc.present()
    elif url_action == 'copy_repo':
        xc.copy_repo(url_args[0], url_args[1])
    elif url_action == 'overwrite_file':
        xc.copy_file(url_args[0], url_args[1])
    elif url_action == 'repo_list':
        wc.copy_repo_from_wc(
            repo_list=[repo['name'] for repo in json.loads(url_args[0])])
    else:
        msg = "Not a valid URL scheme action. Now say you're sorry."
        console.alert(msg, button1="I'm sorry.", hide_cancel_button=True)
Example #2
0
    def handle(self, **options):
        # ignore_errors = options.get('ignore_errors')
        config = Config(options)
        force_exec = options.get('force_exec')
        backup_dir = options.get('backup_dir')
        skip_geoserver = options.get('skip_geoserver')

        if not backup_dir or len(backup_dir) == 0:
            raise CommandError("Destination folder '--backup-dir' is mandatory")

        print "Before proceeding with the Backup, please ensure that:"
        print " 1. The backend (DB or whatever) is accessible and you have rights"
        print " 2. The GeoServer is up and running and reachable from this machine"
        message = 'You want to proceed?'

        if force_exec or helpers.confirm(prompt=message, resp=False):

            # Create Target Folder
            dir_time_suffix = get_dir_time_suffix()
            target_folder = os.path.join(backup_dir, dir_time_suffix)
            if not os.path.exists(target_folder):
                os.makedirs(target_folder)
            # Temporary folder to store backup files. It will be deleted at the end.
            os.chmod(target_folder, 0777)

            if not skip_geoserver:
                self.create_geoserver_backup(settings, target_folder)
                self.dump_geoserver_raster_data(config, settings, target_folder)
                self.dump_geoserver_vector_data(config, settings, target_folder)
                print("Duming geoserver external resources")
                self.dump_geoserver_externals(config, settings, target_folder)
            else:
                print("Skipping geoserver backup")

            try:
                # Deactivate GeoNode Signals
                print "Deactivating GeoNode Signals..."
                designals()
                print "...done!"

                # Dump Fixtures
                for app_name, dump_name in zip(config.app_names, config.dump_names):
                    print "Dumping '"+app_name+"' into '"+dump_name+".json'."
                    # Point stdout at a file for dumping data to.
                    output = open(os.path.join(target_folder, dump_name+'.json'), 'w')
                    call_command('dumpdata', app_name, format='json', indent=2, natural=True, stdout=output)
                    output.close()

                # Store Media Root
                media_root = settings.MEDIA_ROOT
                media_folder = os.path.join(target_folder, helpers.MEDIA_ROOT)
                if not os.path.exists(media_folder):
                    os.makedirs(media_folder)

                copy_tree(media_root, media_folder)
                print "Saved Media Files from '"+media_root+"'."

                # Store Static Root
                static_root = settings.STATIC_ROOT
                static_folder = os.path.join(target_folder, helpers.STATIC_ROOT)
                if not os.path.exists(static_folder):
                    os.makedirs(static_folder)

                copy_tree(static_root, static_folder)
                print "Saved Static Root from '"+static_root+"'."

                # Store Static Folders
                static_folders = settings.STATICFILES_DIRS
                static_files_folders = os.path.join(target_folder, helpers.STATICFILES_DIRS)
                if not os.path.exists(static_files_folders):
                    os.makedirs(static_files_folders)

                for static_files_folder in static_folders:
                    static_folder = os.path.join(static_files_folders,
                                                 os.path.basename(os.path.normpath(static_files_folder)))
                    if not os.path.exists(static_folder):
                        os.makedirs(static_folder)

                    copy_tree(static_files_folder, static_folder)
                    print "Saved Static Files from '"+static_files_folder+"'."

                # Store Template Folders
                template_folders = []
                try:
                    template_folders = settings.TEMPLATE_DIRS
                except:
                    try:
                        template_folders = settings.TEMPLATES[0]['DIRS']
                    except:
                        pass
                template_files_folders = os.path.join(target_folder, helpers.TEMPLATE_DIRS)
                if not os.path.exists(template_files_folders):
                    os.makedirs(template_files_folders)

                for template_files_folder in template_folders:
                    template_folder = os.path.join(template_files_folders,
                                                   os.path.basename(os.path.normpath(template_files_folder)))
                    if not os.path.exists(template_folder):
                        os.makedirs(template_folder)

                    copy_tree(template_files_folder, template_folder)
                    print "Saved Template Files from '"+template_files_folder+"'."

                # Store Locale Folders
                locale_folders = settings.LOCALE_PATHS
                locale_files_folders = os.path.join(target_folder, helpers.LOCALE_PATHS)
                if not os.path.exists(locale_files_folders):
                    os.makedirs(locale_files_folders)

                for locale_files_folder in locale_folders:
                    locale_folder = os.path.join(locale_files_folders,
                                                 os.path.basename(os.path.normpath(locale_files_folder)))
                    if not os.path.exists(locale_folder):
                        os.makedirs(locale_folder)

                    copy_tree(locale_files_folder, locale_folder)
                    print "Saved Locale Files from '"+locale_files_folder+"'."

                # Create Final ZIP Archive
                zip_dir(target_folder, os.path.join(backup_dir, dir_time_suffix+'.zip'))

                # Clean-up Temp Folder
                try:
                    shutil.rmtree(target_folder)
                except:
                    print "WARNING: Could not be possible to delete the temp folder: '" + str(target_folder) + "'"

                print "Backup Finished. Archive generated."

                return str(os.path.join(backup_dir, dir_time_suffix+'.zip'))
            finally:
                # Reactivate GeoNode Signals
                print "Reactivating GeoNode Signals..."
                resignals()
                print "...done!"
Example #3
0
    def handle(self, **options):
        # ignore_errors = options.get('ignore_errors')
        config = Config(options)
        force_exec = options.get('force_exec')
        backup_file = options.get('backup_file')
        skip_geoserver = options.get('skip_geoserver')
        backup_dir = options.get('backup_dir')

        if not any([backup_file, backup_dir]):
            raise CommandError("Mandatory option (--backup-file|--backup-dir)")

        if all([backup_file, backup_dir]):
            raise CommandError("Exclusive option (--backup-file|--backup-dir)")

        if backup_file and not os.path.isfile(backup_file):
            raise CommandError("Provided '--backup-file' is not a file")

        if backup_dir and not os.path.isdir(backup_dir):
            raise CommandError("Provided '--backup-dir' is not a directory")

        print "Before proceeding with the Restore, please ensure that:"
        print " 1. The backend (DB or whatever) is accessible and you have rights"
        print " 2. The GeoServer is up and running and reachable from this machine"
        message = 'WARNING: The restore will overwrite ALL GeoNode data. You want to proceed?'
        if force_exec or helpers.confirm(prompt=message, resp=False):
            target_folder = backup_dir

            if backup_file:
                # Create Target Folder
                restore_folder = os.path.join(tempfile.gettempdir(), 'restore')
                if not os.path.exists(restore_folder):
                    os.makedirs(restore_folder)

                # Extract ZIP Archive to Target Folder
                target_folder = extract_archive(backup_file, restore_folder)

                # Write Checks
                media_root = settings.MEDIA_ROOT
                media_folder = os.path.join(target_folder, helpers.MEDIA_ROOT)
                static_root = settings.STATIC_ROOT
                static_folder = os.path.join(target_folder, helpers.STATIC_ROOT)
                static_folders = settings.STATICFILES_DIRS
                static_files_folders = os.path.join(target_folder, helpers.STATICFILES_DIRS)
                template_folders = settings.TEMPLATE_DIRS
                template_files_folders = os.path.join(target_folder, helpers.TEMPLATE_DIRS)
                locale_folders = settings.LOCALE_PATHS
                locale_files_folders = os.path.join(target_folder, helpers.LOCALE_PATHS)

                try:
                    print("[Sanity Check] Full Write Access to '{}' ...".format(media_root))
                    chmod_tree(media_root)
                    print("[Sanity Check] Full Write Access to '{}' ...".format(static_root))
                    chmod_tree(static_root)
                    for static_files_folder in static_folders:
                        print("[Sanity Check] Full Write Access to '{}' ...".format(static_files_folder))
                        chmod_tree(static_files_folder)
                    for template_files_folder in template_folders:
                        print("[Sanity Check] Full Write Access to '{}' ...".format(template_files_folder))
                        chmod_tree(template_files_folder)
                    for locale_files_folder in locale_folders:
                        print("[Sanity Check] Full Write Access to '{}' ...".format(locale_files_folder))
                        chmod_tree(locale_files_folder)
                except:
                    print("...Sanity Checks on Folder failed. Please make sure that the current user has full WRITE access to the above folders (and sub-folders or files).")
                    print("Reason:")
                    raise

            if not skip_geoserver:
                self.restore_geoserver_backup(settings, target_folder)
                self.restore_geoserver_raster_data(config, settings, target_folder)
                self.restore_geoserver_vector_data(config, settings, target_folder)
                print("Restoring geoserver external resources")
                self.restore_geoserver_externals(config, settings, target_folder)
            else:
                print("Skipping geoserver backup restore")

            # Prepare Target DB
            try:
                call_command('migrate', interactive=False, load_initial_data=False)

                db_name = settings.DATABASES['default']['NAME']
                db_user = settings.DATABASES['default']['USER']
                db_port = settings.DATABASES['default']['PORT']
                db_host = settings.DATABASES['default']['HOST']
                db_passwd = settings.DATABASES['default']['PASSWORD']

                helpers.patch_db(db_name, db_user, db_port, db_host, db_passwd, settings.MONITORING_ENABLED)
            except:
                traceback.print_exc()

            try:
                # Deactivate GeoNode Signals
                print "Deactivating GeoNode Signals..."
                designals()
                print "...done!"

                # Flush DB
                try:
                    db_name = settings.DATABASES['default']['NAME']
                    db_user = settings.DATABASES['default']['USER']
                    db_port = settings.DATABASES['default']['PORT']
                    db_host = settings.DATABASES['default']['HOST']
                    db_passwd = settings.DATABASES['default']['PASSWORD']

                    helpers.flush_db(db_name, db_user, db_port, db_host, db_passwd)
                except:
                    try:
                        call_command('flush', interactive=False, load_initial_data=False)
                    except:
                        traceback.print_exc()
                        raise

                # Restore Fixtures
                for app_name, dump_name in zip(config.app_names, config.dump_names):
                    fixture_file = os.path.join(target_folder, dump_name+'.json')

                    print "Deserializing "+fixture_file
                    try:
                        call_command('loaddata', fixture_file, app_label=app_name)
                    except:
                        traceback.print_exc()
                        print "WARNING: No valid fixture data found for '"+dump_name+"'."
                        # helpers.load_fixture(app_name, fixture_file)
                        raise

                # Restore Media Root
                try:
                    shutil.rmtree(media_root)
                except:
                    pass

                if not os.path.exists(media_root):
                    os.makedirs(media_root)

                copy_tree(media_folder, media_root)
                chmod_tree(media_root)
                print "Media Files Restored into '"+media_root+"'."

                # Restore Static Root
                try:
                    shutil.rmtree(static_root)
                except:
                    pass

                if not os.path.exists(static_root):
                    os.makedirs(static_root)

                copy_tree(static_folder, static_root)
                chmod_tree(static_root)
                print "Static Root Restored into '"+static_root+"'."

                # Restore Static Root
                try:
                    shutil.rmtree(static_root)
                except:
                    pass

                if not os.path.exists(static_root):
                    os.makedirs(static_root)

                copy_tree(static_folder, static_root)
                chmod_tree(static_root)
                print "Static Root Restored into '"+static_root+"'."

                # Restore Static Folders
                for static_files_folder in static_folders:
                    try:
                        shutil.rmtree(static_files_folder)
                    except:
                        pass

                    if not os.path.exists(static_files_folder):
                        os.makedirs(static_files_folder)

                    copy_tree(os.path.join(static_files_folders,
                                           os.path.basename(os.path.normpath(static_files_folder))),
                              static_files_folder)
                    chmod_tree(static_files_folder)
                    print "Static Files Restored into '"+static_files_folder+"'."

                # Restore Template Folders
                for template_files_folder in template_folders:
                    try:
                        shutil.rmtree(template_files_folder)
                    except:
                        pass

                    if not os.path.exists(template_files_folder):
                        os.makedirs(template_files_folder)

                    copy_tree(os.path.join(template_files_folders,
                                           os.path.basename(os.path.normpath(template_files_folder))),
                              template_files_folder)
                    chmod_tree(template_files_folder)
                    print "Template Files Restored into '"+template_files_folder+"'."

                # Restore Locale Folders
                for locale_files_folder in locale_folders:
                    try:
                        shutil.rmtree(locale_files_folder)
                    except:
                        pass

                    if not os.path.exists(locale_files_folder):
                        os.makedirs(locale_files_folder)

                    copy_tree(os.path.join(locale_files_folders,
                                           os.path.basename(os.path.normpath(locale_files_folder))),
                              locale_files_folder)
                    chmod_tree(locale_files_folder)
                    print "Locale Files Restored into '"+locale_files_folder+"'."

                call_command('collectstatic', interactive=False)

                # Cleanup DB
                try:
                    db_name = settings.DATABASES['default']['NAME']
                    db_user = settings.DATABASES['default']['USER']
                    db_port = settings.DATABASES['default']['PORT']
                    db_host = settings.DATABASES['default']['HOST']
                    db_passwd = settings.DATABASES['default']['PASSWORD']

                    helpers.cleanup_db(db_name, db_user, db_port, db_host, db_passwd)
                except:
                    traceback.print_exc()

                return str(target_folder)

            finally:
                # Reactivate GeoNode Signals
                print "Reactivating GeoNode Signals..."
                resignals()
                print "...done!"

                call_command('migrate', interactive=False, load_initial_data=False, fake=True)

                print "HINT: If you migrated from another site, do not forget to run the command 'migrate_baseurl' to fix Links"
                print " e.g.:  DJANGO_SETTINGS_MODULE=my_geonode.settings python manage.py migrate_baseurl --source-address=my-host-dev.geonode.org --target-address=my-host-prod.geonode.org"
                print "Restore finished. Please find restored files and dumps into:"
Example #4
0
    def handle(self, **options):
        # ignore_errors = options.get('ignore_errors')
        config = Config(options)
        force_exec = options.get('force_exec')
        backup_file = options.get('backup_file')
        skip_geoserver = options.get('skip_geoserver')
        backup_dir = options.get('backup_dir')

        if not any([backup_file, backup_dir]):
            raise CommandError("Mandatory option (--backup-file|--backup-dir)")

        if all([backup_file, backup_dir]):
            raise CommandError("Exclusive option (--backup-file|--backup-dir)")

        if backup_file and not os.path.isfile(backup_file):
            raise CommandError("Provided '--backup-file' is not a file")

        if backup_dir and not os.path.isdir(backup_dir):
            raise CommandError("Provided '--backup-dir' is not a directory")

        print "Before proceeding with the Restore, please ensure that:"
        print " 1. The backend (DB or whatever) is accessible and you have rights"
        print " 2. The GeoServer is up and running and reachable from this machine"
        message = 'WARNING: The restore will overwrite ALL GeoNode data. You want to proceed?'
        if force_exec or helpers.confirm(prompt=message, resp=False):
            target_folder = backup_dir

            if backup_file:
                # Create Target Folder
                restore_folder = os.path.join(tempfile.gettempdir(), 'restore')
                if not os.path.exists(restore_folder):
                    os.makedirs(restore_folder)

                # Extract ZIP Archive to Target Folder
                target_folder = helpers.unzip_file(backup_file, restore_folder)

            if not skip_geoserver:
                self.restore_geoserver_backup(settings, target_folder)
                self.restore_geoserver_raster_data(config, settings, target_folder)
                self.restore_geoserver_vector_data(config, settings, target_folder)
                print("Restoring geoserver external resources")
                self.restore_geoserver_externals(config, settings, target_folder)
            else:
                print("Skipping geoserver backup restore")

            # Prepare Target DB
            try:
                call_command('migrate', interactive=False, load_initial_data=False)
                call_command('flush', interactive=False, load_initial_data=False)

                db_name = settings.DATABASES['default']['NAME']
                db_user = settings.DATABASES['default']['USER']
                db_port = settings.DATABASES['default']['PORT']
                db_host = settings.DATABASES['default']['HOST']
                db_passwd = settings.DATABASES['default']['PASSWORD']

                helpers.patch_db(db_name, db_user, db_port, db_host, db_passwd)
            except:
                traceback.print_exc()

            try:
                # Deactivate GeoNode Signals
                print "Deactivating GeoNode Signals..."
                designals()
                print "...done!"

                # Restore Fixtures
                for app_name, dump_name in zip(config.app_names, config.dump_names):
                    fixture_file = os.path.join(target_folder, dump_name+'.json')

                    print "Deserializing "+fixture_file
                    try:
                        call_command('loaddata', fixture_file, app_label=app_name)
                    except:
                        traceback.print_exc()
                        print "WARNING: No valid fixture data found for '"+dump_name+"'."
                        # helpers.load_fixture(app_name, fixture_file)

                # Restore Media Root
                media_root = settings.MEDIA_ROOT
                media_folder = os.path.join(target_folder, helpers.MEDIA_ROOT)

                try:
                    shutil.rmtree(media_root)
                except:
                    pass

                if not os.path.exists(media_root):
                    os.makedirs(media_root)

                helpers.copy_tree(media_folder, media_root)
                helpers.chmod_tree(media_root)
                print "Media Files Restored into '"+media_root+"'."

                # Restore Static Root
                static_root = settings.STATIC_ROOT
                static_folder = os.path.join(target_folder, helpers.STATIC_ROOT)

                try:
                    shutil.rmtree(static_root)
                except:
                    pass

                if not os.path.exists(static_root):
                    os.makedirs(static_root)

                helpers.copy_tree(static_folder, static_root)
                helpers.chmod_tree(static_root)
                print "Static Root Restored into '"+static_root+"'."

                # Restore Static Root
                static_root = settings.STATIC_ROOT
                static_folder = os.path.join(target_folder, helpers.STATIC_ROOT)

                try:
                    shutil.rmtree(static_root)
                except:
                    pass

                if not os.path.exists(static_root):
                    os.makedirs(static_root)

                helpers.copy_tree(static_folder, static_root)
                helpers.chmod_tree(static_root)
                print "Static Root Restored into '"+static_root+"'."

                # Restore Static Folders
                static_folders = settings.STATICFILES_DIRS
                static_files_folders = os.path.join(target_folder, helpers.STATICFILES_DIRS)

                for static_files_folder in static_folders:
                    try:
                        shutil.rmtree(static_files_folder)
                    except:
                        pass

                    if not os.path.exists(static_files_folder):
                        os.makedirs(static_files_folder)

                    helpers.copy_tree(os.path.join(static_files_folders,
                                                   os.path.basename(os.path.normpath(static_files_folder))),
                                      static_files_folder)
                    helpers.chmod_tree(static_files_folder)
                    print "Static Files Restored into '"+static_files_folder+"'."

                # Restore Template Folders
                template_folders = settings.TEMPLATE_DIRS
                template_files_folders = os.path.join(target_folder, helpers.TEMPLATE_DIRS)

                for template_files_folder in template_folders:
                    try:
                        shutil.rmtree(template_files_folder)
                    except:
                        pass

                    if not os.path.exists(template_files_folder):
                        os.makedirs(template_files_folder)

                    helpers.copy_tree(os.path.join(template_files_folders,
                                                   os.path.basename(os.path.normpath(template_files_folder))),
                                      template_files_folder)
                    helpers.chmod_tree(template_files_folder)
                    print "Template Files Restored into '"+template_files_folder+"'."

                # Restore Locale Folders
                locale_folders = settings.LOCALE_PATHS
                locale_files_folders = os.path.join(target_folder, helpers.LOCALE_PATHS)

                for locale_files_folder in locale_folders:
                    try:
                        shutil.rmtree(locale_files_folder)
                    except:
                        pass

                    if not os.path.exists(locale_files_folder):
                        os.makedirs(locale_files_folder)

                    helpers.copy_tree(os.path.join(locale_files_folders,
                                                   os.path.basename(os.path.normpath(locale_files_folder))),
                                      locale_files_folder)
                    helpers.chmod_tree(locale_files_folder)
                    print "Locale Files Restored into '"+locale_files_folder+"'."

                # Cleanup DB
                try:
                    db_name = settings.DATABASES['default']['NAME']
                    db_user = settings.DATABASES['default']['USER']
                    db_port = settings.DATABASES['default']['PORT']
                    db_host = settings.DATABASES['default']['HOST']
                    db_passwd = settings.DATABASES['default']['PASSWORD']

                    helpers.cleanup_db(db_name, db_user, db_port, db_host, db_passwd)
                except:
                    traceback.print_exc()

                print "Restore finished. Please find restored files and dumps into:"

                return str(target_folder)

            finally:
                # Reactivate GeoNode Signals
                print "Reactivating GeoNode Signals..."
                resignals()
                print "...done!"
Example #5
0
def main():
    # Read config
    config = Config()
    credentials = Credentials()

    # Set up logging
    format = logging.Formatter("{asctime} [{levelname}] [{module}] {message}",
                               style="{",
                               datefmt="%H:%M:%S")

    # Get the discord logger
    log = logging.getLogger("discord")
    # Set logging level to debug for now
    log.setLevel(logging.INFO)

    # Create a handler that outputs INFO to stdout
    text_handler = logging.StreamHandler()
    text_handler.setLevel(logging.INFO)
    text_handler.setFormatter(format)
    log.addHandler(text_handler)

    # Create a handler that outputs WARNING to the discord webhook
    webhook_handler = DiscordHandler(credentials.bot_log_webhook,
                                     config.bot_log_name)
    webhook_handler.setLevel(logging.WARNING)
    webhook_handler.setFormatter(format)
    log.addHandler(webhook_handler)

    intents = discord.Intents.all()
    intents.typing = True
    intents.presences = True

    # Initialize bot object
    bot = commands.Bot(command_prefix=config.bot_prefix,
                       intents=intents,
                       help_command=None)

    # Load cogs
    bot.add_cog(clean_old_messages.CleanOldMessages(bot, config, log))
    bot.add_cog(ping_for_messages.PingForMessages(bot, config, log))
    bot.add_cog(disable_reacts.DisableReacts(bot, config, log))
    bot.add_cog(vet_ping_unofficials.VetPingUnofficials(bot, config, log))
    bot.add_cog(purge_channel.PurgeChannel(bot, config, log))
    bot.add_cog(reacts_required.ReactsRequired(bot, config, log))
    bot.add_cog(delete_match_regex.DeleteMatchRegEx(bot, config, log))
    bot.add_cog(warn_wrong_battletags.WarnWrongBattletags(bot, config, log))
    bot.add_cog(auto_question_answer.AutoQuestionAnswer(bot, config, log))
    bot.add_cog(
        announce.Announce(bot, config, log, credentials.airtable_api_key))
    bot.add_cog(alert_unofficial_start.AlertUnofficialStart(bot, config, log))
    # bot.add_cog(analytics.Analytics(bot, config, log, credentials.airtable_api_key))
    bot.add_cog(
        most_active.MostActive(bot, config, log, credentials.airtable_api_key))
    bot.add_cog(jts_bnets.JTSBnets(bot, config, log))
    bot.add_cog(jts_lock.JTSLock(bot, config, log))
    # bot.add_cog(pug_points.PugPoints(bot, config, log, credentials.airtable_api_key))
    bot.add_cog(cerberus.Cerberus(bot, config, log))
    bot.add_cog(coached_timer.CoachedTimer(bot, config, log))
    bot.add_cog(help.Help(bot, config, log))
    bot.add_cog(initialize.Initialize(bot, config, log))

    # Start bot
    log.warning("Starting Bot...")
    bot.run(credentials.token)