def get_team(team):
    """
        We're asking user for names and address of the rest of the team,
        and append that to a list we got from get_main_organizer
    """
    add_team = click.confirm(click.style(
        "Do you want to add additional team members?", bold=True, fg='yellow'), default=False)
    i = 1
    while add_team:
        i += 1
        name = click.prompt(click.style(
            "First and last name of #{0} member".format(i), bold=True, fg='yellow'))
        email = click.prompt(click.style(
            "E-mail address of #{0} member".format(i), bold=True, fg='yellow'))
        if len(name) > 0:
            try:
                team.append({'first_name': name.split(' ')[
                            0], 'last_name': name.split(' ')[1], 'email': email})
            except IndexError:
                team.append(
                    {'first_name': name, 'last_name': '', 'email': email})
            click.echo("All right, the #{0} team member of Django Girls is {1} ({2})".format(
                i, name, email))
        add_team = click.confirm(click.style(
            "Do you want to add additional team members?", bold=True, fg='yellow'), default=False)

    return team
Exemple #2
0
def get_team(team):
    """
        We're asking user for names and address of the rest of the team,
        and append that to a list we got from get_main_organizer
    """
    add_team = click.confirm(click.style(
        "Do you want to add additional team members?", bold=True, fg='yellow'),
                             default=False)
    i = 1
    while add_team:
        i += 1
        name = click.prompt(
            click.style("First and last name of #{0} member".format(i),
                        bold=True,
                        fg='yellow'))
        email = click.prompt(
            click.style("E-mail address of #{0} member".format(i),
                        bold=True,
                        fg='yellow'))
        if len(name) > 0:
            team.append({'name': name, 'email': email})
            click.echo(
                "All right, the #{0} team member of Django Girls is {1} ({2})".
                format(i, name, email))
        add_team = click.confirm(click.style(
            "Do you want to add additional team members?",
            bold=True,
            fg='yellow'),
                                 default=False)

    return team
Exemple #3
0
def start(path, limit, username, collection):
    """Start an import session"""

    click.secho(
        '--------------------------------------------------------------------',
        bold=True)
    click.echo('Username:\t {}'.format(username))
    click.echo('Collection:\t {}'.format(collection))
    click.echo('Limit:\t\t {}'.format(limit))
    click.echo('Path:\t\t {}'.format(path))
    click.secho(
        '--------------------------------------------------------------------',
        bold=True)
    click.echo('')

    if not os.path.isdir(path):
        click.secho('Directory does not exist: {}'.format(path),
                    bold=True,
                    fg='red')
        return

    if not path.endswith('/'):
        path += '/'

    if not get_user_model().objects.filter(username=username).exists():
        click.secho('User does not exist: {}'.format(username),
                    bold=True,
                    fg='red')
        return

    if Massimport.objects.filter(directory=path).exists():
        click.secho('Import session already exists: {}'.format(path),
                    bold=True,
                    fg='red')
        return

    massimport = Massimport(
        directory=path,
        user=get_user_model().objects.get(username=username),
        collection_name=collection)

    massimport.save()

    if click.confirm('Continue with scanning directories?', default='Y'):
        massimport.scan()

    if click.confirm('Continue with enqueuing files?'):

        for item in massimport.files.filter(status=0)[0:limit]:
            item.enqueue()
Exemple #4
0
def create_service_user(email, password, token):

    click.secho('Create service user: {}'.format(email), fg='green')

    try:
        user = User.objects.get(email=email)
        click.secho('User already exists: {}'.format(user), fg='yellow')
        click.secho('Current auth token:  {}'.format(user.auth_token), fg='white')

        if click.confirm('Do you want to delete this user account?'):
            user.delete()
        else:
            click.secho('Exiting.', fg='yellow')
            sys.exit()


    except User.DoesNotExist:
        pass

    user = User.objects.create_user(email=email, password=password)
    click.secho('Created user: {}'.format(user), fg='green')

    if token:
        Token.objects.filter(user=user).update(key=token)

    auth_token = Token.objects.get(user=user)

    click.secho('Current auth token:  {}'.format(auth_token), fg='green')
Exemple #5
0
def gather_information():
    click.echo("Hello there sunshine! We're gonna copy an event website now.")

    event = get_event(
        click.prompt(
            click.style(
                "First, give me the latest ID of the Event "
                "object you want to copy",
                bold=True,
                fg='yellow')))

    while not event:
        event = get_event(click.prompt("Wrong ID! Try again"))

    click.echo("Ok, we're copying {}, {}".format(event.city, event.country))

    number = click.prompt(
        click.style(
            "What is the number of the event in this city? "
            "If this is a second event, write 2. If third, then 3. You got it",
            bold=True,
            fg='yellow'))

    date = gather_event_date_from_prompt()

    click.echo("The current team is: " +
               ", ".join(str(organizer) for organizer in event.team.all()))

    new_team = click.confirm(click.style(
        "Do you need to change the whole team?", bold=True, fg='yellow'),
                             default=False)

    return (event, number, date, new_team)
Exemple #6
0
def command(reset):
    """
    Create and initialize Simpl Calc game.
    Create a "default" Simpl Calc run.
    Set the run phase to "Play".
    Add 1 leader ("leader") to the run
    Add 2 players ("s1", "s2") to the run.
    Add a scenario and period 1 for each player.
    """

    # Handle resetting the game
    if reset:
        if click.confirm(
                'Are you sure you want to delete the default game run and recreate from scratch?'
        ):
            delete_default_run(games_client)

    # Create a Game
    game = games_client.games.get_or_create(name='Simpl Calc',
                                            slug='simpl-calc')
    echo('getting or creating game: ', game.name)

    # Create game Phases ("Play")
    play_phase = games_client.phases.get_or_create(
        game=game.id,
        name='Play',
        order=1,
    )
    echo('getting or creating phase: ', play_phase.name)

    # Add run with 2 players ready to play
    run = add_run(game, 'default', 2, play_phase, games_client)

    echo('Completed setting up run: id=', run.id)
Exemple #7
0
def gather_information():
    click.echo("Hello there sunshine! We're gonna copy an event website now.")

    event = get_event(
        click.prompt(click.style("First, give me the latest ID of the Event "
        "object you want to copy", bold=True, fg='yellow'))
    )

    while not event:
        event = get_event(click.prompt("Wrong ID! Try again"))

    click.echo("Ok, we're copying {}, {}".format(
        event.city, event.country))

    number = click.prompt(click.style("What is the number of the event in this city? "
        "If this is a second event, write 2. If third, then 3. You got it", bold=True, fg='yellow')
    )

    date = gather_event_date_from_prompt()

    click.echo("The current team is: " + ", ".join(
        str(organizer) for organizer in event.team.all()))

    new_team = click.confirm(click.style(
        "Do you need to change the whole team?", bold=True, fg='yellow'), default=False
    )

    return (event, number, date, new_team)
def command():
    """Creates new Django Girls organizer"""
    event_id = click.prompt(
        click.style(
            "What's the event ID? NOT the event page ID. We want EVENT ID here",
            bold=True,
            fg='yellow'))
    event = Event.objects.get(id=event_id)
    click.echo("Ok, we're adding to an event in {}, {}".format(
        event.city, event.country))
    team = [get_organizer_data()]

    while click.confirm(
            click.style("Do you want to add additional team members?",
                        bold=True,
                        fg='yellow'),
            default=False):
        team.append(get_organizer_data())

    click.echo("OK! That's it. Now I'll add your organizers.")

    members = create_users(team, event)

    for member in members:
        click.echo("User {} has been added and notified".format(member.email))

    click.echo(DELIMITER)

    click.echo("You still need to invite people to Google Group!")
def view_files(username):
    # get files - CHANGE URL TO HEROKU AFTER DEVELOPMENT
    r = requests.post(
        'http://afternoon-fortress-38321.herokuapp.com/fda_view_all_files/',
        {'username': username})

    if r.status_code == 200:
        json_str = r.text
        list = json.loads(json_str)
        reports_list = list['reports_list']

        click.echo('Here is a list of your reports:')
        for report in reports_list:
            click.echo(report['report_title'])

        if click.confirm('Do you want to display a report?'):
            view_report_contents(reports_list)
        else:
            click.echo('Goodbye then.')
            exit()
    elif r.status_code == 404:
        click.echo('You do not currently have any reports. Goodbye.')
    else:
        click.echo('Error. Please contact site manager.')
        exit()
Exemple #10
0
def command():
    """Creates new Django Girls organizer"""
    event_id = click.prompt(click.style(
        "What's the event ID? NOT the event page ID. We want EVENT ID here",
        bold=True, fg='yellow'))
    event = Event.objects.get(id=event_id)
    click.echo("Ok, we're adding to an event in {}, {}".format(
        event.city, event.country))
    team = [get_organizer_data()]

    while click.confirm(
        click.style("Do you want to add additional team members?",
                    bold=True, fg='yellow'), default=False):
        team.append(get_organizer_data())

    click.echo("OK! That's it. Now I'll add your organizers.")

    members = create_users(team, event)

    for member in members:
        click.echo(
                "User {} has been added and notified".format(member.email))

    click.echo(DELIMITER)

    click.echo("You still need to invite people to Google Group!")
def command(reset):
    """
    Create and initialize Blackjack game.
    Create a "default" Blackjack run.
    Set the run phase to "Play".
    Add 1 player "*****@*****.**" with password "demo"
    Add a scenario and period 1 for each player.
    """

    # Handle resetting the game
    if reset:
        if click.confirm(
            "Are you sure you want to delete the default game run and recreate from scratch?"
        ):
            delete_default_run(games_client)

    # Create a Game
    game = games_client.games.get_or_create(name="Blackjack", slug="blackjack")
    echo("getting or creating game: ", game.name)

    # Create game Phases ("Play")
    play_phase = games_client.phases.get_or_create(game=game.id, name="Play", order=1)
    echo("getting or creating phase: ", play_phase.name)

    run = games_client.runs.get_or_create(game=game.id, name="default")
    echo("getting or creating run: ", run.name)

    # Set run to phase
    run.phase = play_phase.id
    run.save()
    echo("setting run to phase: ", play_phase.name)

    player = games_client.users.get_or_create(
        password="******",
        first_name="Blackjack",
        last_name="Demo",
        email="*****@*****.**",
    )
    echo("getting or creating user: "******"Scenario 1"
    )
    click.echo(
        "getting or creating runuser {} scenario: {}".format(runuser.id, scenario.id)
    )

    period = games_client.periods.get_or_create(scenario=scenario.id, order=1)
    click.echo(
        "getting or creating runuser {} period 1 for scenario: {}".format(
            runuser.id, scenario.id
        )
    )
    echo("Completed setting up run: id=", run.id)
def check_download(files, report_id, reports_list, files_encrypted):
    if click.confirm("Would you like to download a file from this report?"):
        file = click.prompt('Please enter the name of the file')
        check = False
        for f in files:
            if f == file:
                check = True
        if check is False:
            click.echo('No existing file with that name.')
            check_download(files, report_id, reports_list, files_encrypted)
        else:
            download_files(report_id, reports_list, file, files_encrypted)
    else:
        if click.confirm("Would you like to view another report's contents?"):
            view_report_contents(reports_list)
        else:
            click.echo('Goodbye then.')
            exit()
Exemple #13
0
def delete_invalid_feeds(days=1, limit=100, threshold=99):
    sql = """
    SELECT feed_id, title, link, url, status_code, count FROM (
        SELECT feed_id, status_code, count(1) as count FROM rssant_api_rawfeed
        WHERE dt_created >= %s and (status_code < 200 or status_code >= 400)
        group by feed_id, status_code
        having count(1) > 3
        order by count desc
        limit %s
    ) error_feed
    join rssant_api_feed
        on error_feed.feed_id = rssant_api_feed.id
    order by feed_id, status_code, count;
    """
    sql_ok_count = """
    SELECT feed_id, count(1) as count FROM rssant_api_rawfeed
    WHERE dt_created >= %s and (status_code >= 200 and status_code < 400)
        AND feed_id=ANY(%s)
    group by feed_id
    """
    t_begin = timezone.now() - timezone.timedelta(days=days)
    error_feeds = defaultdict(dict)
    with connection.cursor() as cursor:
        cursor.execute(sql, [t_begin, limit])
        for feed_id, title, link, url, status_code, count in cursor.fetchall():
            error_feeds[feed_id].update(feed_id=feed_id,
                                        title=title,
                                        link=link,
                                        url=url)
            error = error_feeds[feed_id].setdefault('error', {})
            error_name = FeedResponseStatus.name_of(status_code)
            error[error_name] = count
            error_feeds[feed_id]['error_count'] = sum(error.values())
            error_feeds[feed_id].update(ok_count=0, error_percent=100)
        cursor.execute(sql_ok_count, [t_begin, list(error_feeds)])
        for feed_id, ok_count in cursor.fetchall():
            feed = error_feeds[feed_id]
            total = feed['error_count'] + ok_count
            error_percent = round((feed['error_count'] / total) * 100)
            feed.update(ok_count=ok_count, error_percent=error_percent)
    error_feeds = list(
        sorted(error_feeds.values(),
               key=lambda x: x['error_percent'],
               reverse=True))
    delete_feed_ids = []
    for feed in error_feeds:
        if feed['error_percent'] >= threshold:
            delete_feed_ids.append(feed['feed_id'])
            click.echo(pretty_format_json(feed))
    if delete_feed_ids:
        confirm_delete = click.confirm(f'Delete {len(delete_feed_ids)} feeds?')
        if not confirm_delete:
            click.echo('Abort!')
        else:
            UnionFeed.bulk_delete(delete_feed_ids)
            click.echo('Done!')
    return error_feeds
def delete(id):
    """Delete session"""
    try:
        massimport = Massimport.objects.get(pk=id)
    except Massimport.DoesNotExist as e:
        click.secho('Massimport session with id: {} does not exist.'.format(id), bold=True, fg='red')
        return

    if click.confirm('Do you want to delete session id: {} ?'.format(id), default='Y'):
        massimport.delete()
def start(path, limit, username, collection):
    """Start an import session"""

    click.secho('--------------------------------------------------------------------', bold=True)
    click.echo('Username:\t {}'.format(username))
    click.echo('Collection:\t {}'.format(collection))
    click.echo('Limit:\t\t {}'.format(limit))
    click.echo('Path:\t\t {}'.format(path))
    click.secho('--------------------------------------------------------------------', bold=True)
    click.echo('')

    if not os.path.isdir(path):
        click.secho('Directory does not exist: {}'.format(path), bold=True, fg='red')
        return

    if not path.endswith('/'):
        path += '/'

    if not User.objects.filter(username=username).exists():
        click.secho('User does not exist: {}'.format(username), bold=True, fg='red')
        return

    if Massimport.objects.filter(directory=path).exists():
        click.secho('Import session already exists: {}'.format(path), bold=True, fg='red')
        return


    massimport = Massimport(
        directory=path,
        user=User.objects.get(username=username),
        collection_name=collection
    )

    massimport.save()

    if click.confirm('Continue with scanning directories?', default='Y'):
        massimport.scan()

    if click.confirm('Continue with enqueuing files?'):

        for item in massimport.files.filter(status=0)[0:limit]:
            item.enqueue()
def download_files(report_id, reports_list, file_name, files_encrypted):
    # get report files - CHANGE URL TO HEROKU AFTER DEVELOPMENT
    r = requests.post(
        'http://afternoon-fortress-38321.herokuapp.com/fda_get_files/', {
            'report_id': report_id,
            'file_name': file_name
        })
    if files_encrypted:
        if click.confirm("Do you have the private key?"):
            key_file = click.prompt(
                'Enter the path to the keyfile for this file')
            key = None
            try:
                with open(key_file, 'rb') as f:
                    key = b64decode(f.read())
                out_name = file_name
                if file_name.split('.')[-1] == 'enc':
                    out_name = file_name[:-4]
                    decrypt_file(file_name, key, out_name)
            except FileNotFoundError:
                click.echo('ERROR: Keyfile not found')
                exit()
            click.echo('File saved as {}'.format(out_name))
        else:
            click.echo('You cannot download the file.')
            if click.confirm(
                    "Would you like to view another report's contents?"):
                view_report_contents(reports_list)
            else:
                click.echo('Goodbye then.')
                exit()
    else:
        file = open(file_name, 'wb')
        file.write(r.content)
        file.close()
        click.echo('Success. Your file has been downloaded.')

    if click.confirm("Would you like to view another report's contents?"):
        view_report_contents(reports_list)
    else:
        click.echo('Goodbye then.')
        exit()
Exemple #17
0
def migrate_published_version_metadata(dandiset: str, published_version: str,
                                       to_version: str):
    click.echo(
        f'Migrating published version {dandiset}/{published_version} metadata to version {to_version}'  # noqa: E501
    )
    version = Version.objects.filter(~Q(version='draft')).get(
        dandiset=dandiset, version=published_version)
    metadata = version.metadata

    # If there is no schemaVersion, assume to_version
    if 'schemaVersion' not in metadata:
        metadata['schemaVersion'] = to_version

    try:
        metanew = migrate(metadata,
                          to_version=to_version,
                          skip_validation=False)
    except Exception as e:
        click.echo(f'Failed to migrate {dandiset}/{published_version}')
        click.echo(e)
        raise click.Abort()

    if metadata == metanew:
        click.echo('No changes detected')
    else:
        click.echo('Diff of changes to be saved:')
        click.echo(''.join(
            ndiff(
                pformat(metadata).splitlines(keepends=True),
                pformat(metanew).splitlines(keepends=True),
            )))

        click.confirm('Do you want to save these changes?', abort=True)

        version.metadata = metanew
        version.save()

        write_manifest_files.delay(version.id)
def delete(id):
    """Delete session"""
    try:
        massimport = Massimport.objects.get(pk=id)
    except Massimport.DoesNotExist as e:
        click.secho(
            'Massimport session with id: {} does not exist.'.format(id),
            bold=True,
            fg='red')
        return

    if click.confirm('Do you want to delete session id: {} ?'.format(id),
                     default='Y'):
        massimport.delete()
def view_report_contents(reports_list):
    report_name = click.prompt('Which report would you like to display?')
    # get report ID
    check = False
    report_id = 0
    for r in reports_list:
        if r['report_title'] == report_name:
            report_id = r['report_id']
            check = True
    if check is False:
        click.echo(
            'No existing report with that name. Please enter another report.')
        view_report_contents(reports_list)
    else:
        # get report contents - CHANGE URL TO HEROKU AFTER DEVELOPMENT
        r = requests.post(
            'http://afternoon-fortress-38321.herokuapp.com/fda_view_report_contents/',
            {'report_id': report_id})
        if r.status_code == 200:
            json_str = r.text
            r_info = json.loads(json_str)
            report_info = r_info['report_info']
            # display report contents
            click.echo('Title: ' + report_info['title'])
            click.echo('Owner: ' + report_info['owner'])
            click.echo('Short Description: ' + report_info['short_desc'])
            click.echo('Summary: ' + report_info['long_desc'])
            click.echo('Shared With: ' + report_info['shared_with'])
            click.echo('Created: ' + report_info['timestamp'])

            click.echo('Files: ')
            for r in report_info['files']:
                click.echo(r)

            if report_info['files'] == []:
                if click.confirm(
                        "Would you like to view another report's contents?"):
                    view_report_contents(reports_list)
                else:
                    click.echo('Goodbye then.')
                    exit()
            else:
                check_download(report_info['files'], report_id, reports_list,
                               report_info['files_encrypted'])
        elif r.status_code == 404:
            click.echo('You do not currently have any reports. Goodbye.')
        else:
            click.echo('Error. Please contact site manager.')
            exit()
def encrypt_file():
    key = random_generator(256)
    file_name = click.prompt('Enter the path of the file you wish to encrypt')
    try:
        encrypt(file_name, key)
    except FileNotFoundError:
        click.echo('ERROR: File not found')
        encrypt_file()
    pk_file_name = file_name + '.pem'
    with open(pk_file_name, 'wb') as f:
        f.write(b64encode(key))
    click.echo('Encrypted file saved as {}.enc'.format(file_name))
    click.echo('Keyfile saved as {}'.format(pk_file_name))
    if click.confirm('Would you like to encrypt another file?'):
        encrypt_file()
    exit()
def command(csv_file):
    csv_reader = csv.reader(csv_file)

    appliances = [
        Appliance(
            product_line=ProductLine.objects.get_or_create(
                name=product_line)[0],
            serial_number=serial_number,
            model_number=model_number,
        ) for product_line, model_number, serial_number in csv_reader
    ]

    print("Here are the appliances extracted:")
    pprint(appliances)

    if click.confirm("Proceed?"):
        Appliance.objects.bulk_create(appliances)
def login():
    click.echo('Please login.')
    username = click.prompt('Username', type=str)
    password = click.prompt('Password', type=str)
    # authenticate user using requests - CHANGE URL TO HEROKU AFTER DEVELOPMENT
    r = requests.post(
        'http://afternoon-fortress-38321.herokuapp.com/fda_login/',
        data={
            'username': username,
            'password': password
        })
    if (r.status_code == requests.codes.ok):
        if click.confirm('Would you like to encrypt a file?'):
            encrypt_file()
        view_files(username)
    else:
        click.echo('Invalid username and password')
        login()
def command():
    """Creates new Django Girls organizer"""
    event_id = click.prompt(
        click.style(
            "What's the event ID? NOT the event page ID. We want EVENT ID here",
            bold=True, fg='yellow'))
    event = Event.objects.get(id=event_id)
    click.echo("Ok, we're adding to an event in {}, {}".format(
        event.city, event.country))

    team = [get_organizer_data()]

    while click.confirm(
        click.style("Do you want to add additional team members?",
                    bold=True, fg='yellow'), default=False):
        team.append(get_organizer_data())

    click.echo("OK! That's it. Now I'll add your organizers.")

    members, members_as_list = create_users(team)

    for member in members:
        event.team.add(member)

    event.save()

    for member in members_as_list:
        if 'password' in member:
            click.echo("{} - email: {} password {}".format(
                member['first_name'], member['email'], member['password']))
        else:
            click.echo(
                "{} - email: {} already has account".format(
                    member['first_name'], member['email']))

    click.echo(DELIMITER)

    invite_team_to_slack(members)

    click.echo(DELIMITER)

    click.echo("You still need to invite people to Google Group!")
Exemple #24
0
def command(reset, name):
    """
    Create and initialize Simpl Calc game.
    Create a "default" Simpl Calc run.
    Set the run phase to "Play".
    Add 1 leader ("leader") to the run
    Add 2 players ("s1", "s2") to the run.
    Add a scenario and period 1 for each player.
    """

    # Create a Game
    game = games_client.games.get_or_create(name='Simpl Calc',
                                            slug='simpl-calc')
    echo('getting or creating game: ', game.name)

    if reset:  # Handle resetting the run
        lookup = {'game': game.id, 'name': name}
        try:
            run = games_client.runs.get(**lookup)
        except games_client.ResourceNotFound:
            run = None
        except games_client.TypeError:
            run = None

        if run is not None:
            if click.confirm(
                    'Are you sure you want to delete the "{0}" game run and '
                    'recreate from scratch?'.format(name)):
                delete_run(run)

    # Create game Phases ("Play")
    play_phase = games_client.phases.get_or_create(
        game=game.id,
        name='Play',
        order=1,
    )
    echo('getting or creating phase: ', play_phase.name)

    # Add run with 2 players ready to play
    run = add_run(game, name, 2, play_phase, games_client)

    echo('Completed setting up run: id=', run.id)
Exemple #25
0
def command():
    """Creates new Django Girls organizer"""
    event_id = click.prompt(click.style(
        "What's the event ID? NOT the event page ID. We want EVENT ID here",
        bold=True, fg='yellow'))
    event = Event.objects.get(id=event_id)
    click.echo("Ok, we're adding to an event in {}, {}".format(
        event.city, event.country))

    team = [get_organizer_data()]

    while click.confirm(
        click.style("Do you want to add additional team members?",
                    bold=True, fg='yellow'), default=False):
        team.append(get_organizer_data())

    click.echo("OK! That's it. Now I'll add your organizers.")

    members, members_as_list = create_users(team)

    for member in members:
        event.team.add(member)

    event.save()

    for member in members_as_list:
        if 'password' in member:
            click.echo("{} - email: {} password {}".format(
                member['first_name'], member['email'], member['password']))
        else:
            click.echo(
                "{} - email: {} already has account".format(
                    member['first_name'], member['email']))

    click.echo(DELIMITER)

    invite_team_to_slack(members)

    click.echo(DELIMITER)

    click.echo("You still need to invite people to Google Group!")
def command():
    click.echo("=== Setup Simpl API ===")
    click.echo("")

    # Deal with super users
    superusers = find_super_users()
    if not superusers:
        click.echo("We found no superusers, please set one up below")
        create_super_user()
    else:
        click.echo(
            "We found the following superusers that were already setup:")
        for u in superusers:
            click.echo(f" - {u}")
        click.echo("")
        click.echo(
            "If you would like to setup another super user for yourself run `./manage.py createsuperuser`"
        )

    click.echo("")

    # Deal with staff users
    staffusers = find_staff_users()
    if not staffusers:
        click.echo(
            "We found no staff users, you will need one of these for your model service to connect to the Simpl API"
        )
        click.echo(
            "This is typically setup as a setting `SIMPL_GAMES_AUTH` from the environment variables `SIMPL_USER` and `SIMPL_PASS`"
        )
        click.echo("")
        create_staff_user()
    else:
        click.echo(
            "We found the following staff users that are already setup:")
        for u in staffusers:
            click.echo(f" - {u}")
        click.echo("")

        if click.confirm("Would you like to setup another staff user?"):
            create_staff_user()
Exemple #27
0
def collect_garbage(assets: bool, assetblobs: bool, uploads: bool,
                    s3blobs: bool):
    """Manually run garbage collection on the database."""
    # Log how many things there are before deleting them
    doing_deletes = assets or assetblobs or uploads or s3blobs
    if doing_deletes:
        echo_report()

    if assetblobs:
        raise click.NoSuchOption('Deleting AssetBlobs is not yet implemented')
    if uploads:
        raise click.NoSuchOption('Deleting Uploads is not yet implemented')
    if s3blobs:
        raise click.NoSuchOption('Deleting S3 Blobs is not yet implemented')
    if assets:
        assets_to_delete = stale_assets()
        if click.confirm(
                f'This will delete {assets_to_delete.count()} assets. Are you sure?'
        ):
            assets_to_delete.delete()

    # Log how many things there are, either after deletion
    # or if the user forgot to specify anything to delete
    echo_report()
Exemple #28
0
def import_from_file(osm_data, resume: bool, rerun: bool):
    Settings = IngestSettings(
        max_distance=Distance(km=50),
        max_segments=300,
        max_concurrent=40,
        quality_settings=DefaultQualitySettings,
        location_filter=None,
    )
    digest = sha256_digest(osm_data)
    print('Digest: ', digest)
    previous_import = e.Import.objects.filter(
        complete=False, sha256_sum=digest).order_by('-updated_at').first()
    if not resume or previous_import is None:
        if e.Import.objects.filter(sha256_sum=digest, complete=True):
            print('Import already done!')
            if rerun:
                e.Import.objects.filter(sha256_sum=digest,
                                        complete=True).delete()
            else:
                return
        #e.Import.objects.all().update(active=False)
        import_obj = e.Import(active=True,
                              complete=False,
                              border=Polygon(),
                              name=str(osm_data),
                              sha256_sum=digest)
        import_obj.save()
        digests = set()
    else:
        import_obj = previous_import
        if not click.confirm(
                f'Resuming import {import_obj.name}, last modified {import_obj.updated_at} currently containing {import_obj.networks.count()} trail networks'
        ):
            return 1
        # TODO: probably n queries
        digests = {n.digest for n in import_obj.networks.all()}
    print(f'{len(digests)} loaded')
    loader = OSMIngestor(Settings)
    loader.load_osm(osm_data, extra_links=[(885729040, 827103027)])
    networks = []
    for network in tqdm(loader.trail_networks(already_processed=digests)):
        try:
            multiline_strs = MultiLineString([
                LineString(trail.points())
                for trail in network.trail_segments()
            ])

            border = multiline_strs.convex_hull
            simplified = multiline_strs  # .simplify(tolerance=0.01)
            if isinstance(simplified, LineString):
                simplified = MultiLineString([simplified])
            # TODO: look for polygons that intersect this one

            trailheads = MultiPoint(
                [t.node.to_point() for t in network.trailheads])

            est_network = e.TrailNetwork(name=network.name or '',
                                         source=import_obj,
                                         trails=simplified,
                                         poly=border,
                                         total_length=network.total_length(),
                                         graph=pickle.dumps(network.graph),
                                         area=border.area,
                                         trailheads=trailheads,
                                         digest=network.digest)
            est_network.save()
            networks.append(est_network)
        except Exception as ex:
            import pdb
            pdb.set_trace()
            print(ex)
    import_obj.complete = True
    if networks:
        import_border = MultiPolygon([n.poly for n in networks])
        import_obj.border = import_border.convex_hull
        import_obj.save()
Exemple #29
0
def repair_durations(limit_range, dump_to, load_from, tolerance, log_file):
    """
    Repair/reprocess master durations.
    """
    from base.audio.fileinfo import FileInfoProcessor

    items_to_reprocess = []
    affected_playlists = []
    affected_playlist_ids = []

    # invalidate cache for Media
    invalidate_model(Media)

    if load_from:

        if limit_range:
            raise NotImplementedError(
                '--limit-range option not allowed in combination with --load-from'
            )

        # using `set` to remove duplicate ids
        item_ids = set([
            int(l.strip().split(',')[0]) for l in load_from.readlines()
            if float(l.strip().split(',')[1]) > tolerance
        ])

        click.echo('loaded {} ids from dump file'.format(len(item_ids)))

        items_to_reprocess = Media.objects.filter(pk__in=item_ids)

    else:

        # mysql does not support remote/streaming cursors
        # to save memory items are loaded from db individually
        values = Media.objects.order_by('pk').values('id').nocache()

        if limit_range:
            _limits = limit_range.split(':')
            values = values[_limits[0]:_limits[1]]

        item_ids = [i['id'] for i in values]

        with click.progressbar(item_ids,
                               show_pos=True,
                               width=48,
                               label='Reprocessing {} tracks'.format(
                                   len(item_ids))) as bar:

            for item_pk in bar:

                close_old_connections()

                item = Media.objects.get(pk=item_pk)

                if item.master and item.master.path:

                    p = FileInfoProcessor(item.master.path)

                    current_duration = item.master_duration
                    new_duration = p.duration

                    try:
                        diff = abs(current_duration - new_duration)
                    except TypeError:
                        diff = 100.0

                    if diff > tolerance:
                        items_to_reprocess.append(item)

                    # add to csv log
                    if diff > tolerance and dump_to:
                        dump_to.write('{pk},{diff}\n'.format(pk=item.pk,
                                                             diff=diff))
                        dump_to.flush()

    click.echo('{} tracks have differences in duration'.format(
        len(items_to_reprocess)))

    if click.confirm(
            'Do you want to update/repair the durations on {} tracks?'.format(
                len(items_to_reprocess))):

        base_url = 'http://{}'.format(Site.objects.get_current().domain)

        tpl = u'''id: {id} - "{name}"
{url}
old: {current_duration} new: {new_duration} diff: {diff}
        '''

        tpl_log = u'{ct},{pk},{type},{current_duration},{new_duration},{diff},{url}\n'

        # write column header
        if log_file:
            log_file.write(
                tpl_log.format(
                    ct='content-type',
                    pk='id',
                    url='url',
                    type='type',
                    #
                    current_duration='old_duration',
                    new_duration='new_duration',
                    diff='diff',
                ))

        # loop affected media, fix durations, get playlist appearances & print/log info
        for item in items_to_reprocess:

            p = FileInfoProcessor(item.master.path)

            current_duration = item.master_duration
            new_duration = p.duration

            try:
                diff = current_duration - new_duration
            except TypeError:
                diff = '-'

            click.echo(
                tpl.format(
                    id=item.id,
                    name=item.name,
                    url=base_url + item.get_absolute_url(),
                    #
                    current_duration=current_duration,
                    new_duration=new_duration,
                    diff=diff))

            if log_file:
                log_file.write(
                    tpl_log.format(
                        ct='media',
                        pk=item.pk,
                        url=base_url + item.get_absolute_url(),
                        type=item.get_mediatype_display(),
                        #
                        current_duration=current_duration,
                        new_duration=new_duration,
                        diff=diff))
                log_file.flush()

            for p in item.get_appearances():
                if not p.pk in affected_playlist_ids:
                    affected_playlist_ids.append(p.pk)
                    # we need to store the 'current' value of the duration
                    affected_playlists.append({
                        'obj':
                        p,
                        'current_duration':
                        p.get_duration()
                    })

            # update media duration
            Media.objects.filter(pk=item.pk).update(
                master_duration=new_duration)
            invalidate_obj(item)

        # loop playlists & print/log info
        for item in affected_playlists:

            invalidate_obj(item['obj'])

            current_duration = float(item['current_duration']) / 1000
            new_duration = float(item['obj'].get_duration()) / 1000

            try:
                diff = current_duration - new_duration
            except TypeError:
                diff = '-'

            click.echo(
                tpl.format(
                    id=item['obj'].id,
                    name=item['obj'].name,
                    url=base_url + item['obj'].get_absolute_url(),
                    #
                    current_duration=current_duration,
                    new_duration=new_duration,
                    diff=diff))

            if log_file:
                log_file.write(
                    tpl_log.format(
                        ct='playlist',
                        pk=item['obj'].pk,
                        url=base_url + item['obj'].get_absolute_url(),
                        type=item['obj'].get_type_display(),
                        #
                        current_duration=current_duration,
                        new_duration=new_duration,
                        diff=diff))
                log_file.flush()

            # update playlist duration
            Playlist.objects.filter(pk=item['obj'].pk).update(
                duration=new_duration * 1000)
            invalidate_obj(item)
Exemple #30
0
def repair_durations(limit_range, dump_to, load_from, tolerance, log_file):
    """
    Repair/reprocess master durations.
    """
    from base.audio.fileinfo import FileInfoProcessor

    items_to_reprocess = []
    affected_playlists = []
    affected_playlist_ids = []

    # invalidate cache for Media
    invalidate_model(Media)

    if load_from:

        if limit_range:
            raise NotImplementedError('--limit-range option not allowed in combination with --load-from')

        # using `set` to remove duplicate ids
        item_ids = set([
            int(l.strip().split(',')[0]) for l in load_from.readlines() if float(l.strip().split(',')[1]) > tolerance
        ])


        click.echo('loaded {} ids from dump file'.format(len(item_ids)))

        items_to_reprocess = Media.objects.filter(pk__in=item_ids)

    else:

        # mysql does not support remote/streaming cursors
        # to save memory items are loaded from db individually
        values = Media.objects.order_by('pk').values('id').nocache()

        if limit_range:
            _limits = limit_range.split(':')
            values = values[_limits[0]:_limits[1]]

        item_ids = [i['id'] for i in values]

        with click.progressbar(item_ids, show_pos=True, width=48, label='Reprocessing {} tracks'.format(len(item_ids))) as bar:

            for item_pk in bar:

                close_old_connections()

                item = Media.objects.get(pk=item_pk)

                if item.master and item.master.path:

                    p = FileInfoProcessor(item.master.path)

                    current_duration = item.master_duration
                    new_duration = p.duration

                    try:
                        diff = abs(current_duration - new_duration)
                    except TypeError:
                        diff = 100.0

                    if diff > tolerance:
                        items_to_reprocess.append(item)

                    # add to csv log
                    if diff > tolerance and dump_to:
                        dump_to.write('{pk},{diff}\n'.format(pk=item.pk, diff=diff))
                        dump_to.flush()


    click.echo('{} tracks have differences in duration'.format(len(items_to_reprocess)))


    if click.confirm('Do you want to update/repair the durations on {} tracks?'.format(len(items_to_reprocess))):

        base_url = 'http://{}'.format(Site.objects.get_current().domain)


        tpl = u'''id: {id} - "{name}"
{url}
old: {current_duration} new: {new_duration} diff: {diff}
        '''


        tpl_log = u'{ct},{pk},{type},{current_duration},{new_duration},{diff},{url}\n'

        # write column header
        if log_file:
            log_file.write(tpl_log.format(
                ct='content-type',
                pk='id',
                url='url',
                type='type',
                #
                current_duration='old_duration',
                new_duration='new_duration',
                diff='diff',
            ))


        # loop affected media, fix durations, get playlist appearances & print/log info
        for item in items_to_reprocess:

            p = FileInfoProcessor(item.master.path)

            current_duration = item.master_duration
            new_duration = p.duration

            try:
                diff = current_duration - new_duration
            except TypeError:
                diff = '-'


            click.echo(tpl.format(
                id=item.id,
                name=item.name,
                url=base_url + item.get_absolute_url(),
                #
                current_duration=current_duration,
                new_duration=new_duration,
                diff=diff
            ))

            if log_file:
                log_file.write(tpl_log.format(
                    ct='media',
                    pk=item.pk,
                    url=base_url + item.get_absolute_url(),
                    type=item.get_mediatype_display(),
                    #
                    current_duration=current_duration,
                    new_duration=new_duration,
                    diff=diff
                ))
                log_file.flush()

            for p in item.get_appearances():
                if not p.pk in affected_playlist_ids:
                    affected_playlist_ids.append(p.pk)
                    # we need to store the 'current' value of the duration
                    affected_playlists.append({
                        'obj': p,
                        'current_duration': p.get_duration()
                    })

            # update media duration
            Media.objects.filter(pk=item.pk).update(master_duration=new_duration)
            invalidate_obj(item)


        # loop playlists & print/log info
        for item in affected_playlists:

            invalidate_obj(item['obj'])

            current_duration = float(item['current_duration']) / 1000
            new_duration = float(item['obj'].get_duration()) / 1000

            try:
                diff = current_duration - new_duration
            except TypeError:
                diff = '-'

            click.echo(tpl.format(
                id=item['obj'].id,
                name=item['obj'].name,
                url=base_url + item['obj'].get_absolute_url(),
                #
                current_duration=current_duration,
                new_duration=new_duration,
                diff=diff
            ))

            if log_file:
                log_file.write(tpl_log.format(
                    ct='playlist',
                    pk=item['obj'].pk,
                    url=base_url + item['obj'].get_absolute_url(),
                    type=item['obj'].get_type_display(),
                    #
                    current_duration=current_duration,
                    new_duration=new_duration,
                    diff=diff
                ))
                log_file.flush()

            # update playlist duration
            Playlist.objects.filter(pk=item['obj'].pk).update(duration=new_duration * 1000)
            invalidate_obj(item)