Ejemplo n.º 1
0
def preview_from_config(
    name,
    url,
    backend,
    description=None,
    frequency=DEFAULT_HARVEST_FREQUENCY,
    owner=None,
    organization=None,
    config=None,
):
    '''Preview an harvesting from a source created with the given parameters'''
    if owner and not isinstance(owner, User):
        owner = User.get(owner)

    if organization and not isinstance(organization, Organization):
        organization = Organization.get(organization)

    source = HarvestSource(
        name=name,
        url=url,
        backend=backend,
        description=description,
        frequency=frequency or DEFAULT_HARVEST_FREQUENCY,
        owner=owner,
        organization=organization,
        config=config,
    )
    cls = backends.get(current_app, source.backend)
    max_items = current_app.config['HARVEST_PREVIEW_MAX_ITEMS']
    backend = cls(source, dryrun=True, max_items=max_items)
    return backend.harvest()
Ejemplo n.º 2
0
def create_source(name, url, backend,
                  description=None,
                  frequency=DEFAULT_HARVEST_FREQUENCY,
                  owner=None,
                  organization=None,
                  config=None,
                  ):
    '''Create a new harvest source'''
    if owner and not isinstance(owner, User):
        owner = User.get(owner)

    if organization and not isinstance(organization, Organization):
        organization = Organization.get(organization)

    source = HarvestSource.objects.create(
        name=name,
        url=url,
        backend=backend,
        description=description,
        frequency=frequency or DEFAULT_HARVEST_FREQUENCY,
        owner=owner,
        organization=organization,
        config=config,
    )
    signals.harvest_source_created.send(source)
    return source
Ejemplo n.º 3
0
def preview_from_config(name, url, backend,
                        description=None,
                        frequency=DEFAULT_HARVEST_FREQUENCY,
                        owner=None,
                        organization=None,
                        config=None,
                        ):
    '''Preview an harvesting from a source created with the given parameters'''
    if owner and not isinstance(owner, User):
        owner = User.get(owner)

    if organization and not isinstance(organization, Organization):
        organization = Organization.get(organization)

    source = HarvestSource(
        name=name,
        url=url,
        backend=backend,
        description=description,
        frequency=frequency or DEFAULT_HARVEST_FREQUENCY,
        owner=owner,
        organization=organization,
        config=config,
    )
    cls = backends.get(current_app, source.backend)
    max_items = current_app.config['HARVEST_PREVIEW_MAX_ITEMS']
    backend = cls(source, dryrun=True, max_items=max_items)
    return backend.harvest()
Ejemplo n.º 4
0
def create_source(
    name,
    url,
    backend,
    description=None,
    frequency=DEFAULT_HARVEST_FREQUENCY,
    owner=None,
    organization=None,
    config=None,
):
    '''Create a new harvest source'''
    if owner and not isinstance(owner, User):
        owner = User.get(owner)

    if organization and not isinstance(organization, Organization):
        organization = Organization.get(organization)

    source = HarvestSource.objects.create(
        name=name,
        url=url,
        backend=backend,
        description=description,
        frequency=frequency or DEFAULT_HARVEST_FREQUENCY,
        owner=owner,
        organization=organization,
        config=config,
    )
    signals.harvest_source_created.send(source)
    return source
Ejemplo n.º 5
0
def delete():
    '''Delete an existing user'''
    email = click.prompt('Email')
    user = User.objects(email=email).first()
    if not user:
        exit_with_error('Invalid user')
    user.delete()
    success('User deleted successfully')
Ejemplo n.º 6
0
def delete():
    '''Delete an existing user'''
    email = click.prompt('Email')
    user = User.objects(email=email).first()
    if not user:
        exit_with_error('Invalid user')
    user.delete()
    success('User deleted successfully')
Ejemplo n.º 7
0
 def run(self):
     email = prompt('Email')
     user = User.objects(email=email).first()
     if not user:
         print 'Invalid user'
         return
     user.delete()
     print 'User deleted successfully'
Ejemplo n.º 8
0
def delete():
    '''Delete an existing user'''
    email = prompt('Email')
    user = User.objects(email=email).first()
    if not user:
        print 'Invalid user'
        return
    user.delete()
    print 'User deleted successfully'
Ejemplo n.º 9
0
def delete():
    '''Delete an existing user'''
    email = prompt('Email')
    user = User.objects(email=email).first()
    if not user:
        print 'Invalid user'
        return
    user.delete()
    print 'User deleted successfully'
Ejemplo n.º 10
0
def delete():
    """Delete an existing user"""
    email = prompt("Email")
    user = User.objects(email=email).first()
    if not user:
        print "Invalid user"
        return
    user.delete()
    print "User deleted successfully"
Ejemplo n.º 11
0
def activate():
    '''Activate an existing user (validate their email confirmation)'''
    email = click.prompt('Email')
    user = User.objects(email=email).first()
    if not user:
        exit_with_error('Invalid user')
    if user.confirmed_at is not None:
        exit_with_error('User email address already confirmed')
        return
    user.confirmed_at = datetime.utcnow()
    user.save()
    success('User activated successfully')
Ejemplo n.º 12
0
def activate():
    '''Activate an existing user (validate their email confirmation)'''
    email = click.prompt('Email')
    user = User.objects(email=email).first()
    if not user:
        exit_with_error('Invalid user')
    if user.confirmed_at is not None:
        exit_with_error('User email address already confirmed')
        return
    user.confirmed_at = datetime.utcnow()
    user.save()
    success('User activated successfully')
Ejemplo n.º 13
0
def activate():
    '''Activate an existing user (validate their email confirmation)'''
    email = prompt('Email')
    user = User.objects(email=email).first()
    if not user:
        print 'Invalid user'
        return
    if user.confirmed_at is not None:
        print 'User email address already confirmed'
        return
    user.confirmed_at = datetime.utcnow()
    user.save()
    print 'User activated successfully'
Ejemplo n.º 14
0
 def get(self):
     '''Suggest users'''
     args = suggest_parser.parse_args()
     users = User.objects(deleted=None,
                          slug__icontains=slugify(args['q'],
                                                  separator='-',
                                                  to_lower=True))
     return [{
         'id': user.id,
         'first_name': user.first_name,
         'last_name': user.last_name,
         'avatar_url': user.avatar_url,
         'slug': user.slug,
     } for user in users.order_by(DEFAULT_SORTING).limit(args['size'])]
Ejemplo n.º 15
0
 def get(self):
     '''List all users'''
     args = user_parser.parse()
     users = User.objects(deleted=None)
     if args['q']:
         search_users = users.search_text(args['q'])
         if args['sort']:
             return search_users.order_by(args['sort']).paginate(
                 args['page'], args['page_size'])
         else:
             return search_users.order_by('$text_score').paginate(
                 args['page'], args['page_size'])
     if args['sort']:
         return users.order_by(args['sort']).paginate(
             args['page'], args['page_size'])
     return users.order_by(DEFAULT_SORTING).paginate(
         args['page'], args['page_size'])
Ejemplo n.º 16
0
def c3_badges(filename):
    '''Toggle C3 badges from an organization list'''
    with open(filename, 'r') as titles:
        user = User.objects(first_name='Etalab', last_name='Bot').first()
        badge = DatasetBadge(kind=C3, created_by=user)
        for title in titles:
            title = title.decode('utf-8').strip(u'\n')
            if title.startswith(u'*'):
                continue
            slug = slugify.slugify(title.lower())
            dataset = (Dataset.objects(title=title).first()
                       or Dataset.objects(slug=slug).first())
            if dataset is None:
                log.info(u'{title} not found'.format(title=title))
            else:
                dataset.badges.append(badge)
                dataset.save()
    log.info('Done')
Ejemplo n.º 17
0
    def serialize(cls, reuse):
        """By default use the ``to_dict`` method

        and exclude ``_id``, ``_cls`` and ``owner`` fields.
        """
        datasets = Dataset.objects(id__in=[r.id for r in reuse.datasets])
        datasets = list(datasets.only('id', 'title').no_dereference())
        organization = None
        owner = None
        if reuse.organization:
            organization = Organization.objects(
                id=reuse.organization.id).first()
        elif reuse.owner:
            owner = User.objects(id=reuse.owner.id).first()
        return {
            'title': reuse.title,
            'description': reuse.description,
            'url': reuse.url,
            'organization': str(organization.id) if organization else None,
            'owner': str(owner.id) if owner else None,
            'type': reuse.type,
            'topic': reuse.topic,
            'tags': reuse.tags,
            'tag_suggest': reuse.tags,
            'badges': [badge.kind for badge in reuse.badges],
            'created': to_iso_datetime(reuse.created_at),
            'last_modified': to_iso_datetime(reuse.last_modified),
            'dataset': [{
                'id': str(d.id),
                'title': d.title
            } for d in datasets],
            'metrics': reuse.metrics,
            'featured': reuse.featured,
            'extras': reuse.extras,
            'reuse_suggest': {
                'input': cls.completer_tokenize(reuse.title) + [reuse.id],
                'output': str(reuse.id),
                'payload': {
                    'title': reuse.title,
                    'slug': reuse.slug,
                    'image_url': reuse.image(500, external=True),
                },
            },
        }
Ejemplo n.º 18
0
    def serialize(cls, reuse):
        """By default use the ``to_dict`` method

        and exclude ``_id``, ``_cls`` and ``owner`` fields.
        """
        datasets = Dataset.objects(id__in=[r.id for r in reuse.datasets])
        datasets = list(datasets.only('id', 'title').no_dereference())
        organization = None
        owner = None
        if reuse.organization:
            organization = Organization.objects(id=reuse.organization.id).first()
        elif reuse.owner:
            owner = User.objects(id=reuse.owner.id).first()
        return {
            'title': reuse.title,
            'description': reuse.description,
            'url': reuse.url,
            'organization': str(organization.id) if organization else None,
            'owner': str(owner.id) if owner else None,
            'type': reuse.type,
            'tags': reuse.tags,
            'tag_suggest': reuse.tags,
            'badges': [badge.kind for badge in reuse.badges],
            'created': reuse.created_at.strftime('%Y-%m-%dT%H:%M:%S'),
            'last_modified': reuse.last_modified.strftime('%Y-%m-%dT%H:%M:%S'),
            'dataset': [{
                'id': str(d.id),
                'title': d.title
            } for d in datasets],
            'metrics': reuse.metrics,
            'featured': reuse.featured,
            'extras': reuse.extras,
            'reuse_suggest': {
                'input': cls.completer_tokenize(reuse.title) + [reuse.id],
                'output': str(reuse.id),
                'payload': {
                    'title': reuse.title,
                    'slug': reuse.slug,
                    'image_url': reuse.image(40, external=True),
                },
            },
        }
Ejemplo n.º 19
0
def create_oauth_client(client_name, user_email, uri, grant_types, scope,
                        response_types):
    '''Creates an OAuth2Client instance in DB'''
    user = User.objects(email=user_email).first()
    if user is None:
        exit_with_error('No matching user to email')

    client = OAuth2Client.objects.create(name=client_name,
                                         owner=user,
                                         grant_types=grant_types,
                                         scope=scope,
                                         response_types=response_types,
                                         redirect_uris=uri)

    click.echo(f'New OAuth client: {client.name}')
    click.echo(f'Client\'s ID {client.id}')
    click.echo(f'Client\'s secret {client.secret}')
    click.echo(f'Client\'s grant_types {client.grant_types}')
    click.echo(f'Client\'s response_types {client.response_types}')
    click.echo(f'Client\'s URI {client.redirect_uris}')
Ejemplo n.º 20
0
    def serialize(cls, dataset):
        organization = None
        owner = None
        image_url = None
        spatial_weight = DEFAULT_SPATIAL_WEIGHT
        temporal_weight = DEFAULT_TEMPORAL_WEIGHT

        if dataset.organization:
            organization = Organization.objects(
                id=dataset.organization.id).first()
            image_url = organization.logo(40, external=True)
        elif dataset.owner:
            owner = User.objects(id=dataset.owner.id).first()
            image_url = owner.avatar(40, external=True)

        certified = organization and organization.certified

        document = {
            'title':
            dataset.title,
            'description':
            dataset.description,
            'license':
            getattr(dataset.license, 'id', None),
            'tags':
            dataset.tags,
            'badges': [badge.kind for badge in dataset.badges],
            'tag_suggest':
            dataset.tags,
            'resources': [{
                'title': r.title,
                'description': r.description,
                'format': r.format,
            } for r in dataset.resources],
            'format_suggest':
            [r.format.lower() for r in dataset.resources if r.format],
            'frequency':
            dataset.frequency,
            'organization':
            str(organization.id) if organization else None,
            'owner':
            str(owner.id) if owner else None,
            'dataset_suggest': {
                'input': cls.completer_tokenize(dataset.title) + [dataset.id],
                'output': dataset.title,
                'payload': {
                    'id': str(dataset.id),
                    'slug': dataset.slug,
                    'acronym': dataset.acronym,
                    'image_url': image_url,
                },
            },
            'created':
            dataset.created_at.strftime('%Y-%m-%dT%H:%M:%S'),
            'last_modified':
            dataset.last_modified.strftime('%Y-%m-%dT%H:%M:%S'),
            'metrics':
            dataset.metrics,
            'featured':
            dataset.featured,
            'from_certified':
            certified,
        }
        if (dataset.temporal_coverage is not None
                and dataset.temporal_coverage.start
                and dataset.temporal_coverage.end):
            start = dataset.temporal_coverage.start.toordinal()
            end = dataset.temporal_coverage.end.toordinal()
            temporal_weight = min((end - start) / 365, MAX_TEMPORAL_WEIGHT)
            document.update({
                'temporal_coverage': {
                    'start': start,
                    'end': end
                },
                'temporal_weight': temporal_weight,
            })

        if dataset.spatial is not None:
            # Index precise zone labels and parents zone identifiers
            # to allow fast filtering.
            zone_ids = [z.id for z in dataset.spatial.zones]
            zones = GeoZone.objects(id__in=zone_ids).exclude('geom')
            parents = set()
            geozones = []
            coverage_level = ADMIN_LEVEL_MAX
            for zone in zones:
                geozones.append({
                    'id': zone.id,
                    'name': zone.name,
                    'keys': zone.keys_values
                })
                parents |= set(zone.parents)
                coverage_level = min(coverage_level, admin_levels[zone.level])

            geozones.extend([{'id': p} for p in parents])

            spatial_weight = ADMIN_LEVEL_MAX / coverage_level
            document.update({
                'geozones': geozones,
                'granularity': dataset.spatial.granularity,
                'spatial_weight': spatial_weight,
            })

        document['dataset_suggest']['weight'] = cls.get_suggest_weight(
            temporal_weight, spatial_weight, dataset.featured)

        if dataset.acronym:
            document['dataset_suggest']['input'].append(dataset.acronym)

        return document
Ejemplo n.º 21
0
    def serialize(cls, dataset):
        organization = None
        owner = None
        image_url = None
        spatial_weight = DEFAULT_SPATIAL_WEIGHT
        temporal_weight = DEFAULT_TEMPORAL_WEIGHT

        if dataset.organization:
            organization = Organization.objects(id=dataset.organization.id).first()
            image_url = organization.logo(40, external=True)
        elif dataset.owner:
            owner = User.objects(id=dataset.owner.id).first()
            image_url = owner.avatar(40, external=True)

        certified = organization and organization.certified

        document = {
            'title': dataset.title,
            'description': dataset.description,
            'license': getattr(dataset.license, 'id', None),
            'tags': dataset.tags,
            'badges': [badge.kind for badge in dataset.badges],
            'tag_suggest': dataset.tags,
            'resources': [
                {
                    'title': r.title,
                    'description': r.description,
                    'format': r.format,
                    'type': r.type,
                }
                for r in dataset.resources],
            'format_suggest': [r.format.lower()
                               for r in dataset.resources
                               if r.format],
            'frequency': dataset.frequency,
            'organization': str(organization.id) if organization else None,
            'owner': str(owner.id) if owner else None,
            'dataset_suggest': {
                'input': cls.completer_tokenize(dataset.title) + [str(dataset.id)],
                'output': dataset.title,
                'payload': {
                    'id': str(dataset.id),
                    'slug': dataset.slug,
                    'acronym': dataset.acronym,
                    'image_url': image_url,
                },
            },
            'created': dataset.created_at.strftime('%Y-%m-%dT%H:%M:%S'),
            'last_modified': dataset.last_modified.strftime(
                '%Y-%m-%dT%H:%M:%S'),
            'metrics': dataset.metrics,
            'featured': dataset.featured,
            'from_certified': certified,
        }
        if (dataset.temporal_coverage is not None and
                dataset.temporal_coverage.start and
                dataset.temporal_coverage.end):
            start = dataset.temporal_coverage.start.toordinal()
            end = dataset.temporal_coverage.end.toordinal()
            temporal_weight = min((end - start) / 365, MAX_TEMPORAL_WEIGHT)
            document.update({
                'temporal_coverage': {'start': start, 'end': end},
                'temporal_weight': temporal_weight,
            })

        if dataset.spatial is not None:
            # Index precise zone labels and parents zone identifiers
            # to allow fast filtering.
            zone_ids = [z.id for z in dataset.spatial.zones]
            zones = GeoZone.objects(id__in=zone_ids).exclude('geom')
            parents = set()
            geozones = []
            coverage_level = ADMIN_LEVEL_MAX
            for zone in zones:
                geozones.append({
                    'id': zone.id,
                    'name': zone.name,
                    'keys': zone.keys_values
                })
                parents |= set(zone.parents)
                coverage_level = min(coverage_level, admin_levels[zone.level])

            geozones.extend([{'id': p} for p in parents])

            spatial_weight = ADMIN_LEVEL_MAX / coverage_level
            document.update({
                'geozones': geozones,
                'granularity': dataset.spatial.granularity,
                'spatial_weight': spatial_weight,
            })

        document['dataset_suggest']['weight'] = cls.get_suggest_weight(
            temporal_weight, spatial_weight, dataset.featured)

        if dataset.acronym:
            document['dataset_suggest']['input'].append(dataset.acronym)

        return document
Ejemplo n.º 22
0
 def get_value(self):
     return User.objects(starred_reuses=self.target).count()
Ejemplo n.º 23
0
 def get_value(self):
     return User.objects(starred_datasets=self.target).count()
Ejemplo n.º 24
0
    def serialize(cls, dataset):
        organization = None
        owner = None

        if dataset.organization:
            org = Organization.objects(id=dataset.organization.id).first()
            organization = {
                'id': str(org.id),
                'name': org.name,
                'public_service': 1 if org.public_service else 0,
                'followers': org.metrics.get('followers', 0)
            }
        elif dataset.owner:
            owner = User.objects(id=dataset.owner.id).first()

        document = {
            'id': str(dataset.id),
            'title': dataset.title,
            'description': dataset.description,
            'acronym': dataset.acronym or None,
            'url': dataset.display_url,
            'tags': dataset.tags,
            'license': getattr(dataset.license, 'id', None),
            'badges': [badge.kind for badge in dataset.badges],
            'frequency': dataset.frequency,
            'created_at': to_iso_datetime(dataset.created_at),
            'views': dataset.metrics.get('views', 0),
            'followers': dataset.metrics.get('followers', 0),
            'reuses': dataset.metrics.get('reuses', 0),
            'featured': 1 if dataset.featured else 0,
            'resources_count': len(dataset.resources),
            'organization': organization,
            'owner': str(owner.id) if owner else None,
            'format':
            [r.format.lower() for r in dataset.resources if r.format],
            'schema':
            [r.schema.get('name') for r in dataset.resources if r.schema]
        }
        extras = {}
        for key, value in dataset.extras.items():
            extras[key] = to_iso_datetime(value) if isinstance(
                value, datetime.datetime) else value
        document.update({'extras': extras})

        if (dataset.temporal_coverage is not None
                and dataset.temporal_coverage.start
                and dataset.temporal_coverage.end):
            start = to_iso_datetime(dataset.temporal_coverage.start)
            end = to_iso_datetime(dataset.temporal_coverage.end)
            document.update({
                'temporal_coverage_start': start,
                'temporal_coverage_end': end,
            })

        if dataset.spatial is not None:
            # Index precise zone labels and parents zone identifiers
            # to allow fast filtering.
            zone_ids = [z.id for z in dataset.spatial.zones]
            zones = GeoZone.objects(id__in=zone_ids).exclude('geom')
            parents = set()
            geozones = []
            coverage_level = ADMIN_LEVEL_MAX
            for zone in zones:
                geozones.append({
                    'id': zone.id,
                    'name': zone.name,
                    'keys': zone.keys_values
                })
                parents |= set(zone.parents)
                coverage_level = min(coverage_level, admin_levels[zone.level])

            geozones.extend([{'id': p} for p in parents])
            document.update({
                'geozones': geozones,
                'granularity': dataset.spatial.granularity,
            })
        return document
Ejemplo n.º 25
0
 def get_value(self):
     return User.objects(starred_organizations=self.target).count()
Ejemplo n.º 26
0
 def count_users(self):
     from udata.models import User
     self.metrics['users'] = User.objects(confirmed_at__ne=None,
                                          deleted=None).count()
     self.save()