Beispiel #1
0
from __future__ import unicode_literals

import logging

from datetime import date, timedelta

from udata.commands import submanager

from .counter import counter


log = logging.getLogger(__name__)


m = submanager('piwik',
    help='Piwik related operations',
    description='Handle all Piwik related operations'
)


def parse_date(string):
    if string:
        parts = [int(s) for s in string.strip().split('-')]
        return date(*parts)


@m.command
def fill(start=None, end=None):
    '''Fill the piwik metrics'''
    end = parse_date(end) or date.today()
    start = parse_date(start) or end
    log.info('Loading metrics from {start} to {end}'.format(start=start, end=end))
Beispiel #2
0
from __future__ import unicode_literals

import logging

# from flask_script import Manager

from udata.commands import submanager
from udata.models import User, Dataset, Reuse, Organization

from . import metric_catalog
from .tasks import update_metrics_for, update_site_metrics

log = logging.getLogger(__name__)

m = submanager(
    'metrics',
    help='Metrics related operations',
    description='Handle all metrics related operations and maintenance')


def iter_catalog(*models):
    for model, metrics in metric_catalog.items():
        if not models or model.__name__.lower() in models:
            yield (model, metrics)


@m.option('-s',
          '--site',
          action='store_true',
          default=False,
          help='Update site metrics')
@m.option('-o',
Beispiel #3
0
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

from argparse import SUPPRESS

from . import actions

from udata.commands import submanager

log = logging.getLogger(__name__)

m = submanager('harvest',
               help='Remote repositories harvesting operations',
               description='Handle remote repositories harvesting operations'
               )


@m.option('backend')
@m.option('url')
@m.option('name')
@m.option('-f', '--frequency', default=None)
@m.option('-u', '--owner', dest='owner', default=None)
@m.option('-o', '--org', dest='org', default=None)
def create(name, url, backend, frequency=None, owner=None, org=None):
    '''Create a new harvest source'''
    log.info('Creating a new Harvest source "%s"', name)
    source = actions.create_source(name, url, backend,
                                   frequency=frequency,
                                   owner=owner,
Beispiel #4
0
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

from os.path import exists

from udata.commands import submanager
from udata.models import Organization, OrganizationBadge

log = logging.getLogger(__name__)

m = submanager(
    'badges',
    help='Badges related operations',
    description='Handle all badges related operations and maintenance')


def toggle_badge(id_or_slug, badge_kind):
    organization = Organization.objects.get_by_id_or_slug(id_or_slug)
    if not organization:
        log.error('No organization found for %s', id_or_slug)
        return
    log.info('Toggling badge {kind} for organization {org}'.format(
        kind=badge_kind, org=organization.name))
    existed = False
    for badge in organization.badges:
        if badge.kind == badge_kind:
            organization.badges.remove(badge)
            existed = True
            break
Beispiel #5
0
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

from udata.commands import submanager
from udata.app import cache

log = logging.getLogger(__name__)


m = submanager(
    'cache',
    help='Cache related operations',
    description='Handle all cache related operations and maintenance'
)


@m.command
def flush():
    '''Flush the cache'''
    print('Flusing cache')
    cache.clear()
    print('Cache flushed')
Beispiel #6
0
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

from udata.commands import submanager
from udata.app import cache

log = logging.getLogger(__name__)


m = submanager('cache',
    help='Cache related operations',
    description='Handle all cache related operations and maintenance'
)


@m.command
def flush():
    '''Flush the cache'''
    print('Flusing cache')
    cache.clear()
    print('Cache flushed')
Beispiel #7
0
import logging

from flask.ext.script import prompt, prompt_pass
from flask.ext.security.forms import RegisterForm
from flask.ext.security.utils import encrypt_password
from werkzeug.datastructures import MultiDict

from udata.models import User, datastore

from udata.commands import submanager

log = logging.getLogger(__name__)

m = submanager(
    'user',
    help='User related operations',
    description='Handle all user related operations and maintenance')


@m.command
def create():
    '''Create a new user'''
    data = {
        'first_name': prompt('First name'),
        'last_name': prompt('Last name'),
        'email': prompt('Email'),
        'password': prompt_pass('Password'),
        'password_confirm': prompt_pass('Confirm Password'),
    }
    form = RegisterForm(MultiDict(data), csrf_enabled=False)
    if form.validate():
Beispiel #8
0
import requests

from flask import current_app

from udata.models import Dataset, ResourceBasedTerritoryDataset
from udata.commands import submanager
from udata.core.storages import logos, references, tmp


log = logging.getLogger(__name__)


m = submanager(
    'territories',
    help='Territories specifics operations',
    description='Handle all territories related operations and maintenance'
)


@m.command
def load_logos(filename):
    """Load logos from geologos archive file."""
    if filename.startswith('http'):
        log.info('Downloading GeoLogos bundle: %s', filename)
        filename, _ = urlretrieve(filename, tmp.path('geologos.tar.xz'))

    log.info('Extracting GeoLogos bundle')
    with contextlib.closing(lzma.LZMAFile(filename)) as xz:
        with tarfile.open(fileobj=xz) as f:
            f.extractall(tmp.root)
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

from udata.commands import submanager

log = logging.getLogger(__name__)

m = submanager(
    'gouvfr',
    help='Data.gouv.fr specifics operations',
    description='Handle all Data.gouv.fr related operations and maintenance')
Beispiel #10
0
from __future__ import unicode_literals

import logging
import os

from flask import json

from udata.api import api
from udata.commands import submanager

log = logging.getLogger(__name__)


m = submanager(
    'api',
    help='API related operations',
    description='Handle all API related operations and maintenance'
)


def json_to_file(data, filename, pretty=False):
    '''Dump JSON data to a file'''
    kwargs = dict(indent=4) if pretty else {}
    dirname = os.path.dirname(filename)
    if not os.path.exists(dirname):
        os.makedirs(dirname)
    dump = json.dumps(api.__schema__, **kwargs)
    with open(filename, 'wb') as f:
        f.write(dump.encode('utf-8'))

Beispiel #11
0
import logging

from datetime import date

from flask import json
from flask.ext.script import prompt_bool

from udata.commands import manager
from udata.search import es, adapter_catalog, ANALYSIS_JSON

log = logging.getLogger(__name__)

from udata.commands import submanager

m = submanager('search',
    help='Search/Indexation related operations',
    description='Handle search and indexation related operations'
)


@m.option('-t', '--type', dest='doc_type', default=None, help='Only reindex a given type')
@m.option('-n', '--name', default=None, help='Optionnal index name')
def reindex(name=None, doc_type=None):
    '''Reindex models'''
    for model, adapter in adapter_catalog.items():
        if not doc_type or doc_type == adapter.doc_type():
            log.info('Reindexing {0} objects'.format(model.__name__))
            if es.indices.exists_type(index=es.index_name, doc_type=adapter.doc_type()):
                es.indices.delete_mapping(index=es.index_name, doc_type=adapter.doc_type())
            es.indices.put_mapping(index=es.index_name, doc_type=adapter.doc_type(), body=adapter.mapping)
            qs = model.objects.visible() if hasattr(model.objects, 'visible') else model.objects
            for obj in qs.timeout(False):
Beispiel #12
0
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

from argparse import SUPPRESS

from . import actions

from udata.commands import submanager

log = logging.getLogger(__name__)

m = submanager(
    "harvest",
    help="Remote repositories harvesting operations",
    description="Handle remote repositories harvesting operations",
)


@m.option("backend")
@m.option("url")
@m.option("name")
@m.option("-f", "--frequency", default=None)
@m.option("-u", "--owner", dest="owner", default=None)
@m.option("-o", "--org", dest="org", default=None)
def create(name, url, backend, frequency=None, owner=None, org=None):
    """Create a new harvest source"""
    log.info('Creating a new Harvest source "%s"', name)
    source = actions.create_source(name, url, backend, frequency=frequency, owner=owner, organization=org)
    log.info(
Beispiel #13
0
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

from flask import json

from udata.api import api
from udata.commands import submanager

from flask.ext.restplus.swagger import Swagger

log = logging.getLogger(__name__)


m = submanager(
    'api',
    help='API related operations',
    description='Handle all API related operations and maintenance'
)


@m.option('-p', '--pretty', action='store_true', default=False,
          help='Pretty print (sort')
def swagger(pretty):
    '''Dump the swagger specifications'''
    data = Swagger(api).as_dict()
    kwargs = dict(indent=4) if pretty else {}
    print(json.dumps(data, **kwargs))
Beispiel #14
0
from os.path import join
from pkg_resources import resource_isdir, resource_listdir, resource_string

from flask import current_app

from pymongo.errors import PyMongoError, OperationFailure
from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME

from udata.commands import submanager, green, yellow, cyan, purple, red

log = logging.getLogger(__name__)


m = submanager(
    'db',
    help='Databse related operations',
    description='Handle all databse related operations and maintenance'
)

# A migration script wrapper recording the stdout lines
SCRIPT_WRAPPER = '''
function(plugin, filename, script) {{
    var stdout = [];
    function print() {{
        var args = Array.prototype.slice.call(arguments);
        stdout.push(args.join(' '));
    }}

    {0}

    db.migrations.insert({{
Beispiel #15
0
# -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import

import logging

from udata.commands import submanager
from udata.tasks import schedulables, celery

log = logging.getLogger(__name__)


m = submanager(
    'job',
    help='Jobs related operations',
    description='Handle all jobs related operations and maintenance'
)


@m.option('name', help='The job name')
@m.option('-d', '--delay', action='store_true', default=False,
          help='Run the job asynchronously on a worker')
@m.option('-a', '--args', nargs='*', help='job arguments')
@m.option('-k', '--kwargs', nargs='*', help='job keyword arguments')
def run(name, delay, args, kwargs):
    args = args or []
    kwargs = dict(k.split() for k in kwargs) if kwargs else {}
    if name not in celery.tasks:
        log.error('Job %s not found', name)
    job = celery.tasks[name]
    if delay:
        log.info('Sending job %s', name)
Beispiel #16
0
import logging

from flask.ext.script import prompt, prompt_pass
from flask.ext.security.forms import RegisterForm
from flask.ext.security.utils import encrypt_password
from werkzeug.datastructures import MultiDict

from udata.models import User, datastore

from udata.commands import submanager

log = logging.getLogger(__name__)


m = submanager('user',
    help='User related operations',
    description='Handle all user related operations and maintenance'
)


@m.command
def create():
    '''Create a new user'''
    data = {
        'first_name': prompt('First name'),
        'last_name': prompt('Last name'),
        'email': prompt('Email'),
        'password': prompt_pass('Password'),
        'password_confirm': prompt_pass('Confirm Password'),
    }
    form = RegisterForm(MultiDict(data), csrf_enabled=False)
    if form.validate():
Beispiel #17
0
from collections import Counter
from urlparse import urlparse

import redis

from celery.task.control import inspect
from flask import current_app

from udata.app import cache
from udata.commands import submanager, red
from udata.tasks import celery

log = logging.getLogger(__name__)

m = submanager(
    'worker',
    help='Worker related operations',
    description='Handle all worker related operations and maintenance')

TASKS_LIST_CACHE_KEY = 'worker-status-tasks'
# we're using an aggressive cache in order not to hit Celery every 5 min
TASKS_LIST_CACHE_DURATION = 60 * 60 * 24  # in seconds


@m.command
def start():
    """Start a worker"""
    celery.start()


def status_print_task(count, biggest_task_name, munin=False):
    if not munin:
Beispiel #18
0
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

from os.path import exists

from udata.commands import submanager
from udata.models import Organization

log = logging.getLogger(__name__)


m = submanager(
    "badges", help="Badges related operations", description="Handle all badges related operations and maintenance"
)


def toggle_badge(id_or_slug, badge_kind):
    organization = Organization.objects.get_by_id_or_slug(id_or_slug)
    if not organization:
        log.error("No organization found for %s", id_or_slug)
        return
    log.info("Toggling badge {kind} for organization {org}".format(kind=badge_kind, org=organization.name))
    organization.toggle_badge(badge_kind)


@m.command
def toggle(path_or_id, badge_kind):
    """Toggle a `badge_kind` for a given `path_or_id`
Beispiel #19
0
from pkg_resources import resource_isdir, resource_listdir, resource_string

from flask import current_app

from pymongo.errors import PyMongoError, OperationFailure
from flask_mongoengine.connection import get_db
from flask_mongoengine.connection import DEFAULT_CONNECTION_NAME

from udata.commands import submanager, green, yellow, cyan, purple, red

log = logging.getLogger(__name__)


m = submanager(
    'db',
    help='Database related operations',
    description='Handle all database related operations and maintenance'
)

# A migration script wrapper recording the stdout lines
SCRIPT_WRAPPER = '''
function(plugin, filename, script) {{
    var stdout = [];
    function print() {{
        var args = Array.prototype.slice.call(arguments);
        stdout.push(args.join(' '));
    }}

    {0}

    db.migrations.insert({{
Beispiel #20
0
from __future__ import unicode_literals

import logging

from collections import Counter

from udata.commands import submanager, green, cyan, white
from udata.core.organization.models import Organization
from udata.core.post.models import Post
from udata.core.reuse.models import Reuse
from udata.core.user.models import User

log = logging.getLogger(__name__)

m = submanager(
    'images',
    help='Images related operations',
    description='Handle all images related operations and maintenance')


def render_or_skip(obj, attr):
    try:
        getattr(obj, attr).rerender()
        obj.save()
        return 1
    except Exception as e:
        log.warning('Skipped "%s": %s(%s)', obj, e.__class__.__name__, e)
        return 0


@m.command
def render():
Beispiel #21
0
import tempfile
import tarfile
import shutil

from os.path import join
from urllib import urlretrieve

from udata.commands import submanager
from udata.models import GeoLevel, GeoZone

log = logging.getLogger(__name__)


m = submanager(
    'spatial',
    help='Geospatial related operations',
    description='Handle all geospatial related operations and maintenance'
)


@m.command
def load(filename, drop=False):
    '''Load a GeoZones Bundle'''
    tmp = tempfile.mkdtemp()

    if filename.startswith('http'):
        log.info('Downloading GeoZones bundle: %s', filename)
        filename, _ = urlretrieve(filename, join(tmp, 'geozones.tar.xz'))

    log.info('Extracting GeoZones bundle')
    with contextlib.closing(lzma.LZMAFile(filename)) as xz:
Beispiel #22
0
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

from udata.commands import submanager
from udata.linkchecker.tasks import check_resources


log = logging.getLogger(__name__)

m = submanager('linkchecker',
               help='Link checking operations',
               description='Handle link checking operations'
               )


@m.option('-n', '--number', type=int, dest='number', default=5000,
          help='Number of URLs to check')
def check(number):
    '''Check <number> of URLs that have not been (recently) checked'''
    check_resources(number)
Beispiel #23
0
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

from argparse import SUPPRESS

from . import actions

from udata.commands import submanager

log = logging.getLogger(__name__)

m = submanager('harvest',
               help='Remote repositories harvesting operations',
               description='Handle remote repositories harvesting operations'
               )


@m.option('backend')
@m.option('url')
@m.option('name')
@m.option('-f', '--frequency', default=None)
@m.option('-u', '--owner', dest='owner', default=None)
@m.option('-o', '--org', dest='org', default=None)
def create(name, url, backend, frequency=None, owner=None, org=None):
    '''Create a new harvest source'''
    log.info('Creating a new Harvest source "%s"', name)
    source = actions.create_source(name, url, backend,
                                   frequency=frequency,
                                   owner=owner,
Beispiel #24
0

# from flask.ext.script import Manager

from udata.commands import submanager
from udata.models import User, Dataset, Reuse, Organization

from . import metric_catalog
from .tasks import update_metrics_for, update_site_metrics

log = logging.getLogger(__name__)


m = submanager(
    'metrics',
    help='Metrics related operations',
    description='Handle all metrics related operations and maintenance'
)


def iter_catalog(*models):
    for model, metrics in metric_catalog.items():
        if not models or model.__name__.lower() in models:
            yield (model, metrics)


@m.option(
    '-s', '--site', action='store_true', default=False,
    help='Update site metrics')
@m.option(
    '-o', '--organizations', action='store_true', default=False,
Beispiel #25
0
import logging

from datetime import datetime

from flask_script import prompt_bool
from flask_script.commands import InvalidCommand

from udata.commands import submanager
from udata.search import es, adapter_catalog

from elasticsearch.helpers import reindex as es_reindex

log = logging.getLogger(__name__)

m = submanager('search',
               help='Search/Indexation related operations',
               description='Handle search and indexation related operations')

TIMESTAMP_FORMAT = '%Y-%m-%d-%H-%M'


def default_index_name():
    '''Build a time based index name'''
    return '-'.join([es.index_name, datetime.now().strftime(TIMESTAMP_FORMAT)])


def iter_adapters():
    '''Iter over adapter in predictable way'''
    adapters = adapter_catalog.values()
    return sorted(adapters, key=lambda a: a.model.__name__)
Beispiel #26
0
# -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import

import logging

from udata.commands import submanager
from udata.tasks import schedulables, celery

log = logging.getLogger(__name__)

m = submanager(
    'job',
    help='Jobs related operations',
    description='Handle all jobs related operations and maintenance')


@m.option('name', help='The job name')
@m.option('-d',
          '--delay',
          action='store_true',
          default=False,
          help='Run the job asynchronously on a worker')
@m.option('-a', '--args', nargs='*', help='job arguments')
@m.option('-k', '--kwargs', nargs='*', help='job keyword arguments')
def run(name, delay, args, kwargs):
    args = args or []
    kwargs = dict(k.split('=') for k in kwargs) if kwargs else {}
    if name not in celery.tasks:
        log.error('Job %s not found', name)
    job = celery.tasks[name]
    if delay:
Beispiel #27
0
import logging

from flask.ext.script import prompt, prompt_pass
from flask.ext.security.forms import RegisterForm
from flask.ext.security.utils import encrypt_password
from werkzeug.datastructures import MultiDict

from udata.models import User, datastore

from udata.commands import submanager

log = logging.getLogger(__name__)


m = submanager("user", help="User related operations", description="Handle all user related operations and maintenance")


@m.command
def create():
    """Create a new user"""
    data = {
        "first_name": prompt("First name"),
        "last_name": prompt("Last name"),
        "email": prompt("Email"),
        "password": prompt_pass("Password"),
        "password_confirm": prompt_pass("Confirm Password"),
    }
    form = RegisterForm(MultiDict(data), csrf_enabled=False)
    if form.validate():
        data["password"] = encrypt_password(data["password"])
Beispiel #28
0
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

from udata.commands import submanager
from udata.models import GeoZone, Organization

log = logging.getLogger(__name__)

m = submanager(
    "organizations",
    help="Organizations related operations",
    description="Handle all organizations related operations and maintenance",
)


@m.command
def attach_zone(insee_code, organization_id_or_slug, level="fr/town"):
    """Attach a zone restricted to level for a given organization."""
    organization = Organization.objects.get_by_id_or_slug(organization_id_or_slug)
    geozone = GeoZone.objects(code=insee_code, level=level).first()
    log.info("Attaching {organization} with {geozone.name}".format(organization=organization, geozone=geozone))
    organization.zone = geozone.id
    organization.save()
    log.info("Done")


@m.command
def detach_zone(organization_id_or_slug):
    """Detach the zone of a given organization."""
Beispiel #29
0
import logging
import lzma
import tarfile
import shutil
from urllib import urlretrieve

import msgpack

from udata.commands import submanager
from udata.core.storages import tmp
from udata.models import GeoLevel, GeoZone

log = logging.getLogger(__name__)

m = submanager(
    'spatial',
    help='Geospatial related operations',
    description='Handle all geospatial related operations and maintenance')


@m.command
def load(filename, drop=False):
    '''Load a GeoZones Bundle'''
    if filename.startswith('http'):
        log.info('Downloading GeoZones bundle: %s', filename)
        filename, _ = urlretrieve(filename, tmp.path('geozones.tar.xz'))

    log.info('Extracting GeoZones bundle')
    with contextlib.closing(lzma.LZMAFile(filename)) as xz:
        with tarfile.open(fileobj=xz) as f:
            f.extractall(tmp.root)
Beispiel #30
0
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

from os.path import exists

from udata.commands import submanager
from udata.models import Organization, Badge

log = logging.getLogger(__name__)


m = submanager(
    'badges',
    help='Badges related operations',
    description='Handle all badges related operations and maintenance'
)


def toggle_badge(id_or_slug, badge_kind):
    organization = Organization.objects.get_by_id_or_slug(id_or_slug)
    if not organization:
        log.error('No organization found for %s', id_or_slug)
        return
    log.info('Toggling badge {kind} for organization {org}'.format(
        kind=badge_kind,
        org=organization.name
    ))
    existed = False
    for badge in organization.badges:
Beispiel #31
0
import shutil
import tarfile

from collections import Counter
from datetime import date
from string import Formatter
from urllib import urlretrieve

from udata.models import Dataset, GeoZone, SpatialCoverage
from udata.commands import submanager
from udata.core.storages import logos, tmp

log = logging.getLogger(__name__)

m = submanager(
    'territories',
    help='Territories specifics operations',
    description='Handle all territories related operations and maintenance')


@m.command
def load_logos(filename):
    """Load logos from geologos archive file."""
    if filename.startswith('http'):
        log.info('Downloading GeoLogos bundle: %s', filename)
        filename, _ = urlretrieve(filename, tmp.path('geologos.tar.xz'))

    log.info('Extracting GeoLogos bundle')
    with contextlib.closing(lzma.LZMAFile(filename)) as xz:
        with tarfile.open(fileobj=xz) as f:
            f.extractall(tmp.root)
Beispiel #32
0
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging
import slugify

from udata.models import Dataset, DatasetBadge, User, C3
from udata.commands import submanager

log = logging.getLogger(__name__)


m = submanager(
    'gouvfr',
    help='Data.gouv.fr specifics operations',
    description='Handle all Data.gouv.fr related operations and maintenance'
)


@m.command
def c3_badges(filename):
    '''Toggle C3 badges from an organization list'''
    with open(filename, 'r') as titles:
        user = User.objects(first_name='Etalab', last_name='Bot').first()
        badge = DatasetBadge(kind=C3, created_by=user)
        for title in titles:
            title = title.decode('utf-8').strip(u'\n')
            if title.startswith(u'*'):
                continue
            slug = slugify.slugify(title.lower())
            dataset = (Dataset.objects(title=title).first()
Beispiel #33
0
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

from udata.commands import submanager
from udata.models import GeoZone, Organization

log = logging.getLogger(__name__)

m = submanager(
    'organizations',
    help='Organizations related operations',
    description='Handle all organizations related operations and maintenance')


@m.command
def attach_zone(geoid, organization_id_or_slug):
    '''Attach a zone restricted to level for a given organization.'''
    organization = Organization.objects.get_by_id_or_slug(
        organization_id_or_slug)
    if not organization:
        log.error('No organization found for %s', organization_id_or_slug)
    geozone = GeoZone.objects.get(id=geoid)
    if not geozone:
        log.error('No geozone found for %s', geoid)
    log.info('Attaching {organization} with {geozone.name}'.format(
        organization=organization, geozone=geozone))
    organization.zone = geozone.id
    organization.save()
    log.info('Done')
Beispiel #34
0
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import logging

from udata.commands import submanager

log = logging.getLogger(__name__)


m = submanager(
    "gouvfr",
    help="Data.gouv.fr specifics operations",
    description="Handle all Data.gouv.fr related operations and maintenance",
)