from __future__ import unicode_literals
import logging
from datetime import date, timedelta
from udata.commands import submanager
from .counter import counter
log = logging.getLogger(__name__)
m = submanager('piwik',
help='Piwik related operations',
description='Handle all Piwik related operations'
)
def parse_date(string):
if string:
parts = [int(s) for s in string.strip().split('-')]
return date(*parts)
@m.command
def fill(start=None, end=None):
'''Fill the piwik metrics'''
end = parse_date(end) or date.today()
start = parse_date(start) or end
log.info('Loading metrics from {start} to {end}'.format(start=start, end=end))
from __future__ import unicode_literals
import logging
# from flask_script import Manager
from udata.commands import submanager
from udata.models import User, Dataset, Reuse, Organization
from . import metric_catalog
from .tasks import update_metrics_for, update_site_metrics
log = logging.getLogger(__name__)
m = submanager(
'metrics',
help='Metrics related operations',
description='Handle all metrics related operations and maintenance')
def iter_catalog(*models):
for model, metrics in metric_catalog.items():
if not models or model.__name__.lower() in models:
yield (model, metrics)
@m.option('-s',
'--site',
action='store_true',
default=False,
help='Update site metrics')
@m.option('-o',
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
from argparse import SUPPRESS
from . import actions
from udata.commands import submanager
log = logging.getLogger(__name__)
m = submanager('harvest',
help='Remote repositories harvesting operations',
description='Handle remote repositories harvesting operations'
)
@m.option('backend')
@m.option('url')
@m.option('name')
@m.option('-f', '--frequency', default=None)
@m.option('-u', '--owner', dest='owner', default=None)
@m.option('-o', '--org', dest='org', default=None)
def create(name, url, backend, frequency=None, owner=None, org=None):
'''Create a new harvest source'''
log.info('Creating a new Harvest source "%s"', name)
source = actions.create_source(name, url, backend,
frequency=frequency,
owner=owner,
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
from os.path import exists
from udata.commands import submanager
from udata.models import Organization, OrganizationBadge
log = logging.getLogger(__name__)
m = submanager(
'badges',
help='Badges related operations',
description='Handle all badges related operations and maintenance')
def toggle_badge(id_or_slug, badge_kind):
organization = Organization.objects.get_by_id_or_slug(id_or_slug)
if not organization:
log.error('No organization found for %s', id_or_slug)
return
log.info('Toggling badge {kind} for organization {org}'.format(
kind=badge_kind, org=organization.name))
existed = False
for badge in organization.badges:
if badge.kind == badge_kind:
organization.badges.remove(badge)
existed = True
break
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
from udata.commands import submanager
from udata.app import cache
log = logging.getLogger(__name__)
m = submanager(
'cache',
help='Cache related operations',
description='Handle all cache related operations and maintenance'
)
@m.command
def flush():
'''Flush the cache'''
print('Flusing cache')
cache.clear()
print('Cache flushed')
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
from udata.commands import submanager
from udata.app import cache
log = logging.getLogger(__name__)
m = submanager('cache',
help='Cache related operations',
description='Handle all cache related operations and maintenance'
)
@m.command
def flush():
'''Flush the cache'''
print('Flusing cache')
cache.clear()
print('Cache flushed')
import logging
from flask.ext.script import prompt, prompt_pass
from flask.ext.security.forms import RegisterForm
from flask.ext.security.utils import encrypt_password
from werkzeug.datastructures import MultiDict
from udata.models import User, datastore
from udata.commands import submanager
log = logging.getLogger(__name__)
m = submanager(
'user',
help='User related operations',
description='Handle all user related operations and maintenance')
@m.command
def create():
'''Create a new user'''
data = {
'first_name': prompt('First name'),
'last_name': prompt('Last name'),
'email': prompt('Email'),
'password': prompt_pass('Password'),
'password_confirm': prompt_pass('Confirm Password'),
}
form = RegisterForm(MultiDict(data), csrf_enabled=False)
if form.validate():
import requests
from flask import current_app
from udata.models import Dataset, ResourceBasedTerritoryDataset
from udata.commands import submanager
from udata.core.storages import logos, references, tmp
log = logging.getLogger(__name__)
m = submanager(
'territories',
help='Territories specifics operations',
description='Handle all territories related operations and maintenance'
)
@m.command
def load_logos(filename):
"""Load logos from geologos archive file."""
if filename.startswith('http'):
log.info('Downloading GeoLogos bundle: %s', filename)
filename, _ = urlretrieve(filename, tmp.path('geologos.tar.xz'))
log.info('Extracting GeoLogos bundle')
with contextlib.closing(lzma.LZMAFile(filename)) as xz:
with tarfile.open(fileobj=xz) as f:
f.extractall(tmp.root)
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
from udata.commands import submanager
log = logging.getLogger(__name__)
m = submanager(
'gouvfr',
help='Data.gouv.fr specifics operations',
description='Handle all Data.gouv.fr related operations and maintenance')
from __future__ import unicode_literals
import logging
import os
from flask import json
from udata.api import api
from udata.commands import submanager
log = logging.getLogger(__name__)
m = submanager(
'api',
help='API related operations',
description='Handle all API related operations and maintenance'
)
def json_to_file(data, filename, pretty=False):
'''Dump JSON data to a file'''
kwargs = dict(indent=4) if pretty else {}
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
dump = json.dumps(api.__schema__, **kwargs)
with open(filename, 'wb') as f:
f.write(dump.encode('utf-8'))
import logging
from datetime import date
from flask import json
from flask.ext.script import prompt_bool
from udata.commands import manager
from udata.search import es, adapter_catalog, ANALYSIS_JSON
log = logging.getLogger(__name__)
from udata.commands import submanager
m = submanager('search',
help='Search/Indexation related operations',
description='Handle search and indexation related operations'
)
@m.option('-t', '--type', dest='doc_type', default=None, help='Only reindex a given type')
@m.option('-n', '--name', default=None, help='Optionnal index name')
def reindex(name=None, doc_type=None):
'''Reindex models'''
for model, adapter in adapter_catalog.items():
if not doc_type or doc_type == adapter.doc_type():
log.info('Reindexing {0} objects'.format(model.__name__))
if es.indices.exists_type(index=es.index_name, doc_type=adapter.doc_type()):
es.indices.delete_mapping(index=es.index_name, doc_type=adapter.doc_type())
es.indices.put_mapping(index=es.index_name, doc_type=adapter.doc_type(), body=adapter.mapping)
qs = model.objects.visible() if hasattr(model.objects, 'visible') else model.objects
for obj in qs.timeout(False):
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
from argparse import SUPPRESS
from . import actions
from udata.commands import submanager
log = logging.getLogger(__name__)
m = submanager(
"harvest",
help="Remote repositories harvesting operations",
description="Handle remote repositories harvesting operations",
)
@m.option("backend")
@m.option("url")
@m.option("name")
@m.option("-f", "--frequency", default=None)
@m.option("-u", "--owner", dest="owner", default=None)
@m.option("-o", "--org", dest="org", default=None)
def create(name, url, backend, frequency=None, owner=None, org=None):
"""Create a new harvest source"""
log.info('Creating a new Harvest source "%s"', name)
source = actions.create_source(name, url, backend, frequency=frequency, owner=owner, organization=org)
log.info(
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
from flask import json
from udata.api import api
from udata.commands import submanager
from flask.ext.restplus.swagger import Swagger
log = logging.getLogger(__name__)
m = submanager(
'api',
help='API related operations',
description='Handle all API related operations and maintenance'
)
@m.option('-p', '--pretty', action='store_true', default=False,
help='Pretty print (sort')
def swagger(pretty):
'''Dump the swagger specifications'''
data = Swagger(api).as_dict()
kwargs = dict(indent=4) if pretty else {}
print(json.dumps(data, **kwargs))
from os.path import join
from pkg_resources import resource_isdir, resource_listdir, resource_string
from flask import current_app
from pymongo.errors import PyMongoError, OperationFailure
from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME
from udata.commands import submanager, green, yellow, cyan, purple, red
log = logging.getLogger(__name__)
m = submanager(
'db',
help='Databse related operations',
description='Handle all databse related operations and maintenance'
)
# A migration script wrapper recording the stdout lines
SCRIPT_WRAPPER = '''
function(plugin, filename, script) {{
var stdout = [];
function print() {{
var args = Array.prototype.slice.call(arguments);
stdout.push(args.join(' '));
}}
{0}
db.migrations.insert({{
# -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import
import logging
from udata.commands import submanager
from udata.tasks import schedulables, celery
log = logging.getLogger(__name__)
m = submanager(
'job',
help='Jobs related operations',
description='Handle all jobs related operations and maintenance'
)
@m.option('name', help='The job name')
@m.option('-d', '--delay', action='store_true', default=False,
help='Run the job asynchronously on a worker')
@m.option('-a', '--args', nargs='*', help='job arguments')
@m.option('-k', '--kwargs', nargs='*', help='job keyword arguments')
def run(name, delay, args, kwargs):
args = args or []
kwargs = dict(k.split() for k in kwargs) if kwargs else {}
if name not in celery.tasks:
log.error('Job %s not found', name)
job = celery.tasks[name]
if delay:
log.info('Sending job %s', name)
import logging
from flask.ext.script import prompt, prompt_pass
from flask.ext.security.forms import RegisterForm
from flask.ext.security.utils import encrypt_password
from werkzeug.datastructures import MultiDict
from udata.models import User, datastore
from udata.commands import submanager
log = logging.getLogger(__name__)
m = submanager('user',
help='User related operations',
description='Handle all user related operations and maintenance'
)
@m.command
def create():
'''Create a new user'''
data = {
'first_name': prompt('First name'),
'last_name': prompt('Last name'),
'email': prompt('Email'),
'password': prompt_pass('Password'),
'password_confirm': prompt_pass('Confirm Password'),
}
form = RegisterForm(MultiDict(data), csrf_enabled=False)
if form.validate():
from collections import Counter
from urlparse import urlparse
import redis
from celery.task.control import inspect
from flask import current_app
from udata.app import cache
from udata.commands import submanager, red
from udata.tasks import celery
log = logging.getLogger(__name__)
m = submanager(
'worker',
help='Worker related operations',
description='Handle all worker related operations and maintenance')
TASKS_LIST_CACHE_KEY = 'worker-status-tasks'
# we're using an aggressive cache in order not to hit Celery every 5 min
TASKS_LIST_CACHE_DURATION = 60 * 60 * 24 # in seconds
@m.command
def start():
"""Start a worker"""
celery.start()
def status_print_task(count, biggest_task_name, munin=False):
if not munin:
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
from os.path import exists
from udata.commands import submanager
from udata.models import Organization
log = logging.getLogger(__name__)
m = submanager(
"badges", help="Badges related operations", description="Handle all badges related operations and maintenance"
)
def toggle_badge(id_or_slug, badge_kind):
organization = Organization.objects.get_by_id_or_slug(id_or_slug)
if not organization:
log.error("No organization found for %s", id_or_slug)
return
log.info("Toggling badge {kind} for organization {org}".format(kind=badge_kind, org=organization.name))
organization.toggle_badge(badge_kind)
@m.command
def toggle(path_or_id, badge_kind):
"""Toggle a `badge_kind` for a given `path_or_id`
from pkg_resources import resource_isdir, resource_listdir, resource_string
from flask import current_app
from pymongo.errors import PyMongoError, OperationFailure
from flask_mongoengine.connection import get_db
from flask_mongoengine.connection import DEFAULT_CONNECTION_NAME
from udata.commands import submanager, green, yellow, cyan, purple, red
log = logging.getLogger(__name__)
m = submanager(
'db',
help='Database related operations',
description='Handle all database related operations and maintenance'
)
# A migration script wrapper recording the stdout lines
SCRIPT_WRAPPER = '''
function(plugin, filename, script) {{
var stdout = [];
function print() {{
var args = Array.prototype.slice.call(arguments);
stdout.push(args.join(' '));
}}
{0}
db.migrations.insert({{
from __future__ import unicode_literals
import logging
from collections import Counter
from udata.commands import submanager, green, cyan, white
from udata.core.organization.models import Organization
from udata.core.post.models import Post
from udata.core.reuse.models import Reuse
from udata.core.user.models import User
log = logging.getLogger(__name__)
m = submanager(
'images',
help='Images related operations',
description='Handle all images related operations and maintenance')
def render_or_skip(obj, attr):
try:
getattr(obj, attr).rerender()
obj.save()
return 1
except Exception as e:
log.warning('Skipped "%s": %s(%s)', obj, e.__class__.__name__, e)
return 0
@m.command
def render():
import tempfile
import tarfile
import shutil
from os.path import join
from urllib import urlretrieve
from udata.commands import submanager
from udata.models import GeoLevel, GeoZone
log = logging.getLogger(__name__)
m = submanager(
'spatial',
help='Geospatial related operations',
description='Handle all geospatial related operations and maintenance'
)
@m.command
def load(filename, drop=False):
'''Load a GeoZones Bundle'''
tmp = tempfile.mkdtemp()
if filename.startswith('http'):
log.info('Downloading GeoZones bundle: %s', filename)
filename, _ = urlretrieve(filename, join(tmp, 'geozones.tar.xz'))
log.info('Extracting GeoZones bundle')
with contextlib.closing(lzma.LZMAFile(filename)) as xz:
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
from udata.commands import submanager
from udata.linkchecker.tasks import check_resources
log = logging.getLogger(__name__)
m = submanager('linkchecker',
help='Link checking operations',
description='Handle link checking operations'
)
@m.option('-n', '--number', type=int, dest='number', default=5000,
help='Number of URLs to check')
def check(number):
'''Check <number> of URLs that have not been (recently) checked'''
check_resources(number)
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
from argparse import SUPPRESS
from . import actions
from udata.commands import submanager
log = logging.getLogger(__name__)
m = submanager('harvest',
help='Remote repositories harvesting operations',
description='Handle remote repositories harvesting operations'
)
@m.option('backend')
@m.option('url')
@m.option('name')
@m.option('-f', '--frequency', default=None)
@m.option('-u', '--owner', dest='owner', default=None)
@m.option('-o', '--org', dest='org', default=None)
def create(name, url, backend, frequency=None, owner=None, org=None):
'''Create a new harvest source'''
log.info('Creating a new Harvest source "%s"', name)
source = actions.create_source(name, url, backend,
frequency=frequency,
owner=owner,
# from flask.ext.script import Manager
from udata.commands import submanager
from udata.models import User, Dataset, Reuse, Organization
from . import metric_catalog
from .tasks import update_metrics_for, update_site_metrics
log = logging.getLogger(__name__)
m = submanager(
'metrics',
help='Metrics related operations',
description='Handle all metrics related operations and maintenance'
)
def iter_catalog(*models):
for model, metrics in metric_catalog.items():
if not models or model.__name__.lower() in models:
yield (model, metrics)
@m.option(
'-s', '--site', action='store_true', default=False,
help='Update site metrics')
@m.option(
'-o', '--organizations', action='store_true', default=False,
import logging
from datetime import datetime
from flask_script import prompt_bool
from flask_script.commands import InvalidCommand
from udata.commands import submanager
from udata.search import es, adapter_catalog
from elasticsearch.helpers import reindex as es_reindex
log = logging.getLogger(__name__)
m = submanager('search',
help='Search/Indexation related operations',
description='Handle search and indexation related operations')
TIMESTAMP_FORMAT = '%Y-%m-%d-%H-%M'
def default_index_name():
'''Build a time based index name'''
return '-'.join([es.index_name, datetime.now().strftime(TIMESTAMP_FORMAT)])
def iter_adapters():
'''Iter over adapter in predictable way'''
adapters = adapter_catalog.values()
return sorted(adapters, key=lambda a: a.model.__name__)
# -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import
import logging
from udata.commands import submanager
from udata.tasks import schedulables, celery
log = logging.getLogger(__name__)
m = submanager(
'job',
help='Jobs related operations',
description='Handle all jobs related operations and maintenance')
@m.option('name', help='The job name')
@m.option('-d',
'--delay',
action='store_true',
default=False,
help='Run the job asynchronously on a worker')
@m.option('-a', '--args', nargs='*', help='job arguments')
@m.option('-k', '--kwargs', nargs='*', help='job keyword arguments')
def run(name, delay, args, kwargs):
args = args or []
kwargs = dict(k.split('=') for k in kwargs) if kwargs else {}
if name not in celery.tasks:
log.error('Job %s not found', name)
job = celery.tasks[name]
if delay:
import logging
from flask.ext.script import prompt, prompt_pass
from flask.ext.security.forms import RegisterForm
from flask.ext.security.utils import encrypt_password
from werkzeug.datastructures import MultiDict
from udata.models import User, datastore
from udata.commands import submanager
log = logging.getLogger(__name__)
m = submanager("user", help="User related operations", description="Handle all user related operations and maintenance")
@m.command
def create():
"""Create a new user"""
data = {
"first_name": prompt("First name"),
"last_name": prompt("Last name"),
"email": prompt("Email"),
"password": prompt_pass("Password"),
"password_confirm": prompt_pass("Confirm Password"),
}
form = RegisterForm(MultiDict(data), csrf_enabled=False)
if form.validate():
data["password"] = encrypt_password(data["password"])
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
from udata.commands import submanager
from udata.models import GeoZone, Organization
log = logging.getLogger(__name__)
m = submanager(
"organizations",
help="Organizations related operations",
description="Handle all organizations related operations and maintenance",
)
@m.command
def attach_zone(insee_code, organization_id_or_slug, level="fr/town"):
"""Attach a zone restricted to level for a given organization."""
organization = Organization.objects.get_by_id_or_slug(organization_id_or_slug)
geozone = GeoZone.objects(code=insee_code, level=level).first()
log.info("Attaching {organization} with {geozone.name}".format(organization=organization, geozone=geozone))
organization.zone = geozone.id
organization.save()
log.info("Done")
@m.command
def detach_zone(organization_id_or_slug):
"""Detach the zone of a given organization."""
import logging
import lzma
import tarfile
import shutil
from urllib import urlretrieve
import msgpack
from udata.commands import submanager
from udata.core.storages import tmp
from udata.models import GeoLevel, GeoZone
log = logging.getLogger(__name__)
m = submanager(
'spatial',
help='Geospatial related operations',
description='Handle all geospatial related operations and maintenance')
@m.command
def load(filename, drop=False):
'''Load a GeoZones Bundle'''
if filename.startswith('http'):
log.info('Downloading GeoZones bundle: %s', filename)
filename, _ = urlretrieve(filename, tmp.path('geozones.tar.xz'))
log.info('Extracting GeoZones bundle')
with contextlib.closing(lzma.LZMAFile(filename)) as xz:
with tarfile.open(fileobj=xz) as f:
f.extractall(tmp.root)
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
from os.path import exists
from udata.commands import submanager
from udata.models import Organization, Badge
log = logging.getLogger(__name__)
m = submanager(
'badges',
help='Badges related operations',
description='Handle all badges related operations and maintenance'
)
def toggle_badge(id_or_slug, badge_kind):
organization = Organization.objects.get_by_id_or_slug(id_or_slug)
if not organization:
log.error('No organization found for %s', id_or_slug)
return
log.info('Toggling badge {kind} for organization {org}'.format(
kind=badge_kind,
org=organization.name
))
existed = False
for badge in organization.badges:
import shutil
import tarfile
from collections import Counter
from datetime import date
from string import Formatter
from urllib import urlretrieve
from udata.models import Dataset, GeoZone, SpatialCoverage
from udata.commands import submanager
from udata.core.storages import logos, tmp
log = logging.getLogger(__name__)
m = submanager(
'territories',
help='Territories specifics operations',
description='Handle all territories related operations and maintenance')
@m.command
def load_logos(filename):
"""Load logos from geologos archive file."""
if filename.startswith('http'):
log.info('Downloading GeoLogos bundle: %s', filename)
filename, _ = urlretrieve(filename, tmp.path('geologos.tar.xz'))
log.info('Extracting GeoLogos bundle')
with contextlib.closing(lzma.LZMAFile(filename)) as xz:
with tarfile.open(fileobj=xz) as f:
f.extractall(tmp.root)
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
import slugify
from udata.models import Dataset, DatasetBadge, User, C3
from udata.commands import submanager
log = logging.getLogger(__name__)
m = submanager(
'gouvfr',
help='Data.gouv.fr specifics operations',
description='Handle all Data.gouv.fr related operations and maintenance'
)
@m.command
def c3_badges(filename):
'''Toggle C3 badges from an organization list'''
with open(filename, 'r') as titles:
user = User.objects(first_name='Etalab', last_name='Bot').first()
badge = DatasetBadge(kind=C3, created_by=user)
for title in titles:
title = title.decode('utf-8').strip(u'\n')
if title.startswith(u'*'):
continue
slug = slugify.slugify(title.lower())
dataset = (Dataset.objects(title=title).first()
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
from udata.commands import submanager
from udata.models import GeoZone, Organization
log = logging.getLogger(__name__)
m = submanager(
'organizations',
help='Organizations related operations',
description='Handle all organizations related operations and maintenance')
@m.command
def attach_zone(geoid, organization_id_or_slug):
'''Attach a zone restricted to level for a given organization.'''
organization = Organization.objects.get_by_id_or_slug(
organization_id_or_slug)
if not organization:
log.error('No organization found for %s', organization_id_or_slug)
geozone = GeoZone.objects.get(id=geoid)
if not geozone:
log.error('No geozone found for %s', geoid)
log.info('Attaching {organization} with {geozone.name}'.format(
organization=organization, geozone=geozone))
organization.zone = geozone.id
organization.save()
log.info('Done')
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import logging
from udata.commands import submanager
log = logging.getLogger(__name__)
m = submanager(
"gouvfr",
help="Data.gouv.fr specifics operations",
description="Handle all Data.gouv.fr related operations and maintenance",
)
