def cmd_update_tags(fetcher=None, dataset=None, max_bulk=100,
update_only=False, async_mode=None,
dry_mode=False, **kwargs):
"""Create or Update field tags"""
"""
Examples:
dlstats fetchers tag -f BIS -d CNFS -S
dlstats fetchers tag -f BEA -d "10101 Ann" -S
dlstats fetchers tag -f BEA -d "10101 Ann" -S
dlstats fetchers tag -f Eurostat -d nama_10_a10 -S
dlstats fetchers tag -f OECD -d MEI -S
"""
ctx = client.Context(**kwargs)
if ctx.silent or click.confirm('Do you want to continue?', abort=True):
db = ctx.mongo_database()
if fetcher:
fetchers = [fetcher]
else:
fetchers = FETCHERS.keys()
for provider in fetchers:
provider_name = provider
_update_tags(ctx, db, provider_name, dataset, max_bulk, update_only, dry_mode, async_mode)
def cmd_providers(fetcher=None, **kwargs):
"""Create or Update fetcher Providers"""
ctx = client.Context(**kwargs)
ctx.log_ok("Create/Update Provider for %s fetcher:" % fetcher)
if ctx.silent or click.confirm('Do you want to continue?', abort=True):
_fetchers = []
if fetcher:
_fetchers = [fetcher]
else:
_fetchers = list(FETCHERS.keys())
for _fetcher in _fetchers:
ctx.log("Run provider for [%s]" % _fetcher)
f = FETCHERS[_fetcher](db=ctx.mongo_database())
result = f.provider.update_database()
if result:
ctx.log_ok("Provider [%s] updated." % _fetcher)
else:
ctx.log_error("Provider [%s] update ERROR." % _fetcher)
def cmd_providers(fetcher=None, **kwargs):
"""Create or Update fetcher Providers"""
ctx = client.Context(**kwargs)
ctx.log_ok("Create/Update Provider for %s fetcher:" % fetcher)
if ctx.silent or click.confirm('Do you want to continue?', abort=True):
_fetchers = []
if fetcher:
_fetchers = [fetcher]
else:
_fetchers = list(FETCHERS.keys())
for _fetcher in _fetchers:
ctx.log("Run provider for [%s]" % _fetcher)
f = FETCHERS[_fetcher](db=ctx.mongo_database())
result = f.provider.update_database()
if result:
ctx.log_ok("Provider [%s] updated." % _fetcher)
else:
ctx.log_error("Provider [%s] update ERROR." % _fetcher)
def cmd_update_tags(fetcher=None,
dataset=None,
max_bulk=100,
update_only=False,
async_mode=None,
dry_mode=False,
**kwargs):
"""Create or Update field tags"""
"""
Examples:
dlstats fetchers tag -f BIS -d CNFS -S
dlstats fetchers tag -f BEA -d "10101 Ann" -S
dlstats fetchers tag -f BEA -d "10101 Ann" -S
dlstats fetchers tag -f Eurostat -d nama_10_a10 -S
dlstats fetchers tag -f OECD -d MEI -S
"""
ctx = client.Context(**kwargs)
if ctx.silent or click.confirm('Do you want to continue?', abort=True):
db = ctx.mongo_database()
if fetcher:
fetchers = [fetcher]
else:
fetchers = FETCHERS.keys()
for provider in fetchers:
provider_name = provider
_update_tags(ctx, db, provider_name, dataset, max_bulk,
update_only, dry_mode, async_mode)
def cmd_list():
"""Show fetchers list"""
print("----------------------------------------------------")
for key in FETCHERS.keys():
print(key)
print("----------------------------------------------------")
from operator import itemgetter
import click
from widukind_common import tags
from widukind_common.tasks import consolidate
from widukind_common import errors
from dlstats import constants
from dlstats.fetchers import FETCHERS
from dlstats import client
from dlstats.utils import last_error
async_frameworks = ["future"]#, "gevent", "mp", "tornado"]
opt_fetcher = click.option('--fetcher', '-f',
required=True, type=click.Choice(FETCHERS.keys()),
help='Fetcher choice')
opt_dataset = click.option('--dataset', '-d',
required=False,
help='Run selected dataset only')
opt_dataset_multiple = click.option('--dataset', '-d',
required=False, multiple=True,
help='Run selected dataset(s) only')
opt_fetcher_not_required = click.option('--fetcher', '-f',
type=click.Choice(FETCHERS.keys()),
help='Fetcher choice')
opt_async_mode = click.option('--async-mode',
def cmd_update_tags(fetcher=None, dataset=None, max_bulk=100,
update_only=False, async_mode=None,
dry_mode=False, **kwargs):
"""Create or Update field tags"""
"""
Examples:
dlstats fetchers tag -f BIS -d CNFS -S
dlstats fetchers tag -f BEA -d "10101 Ann" -S
dlstats fetchers tag -f BEA -d "10101 Ann" -S
dlstats fetchers tag -f Eurostat -d nama_10_a10 -S
dlstats fetchers tag -f OECD -d MEI -S
"""
ctx = client.Context(**kwargs)
if ctx.silent or click.confirm('Do you want to continue?', abort=True):
start = time.time()
db = ctx.mongo_database()
if fetcher:
fetchers = [fetcher]
else:
fetchers = FETCHERS.keys()
for provider in fetchers:
provider_name = provider
ctx.log("START update tags for [%s]" % provider_name)
"""
ctx.log("Update provider[%s] Categories tags..." % provider_name)
try:
result = tags.update_tags_categories(db,
provider_name=provider_name,
max_bulk=max_bulk,
update_only=update_only,
dry_mode=dry_mode)
ctx.log_ok("Update provider[%s] Categories tags Success. Docs Updated[%s]" % (provider_name, result["nModified"]))
except Exception as err:
ctx.log_error("Update Categories tags Fail - provider[%s] - [%s]" % (provider_name, str(err)))
"""
ctx.log("Update provider[%s] Datasets tags..." % provider_name)
try:
result = tags.update_tags_datasets(db,
provider_name=provider_name,
dataset_code=dataset,
max_bulk=max_bulk,
update_only=update_only,
dry_mode=dry_mode)
ctx.log_ok("Update provider[%s] Datasets tags Success. Docs Updated[%s]" % (provider_name, result["nModified"]))
except Exception as err:
ctx.log_error("Update Datasets tags Fail - provider[%s] - [%s]" % (provider_name, str(err)))
ctx.log("Update provider[%s] Series tags..." % provider_name)
try:
result = tags.update_tags_series(db,
provider_name=provider_name,
dataset_code=dataset,
max_bulk=max_bulk,
update_only=update_only,
async_mode=async_mode,
dry_mode=dry_mode)
if not async_mode:
ctx.log_ok("Update provider[%s] Series tags Success. Docs Updated[%s]" % (provider_name, result["nModified"]))
except Exception:
ctx.log_error("Update Series tags Fail - provider[%s]: %s" % (provider_name, last_error()))
end = time.time() - start
ctx.log("END update tags for [%s] - time[%.3f]" % (provider_name, end))
# -*- coding: utf-8 -*-
import click
from widukind_common.tasks import export_files
from dlstats import client
from dlstats.fetchers import FETCHERS
opt_provider = click.option('--provider',
'-p',
required=True,
type=click.Choice(FETCHERS.keys()),
help='Provider Name')
opt_dataset = click.option('--dataset',
'-d',
required=True,
help='Run selected dataset only')
opt_export_filepath = click.option(
'--filepath',
'-P',
type=click.Path(exists=False),
#required=True,
help='Export filepath')
@click.group()
def cli():
"""Export File commands."""
# -*- coding: utf-8 -*-
import click
from widukind_common.tasks import export_files
from dlstats import client
from dlstats.fetchers import FETCHERS
opt_provider = click.option('--provider', '-p',
required=True,
type=click.Choice(FETCHERS.keys()),
help='Provider Name')
opt_dataset = click.option('--dataset', '-d',
required=True,
help='Run selected dataset only')
opt_export_filepath = click.option('--filepath', '-P',
type=click.Path(exists=False),
#required=True,
help='Export filepath')
@click.group()
def cli():
"""Export File commands."""
pass
@cli.command('csvfile', context_settings=client.DLSTATS_SETTINGS)
@client.opt_verbose
@client.opt_silent
def options():
parser = argparse.ArgumentParser(
description='dlstats client',
formatter_class=argparse.RawTextHelpFormatter,
add_help=True)
parser.add_argument('-D', '--debug', action="store_true")
subparsers = parser.add_subparsers(title='subcommands',
description='valid subcommands',
help='additional help',
dest="subcommand")
#TODO: liste des tâches en cours et leur état ou existe déjà ?
tools_parser = subparsers.add_parser('tools', help="Utils commands")
"""
tools_parser.add_argument('--dry',
action="store_true",
help="Dry mode")
"""
fetchers_parser = subparsers.add_parser(
'fetchers',
help="Fetcher commands",
#aliases=['fetch'],
#dest="fetch_cmd"
)
fetchers_parser.add_argument(choices=FETCHERS.keys(),
dest='fetcher',
help="Choice fetcher.")
fetchers_parser.add_argument(choices=['run', 'datasets'],
dest='action',
help="Action.")
fetchers_parser.add_argument('--dataset',
dest='dataset',
help='Dataset choice')
"""
fetchers_parser.add_argument('--dry',
action="store_true",
help="Dry mode")
"""
"""
print(dir(fetcher_choice))
python -m dlstats.cli fetchers --dry BIS run
_StoreAction(option_strings=[], dest='fetcher', nargs=None, const=None, default=None, type=None, choices=dict_keys(['BIS']), help='Choice fetcher.', metavar=None)
"""
"""
parser.add_argument('--reset',
dest="reset",
action="store_true",
help="Reset fixtures")
parser.add_argument('-G', '--greenlets',
default=1,
dest="greenlets",
type=int)
parser.add_argument('-C', '--config',
dest='config_path',
default=os.environ.get('WIDUKIND_SETTINGS', default_config_path),
help='Config filepath. default[%(default)s]')
parser.add_argument('--log-config',
dest="log_config",
help='Log config from file')
"""
return dict(parser.parse_args()._get_kwargs())
def options():
parser = argparse.ArgumentParser(description='dlstats client',
formatter_class=argparse.RawTextHelpFormatter,
add_help=True)
parser.add_argument('-D', '--debug', action="store_true")
subparsers = parser.add_subparsers(title='subcommands',
description='valid subcommands',
help='additional help',
dest="subcommand")
#TODO: liste des tâches en cours et leur état ou existe déjà ?
tools_parser = subparsers.add_parser('tools',
help="Utils commands")
"""
tools_parser.add_argument('--dry',
action="store_true",
help="Dry mode")
"""
fetchers_parser = subparsers.add_parser('fetchers',
help="Fetcher commands",
#aliases=['fetch'],
#dest="fetch_cmd"
)
fetchers_parser.add_argument(choices=FETCHERS.keys(),
dest='fetcher',
help="Choice fetcher.")
fetchers_parser.add_argument(choices=['run', 'datasets'],
dest='action',
help="Action.")
fetchers_parser.add_argument('--dataset',
dest='dataset',
help='Dataset choice')
"""
fetchers_parser.add_argument('--dry',
action="store_true",
help="Dry mode")
"""
"""
print(dir(fetcher_choice))
python -m dlstats.cli fetchers --dry BIS run
_StoreAction(option_strings=[], dest='fetcher', nargs=None, const=None, default=None, type=None, choices=dict_keys(['BIS']), help='Choice fetcher.', metavar=None)
"""
"""
parser.add_argument('--reset',
dest="reset",
action="store_true",
help="Reset fixtures")
parser.add_argument('-G', '--greenlets',
default=1,
dest="greenlets",
type=int)
parser.add_argument('-C', '--config',
dest='config_path',
default=os.environ.get('WIDUKIND_SETTINGS', default_config_path),
help='Config filepath. default[%(default)s]')
parser.add_argument('--log-config',
dest="log_config",
help='Log config from file')
"""
return dict(parser.parse_args()._get_kwargs())
def cmd_list():
"""Show fetchers list"""
print("----------------------------------------------------")
for key in FETCHERS.keys():
print(key)
print("----------------------------------------------------")
