def get_config():
config = {}
config['input_ds'] = dataiku.Dataset(get_input_names_for_role('input_ds')[0])
config['output_ds'] = dataiku.Dataset(get_output_names_for_role('output_ds')[0])
for param in ['lat_column', 'lng_column', 'provider', 'cache_enabled', 'api_key', 'here_app_id', 'here_app_code', 'google_client', 'google_client_secret']:
config[param] = get_recipe_config().get(param, None)
config['batch_enabled'] = get_recipe_config().get('batch_enabled', False) \
and (config['provider'] == 'bing')
config['batch_size'] = get_recipe_config().get('batch_size_bing', 50)
config['features'] = []
prefix = get_recipe_config().get('column_prefix', '')
for feature in ['address', 'city', 'postal', 'state', 'country']:
if get_recipe_config().get(feature, False):
config['features'].append({'name': feature, 'column': prefix + feature})
if get_plugin_config().get('cache_location', 'original') == 'original':
config['cache_location'] = os.environ["DIP_HOME"] + '/caches/plugins/geocoder/reverse'
else:
config['cache_location'] = get_plugin_config().get('cache_location_custom', '')
config['cache_size'] = get_plugin_config().get('reverse_cache_size', 1000) * 1000
config['cache_eviction'] = get_plugin_config().get('reverse_cache_policy', 'least-recently-stored')
if len(config['features']) == 0:
raise AttributeError('Please select at least one feature to extract.')
if config['provider'] is None:
raise AttributeError('Please select a geocoding provider.')
return config
def get_neo4jhandle():
neo4jhandle = Neo4jHandle(
get_plugin_config().get("neo4jUri"),
get_plugin_config().get("neo4jUsername"),
get_plugin_config().get("neo4jPassword"),
)
neo4jhandle.check()
return neo4jhandle
def load_cache_config(config):
plugin_config = get_plugin_config()
recipe_config = get_recipe_config()
config.cache_location = utils.get_cache_location_from_configs(
cache_location=plugin_config.get("cache_location"),
default=plugin_config.get("cache_location_custom", "")
)
config.cache_size = plugin_config.get("cache_size", 1000) * 1000
config.cache_policy = plugin_config.get("cache_policy", "least-recently-stored")
config.cache_enabled = recipe_config.get("cache_enabled") and config.cache_location
def get_config():
config = {}
config['input_ds'] = dataiku.Dataset(get_input_names_for_role('input_ds')[0])
config['output_ds'] = dataiku.Dataset(get_output_names_for_role('output_ds')[0])
for param in ['address_column', 'cache_enabled', 'provider', 'api_key', 'here_app_id', 'here_app_code', 'google_client', 'google_client_secret']:
config[param] = get_recipe_config().get(param, None)
config['batch_enabled'] = get_recipe_config().get('batch_enabled', False) \
and (config['provider'] == 'bing' or config['provider'] == 'mapquest' or config['provider'] == 'uscensus')
config['batch_size'] = {
'bing': get_recipe_config().get('batch_size_bing', 50),
'mapquest': 100,
'uscensus': get_recipe_config().get('batch_size_uscensus', 1000)
}.get(config['provider'], 0)
config['batch_timeout'] = {
'bing': 10,
'mapquest': 30,
'uscensus': 1800
}.get(config['provider'], 0)
if get_plugin_config().get('cache_location', 'original') == 'original':
config['cache_location'] = os.environ["DIP_HOME"] + '/caches/plugins/geocoder/forward'
else:
config['cache_location'] = get_plugin_config().get('cache_location_custom', '')
config['cache_size'] = get_plugin_config().get('forward_cache_size', 1000) * 1000
config['cache_eviction'] = get_plugin_config().get('forward_cache_policy', 'least-recently-stored')
prefix = get_recipe_config().get('column_prefix', '')
for column_name in ['latitude', 'longitude']:
config[column_name] = prefix + column_name
if config['provider'] is None:
raise AttributeError('Please select a geocoding provider.')
return config
get_recipe_config,
get_plugin_config,
)
from meaningcloud_common import setRequestSource, isBlockingErrorType
# ==============================================================================
# PLUGIN + RECIPE SETTINGS
# ==============================================================================
input_name = get_input_names_for_role("input_dataset")[0]
output_name = get_output_names_for_role("output_dataset")[0]
input_dataset = dataiku.Dataset(input_name)
output_dataset = dataiku.Dataset(output_name)
meaningcloud_connection = get_plugin_config().get("meaningcloud_connection")
license_key = meaningcloud_connection.get("license_key", None)
server = meaningcloud_connection.get("meaningcloud_server",
"https://api.meaningcloud.com")
sentences = int(get_recipe_config().get("sentences", 5))
text_column = get_recipe_config().get("column_name", None)
# ==============================================================================
# AUXILIARY FUNCTIONS
# ==============================================================================
# Analyzes the text passed as a parameter
def analyzeText(text):