def __init__(self, url, doc, name):
global inflector
self.url = url
if doc is not None:
self.doc = markdown.markdown(doc.replace(" ", ""))
else:
self.doc = ""
inflector = Inflector(English)
self.name = inflector.humanize(word=name.replace("-", " "))
def process_spanish_owned():
from inflector import Inflector, Spanish
inflector = Inflector(Spanish)
from nltk.stem import SnowballStemmer
stemmer = SnowballStemmer("spanish")
file_valid = open('valid_words.txt', "r")
lines = file_valid.readlines()
valid_words = lines[0].split(' ')
print len(valid_words)
file_valid.close()
#valid_words = set(valid_words)
owned_words = ['cúster', 'custer', 'cústers', 'custers', 'combi', 'combis', 'susana', 'villaran', 'villarán', 'castañeda']
file = open("raw_words.txt", 'r')
fileout = open("spanish_words_owned.txt", 'w')
fout_sing = open("spanish_words_sing.txt", 'w')
fout_stem = open("spanish_words_stem.txt", 'w')
nline = 0
for line in file:
nline += 1
words = line.split(' ')
processed = []
ini_line = True
for word in words:
if (word != '') & (word != '\n') & (word != 'servicio') & (word != 'servicio\n'):
word = word.replace('\n', '')
if (word in valid_words) | (word in owned_words):
processed.append(word)
if word != 'bus':
word_singular = inflector.singularize(word)
#word_singular = word_singular.replace(u'\xF3'.encode('utf-8'), 'o')
else:
word_singular = word
word_stemmed = stemmer.stem(word.decode('utf-8')).encode('utf-8')
if ini_line:
fileout.write(word)
fout_sing.write(word_singular)
fout_stem.write(word_stemmed)
ini_line = False
else:
fileout.write(' ' + word)
fout_sing.write(' ' + word_singular)
fout_stem.write(' ' + word_stemmed)
print nline, word, word_singular, word_stemmed
fileout.write('\n')
fout_sing.write('\n')
fout_stem.write('\n')
file.close()
fileout.close()
fout_sing.close()
fout_stem.close()
def get_date_type(name1):
""" Inherit class must be in TaskBase{Day,Week,Month,Range} style. """
assert isinstance(name1, (str, unicode))
str1 = Inflector().underscore(name1).split("_")[-1].lower()
assert str1 in luiti_config.DateTypes, [str1, luiti_config.DateTypes]
return str1
class EnglishInflectorTestCase(unittest.TestCase):
singular_to_plural = {
"search" : "searches",
"switch" : "switches",
"fix" : "fixes",
"box" : "boxes",
"process" : "processes",
"address" : "addresses",
"case" : "cases",
"stack" : "stacks",
"wish" : "wishes",
"fish" : "fish",
"category" : "categories",
"query" : "queries",
"ability" : "abilities",
"agency" : "agencies",
"movie" : "movies",
"archive" : "archives",
"index" : "indices",
"wife" : "wives",
"safe" : "saves",
"half" : "halves",
"move" : "moves",
"salesperson" : "salespeople",
"person" : "people",
"spokesman" : "spokesmen",
"man" : "men",
"woman" : "women",
"basis" : "bases",
"diagnosis" : "diagnoses",
"datum" : "data",
"medium" : "media",
"analysis" : "analyses",
"node_child" : "node_children",
"child" : "children",
"experience" : "experiences",
"day" : "days",
"comment" : "comments",
"foobar" : "foobars",
"newsletter" : "newsletters",
"old_news" : "old_news",
"news" : "news",
"series" : "series",
"species" : "species",
"quiz" : "quizzes",
"perspective" : "perspectives",
"ox" : "oxen",
"photo" : "photos",
"buffalo" : "buffaloes",
"tomato" : "tomatoes",
"dwarf" : "dwarves",
"elf" : "elves",
"information" : "information",
"equipment" : "equipment",
"bus" : "buses",
"status" : "statuses",
"mouse" : "mice",
"louse" : "lice",
"house" : "houses",
"octopus" : "octopi",
"virus" : "viri",
"alias" : "aliases",
"portfolio" : "portfolios",
"vertex" : "vertices",
"matrix" : "matrices",
"axis" : "axes",
"testis" : "testes",
"crisis" : "crises",
"rice" : "rice",
"shoe" : "shoes",
"horse" : "horses",
"prize" : "prizes",
"edge" : "edges"
}
def setUp(self):
self.inflector = Inflector(English)
def tearDown(self):
self.inflector = None
def test_pluralize(self) :
for singular in self.singular_to_plural.keys() :
assert self.inflector.pluralize(singular) == self.singular_to_plural[singular], \
'English Inlector pluralize(%s) should produce "%s" and NOT "%s"' % (singular, self.singular_to_plural[singular], self.inflector.pluralize(singular))
def test_singularize(self) :
for singular in self.singular_to_plural.keys() :
assert self.inflector.singularize(self.singular_to_plural[singular]) == singular, \
'English Inlector singularize(%s) should produce "%s" and NOT "%s"' % (self.singular_to_plural[singular], singular, self.inflector.singularize(self.singular_to_plural[singular]))
def wizard(target_model,
serializer=None,
icon_class=None,
btn_class=None,
text=None,
meta_type='custom',
**kwargs):
if serializer is None and target_model is not None:
serializer = target_model
target_model = None
assert serializer is not None, "You need to pass a serializer to the wizard decorator"
assert meta_type in ['custom', 'list']
inflector_language = import_string(settings.INFLECTOR_LANGUAGE)
inflector = Inflector(inflector_language)
_kwargs = {
'type': 'wizard',
'params': {},
}
_kwargs.update(kwargs)
kwargs = _kwargs
kwargs['params']['fieldsets'] = kwargs.pop('fieldsets', None)
serializer_instance = serializer()
needs = []
fields = []
Adapter = import_string(settings.METADATA_ADAPTER)
for field_name, field in serializer_instance.fields.items():
if isinstance(field, PrimaryKeyRelatedField):
model = field.queryset.model
needs.append({
'app':
model._meta.app_label,
'singular':
model._meta.model_name.lower(),
'plural':
inflector.pluralize(model._meta.model_name.lower()),
})
fields.append(
Adapter.adapt_field(get_field_dict(field_name, serializer)))
kwargs['params']['needs'] = needs
kwargs['params']['fields'] = fields
kwargs['languages'] = get_languages()
def decorator(func):
def wizard_func(self, request, *args, **kwargs):
Serializer = serializer
serializer_instance = Serializer(data=request.data)
if not serializer_instance.is_valid():
return Response(serializer_instance.errors, status=400)
request.validated_data = serializer_instance.validated_data
return func(self, request, *args, **kwargs)
wizard_func.__name__ = func.__name__
if meta_type == 'custom':
detail = True
else:
detail = False
wizard_func = action(methods=[kwargs.pop('method', 'post')],
detail=detail,
**kwargs)(wizard_func) # NoQA
wizard_func.__name__ = func.__name__
wizard_func.action_type = meta_type
wizard_func.wizard = True
wizard_func.action_kwargs = action_kwargs(icon_class, btn_class, text,
wizard_func, kwargs)
wizard_func.kwargs = {}
if target_model is not None:
wizard_func.action_kwargs['params']['model'] = '{}/{}/{}'.format(
target_model._meta.app_label.lower().replace('_', '-'),
inflector.pluralize(target_model._meta.model_name.lower()),
wizard_func.__name__)
wizard_func.serializer = serializer
return Adapter.adapt_wizard(wizard_func)
return decorator
def get_question_type(q_word, question):
q_word = q_word.lower()
question = question.lower()
inf = Inflector()
question = inf.singularize(question)
if q_word == 'what' or q_word == 'which':
if 'what country' in question or \
'what state' in question or \
'what continental' in question or \
'what place' in question or \
'what city' in question or \
'what province' in question or \
'what river' in question or \
'what region' in question or \
'what area' in question or \
'what nationality' in question or \
'what town' in question or \
'what borough' in question or \
'what location' in question:
return set(['LOCATION'])
if 'what year' in question or \
'what month' in question or \
'what day' in question or \
'what date' in question:
return set(['DATE'])
if 'what percentage' in question or 'what percent' in question:
return set(['PERCENT'])
if 'what company' in question or \
'what group' in question or \
'what organization' in question or \
'what university' in question or \
'what school' in question or \
'what team' in question or \
'what program' in question or \
'what party' in question:
return set(['ORGANIZATION'])
if 'what artist' in question or \
'what actor' in question or \
'what actress' in question or \
'what doctor' in question or \
'what president' in question or \
'what person' in question:
return set(['PERSON'])
if 'which country' in question or \
'which state' in question or \
'which continental' in question or \
'which place' in question or \
'which city' in question or \
'which province' in question or \
'which river' in question or \
'which region' in question or \
'which area' in question or \
'which nationality' in question or \
'which town' in question or \
'which borough' in question or \
'which location' in question:
return set(['LOCATION'])
if 'which year' in question or \
'which month' in question or \
'which day' in question or \
'which date' in question:
return set(['DATE'])
if 'which percentage' in question or 'which percent' in question:
return set(['PERCENT'])
if 'which company' in question or \
'which group' in question or \
'which organization' in question or \
'which university' in question or \
'which school' in question or \
'which team' in question or \
'which program' in question or \
'which party' in question:
return set(['ORGANIZATION'])
if 'which artist' in question or \
'which actor' in question or \
'which actress' in question or \
'which doctor' in question or \
'which president' in question or \
'which person' in question:
return set(['PERSON'])
elif q_word == 'how':
if 'how much' in question:
return set(['MONEY', 'NUMBER'])
if 'how long' in question or 'how old' in question:
return set(['TIME', 'DURATION'])
if 'how many' in question or 'how far' in question:
return set(['NUMBER'])
elif q_word == 'where':
return set(['LOCATION', 'ORGANIZATION'])
elif q_word == 'when':
return set(['DATE', 'TIME', 'DURATION'])
elif q_word == 'who':
return set(['PERSON'])
return set(['O'])
def __new__(cls, name, bases, attrs):
new_class = super(EndpointMetaClass,
cls).__new__(cls, name, bases, attrs)
inflector = None
processed = []
black_list = dir(BaseEndpoint)
model = getattr(new_class, 'model', None)
for base in reversed(new_class.__mro__):
for key, value in list(base.__dict__.items()):
if key not in black_list and key not in processed and hasattr(
value, 'wizard') and value.wizard:
if getattr(value, 'action_kwargs', {}).get(
'params', {}).get('model', None) is None:
if model is not None:
if inflector is None:
inflector_language = import_string(
settings.INFLECTOR_LANGUAGE)
inflector = Inflector(inflector_language)
getattr(new_class, key).action_kwargs['params'][
'model'] = '{}/{}/{}'.format(
model._meta.app_label.lower().replace(
'_', '-'),
inflector.pluralize(
model._meta.model_name.lower()),
value.__name__)
processed.append(key)
if getattr(value, 'action_kwargs', {}).get('params', {}).get('fieldsets', None) is None \
and getattr(new_class, 'model', None) is not None and hasattr(value, 'serializer'):
fieldsets_path = os.path.join(
django_settings.BASE_DIR,
new_class._fieldsets_location,
new_class.__module__.rsplit('.', 1)[0],
'fieldsets.json')
try:
with open(fieldsets_path, 'r') as f:
fieldsets = json.load(f)
value.action_kwargs['params']['fieldsets'] = \
fieldsets['{}_{}'.format(new_class.model.__name__, key)]
except FileNotFoundError:
pass
except KeyError:
pass
if getattr(value, 'action_kwargs', {}).get(
'params', {}).get('fieldsets', None) is None:
value.action_kwargs = getattr(
value, 'action_kwargs', {})
value.action_kwargs[
'params'] = value.action_kwargs.get(
'params', {})
value.action_kwargs['params']['fieldsets'] = [{
'name':
field
} for field in value.serializer().fields.keys()]
if new_class.fieldsets is None and new_class.model is not None:
fieldsets_path = os.path.join(
django_settings.BASE_DIR,
new_class.__module__.rsplit('.', 1)[0], 'fieldsets.json')
try:
with open(fieldsets_path, 'r') as f:
fieldsets = json.load(f)
new_class.fieldsets = fieldsets[new_class.model.__name__]
except FileNotFoundError:
pass
except KeyError:
pass
return new_class
class SpanishInflectorTestCase(unittest.TestCase):
singular_to_plural = {
"álbum": "álbumes",
"almacén": "almacenes",
"androide": "androides",
"antifaz": "antifaces",
"árbol": "árboles",
"atlas": "atlas",
"autobús": "autobuses",
"base": "bases",
"bebé": "bebés",
"camión": "camiones",
"casa": "casas",
"ceutí": "ceutíes",
"chimpancé": "chimpancés",
"clan": "clanes",
"compás": "compases",
"convoy": "convoyes",
"coxis": "coxis",
"crisis": "crisis",
"déficit": "déficits",
"eje": "ejes",
"espíritu": "espíritus",
"flash": "flashes",
"frac": "fracs",
"gafas": "gafas",
"hipótesis": "hipótesis",
"inglés": "ingleses",
"lápiz": "lápices",
"luz": "luces",
"montaje": "montajes",
"no": "noes",
"otitis": "otitis",
"padre": "padres",
"país": "países",
"papá": "papás",
"parking": "parkings",
"portaequipaje": "portaequipajes",
"radiocasete": "radiocasetes",
"show": "shows",
"si": "sis",
"sí": "síes",
"tabú": "tabúes",
"tamiz": "tamices",
"tanque": "tanques",
"taxi": "taxis",
"tijeras": "tijeras",
"tren": "trenes",
"virus": "virus",
}
def setUp(self):
self.inflector = Inflector(Spanish)
def tearDown(self):
self.inflector = None
def test_pluralize(self):
for singular, plural in self.singular_to_plural.iteritems():
inflector_pluralize = self.inflector.pluralize(singular)
assert inflector_pluralize == plural, \
'Spanish Inflector pluralize(%s) should produce "%s" and NOT "%s"' % (
singular, plural, inflector_pluralize)
def test_singularize(self):
for singular, plural in self.singular_to_plural.iteritems():
inflector_singularize = self.inflector.singularize(plural)
assert inflector_singularize == singular, \
'Spanish Inflector singularize(%s) should produce "%s" and NOT "%s"' % (
plural, singular, inflector_singularize)
def wizard(target_model,
serializer,
icon_class=None,
btn_class=None,
text=None,
**kwargs):
inflector_language = import_string(settings.INFLECTOR_LANGUAGE)
inflector = Inflector(inflector_language)
_kwargs = {
'type': 'wizard',
'params': {},
}
_kwargs.update(kwargs)
kwargs = _kwargs
kwargs['params']['fieldsets'] = kwargs.pop('fieldsets', None)
serializer_instance = serializer()
needs = []
fields = []
Adapter = import_string(settings.METADATA_ADAPTER)
for field in serializer.Meta.fields:
field_instance = serializer_instance.fields[field]
if isinstance(field_instance, PrimaryKeyRelatedField):
model = field_instance.queryset.model
needs.append({
'app':
model._meta.app_label,
'singular':
model._meta.model_name.lower(),
'plural':
inflector.pluralize(model._meta.model_name.lower()),
})
fields.append(Adapter.adapt_field(get_field_dict(field, serializer)))
kwargs['params']['needs'] = needs
kwargs['params']['fields'] = fields
kwargs['languages'] = get_languages()
def decorator(func):
# cls = getattr(inspect.getmodule(func),
# func.__qualname__.split('.<locals>', 1)[0].rsplit('.', 1)[0])
func.bind_to_methods = [
kwargs.pop('method', 'POST'),
]
func.detail = True
func.wizard = True
func.action_type = 'custom'
func.action_kwargs = action_kwargs(icon_class, btn_class, text, func,
kwargs)
func.kwargs = {}
if target_model is not None:
func.action_kwargs['params']['model'] = '{}/{}/{}'.format(
target_model._meta.app_label.lower(),
inflector.pluralize(target_model._meta.model_name.lower()),
func.__name__)
func.serializer = serializer
return Adapter.adapt_wizard(func)
return decorator
# Copyright 2019, GeoSolutions SAS.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
#
#########################################################################
"""Script defined to create helper functions for graphql schema."""
from django.conf import settings
from django.utils.text import slugify
from inflector import Inflector, English
from graphql_relay.node.node import from_global_id
INFLECTOR = Inflector(English)
def is_RUP(user):
if user and user.is_authenticated:
_memberships = user.memberships
if _memberships:
for _m in _memberships.all():
if _m.type.code == settings.RUP_CODE:
return True
return False
def get_object(object_name, relayId, otherwise=None):
try:
return object_name.objects.get(pk=from_global_id(relayId)[1])
class Endpoint(with_metaclass(EndpointMetaClass, object)):
base_serializer = import_string(settings.BASE_SERIALIZER)
base_viewset = import_string(settings.BASE_VIEWSET)
base_readonly_viewset = import_string(settings.BASE_READONLY_VIEWSET)
model = None
fields = None
exclude_fields = ()
serializer = None
fieldsets = None
list_display = None
list_editable = None
extra_fields = None
permission_classes = None
filter_fields = None
search_fields = None
ordering_fields = None
page_size = None
viewset = None
read_only = False
include_str = True
list_me = True
save_twice = False
sortable_by = None
custom_actions = None
bulk_actions = None
inflector_language = import_string(settings.INFLECTOR_LANGUAGE)
_translated_fields = None
_translated_field_names = None
_default_language_field_names = None
def __init__(self, model=None, **kwargs):
self.inflector = Inflector(self.inflector_language)
if model is not None:
self.model = model
arg_names = ('fields', 'serializer', 'permission_classes',
'filter_fields', 'search_fields', 'viewset', 'read_only',
'include_str', 'ordering_fields', 'page_size',
'base_viewset', 'fields_annotation', 'fieldsets',
'base_serializer', 'list_me')
for arg_name in arg_names:
setattr(self, arg_name,
kwargs.pop(arg_name, getattr(self, arg_name, None)))
if len(kwargs.keys()) > 0:
raise Exception(
'{} got an unexpected keyword argument: "{}"'.format(
self.__class__.__name__,
list(kwargs.keys())[0]))
if self.serializer is not None:
assert self.fields is None, 'You cannot specify both fields and serializer'
else:
assert self.viewset is not None or self.model is not None, \
'You need to specify at least a model or a viewset'
self.get_serializer()
if self.viewset is not None:
for attr in ('permission_classes', 'filter_fields',
'search_fields', 'ordering_fields', 'page_size'):
assert getattr(self, attr, None) is None, \
'You cannot specify both {} and viewset'.format(attr)
else:
self.get_viewset()
if self.model is None:
self.model = self.get_serializer().Meta.model
def get_languages(self):
return get_languages()
@property
def singular_model_name(self):
return self.model._meta.model_name.lower()
@property
def model_name(self):
return self.inflector.pluralize(self.singular_model_name)
@property
def application_name(self):
return self.model._meta.app_label.lower()
def get_exclude_fields(self):
return self.exclude_fields
def get_fields_for_serializer(self):
if self.fields is None:
if self.serializer is not None:
return self.serializer.Meta.fields
self.fields = tuple([
f for f in get_all_field_names(self.model)
if f not in self.default_language_field_names
and f not in self.get_exclude_fields()
])
if self.extra_fields is not None:
self.fields += tuple(self.extra_fields)
if self.include_str:
self.fields += ('__str__', )
return self.fields
def get_serializer(self, data=None):
if self.serializer is None:
if self.viewset is None:
self.serializer = serializer_factory(self)
else:
self.serializer = self.viewset.serializer_class
if data is None:
return self.serializer
return self.serializer(data)
def get_base_viewset(self):
if not self.read_only:
return self.base_viewset
if '{}.{}'.format(self.base_readonly_viewset.__module__, self.base_readonly_viewset.__name__) \
== settings.BASE_READONLY_VIEWSET and \
'{}.{}'.format(self.base_viewset.__module__, self.base_viewset.__name__) \
!= settings.BASE_VIEWSET:
return self.base_viewset
return self.base_readonly_viewset
def get_viewset(self):
if self.viewset is None:
self.viewset = viewset_factory(self)
return self.viewset
def get_url(self):
return '{}/{}'.format(self.application_name.replace('_', '-'),
self.model_name.replace('_', '-'))
def _get_field_dict(self, field):
return get_field_dict(field, self.get_serializer(),
self.get_translated_fields(),
self.fields_annotation, self.model)
def get_fields(self):
return [
self._get_field_dict(field)
for field in self.get_fields_for_serializer()
]
def get_fieldsets(self):
if self.fieldsets is not None:
return [{
'key': field
} if not isinstance(field, dict) else field
for field in self.fieldsets]
return [{
'key': field
} for field in self.get_fields_for_serializer()
if field != 'id' and field != '__str__'
and field not in self.translated_field_names
and self._get_field_dict(field).get('type', '')[:6] != 'tomany'
]
def get_list_display(self):
if self.list_display is None:
if '__str__' in self.get_fields_for_serializer():
return [
'__str__',
]
return [self.get_fields()[0]['key']]
return self.list_display
def _get_endpoint_list(self, name):
value = getattr(self, name, None)
if value is None:
return []
return value
def get_filter_fields(self):
fields = self._get_endpoint_list('filter_fields')
return fields
@property
def search_enabled(self):
fields = self._get_endpoint_list('search_fields')
return len(fields) > 0
def get_ordering_fields(self):
fields = self._get_endpoint_list('ordering_fields')
return fields
def get_needs(self):
related_models = [
f.related_model if f.related_model else
f.model if f.model and f.model != self.model else None
for f in self.model._meta.get_fields()
if f.is_relation and f.name in self.get_fields_for_serializer()
]
return [{
'app':
model._meta.app_label,
'singular':
model._meta.model_name.lower(),
'plural':
self.inflector.pluralize(model._meta.model_name.lower()),
} for model in related_models if model is not None]
def get_list_editable(self):
if self.list_editable is None:
return []
return self.list_editable
def get_sortable_by(self):
return self.sortable_by
def get_translated_fields(self):
if self._translated_fields is None:
models = translator.get_registered_models()
if self.model in models:
options = translator.get_options_for_model(self.model)
rv = [field for field in options.fields]
self._translated_fields = rv
else:
self._translated_fields = []
return self._translated_fields
@property
def translated_field_names(self):
if self._translated_field_names is None:
rv = []
for field in self.get_translated_fields():
for language in self.get_languages():
l = language.replace('-', '_')
rv.append('{}_{}'.format(field, l))
self._translated_field_names = rv
return self._translated_field_names
@property
def default_language_field_names(self):
from django.conf import settings as django_settings
if self._default_language_field_names is None:
l = django_settings.LANGUAGE_CODE.replace('-', '_')
rv = []
for field in self.get_translated_fields():
rv.append('{}_{}'.format(field, l))
self._default_language_field_names = rv
return self._default_language_field_names
def get_custom_actions(self):
rv = []
viewset = self.get_viewset()
for action_name in dir(viewset):
action = getattr(viewset, action_name)
if getattr(action, 'action_type', None) == 'custom':
custom_action = {
'url':
reverse('{}-{}'.format(
self.get_url(),
action.__name__.lower().replace('_', '-')),
kwargs={'pk': ':id'}),
'verb':
action.bind_to_methods[0],
}
custom_action.update(action.action_kwargs)
rv.append(custom_action)
if self.custom_actions is not None:
rv += self.custom_actions
return rv
def get_bulk_actions(self):
rv = []
viewset = self.get_viewset()
for action_name in dir(viewset):
action = getattr(viewset, action_name)
if getattr(action, 'action_type', None) == 'bulk':
bulk_action = {
'url':
reverse('{}-{}'.format(self.get_url(),
action.__name__.lower())),
'verb':
action.bind_to_methods[0],
}
bulk_action.update(action.action_kwargs)
rv.append(bulk_action)
if self.bulk_actions is not None:
rv += []
return rv
# 2つ目の質問を書き出す
df_second_question = df_input.loc[:,
df_input.columns.str.
contains(SECOND_QUESTION_TEXT)]
df_second_question.to_csv(os.path.join('output', 'second_question.csv'),
index=None)
print('2つ目の質問をcsvで保存しました。')
# Text Analytics APIのインスタンス化
text_analytics_client = TextAnalyticsClient(
endpoint=TEXT_ANALYTICS_ENDPOINT,
credential=TextAnalyticsApiKeyCredential(
TEXT_ANALYTICS_SUBSCRIPTION_KEY))
# WordCloud用の定義
# 英語の複数形を単数形に変換するために必要
infr = Inflector()
# WordCloudのスタイルを定義
wdcl = WordCloud(background_color='white',
width=700,
height=700,
font_path=FONT_PATH)
"""
========= 1つ目の質問 =========
"""
# ========= Sentiment Analytics =========
# null以外のセルのみをリストに変換、そのときのrow, columnも辞書のリストにして保持
first_payload_documents, first_original_position = make_payload_list(
df_first_question)
assert len(first_payload_documents) == len(first_original_position)
if project_dir:
path = join(project_dir, path)
content = write_content_to_file(
a_task_template(Inflector().classify(task_name)),
path,
)
return content
""" 1. Project """
a_project_readme = lambda project_name: u"""
%s
=======================
TODO ...
""".strip() % (Inflector().titleize(project_name), )
a_project_setup = lambda project_name: u"""
# -*-coding:utf-8-*-
from setuptools import setup
setup(
name="%s",
version="0.0.1",
packages=[
"%s",
"%s/luiti_tasks", ],
zip_safe=False,
)
""".strip() % (
def get_time_task(name1):
""" return e.g. TaskDay """
type2 = luiti_config.get_date_type(name1)
return "Task" + Inflector().camelize(type2)
class SpanishInflectorTestCase(unittest.TestCase):
singular_to_plural = {
"álbum": "álbumes",
"almacén": "almacenes",
"androide": "androides",
"antifaz": "antifaces",
"árbol": "árboles",
"atlas": "atlas",
"autobús": "autobuses",
"base": "bases",
"bebé": "bebés",
"camión": "camiones",
"casa": "casas",
"ceutí": "ceutíes",
"chimpancé": "chimpancés",
"clan": "clanes",
"compás": "compases",
"convoy": "convoyes",
"coxis": "coxis",
"crisis": "crisis",
"déficit": "déficits",
"eje": "ejes",
"espíritu": "espíritus",
"flash": "flashes",
"frac": "fracs",
"gafas": "gafas",
"hipótesis": "hipótesis",
"inglés": "ingleses",
"lápiz": "lápices",
"luz": "luces",
"montaje": "montajes",
"no": "noes",
"otitis": "otitis",
"padre": "padres",
"país": "países",
"papá": "papás",
"parking": "parkings",
"portaequipaje": "portaequipajes",
"radiocasete": "radiocasetes",
"show": "shows",
"si": "sis",
"sí": "síes",
"tabú": "tabúes",
"tamiz": "tamices",
"tanque": "tanques",
"taxi": "taxis",
"tijeras": "tijeras",
"tren": "trenes",
"virus": "virus",
}
def setUp(self):
self.inflector = Inflector(Spanish)
def tearDown(self):
self.inflector = None
def test_pluralize(self):
for singular, plural in self.singular_to_plural.iteritems():
inflector_pluralize = self.inflector.pluralize(singular)
assert inflector_pluralize == plural, \
'Spanish Inflector pluralize(%s) should produce "%s" and NOT "%s"' % (
singular, plural, inflector_pluralize)
def test_singularize(self):
for singular, plural in self.singular_to_plural.iteritems():
inflector_singularize = self.inflector.singularize(plural)
assert inflector_singularize == singular, \
'Spanish Inflector singularize(%s) should produce "%s" and NOT "%s"' % (
plural, singular, inflector_singularize)
def setUp(self):
self.inflector = Inflector(Spanish)
class Spider(scrapy.Spider):
inflector = Inflector(English)
def __init__(self, *args, **kwargs):
# Add parameters for feed storage in Chicago time
tz = timezone('America/Chicago')
now = tz.localize(datetime.now())
self.year = now.year
self.month = now.strftime('%m')
self.day = now.strftime('%d')
self.hour_min = now.strftime('%H%M')
super().__init__(*args, **kwargs)
def _generate_id(self, data):
"""
Calulate ID. ID must be unique within the data source being scraped.
"""
name = self.inflector.underscore(data['name']).strip('_')
id = data.get('id', 'x').replace('/', '-')
# Try to get the start date and time in YYYYmmddHHMM
# Replace missing start date or times with 0's
try:
start_date_str = data['start']['date'].strftime('%Y%m%d')
except (KeyError, TypeError):
start_date_str = '00000000'
try:
start_time_str = data['start']['time'].strftime('%H%M')
except (KeyError, TypeError, AttributeError):
start_time_str = '0000'
start_str = '{0}{1}'.format(start_date_str, start_time_str)
parts = [self.name, start_str, id, name]
return '/'.join(parts)
def _generate_status(self, data, text):
"""
Generates one of the following statuses:
* cancelled: the word 'cancelled' or 'rescheduled' is `text`
* tentative: event both does not have an agenda and the event is > 7 days away
* confirmed: either an agenda is posted or the event will happen in <= 7 days
* passed: event happened in the past
"""
if ('cancel' in text.lower()) or ('rescheduled' in text.lower()):
return CANCELED
try:
start = data['start']['date']
# check if event is in the past
if start < date.today():
return PASSED
# check if the event is within 7 days
elif (start - date.today()).days <= 7:
return CONFIRMED
except (KeyError, TypeError):
pass
# look for an agenda
documents = data.get('documents', [])
for doc in documents:
if 'agenda' in doc.get('note', '').lower():
return CONFIRMED
return TENTATIVE
def _naive_datetime_to_tz(self, datetime_object, source_tz='America/Chicago'):
"""
Converts a naive datetime (one without timezone information) by
interpreting it using the source_tz.
"""
tz = timezone(source_tz)
return tz.localize(datetime_object)
class Spider(scrapy.Spider):
inflector = Inflector(English)
def __init__(self, *args, **kwargs):
# Add parameters for feed storage in Chicago time
tz = timezone('America/Chicago')
now = tz.localize(datetime.now())
self.year = now.year
self.month = now.strftime('%m')
self.day = now.strftime('%d')
self.hour_min = now.strftime('%H%M')
super().__init__(*args, **kwargs)
def _clean_name(self, name):
"""Remove canceled strings from name"""
return re.sub(
r'([\s:-]{1,3})?(cancel\w+|rescheduled)([\s:-]{1,3})?',
'',
name,
flags=re.IGNORECASE,
)
def _generate_id(self, data):
"""
Calulate ID. ID must be unique within the data source being scraped.
"""
name = self.inflector.underscore(data['name']).strip('_')
id = data.get('id', 'x').replace('/', '-')
# Try to get the start date and time in YYYYmmddHHMM
# Replace missing start date or times with 0's
try:
start_date_str = data['start']['date'].strftime('%Y%m%d')
except (KeyError, TypeError):
start_date_str = '00000000'
try:
start_time_str = data['start']['time'].strftime('%H%M')
except (KeyError, TypeError, AttributeError):
start_time_str = '0000'
start_str = '{0}{1}'.format(start_date_str, start_time_str)
parts = [self.name, start_str, id, name]
return '/'.join(parts)
def _generate_status(self, data, text):
"""
Generates one of the allowed statuses from constants based on
the name and time of the meeting
"""
if ('cancel' in text.lower()) or ('rescheduled' in text.lower()):
return CANCELED
try:
start = data['start']['date']
# check if event is in the past
if start < date.today():
return PASSED
# check if the event is within 7 days
elif (start - date.today()).days <= 7:
return CONFIRMED
except (KeyError, TypeError):
pass
# look for an agenda
documents = data.get('documents', [])
for doc in documents:
if 'agenda' in doc.get('note', '').lower():
return CONFIRMED
return TENTATIVE
def data_name(self):
return Inflector().underscore(self.__class__.__name__)
def setUp(self):
self.inflector = Inflector(Spanish)
def _add_model(self, table_schema):
return type(Inflector().classify(table_schema['table_name']), (Base, ),
self._gen_model_dict(table_schema))
from repo import Repo
import sys
import os
import types
sys.path.insert(0, os.path.dirname(os.path.abspath(os.path.dirname(__file__))))
from inflector import Inflector, English
inflector = Inflector(English)
def does_not_mutate(func):
"""Prevents methods from mutating the receiver"""
def wrapper(self, *args, **kwargs):
new = self.copy()
return func(new, *args, **kwargs)
wrapper.__name__ = func.__name__
wrapper.__doc__ = func.__doc__
return wrapper
class Query(object):
"""
Generic Query object used for searching a database for records, and
constructing records using the returned values from the database.
"""
def __init__(self, model, record=None):
"""
Instantiates a new Query. +model+ is the lazy_record model (i.e. class)
that the query will be made about: the query's member objects will be
members of +model+'s class. When +record+ is passed, it is used for
def _add_form(self, table_schema):
return type(Inflector().classify(table_schema['table_name']) + "Form",
(FlaskForm, ), self._gen_form_dict(table_schema))
from django.conf import settings as django_settings
from django.contrib.contenttypes.fields import GenericRelation
from django.core.exceptions import FieldDoesNotExist
from django.db.models.fields import NOT_PROVIDED
from django.utils.module_loading import import_string
from django.utils.text import capfirst
from inflector import Inflector
from rest_framework import serializers, relations
from rest_framework.fields import empty
from .app_settings import settings
inflector_language = import_string(settings.INFLECTOR_LANGUAGE)
inflector = Inflector(inflector_language)
def reverse(*args, **kwargs):
try:
exception = ModuleNotFoundError
except:
## py 3.6+
exception = ImportError
try:
from django.core.urlresolvers import reverse
except exception:
# Django 1.11+
from django.urls import reverse
class EnglishInflectorTestCase(unittest.TestCase):
singular_to_plural = {
"search": "searches",
"switch": "switches",
"fix": "fixes",
"box": "boxes",
"process": "processes",
"address": "addresses",
"case": "cases",
"stack": "stacks",
"wish": "wishes",
"fish": "fish",
"category": "categories",
"query": "queries",
"ability": "abilities",
"agency": "agencies",
"movie": "movies",
"archive": "archives",
"index": "indices",
"wife": "wives",
"safe": "saves",
"half": "halves",
"move": "moves",
"salesperson": "salespeople",
"person": "people",
"spokesman": "spokesmen",
"man": "men",
"woman": "women",
"basis": "bases",
"diagnosis": "diagnoses",
"datum": "data",
"medium": "media",
"analysis": "analyses",
"node_child": "node_children",
"child": "children",
"experience": "experiences",
"day": "days",
"comment": "comments",
"foobar": "foobars",
"newsletter": "newsletters",
"old_news": "old_news",
"news": "news",
"series": "series",
"species": "species",
"quiz": "quizzes",
"perspective": "perspectives",
"ox": "oxen",
"photo": "photos",
"buffalo": "buffaloes",
"tomato": "tomatoes",
"dwarf": "dwarves",
"elf": "elves",
"information": "information",
"equipment": "equipment",
"bus": "buses",
"status": "statuses",
"mouse": "mice",
"louse": "lice",
"house": "houses",
"octopus": "octopi",
"virus": "viri",
"alias": "aliases",
"portfolio": "portfolios",
"vertex": "vertices",
"matrix": "matrices",
"axis": "axes",
"testis": "testes",
"crisis": "crises",
"rice": "rice",
"shoe": "shoes",
"horse": "horses",
"prize": "prizes",
"edge": "edges"
}
def setUp(self):
self.inflector = Inflector(English)
def tearDown(self):
self.inflector = None
def test_pluralize(self):
for singular in list(self.singular_to_plural.keys()):
assert self.inflector.pluralize(singular) == self.singular_to_plural[singular], \
'English Inlector pluralize(%s) should produce "%s" and NOT "%s"' % (singular, self.singular_to_plural[singular], self.inflector.pluralize(singular))
def test_singularize(self):
for singular in list(self.singular_to_plural.keys()):
assert self.inflector.singularize(self.singular_to_plural[singular]) == singular, \
'English Inlector singularize(%s) should produce "%s" and NOT "%s"' % (self.singular_to_plural[singular], singular, self.inflector.singularize(self.singular_to_plural[singular]))
def setUp(self):
self.inflector = Inflector(English)
def to_dict(obj: T, *,
flags: SerializationFlags = SerializationFlags(''),
locale: str = DEFAULT_LOCALE,
depth: int = 3,
uuid_hex: bool = True,
inflection: bool = False,
datetime_formatter: Optional[str] = None) -> Union[T, Dict[str, Any], Iterable[Dict[str, Any]], None]:
"""
Converts a Python object to dictionary, with recursion over the inner objects
:param obj: Input Python object
:param flags: Flags applicable for converting, as like allow nulls
:param locale: Localization info as like time-zone, calendar and date/time format
:return: Python dictionary
"""
inflector = Inflector()
if obj is None:
if flags.ReplaceNoneWithEmptyString:
return ''
else:
return
if depth == 0:
return
m = re.match(RegExPatterns.Locale, locale)
t = type(obj)
if m:
locale = m.group()
else:
locale = DEFAULT_LOCALE
if is_dataclass(obj):
annotations = get_type_hints(type(obj))
res = asdict(obj)
return {inflector.underscore(k) if inflection else k: to_dict(res[k], flags=flags,
locale=locale,
depth=depth,
inflection=inflection,
datetime_formatter=datetime_formatter)
for k in res
if res[k] is not None
or flags.IncludeNulls
or (flags.ReplaceNoneWithEmptyString and annotations[k]) == str}
elif issubclass(t, BaseGeometry):
return mapping(obj)
elif t in (int, str, bytes, float, bool):
return obj
elif issubclass(t, Enum):
return obj.value
elif t is Decimal:
return float(obj)
elif t is UUID:
if uuid_hex:
return obj.hex
else:
return obj
elif t in (datetime, date, time):
country: str = locale[-2:]
tz = timezone(country_timezones[country][0])
if locale == 'fa-IR':
if t is datetime:
if flags.IgnoreLocaleCalendar:
if flags.IgnoreLocaleTimeZone:
return obj.isoformat()
if obj.tzinfo is None:
return tz.fromutc(obj).isoformat()
else:
return obj.astimezone(tz)
else:
if datetime_formatter:
formatter = lambda d: '{{:{}}}'.format(datetime_formatter).format(d)
else:
formatter = JalaliDatetime.isoformat
if flags.IgnoreLocaleTimeZone:
return formatter(JalaliDatetime(obj))
if obj.tzinfo is None:
return formatter(JalaliDatetime(tz.fromutc(obj)))
else:
return formatter(JalaliDatetime(obj.astimezone(tz)))
elif t is date:
if flags.IgnoreLocaleCalendar:
return obj.isoformat()
else:
return JalaliDate(obj).isoformat()
elif t is time:
return obj.isoformat()
else:
return obj.isoformat()
elif isinstance(obj, timedelta):
return re.sub(r'0[YMDHS]', '',
'P{year}Y{month}M{day}DT{hour}H{minute}M{second}S'
.format(year=obj.days // 365,
month=(obj.days % 365) // 30,
day=obj.days % 30,
hour=obj.seconds // 3600,
minute=(obj.seconds % 3600) // 60,
second=obj.seconds % 60))
elif isinstance(obj, collections.Mapping):
return {inflector.underscore(k) if inflection else k: to_dict(obj[k],
flags=flags,
locale=locale,
depth=depth - 1,
inflection=inflection,
datetime_formatter=datetime_formatter)
for k in obj
if obj[k] is not None
or flags.IncludeNulls}
elif isinstance(obj, Iterable) or isinstance(obj, collections.Sequence):
gen = (to_dict(item,
flags=flags,
locale=locale,
depth=depth - 1,
inflection=inflection,
datetime_formatter=datetime_formatter)
for item in obj
if item is not None
or flags.IncludeNulls)
return t(gen) if isinstance(obj, collections.Sequence) else tuple(gen)
else:
return {attr: to_dict(getattr(obj, attr),
flags=flags,
locale=locale,
depth=depth - 1,
inflection=inflection,
datetime_formatter=datetime_formatter)
for attr in vars(obj) if not attr.startswith('_')}
def setUp(self):
self.inflector = Inflector(English)
lm_file = lm_name + "-" + lm_file
if lm_dir is not None:
lm_file = os.path.join(lm_dir, lm_file)
augment_lm_file = lm_file
if params["use_lm"]:
if not os.path.exists(lm_file):
data_for_lm = [elem[1:] for elem in data]
dev_data_for_lm = [elem[1:] for elem in dev_data]
with open(lm_config_path, "r", encoding="utf8") as fin:
lm_params = json.load(fin)
lm = NeuralLM(**lm_params)
lm.train(data_for_lm, dev_data_for_lm, save_file=lm_file)
use_lm = params["use_lm"]
else:
use_lm, lm_file = False, None
inflector = Inflector(use_lm=use_lm, lm_file=lm_file, **params["model"])
save_file = os.path.join(save_dir, filename + ".json") if save_dir is not None else None
if to_train:
augmentation_params = params.get("augmentation")
if augmentation_params is not None:
suffix = int(augmentation_params["n"])
generation_params = augmentation_params.get("params", dict())
augment_file = "augmented/{}-{}-{}".format(language, mode, suffix)
if os.path.exists(augment_file) and generation_params.get("to_load", True):
auxiliary_data = read_infile(augment_file)
else:
gen_params = copy.copy(generation_params)
gen_params.pop("to_load")
auxiliary_data = generate_auxiliary(data, dev_data, suffix, augment_lm_file,
augment_file, **gen_params)
else:
