def create_decorator(self):
instance = Inflector().underscore(self.name)
source_code = """
#! /usr/bin/env python3
# -*- encoding: utf-8 -*-
from flask import Blueprint
{instance} = Blueprint('{instances}', __name__)
@{instance}.route('/{instances}', methods=['GET'])
def index():
return 'index'
@{instance}.route('/{instances}/<int:id>', methods=['GET'])
def show(id):
return 'show'
@{instance}.route('/{instances}/', methods=['POST'])
def create():
return 'create'
@{instance}.route('/{instances}/<int:id>', methods=['PUT'])
def update(id):
return 'update'
@{instance}.route('/{instances}/<int:id>', methods=['DELETE'])
def destroy(id):
return 'destroy'
""".format(instance=instance,
instances=Inflector().pluralize(instance))
return dedent(source_code).strip()
def generate_a_task(
task_name,
project_dir=None,
):
path = join('luiti_tasks', Inflector().underscore(task_name) + ".py")
if project_dir:
path = join(project_dir, path)
content = write_content_to_file(
a_task_template(Inflector().classify(task_name)),
path,
)
return content
def create_method_view__init__(self):
source_code = """
#! /usr/bin/env python3
# -*- encoding: utf-8 -*-
from flask import Blueprint
from app.api.v1.{name} import {Name}
api_v1 = Blueprint('api/v1', __name__)
api_v1.add_url_rule('/{names}', view_func={Name}.as_view('{name}'), methods=['GET', 'POST'])
api_v1.add_url_rule('/{names}/<int:id>', view_func={Name}.as_view('{name}'), methods=['GET', 'PUT', 'DELETE'])
""".format(name=Inflector().underscore(self.name),
names=Inflector().pluralize(self.name),
Name=Inflector().camelize(self.name))
return dedent(source_code).strip()
def create_api_test(self):
source_code = """
#! /usr/bin/env python3
# -*- encoding: utf-8 -*-
from test import Experiment
from test.factories.{name}_factory import {Name}Factory
from app.models.{name} import {Name}
class {Name}APITest(Experiment):
def setUp(self):
super().setUp()
{Name}Factory.create_batch(5)
def test_get_list_200(self):
{name} = {Name}.query.first()
url = "/api/v1/{names}"
response = self.client.get(url, data={})
assert response.status_code == 200
def test_get_200(self):
{name} = {Name}.query.first()
url = "/api/v1/{names}/{{}}".format({name}.id)
response = self.client.get(url, data={})
assert response.status_code == 200
def test_post_201(self):
url = "/api/v1/{names}"
response = self.client.post(url, data={})
assert response.status_code == 201
def test_put_204(self):
{name} = {Name}.query.first()
url = "/api/v1/{names}/{{}}".format({name}.id)
response = self.client.put(url, data={})
assert response.status_code == 204
def test_delete_204(self):
{name} = {Name}.query.first()
url = "/api/v1/{names}/{{}}".format({name}.id)
response = self.client.delete(url, data={})
assert response.status_code == 204
""".format(
name=Inflector().underscore(self.name),
names=Inflector().pluralize(self.name),
Name=Inflector().camelize(self.name)
)
return dedent(source_code).strip()
def __init__(self, model=None, **kwargs):
self.inflector = Inflector(self.inflector_language)
if model is not None:
self.model = model
arg_names = ('fields', 'serializer', 'permission_classes', 'filter_fields', 'search_fields',
'viewset', 'read_only', 'include_str', 'ordering_fields', 'page_size',
'base_viewset', 'fields_annotation', 'fieldsets', 'base_serializer', 'list_me')
for arg_name in arg_names:
setattr(self, arg_name, kwargs.pop(arg_name, getattr(self, arg_name, None)))
if len(kwargs.keys()) > 0:
raise Exception('{} got an unexpected keyword argument: "{}"'.format(
self.__class__.__name__,
list(kwargs.keys())[0]
))
if self.serializer is not None:
assert self.fields is None, 'You cannot specify both fields and serializer'
else:
assert self.viewset is not None or self.model is not None, \
'You need to specify at least a model or a viewset'
self.get_serializer()
if self.viewset is not None:
for attr in ('permission_classes', 'filter_fields', 'search_fields', 'ordering_fields',
'page_size'):
assert getattr(self, attr, None) is None, \
'You cannot specify both {} and viewset'.format(attr)
else:
self.get_viewset()
if self.model is None:
self.model = self.get_serializer().Meta.model
def __init__(self):
# Dynamically constructs the JSON schema and the Draft7Validator used in the CommandParser
# IMPORTANT: validate schema with Draft7Validator.check_schema(schema) before using it!
inflector = Inflector()
commands = 'commands'
current_path = dirname(abspath(__file__))
excluded_files = ['__init__.py']
modules = [
file_name[:-3] for file_name in listdir(commands)
if isfile(join(join(current_path, commands), file_name))
and file_name not in excluded_files
]
schema = {
"$schema": "https://json-schema.org/schema#",
"type": "object",
"properties": {
"name": {
"type": "string",
"enum": modules
},
"args": {
"type": "object"
}
},
"required": ["name"]
}
classes = dict()
for module_name in modules:
classes[module_name] = getattr(
import_module(f'{commands}.{module_name}'),
inflector.camelize(module_name))
self.validator = Draft7Validator(schema)
self.classes = classes
def task_clsname_to_package(self):
assert lc.luiti_tasks_packages, "At least have one project!"
result = dict()
for project1 in lc.luiti_tasks_packages:
project_dir2 = project1.__path__[0]
# if it's not a zip file, but a normal package directory
is_zip_file = os.path.exists(
os.path.join(project_dir2, "__init__.py"))
if not is_zip_file:
raise Exception(
"""[setup.py format error] make sure """
"""project "%s" zip_safe=False option exists!""" %
project1.__name__)
task_path_pattern = os.path.join(project_dir2,
"luiti_tasks/[a-z]*.py")
for f2 in glob.glob(task_path_pattern):
task_filename3 = os.path.basename(f2).rsplit(".", 1)[0]
task_clsname4 = Inflector().classify(task_filename3)
result[task_clsname4] = project1
return result
def walk_dir(self, base, ignore_index=False, prefix=''):
from inflector import Inflector
from django.utils.module_loading import import_string
from drf_auto_endpoint.app_settings import settings as auto_settings
inflector_language = import_string(auto_settings.INFLECTOR_LANGUAGE)
inflector = Inflector(inflector_language)
imports = []
for item in os.listdir(base):
filename = os.path.join(base, item)
if os.path.isdir(filename):
imports += self.walk_dir(filename,
prefix=os.path.join(
prefix, item.replace('_', '-')))
elif item == 'index.js' and ignore_index:
continue
else:
try:
base_name, extension = item.rsplit('.', 1)
except ValueError:
# a file without extension
continue
if extension != 'js':
continue
imports.append(
(os.path.join(prefix, inflector.pluralize(base_name)),
os.path.join(prefix,
base_name).replace('/',
'_').replace('-', '_'),
os.path.join('.', prefix.replace('-', '_'), base_name)))
return imports
def ngram_tfidf(text, ngrams=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]):
# data
words = nltk.word_tokenize(text.lower())
# stopwords
stops = set(stopwords.words('english'))
# total number of documents
N = 4379810
ng_tf = dict()
ng_tfidf = dict()
for n in ngrams:
# load of sdf
path = '../pickle/ngram_sdf/' + str(n) + '-gram.pickle'
with open(path, 'rb') as f:
sdf = pickle.load(f)
terms = [words[i:i + n] for i in range(len(words) - n)]
terms = [t for t in terms
if all(w not in t for w in stops)] # stopwords filter
terms = [[Inflector().singularize(w) for w in t]
for t in terms] # singularize
terms = [' '.join(t) for t in terms]
terms = [t for t in terms if t in sdf]
# tf
tf = Counter(terms)
tfidf = dict()
# tfidf
for t in tf.keys():
tfidf[t] = tf[t] * log(N / sdf[t])
ng_tf[n] = tf
ng_tfidf[n] = tfidf
return ng_tf, ng_tfidf
def new_a_project(project_name):
project_name = Inflector().underscore(project_name)
readme_path = join(project_name, "README.markdown")
setup_path = join(project_name, "setup.py")
package_dir = join(project_name, project_name)
package_init = join(package_dir, "__init__.py")
package_luiti_tasks_init = join(package_dir, "luiti_tasks/__init__.py")
package_luiti_tasks_luiti = join(package_dir,
"luiti_tasks/__init_luiti.py")
tests_dir = join(project_name, "tests")
tests_test_main = join(tests_dir, "test_main.py")
write_content_to_file(a_project_readme(project_name), readme_path)
write_content_to_file(a_project_setup(project_name), setup_path)
write_content_to_file(u"", package_init)
write_content_to_file(u"", package_luiti_tasks_init)
write_content_to_file(a_project_init_luiti(),
package_luiti_tasks_luiti)
write_content_to_file(a_project_test_main(project_name),
tests_test_main)
# important files
return [
readme_path, setup_path, package_luiti_tasks_luiti, tests_test_main
]
class Spider(scrapy.Spider):
inflector = Inflector(English)
def __init__(self, *args, **kwargs):
# Add parameters for feed storage in Chicago time
tz = timezone('America/Chicago')
now = tz.localize(datetime.now())
self.year = now.year
self.month = now.strftime('%m')
self.day = now.strftime('%d')
self.hour_min = now.strftime('%H%M')
super(Spider, self).__init__(*args, **kwargs)
def _generate_id(self, data, start_time):
"""
Calulate ID. ID must be unique within the data source being scraped.
"""
name = self.inflector.underscore(data['name']).strip('_')
id = data.get('id', 'x').replace('/', '-')
try:
start_time_str = datetime.strftime(start_time, '%Y%m%d%H%M')
except TypeError:
start_time_str = 'None'
parts = [self.name, start_time_str, id, name]
return '/'.join(parts)
def _naive_datetime_to_tz(self, datetime_object, source_tz='America/Chicago'):
"""
Converts a naive datetime (one without timezone information) by
interpreting it using the source_tz.
"""
tz = timezone(source_tz)
return tz.localize(datetime_object)
def create_restful__init__(self):
name = Inflector().camelize(self.name)
source_code = """
#! /usr/bin/env python3
# -*- encoding: utf-8 -*-
from flask import Blueprint
from flask_restful import Api
from app.api.v1.{name} import {Name}
api_v1 = Blueprint('api/v1', __name__)
api = Api(api_v1)
api.add_resource({Name}, '/{names}', '/{names}/<int:id>')
""".format(name=Inflector().underscore(self.name),
names=Inflector().pluralize(self.name),
Name=Inflector().camelize(self.name))
return dedent(source_code).strip()
def load_a_task_by_name(s1):
task_clsname_1 = Inflector().classify(s1) # force convert
task_filename_1 = Inflector().underscore(s1) # force convert
assert task_clsname_1 in PackageMap.task_clsname_to_package, u"""
"%s" cannt be found. Auto converted class name is "%s", file name
is "luiti_tasks/%s.py", please check it carefully.
Already loaded PackageMap.task_clsname_to_package is %s.
""" % (s1, task_clsname_1, task_filename_1,
PackageMap.task_clsname_to_package)
package_path = PackageMap.task_clsname_to_package[task_clsname_1].__name__ + \
".luiti_tasks." + task_filename_1
task_lib = lc.import2(package_path)
return getattr(task_lib, task_clsname_1)
class Spider(scrapy.Spider):
inflector = Inflector(English)
def _generate_id(self, data, start_time):
"""
Calulate ID. ID must be unique within the data source being scraped.
"""
name = self.inflector.underscore(data['name']).strip('_')
id = data.get('id', 'x').replace('/', '-')
try:
start_time_str = datetime.strftime(start_time, '%Y%m%d%H%M')
except TypeError:
start_time_str = 'None'
parts = [self.name, start_time_str, id, name]
return '/'.join(parts)
def _naive_datetime_to_tz(self,
datetime_object,
source_tz='America/Chicago'):
"""
Converts a naive datetime (one without timezone information) by
interpreting it using the source_tz.
"""
tz = timezone(source_tz)
return tz.localize(datetime_object)
def create_decorator__init__(self):
instance = Inflector().underscore(self.name)
source_code = """
#! /usr/bin/env python3
# -*- encoding: utf-8 -*-
from app.api.v1.{instance} import {instance}
""".format(instance=instance, )
return dedent(source_code).strip()
def wizard(target_model, serializer, icon_class=None, btn_class=None, text=None, **kwargs):
inflector_language = import_string(settings.INFLECTOR_LANGUAGE)
inflector = Inflector(inflector_language)
_kwargs = {
'type': 'wizard',
'params': {},
}
_kwargs.update(kwargs)
kwargs = _kwargs
fieldsets = kwargs.pop('fieldsets', None)
if fieldsets is None:
fieldsets = [
{'name': field}
for field in serializer.Meta.fields
]
kwargs['params']['fieldsets'] = fieldsets
serializer_instance = serializer()
needs = []
fields = []
Adapter = import_string(settings.METADATA_ADAPTER)
for field in serializer.Meta.fields:
field_instance = serializer_instance.fields[field]
if isinstance(field_instance, PrimaryKeyRelatedField):
model = field_instance.queryset.model
needs.append({
'app': model._meta.app_label,
'singular': model._meta.model_name.lower(),
'plural': inflector.pluralize(model._meta.model_name.lower()),
})
fields.append(Adapter.adapt_field(get_field_dict(field, serializer)))
kwargs['params']['needs'] = needs
kwargs['params']['fields'] = fields
kwargs['languages'] = get_languages()
def decorator(func):
# cls = getattr(inspect.getmodule(func),
# func.__qualname__.split('.<locals>', 1)[0].rsplit('.', 1)[0])
func.bind_to_methods = [kwargs.pop('method', 'POST'), ]
func.detail = True
func.action_type = 'custom'
func.action_kwargs = action_kwargs(icon_class, btn_class, text, func, kwargs)
func.kwargs = {}
func.action_kwargs['params']['model'] = '{}/{}/{}'.format(
target_model._meta.app_label.lower(),
inflector.pluralize(target_model._meta.model_name.lower()),
func.__name__
)
func.serializer = serializer
return Adapter.adapt_wizard(func)
return decorator
def create_mongoengine__model(self):
source_code = """
#! /usr/bin/env python3
# -*- encoding: utf-8 -*-
from app.models import Model
class {name}(Model):
pass
""".format(name=Inflector().camelize(self.name))
return dedent(source_code).strip()
def create_model_test(self):
source_code = """
#! /usr/bin/env python3
# -*- encoding: utf-8 -*-
from test import Experiment
from test.factories.{name}_factory import {Name}Factory
from app.models.{name} import {Name}
class {Name}ModelTest(Experiment):
def setUp(self):
{Name}Factory.create_batch(5)
def something(self):
pass
""".format(
name=Inflector().underscore(self.name),
Name=Inflector().camelize(self.name)
)
return dedent(source_code).strip()
def one_to_many_names(name):
"""Generate the names Satellite might use for a one to many field.
Example of usage::
>>> one_to_many_names('person') == {'person', 'person_ids', 'people'}
True
:param name: A field name.
:returns: A set including both ``name`` and variations on ``name``.
"""
return set((name, name + '_ids', Inflector().pluralize(name)))
def create_decorator(self):
source_code = """
def {name}(func):
@wraps(func)
def wrapper(*ar, **kw):
# something hear
return func(*ar, **kw)
return wrapper
""".format(
name=Inflector().underscore(self.name)
)
return dedent(source_code)
class Spider(scrapy.Spider):
inflector = Inflector(English)
def _generate_id(self, item, data, start_time):
"""
Calulate ID. ID must be unique within the data source being scraped.
"""
name = self.inflector.underscore(data['name']).strip('_')
id = data.get('id', 'x').replace('/', '-')
parts = [
self.name,
datetime.strftime(start_time, '%Y%m%d%H%M'), id, name
]
return '/'.join(parts)
def create_config(self, name='config', env='test'):
source_code = """
#! /usr/bin/env python3
# -*- encoding: utf-8 -*-
class {name}(object):
ENV = '{env}'
{db}
{redis}
""".format(
name=Inflector().camelize(name),
env=env,
db=self.create_sqlalchemy(),
redis=self.create_redis(),
)
return dedent(source_code).strip()
def create_mongoengine_factoryboy(self):
source_code = """
#! /usr/bin/env python3
# -*- encoding: utf-8 -*-
from factory import Sequence, SubFactory, Iterator, fuzzy
from factory.mongoengine import MongoEngineFactory
from app.models.{name} import {Name}
class {Name}Factory(MongoEngineFactory):
class Meta:
model = {Name}
""".format(
name=self.name,
Name=Inflector().camelize(self.name),
)
return dedent(source_code).strip()
def create_decorators(self):
source_code = """
#! /usr/bin/env python3
# -*- encoding: utf-8 -*-
from functools import wraps
def {name}(func):
@wraps(func)
def wrapper(*ar, **kw):
# something hear
return func(*ar, **kw)
return wrapper
""".format(
name=Inflector().underscore(self.name)
)
return dedent(source_code).strip()
def _pluralize_periodify(self, adj_time, tm_unit, lkup_unit, abbreviate):
if abbreviate in (
PedanticTimedelta.UNIT_NAME_FULL,
PedanticTimedelta.UNIT_NAME_BRIEF,
PedanticTimedelta.UNIT_NAME_ABBREV,
):
# (lb): I timeit'd Inflector().pluralize vs. inflectr=Inflector();
# inflectr.pluralize. Creating object ahead of time is not faster.
tm_units = Inflector(English).conditional_plural(adj_time, tm_unit)
abbrev_full_i = PedanticTimedelta.UNIT_NAME_FULL
unabbreviated = PedanticTimedelta.UNIT_NAMES[lkup_unit][
abbrev_full_i]
if tm_unit != unabbreviated:
tm_units += '.'
else:
tm_units = tm_unit
return tm_units
def update_titles(titles_dataframe, course_dataframe):
CurricTitles.objects.all().delete()
title_objects = []
undergrad_currics = \
CurricTitles._get_currics_with_undergrad_courses(course_dataframe)
for idx, row in titles_dataframe.iterrows():
if row['curric_abbr'].strip() in CURRIC_BLACKLIST or \
row['curric_abbr'].strip() not in undergrad_currics:
continue
inflect = Inflector(English)
human_name = inflect.titleize(row['curric_full_nm'].strip())
title_objects.append(
CurricTitles(abbrev=row['curric_abbr'].strip(),
name=human_name,
campus=row['curric_branch']))
CurricTitles.objects.bulk_create(title_objects)
class ReadyForConnection(object):
queried_at = datetime.now()
inflicter = Inflector()
sleep_time = timedelta(seconds=2.0)
@classmethod
def call(cls,
objects_kind=None,
object_id=None,
sub_object_kind=None,
method="GET",
test=None,
**kwargs):
"""
Call an A
:param objects_kind: the readyfor object name. e.g. tags, projects, users...etc
:param object_id: id of object which is include in object_name category
:param method: http method.
:param param: specify to get more information such like comments, news , from object page
:param kwargs : kwargs of page. e.g. page...etc
:return: response
"""
domain = "https://readyfor.jp"
delta = datetime.now() - cls.queried_at
if delta < cls.sleep_time:
time_to_sleep = cls.sleep_time - delta
time.sleep(time_to_sleep.seconds +
time_to_sleep.microseconds * 10e-7)
query = "{domain}/{objects_name}/{id}/{sub_object}".format(
domain=domain,
objects_name=objects_kind,
id=object_id,
sub_object=sub_object_kind)
if sub_object_kind is None:
query = "{domain}/{objects_name}/{id}".format(
domain=domain, objects_name=objects_kind, id=object_id)
response = request(method=method, url=query, params=kwargs)
cls.queried_at = datetime.now()
if 400 < response.status_code:
raise AccessException(
"{code} error".format(code=response.status_code))
return response
def create_sqlalchemy_factoryboy(self):
source_code = """
#! /usr/bin/env python3
# -*- encoding: utf-8 -*-
from factory import Sequence, SubFactory, Iterator, fuzzy
from factory.alchemy import SQLAlchemyModelFactory
from app.extensions import db
from app.models.{name} import {Name}
class {Name}Factory(SQLAlchemyModelFactory):
class Meta:
model = {Name}
sqlalchemy_session = db.session
id = Sequence(lambda n: n+1)
""".format(
name=self.name,
Name=Inflector().camelize(self.name),
)
return dedent(source_code).strip()
def create_restful(self):
source_code = """
#! /usr/bin/env python3
# -*- encoding: utf-8 -*-
from flask_restful import Resource, fields, marshal_with, reqparse
resource_fields = {{
'id': fields.Integer,
'created_at': fields.DateTime,
'updated_at': fields.DateTime,
}}
class {name}(Resource):
parser = reqparse.RequestParser()
parser.add_argument('query', type=str, help="query string")
parser.add_argument('body', type=str, help="body string")
@marshal_with(resource_fields)
def get(self, id=None):
args = self.parser.parse_args()
if id is None:
return {{}}, 200
return [], 200
@marshal_with(resource_fields)
def post(self):
args = self.parser.parse_args()
return {{}}, 201
@marshal_with(resource_fields)
def put(self, id=None):
args = self.parser.parse_args()
return {{}}, 204
@marshal_with(resource_fields)
def delete(self, id=None):
return {{}}, 204
""".format(name=Inflector().camelize(self.name))
return dedent(source_code).strip()
def create_method_view(self):
source_code = """
#! /usr/bin/env python3
# -*- encoding: utf-8 -*-
from flask.views import MethodView
class {name}(MethodView):
def get(self, id=None):
if not id: return 'index'
return 'show'
def post(self):
return 'create'
def put(self, id):
return 'update'
def delete(self, id):
return 'destroy'
""".format(name=Inflector().camelize(self.name), )
return dedent(source_code).strip()
