def get_search_point(args):
conf = config.JsonConfig(config_json=args.config)
rakuten_session_manager = rakuten.RakutenSessionManager(args.user, args.password, conf)
text = mimesis.Text()
word_list = [text.word() for _ in range(args.num)]
rakuten_session_manager.search(word_list)
def create_synthetic_data(num_samples):
keywords = {
0: ["bad", "worst", "dirty", "irritating", "disgusting"],
1: ["vague", "vain", "untouchable", "selfish", "rude"],
2: ["perverse", "possessive", "arrogant", "cruel", "calm"],
3: ["clever", "comfortable", "creative", "clean", "gentle"],
4: ["nice", "fantastic", "good", "modern", "quite"],
}
p = mimesis.Text()
data = [
(label, _generate_doc(label, p, keywords))
for label in keywords
for _ in tqdm(range(num_samples // len(keywords)))
]
df = pd.DataFrame.from_records(data, columns=["label", "text"]).sample(
frac=1
)
df.to_csv(f"{DATASET_DIR}/synthetic.csv", index=False)
train_df, testval_df = train_test_split(
df, test_size=0.2, stratify=df.label
)
val_df, test_df = train_test_split(
testval_df, test_size=0.5, stratify=testval_df.label
)
train_df.to_csv(f"{DATASET_DIR}/synthetic_train.csv", index=False)
train_df.append(val_df).to_csv(
f"{DATASET_DIR}/synthetic_trainval.csv", index=False
)
val_df.to_csv(f"{DATASET_DIR}/synthetic_val.csv", index=False)
test_df.to_csv(f"{DATASET_DIR}/synthetic_test.csv", index=False)
def seed():
text = mimesis.Text()
products = models.Product.query.all()
for product in products:
for _ in range(random.randint(1, 3)):
attributes = []
for attribute in product.product_type.product_attributes:
attributes.append(random.choice(attribute.values))
supplier = [
obj for obj in product.product_type.vendors
if obj.is_supplier is True
][0]
product_variant = models.ProductVariant(
description=text.title(),
attribute_values=attributes,
supplier=supplier,
product=product,
)
db.session.add(product_variant)
db.session.commit()
def creating_category(apps, schema_editor):
category = apps.get_model("task_Steel_kiwi", "Category")
products = apps.get_model("task_Steel_kiwi", "Product")
user = apps.get_registered_model("auth", "User")
hardware = mimesis.Hardware()
text = mimesis.Text()
for _ in range(5):
randomize_hardware = random.choice(
[hardware.cpu(), hardware.graphics()])
category.objects.create(
name=randomize_hardware,
description=text.text(quantity=3),
slug=slug_creator(name=randomize_hardware),
)
for _ in range(30):
randomize_hardware = random.choice(
[hardware.cpu(), hardware.graphics()])
category_n = category.objects.order_by('?').first()
products.objects.create(
category=category_n,
name=randomize_hardware,
description=text.text(quantity=3),
price=round(random.uniform(10.00, 1000.00), 2),
slug=slug_creator(name=randomize_hardware),
)
user.objects.create(
username='******',
password=make_password('adminadmin'),
is_staff=True,
is_superuser=True,
)
def gen_tag():
lang = randint(0, 1)
if lang == 0:
lang = 'ru'
else:
lang = 'en'
word_count = randint(1, 3)
words = mimesis.Text(lang).words(quantity=word_count)
return '_'.join(words)
def test(self, tmp_path):
ids_set = set()
for _ in range(self.GEN_FOLDER_NUMBER):
rand_id = str(random.randrange(self.MAX_ID))
ids_set.add(rand_id)
rand_name = ' '.join(mimesis.Text().words())
dir_name = f'{rand_id} {rand_name}'
os.mkdir(os.path.join(tmp_path, dir_name))
assert ids_set == crawler.get_existed_ids(tmp_path)
def populate(authors: Sequence['People'],
count=100) -> Generator['Comment', None, None]:
import mimesis
cid = mimesis.Numbers()
comment = mimesis.Text()
return (Comment(id=cid.between(1, count),
body=comment.sentence(),
author=random.choice(authors)) for _ in range(count))
def seed():
business = mimesis.Business()
datetime = mimesis.Datetime()
text = mimesis.Text()
default_checkout_fields = ["First Name", "Last Name"]
distributors = models.Distributor.query.all()
for distributor in distributors:
campaign = models.Campaign(
name=f"{business.company()} Distributor - {distributor.id}",
company_name=f"Campaign distributor {distributor.id}",
start_date=datetime.datetime(start=2018, end=2019),
end_date=datetime.datetime(start=2020, end=2021),
storefront_pricing=random.choice((True, False)),
company_allowance=random.randrange(30, 200),
company_allowance_personal_pay=random.choice((True, False)),
items_count_limit=1000,
price_limit=10000,
email_shopper=random.choice((True, False)),
checkout_fields=default_checkout_fields,
departments=None,
managers=None,
message=text.sentence(),
bfl_cost=0.0,
distributor_po=None,
pick_pack_partner_message=text.sentence(),
created_by_id=
1, # we must specify this manually, since no user is logged in
pick_pack_partner=None,
distributor=distributor,
)
db.session.add(campaign)
shopper_role = models.Role.query.filter_by(name="Shopper").first()
admin_buyer_role = models.Role.query.filter_by(
name="Admin Buyer").first()
# swap the existing shopper account over to this campaign
shopper = models.Account.query.filter_by(
distributor_id=distributor.id, role=shopper_role).first()
shopper.distributor = None
shopper.campaign = campaign
db.session.add(shopper)
# swap the existing admin buyer account over to this campaign
admin_buyer = models.Account.query.filter_by(
distributor_id=distributor.id, role=admin_buyer_role).first()
admin_buyer.distributor = None
admin_buyer.campaign = campaign
admin_buyer.reports_enabled = True
db.session.add(admin_buyer)
db.session.commit()
def generate_content(posts: int, days: int = 730) -> None:
def _unsafe_file_removal(path: str) -> None:
for file in os.listdir(path):
os.remove(f"{path}/{file}")
dest = _source.joinpath("content/posts")
stream = io.StringIO()
writer = mimesis.Text()
today = date.today()
_unsafe_file_removal(dest)
for _, _ in enumerate(range(posts), 1):
delta = timedelta(days=random.randrange(1, days + 1))
written_date = str(today - delta)
publish_date = str(today - delta)
title = f"{writer.word()}-{writer.word()}-{writer.word()}"
category = random.choice(["python", "data", "docker","go","kubernetes"])
hugo_front_matter = (
f'---\n'
f'title: "{title}"\n'
f'date: {written_date}\n'
f'publishdate: {publish_date}\n'
f'layout: post\n'
f'categories: ["{category}"]\n'
f'---\n'
)
python_snippet = random.choice(
[
method
for method in filter(
lambda x: ismethod(writer.__getattribute__(x)), dir(writer)
)
]
)
code_snippet = (
f"{{{{< highlight python >}}}}\n"
f"{getsource(writer.__getattribute__(python_snippet))}\n"
f"{{{{< /highlight >}}}}\n\n"
)
with open(f"{dest}/{uuid.uuid4().hex}.md", "w") as fd:
fd.write(hugo_front_matter)
for _ in range(3):
fd.write(writer.text() + 2 * "\n")
fd.write(code_snippet)
fd.write(writer.text() + 2 * "\n")
print(f"Post generated: {fd.name}")
def new_article() -> dict:
"""Generate news article
:return: dictionary with title and text
:rtype: dict
"""
random.seed()
_text = mimesis.Text()
return {
'title': _text.title(),
'content': _text.text(random.randint(3, 9))
}
def seed():
text = mimesis.Text()
orders = models.Order.query.all()
accounts = models.Account.query.all()
for order in orders:
for _ in range(3):
order_note = models.OrderNote(
order=order, account=random.choice(accounts), note=text.sentence()
)
db.session.add(order_note)
db.session.commit()
def run(self):
self._start_time = time_util.get_timestamp_now()
options = Options()
add_on_path = os.path.join(self._config.LIB_ROOT_PATH, 'extension_4_655_0_0.crx')
options.add_extension(add_on_path)
driver = webdriver.Chrome(options=options)
with contextlib.ExitStack() as stack:
stack.callback(driver.quit)
driver.get(LOGIN_URL)
logger.info('ログイン開始')
user_id_element = driver.find_element_by_xpath('//*[@id="loginInner_u"]')
user_id_element.send_keys(self._config.RAKUTEN_CONFIG.get('user'))
time.sleep(3)
password_element = driver.find_element_by_xpath('//*[@id="loginInner_p"]')
password_element.send_keys(self._config.RAKUTEN_CONFIG.get('password'))
time.sleep(3)
login_submit = driver.find_element_by_xpath('//*[@id="loginInner"]/p[1]/input')
login_submit.click()
time.sleep(3)
if 'login' in driver.current_url:
raise ValueError('user id or password is incorrect')
logger.info('ログイン完了')
text = mimesis.Text()
logger.info('検索開始')
for i in range(35):
driver.get(WEB_SEARCH_URL)
search_window = driver.find_element_by_xpath('//*[@id="search-input"]')
search_word = text.word()
logger.info('検索ワード: {}'.format(search_word))
search_window.send_keys(search_word)
time.sleep(3)
search_submit = driver.find_element_by_xpath('//*[@id="search-submit"]')
search_submit.click()
time.sleep(3)
logger.info('検索完了')
# slackへ通知
end_time = time_util.get_timestamp_now()
context = {
'start_time': self._start_time,
'end_time': end_time
}
self._slack_notificator.notify_from_file(os.path.join('slack', 'rakuten', 'success.txt'), context)
def seed():
text = mimesis.Text()
# generate some random decorations
decorations = []
for _ in range(15):
decorations.append({
"cost":
random.randint(1, 20),
"location":
random.choice(["top", "bottom", "left", "right"]),
"logo_description":
text.words(3),
})
campaigns = models.Campaign.query
for campaign in campaigns:
products = models.Product.query.filter_by(
distributor_id=campaign.distributor_id)
for product in products:
decorator_ids = [
obj.id for obj in product.product_type.vendors
if obj.is_decorator
]
for variant in product.product_variants:
campaign_product_variant = models.CampaignProductVariant(
product_variant_id=variant.id,
campaign_id=campaign.id,
decorations=json.dumps(
random.sample(decorations, random.randint(1, 4))),
margin=random.uniform(5, 15),
vendor_cost=random.uniform(5, 25),
)
db.session.add(campaign_product_variant)
db.session.commit()
campaign_product_variant.populate_from_variant()
campaign_product_variant.populate_from_decorator(
random.choice(decorator_ids))
db.session.commit()
def next_book(authors: typing.List[Author],
data_locale: str) -> typing.Iterator[Book]:
code = mimesis.Code(data_locale)
date = mimesis.Datetime(data_locale)
text = mimesis.Text(data_locale)
book_id = 0
while True:
book_id += 1
book = Book(book_id)
book.title = text.title()
book.author = random.choice(authors)
book.isbn = code.isbn()
book.release_at = date.date(start=1800, end=2018)
yield book
def seed():
text = mimesis.Text()
product_types = models.ProductType.query.all()
for product_type in product_types:
for distributor in product_type.distributors:
for _ in range(2):
product = models.Product(
name=
f"{text.swear_word()} {product_type.name.rstrip('s')}",
item_number=str(random.randint(0, 10000)),
product_type=product_type,
distributor=distributor,
)
db.session.add(product)
db.session.commit()
def populate(comments: Sequence['Comment'],
authors: Sequence['People'],
count=100) -> Generator['Article', None, None]:
import mimesis
aid = mimesis.Numbers()
article = mimesis.Text()
answers = list(comments)
def get_random_answers(max):
counter = 0
while answers and counter < max:
yield answers.pop(random.randint(0, len(answers) - 1))
counter += 1
return (Article(
id=aid.between(1, count),
title=article.title(),
author=random.choice(authors),
comments=[c for c in get_random_answers(random.randint(1, 10))])
for _ in range(count))
def seed():
# generate 10 random colors and sizes
text = mimesis.Text()
clothing = mimesis.Clothing()
colors = list(set([text.color() for x in range(50)]))[:10]
sizes = list(set([clothing.international_size()
for x in range(50)]))[:10]
# create a single color and size attribute to be shared by all product types
color_attribute = models.ProductAttribute(name="Color")
db.session.add(color_attribute)
for color in colors:
attribute_value = models.ProductAttributeValue(
name=color,
product_attribute_id=color_attribute.id,
)
db.session.add(attribute_value)
size_attribute = models.ProductAttribute(name="Size")
db.session.add(size_attribute)
for size in sizes:
attribute_value = models.ProductAttributeValue(
name=size,
product_attribute_id=size_attribute.id,
)
db.session.add(attribute_value)
# link existing product types with these attributes (and values)
product_types = models.ProductType.query.all()
for product_type in product_types:
product_type.product_attributes = [color_attribute, size_attribute]
db.session.add(product_type)
db.session.commit()
def generate_dummy_data(data_tokens):
"""
Insert dummy data based on data tokens
I - integer id
L - login/username
E - email
P - password
T - piece of text
:return:
"""
token_list = data_tokens.split(",")
samples_count = random.randint(100, 1000)
inserted_data = []
for i in range(samples_count):
values = []
person = mimesis.Person("en")
for token in token_list:
person = mimesis.Person()
if token == "I":
values.append(i)
if token == "L":
data = person.username()
values.append(data)
if token == "E":
data = person.email()
values.append(data)
if token == "P":
data = person.password()
values.append(data)
if token == "T":
sample_length = random.randint(1, 10)
data = mimesis.Text().text(quantity=sample_length)
values.append(data)
inserted_data.append(tuple(values))
return inserted_data, token_list
def setUp(self):
hardware = mimesis.Hardware()
text = mimesis.Text()
for _ in range(5):
randomize_hardware = random.choice(
[hardware.cpu(), hardware.graphics()])
Category.objects.create(
name=randomize_hardware,
description=text.text(quantity=3),
slug=slugify(randomize_hardware + '-' + str(slug_creator())),
)
for _ in range(30):
randomize_hardware = random.choice(
[hardware.cpu(), hardware.graphics()])
category_n = Category.objects.order_by('?').first()
Product.objects.create(
category=category_n,
name=randomize_hardware,
description=text.text(quantity=3),
price=round(random.uniform(10.00, 1000.00), 2),
slug=slugify(randomize_hardware + '-' + str(slug_creator())),
)
def text(request):
return mimesis.Text(request.param)
import unittest
import mimesis
from api.blueprints import create_app
from api.blueprints.models import Post, User
from api.blueprints.tests.api_tests import utils
MODELS = [Post, User]
URL = '/api'
p = mimesis.Person()
t = mimesis.Text()
class AdminRegistrationTests(unittest.TestCase):
link = f'{URL}/admin'
def setUp(self):
self.app, self.db = create_app(testing=True)
username, password = p.username(), p.password()
User.from_dict(dict(username=username, email=p.email(),
password=password, is_admin=True))
self.code, self.admin = utils.post(
self.app,
f'{URL}/auth/login',
username=username,
password=password
)
self.headers = {'x-access-token': self.admin['token']}
def generate_comment():
text = mimesis.Text()
return text.quote()
def handle(self, *args, **options):
get_users_count = User.objects.all().count()
print('users_count:', get_users_count)
sleep(2)
for i in range(users_count - get_users_count):
User.objects.create(
name=mimesis.Person('en').username(),
birthday=mimesis.Datetime().date(start=1950, end=2002),
registration_date=mimesis.Datetime().datetime(),
email=mimesis.Person('en').email(),
description=mimesis.Text('ru').sentence())
this_sleep(i)
print('users_count:', get_users_count + i, ' из ', users_count)
get_publication_count = Publication.objects.all().count()
print('publication_count:', get_publication_count)
sleep(1)
for i in range(publication_count - get_publication_count):
Publication.objects.create(
description=mimesis.Text('ru').sentence(),
author=User.objects.get(pk=randint(1, users_count - 2)),
title=mimesis.Text('ru').title())
this_sleep(i)
print('publication_count:', get_publication_count + i, 'из',
publication_count)
get_comment_count = Comment.objects.all().count()
print('comment_count:', get_comment_count)
sleep(1)
for i in range(comment_count - get_comment_count):
Comment.objects.create(
author=User.objects.get(pk=randint(1, users_count - 2)),
publication=Publication.objects.get(
pk=randint(1, publication_count - 2)),
comment=mimesis.Text('ru').sentence())
this_sleep(i)
print('comment_count:', get_comment_count + i, 'из', comment_count)
get_like_count = Like.objects.all().count()
print('like_count:', get_like_count)
sleep(1)
for i in range(like_count - get_like_count):
if randint(0, 1):
Like.objects.create(
author=User.objects.get(pk=randint(1, users_count - 2)),
publication=Publication.objects.get(
pk=randint(1, publication_count - 2)))
else:
Like.objects.create(
author=User.objects.get(pk=randint(1, users_count - 2)),
comment=Comment.objects.get(randint(1, comment_count - 2)))
this_sleep(i)
print('like_count:', get_like_count + i, 'из', like_count)
get_tags_count = Tag.objects.all().count()
print('tags_count:', get_tags_count)
sleep(1)
for i in range(tags_count - get_tags_count):
tag = Tag(title=gen_tag())
tag.save()
for j in range(1, tags_publ_count):
tag.publication.add(
Publication.objects.get(pk=randint(1, publication_count -
2)))
tag.save()
this_sleep(i)
print('tags_count:', get_tags_count + i, 'из', tags_count)
# need_date = (datetime.now() + timedelta(days=10)).date()
# payouts_types = Payouts_type.objects.filter(date_end=need_date)
# if len(payouts_types) > 0:
# message = "Через 10 дней закончится прием следующих заявлений: " + \
# ', '.join([ payout_type.payouts_type for payout_type in payouts_types])
# self.stdout.write(message)
# admins = Students.objects.filter(proforg__gte=3).exclude(vk_id="")
# if len(admins) > 0:
# message_send({
# 'user_ids': ','.join([ admin.vk_id for admin in admins]),
# 'message': message,
# })
# self.stdout.write('Successfully test')
return
def get_title():
obj = mimesis.Text(locale='ru')
return obj.quote()
int_out.append(m.Person().username())
int_out.append(m.Person().name())
int_out.append(m.Person().last_name())
int_out.append(m.Person().telephone())
int_out.append(m.Address().address())
int_out.append(m.Address().zip_code())
int_out.append(m.Datetime().date())
int_out.append(m.Datetime().date())
int_out.append(m.Hardware().cpu())
int_out.append(m.Hardware().cpu_frequency())
int_out.append(m.Hardware().graphics())
int_out.append(m.Hardware().manufacturer())
int_out.append(m.Hardware().ram_size())
int_out.append(m.Hardware().ram_type())
int_out.append(m.Hardware().resolution())
int_out.append(m.Games().game())
int_out.append(m.Datetime().date())
int_out.append(m.Text().text(2))
int_out.append(m.Text().text(3))
int_out.append(m.Text().level())
full_out.append(int_out)
with open(sys.argv[1], 'w', newline='') as f:
writer = csv.writer(f)
writer.writerows(full_out)
def _text():
return mimesis.Text()
