def generatorFirstName(gend):
person = Person('ru')
if gend == 'M':
name = person.first_name(gender = Gender.MALE)
else:
name = person.first_name(gender = Gender.FEMALE)
return russian_name(name)
def insert_in_batches(col, total, batch_size):
many_people = []
inserted_count = 0
start_time = datetime.utcnow()
previous_time = start_time
p = Person("en", seed=0xBEE)
for i in range(total):
proxy_person = {
"name": p.first_name(),
"surname": p.surname(),
"email": p.email(),
"password": p.password()
}
many_people.append(proxy_person)
if (len(many_people) % batch_size) == 0:
col.insert_many(many_people)
inserted_count = inserted_count + len(many_people)
time_now = datetime.utcnow()
elapsed_time = time_now - previous_time
previous_time = time_now
print(f"Inserted {len(many_people)} people in {elapsed_time}")
many_people.clear()
if len(many_people) > 0:
col.insert_many(many_people)
inserted_count = inserted_count + len(many_people)
end_time = datetime.utcnow()
duration = end_time - start_time
print(f"Inserting {inserted_count} records took {duration}")
def handle(self, *args, **options):
UserProfile.objects.all().delete()
Author.objects.all().delete()
person = Person('en')
for _ in range(COUNT_ELEM):
first_name = person.first_name()
last_name = person.last_name()
login = f'{first_name}.{last_name}'
email = person.email()
birthday_year = datetime.today().year - person.age(minimum=14,
maximum=100)
print(f'Created:{login} - {email}')
user = {
"user_name": login,
"first_name": first_name,
"last_name": last_name,
"gender": person.sex('M', 'F'),
"birthday_year": birthday_year,
"email": email
}
author = {
"first_name": first_name,
"last_name": last_name,
"birthday_year": birthday_year,
}
new_user = UserProfile(**user)
new_user.save()
new_author = Author(**author)
new_author.save()
User.objects.all().delete()
User.objects.create_superuser('admin', 'admin@localhost', 'admin')
def test_sign_up_user(browser):
person = Person()
TEST_DATA = {
'first_name': person.first_name(),
'last_name': person.last_name(),
'email': person.email(),
'telephone': person.telephone(),
'password': person.password(8)
}
sign_up_page = SignUpPage(browser)
sign_up_page.open()
sign_up_page.fill_user_data(first_name=TEST_DATA['first_name'],
last_name=TEST_DATA['last_name'],
email=TEST_DATA['email'],
telephone=TEST_DATA['telephone'],
password=TEST_DATA['password'])
TopMenu(browser).logout()
sign_in_page = SignInPage(browser)
sign_in_page.confirm_logout()
sign_in_page.open()
sign_in_page.login(TEST_DATA['email'], TEST_DATA['password'])
assert sign_in_page.get_available_actions_header() == 'My Account'
def seedDatabase(rows):
person = Person('en')
for _ in range(0, int(rows)):
first_name = person.first_name()
age = person.age()
new_user = User(first_name=first_name, age=age)
Db.session.add(new_user)
Db.session.commit()
return redirect(url_for('index'))
def generate_people():
person = Person('en-gb')
firstname = person.first_name()
surname = person.surname()
university = person.university()
return {
"id": 0,
"firstName": firstname,
"surname": surname,
"uni": university
}
def gen_data_simple_schema(self, data_path, partition_date, num_rows,
file_format):
"""
Input
- data_path: path where the partition will be created (string)
- partition_date: partition date to be created (date)
- num_rows: number of rows to be generated (integer)
- file_format: format of file to be generated (parquet or avro)
This function creates a data sample with a simple schema
"""
person = Person('en')
# Create a simple schema
schema_df = StructType([
StructField('identifier', StringType(), True),
StructField('first_name', StringType(), True),
StructField('last_name', StringType(), True),
StructField('occupation', StringType(), True),
StructField('age', IntegerType(), True),
StructField('date', DateType(), True)
])
# generate data
for _ in range(num_rows):
df_temp = self.spark.createDataFrame([[
person.identifier(),
person.first_name(),
person.last_name(),
person.occupation(),
person.age(), partition_date
]], schema_df)
try:
df = df.union(df_temp)
except:
df = df_temp
df.coalesce(1).write.partitionBy('date').mode('overwrite').format(
file_format).save(data_path)
print('Partition created: {data_path}/date={date}'.format(
data_path=data_path, date=partition_date))
print('# Rows:', df.count())
print('Schema:')
df.printSchema()
print('\n')
return
def handle(self, *args, **kwargs):
person = Person('ru')
total = kwargs['total']
admin = kwargs['admin']
for i in range(total):
password = User.objects.make_random_password()
if admin:
user = User.objects.create_superuser(
username=person.username(),
first_name=person.first_name(),
last_name=person.last_name(),
email=person.email(),
password=password,
)
print(f'Create new Superuser: {user} and password: {password}')
else:
user = User.objects.create_user(
username=person.username(),
first_name=person.first_name(),
last_name=person.last_name(),
email=person.email(),
password=password,
)
print(f'Create new user: {user} and password: {password}')
def insert_one_at_a_time(col, total):
start_time = datetime.utcnow()
p = Person("en", seed=0xBEE)
for i in range(total): # 0..total-1
proxy_person = {
"name": p.first_name(),
"surname": p.surname(),
"email": p.email(),
"password": p.password()
}
col.insert_one(proxy_person)
if ((i + 1) % 10) == 0:
print(f"Inserted {i+1} docs")
end_time = datetime.utcnow()
duration = end_time - start_time
print(f"Inserting {i + 1} records took {duration}")
class CSVData:
def __init__(self):
self.person = Person(locale='zh')
self.address = Address(locale='zh')
self.code = Code()
self.business = Business(locale='zh')
self.text = Text(locale='zh')
self.datetime = Datetime(locale='zh')
self.file = File()
self.path = Path()
self.internet = Internet()
self.structure = Structure()
def mime_data(self):
# col1 = self.person.full_name()
col1 = self.person.last_name() + self.person.first_name()
col2 = self.address.city()
col3 = self.address.street_name()
col4 = self.address.calling_code()
col5 = self.address.longitude()
col6 = self.code.imei()
col7 = self.business.company()
col8 = self.text.hex_color()
col9 = self.datetime.formatted_datetime()
col10 = self.datetime.time()
col11 = self.file.file_name()
col12 = self.path.dev_dir()
col13 = self.internet.ip_v4()
col14 = self.internet.ip_v6()
col15 = self.internet.home_page()
col16 = self.internet.stock_image()
col17 = self.internet.user_agent()
col18 = self.internet.mac_address()
col19 = self.person.email()
col20 = self.person.telephone()
col21 = self.code.issn()
col22 = self.person.social_media_profile()
col23 = self.structure.html()
line = '\"{0}\", \"{1}\", \"{2}\", \"{3}\", {4}, \"{5}\", \"{6}\" , \"{7}\" , \"{8}\" , \"{9}\" , \"{10}\" , \"{11}\" , \"{12}\" , \"{13}\" , \"{14}\" , \"{15}\" , \"{16}\" , \"{17}\" , \"{18}\" , \"{19}\" , \"{20}\" , \"{21}\" , \"{22}\"\n'.format(
col1, col2, col3, col4, col5, col6, col7, col8, col9, col10, col11, col12, col13, col14, col15, col16, col17, col18, col19, col20, col21, col22, col23)
# line = "mime data"
# print(line)
return line
def get_user(number):
"""
Creates random user data
:param number: number of users
:return: user data list
"""
data = []
for i in range(number):
user_gender = random.sample([Gender.MALE, Gender.FEMALE], 1)[0]
person = Person('ru')
data.append(
OrderedDict(first_name=person.first_name(gender=user_gender),
last_name=person.last_name(gender=user_gender),
age=person.age(minimum=18, maximum=66),
email=person.email(domains=('yandex.ru', 'gmail.com')),
username=person.username(template='UU_d')))
return data
def create_app_user(
domains: Optional[List[str]] = None,
organization: Optional[Organization] = None,
) -> AppUser:
"""Опционально можно передать домены, или инстанс организации.
Тогда пользователи будут созданы с emails:
* либо с переданными domains;
* либо с organization.domains.
"""
app_user = Person('ru')
first_name = app_user.first_name()
last_name = app_user.last_name()
if organization:
domains = organization.get_domains()
email = app_user.email(domains=domains)
user = AppUser.objects.create_user(
first_name=first_name,
last_name=last_name,
email=email,
)
return user
for i in range(count_bran):
brabn = Branch(id=i,
organization_id=random.randrange(count_org),
name='Фелиал' + str(i))
session.add(brabn)
session.commit()
for i in range(count_dep):
dep = Department(id=i,
branch_id=random.randrange(count_bran),
name='Отдел' + str(i))
session.add(dep)
session.commit()
for i in range(count_peop):
gender0 = gender[random.randrange(len(gender))]
peop = People(id=i,
first_name=person.first_name(gender=gender0),
last_name=person.last_name(gender=gender0),
department_id=random.randrange(count_dep),
position=positions[random.randrange(len(positions))],
number=gen_number(),
bday=dateTime.date(start=1970, end=2002),
address=address.address(),
gender=gender0.name)
session.add(peop)
session.commit()
session.close()
import pymongo
from mimesis import Person
# Connection object
c = pymongo.MongoClient(
host="mongodb+srv://livedemo:[email protected]/test_db?retryWrites=true&w=majority")
# database object
database = c["test_db"]
# collection/table object
collection = database["users"]
p = Person("en", seed=0xBEEF)
person = {"name": p.first_name(),
"surname": p.surname(),
"email": p.email(),
"password": p.password()}
collection.insert_one(person)
print("Inserted:")
pprint.pprint(person)
collection.update_one({"_id": person["_id"]}, {"$set": {"session_id": 20}})
collection.update_one({"_id": person["_id"]}, {"$push": {"phone": {"cell": "+1 123456789"}}})
doc = collection.find_one({"_id": person["_id"]})
print("Updated:")
pprint.pprint(doc)
collection.update_many({"surname": "Taylor"}, {"$push": {"assets": {"laptop": "MacPro laptop"}}})
def gen_data_remove_column(self, data_path, partition_date, num_rows,
file_format):
"""
Input
- data_path: path where the partition will be created (string)
- partition_date: partition date to be created (date)
- num_rows: number of rows to be generated (integer)
- file_format: format of file to be generated (parquet or avro)
This function creates a data sample removing some columns
"""
person = Person('en')
address = Address('en')
schema_street = StructType([
StructField('street_name', StringType(), True)
# StructField('lat', FloatType(), True), #column removed
# StructField('long', FloatType(), True) #column removed
])
schema_address_details = StructType([
StructField('street', schema_street, True),
StructField('number', IntegerType(), True)
])
schema_address = StructType([
StructField('address_details', schema_address_details, True),
StructField('city', StringType(), True),
StructField('country', StringType(), True),
# StructField('country_code', StringType(), True), #column removed
StructField('state', StringType(), True),
StructField('postal_code', IntegerType(), True)
])
schema_df = StructType([
StructField('identifier', StringType(), True),
StructField('first_name', StringType(), True),
StructField('last_name', StringType(), True),
StructField('occupation', StringType(), True),
StructField('age', IntegerType(), True),
StructField('address', schema_address, True),
# StructField('title_name', StringType(), True), #column removed
StructField('date', DateType(), True)
])
for _ in range(num_rows):
df_temp = self.spark.createDataFrame(
[[
person.identifier(),
person.first_name(),
person.last_name(),
person.occupation(),
person.age(),
[
[
[
address.street_name()
#float(address.latitude()),
#float(address.longitude())
],
int(address.street_number())
],
address.city(),
address.country(),
#address.country_code(),
address.state(),
int(address.postal_code())
],
#person.title(),
partition_date
]],
schema_df)
try:
df = df.union(df_temp)
except:
df = df_temp
df.coalesce(1).write.partitionBy('date').mode('overwrite').format(
file_format).save(data_path)
print('Partition created: {data_path}/date={date}'.format(
data_path=data_path, date=partition_date))
print('# Rows:', df.count())
print('Schema:')
df.printSchema()
print('\n')
return
def person(
cls,
*,
locale=Locales.EN,
qualification=None,
age=None,
blood_type=None,
email=None,
first_name=None,
last_name=None,
gender=None,
height=None,
id=None,
language=None,
nationality=None,
occupation=None,
phone=None,
title=None,
university=None,
weight=None,
work_experience=None,
):
'''
Create an Person Data Entity object.
All individual fields are automatically randomly generated based on locale. If provided, the corresponding values are overriden.
Note:
All individual fields are randomly generated. Don't expect correct correlation e.g. correct postal code for the generated city.
Keyword Arguments:
locale: Approprite Random.locale.<local_name> object. Default is Random.locale.EN
qualification: Educational Qualification
age: Age
blood_type: Blood type
email: Email address
first_name: First name
last_name: Last name
gender: Gender
height: Height
id: Identifier
language: Language
nationality: Nationality
occupation: Occupation
phone: Phone number
title: Title
university: University
weight: Weight
work_experience: Work Experience
'''
person = Person(locale=locale)
from arjuna.engine.data.entity.person import Person as ArjPerson
first_name = first_name is not None and first_name or person.first_name(
)
last_name = last_name is not None and last_name or person.last_name()
return ArjPerson(
qualification=qualification is not None and qualification
or person.academic_degree(),
age=age is not None and age or person.age(),
blood_type=blood_type is not None and blood_type
or person.blood_type(),
email=email is not None and email or person.email(),
first_name=first_name,
last_name=last_name,
name=first_name + " " + last_name,
gender=gender is not None and gender or person.gender(),
height=height is not None and height or person.height(),
id=id is not None and id or person.identifier(),
language=language is not None and language or person.language(),
nationality=nationality is not None and nationality
or person.nationality(),
occupation=occupation is not None and occupation
or person.occupation(),
phone=phone is not None and phone or person.telephone(),
title=title is not None and title or person.title(),
university=university is not None and university
or person.university(),
weight=weight is not None and weight or person.weight(),
work_experience=work_experience is not None and work_experience
or person.work_experience(),
)
day_ob TEXT,
year_ob TEXT,
city_ob TEXT,
city TEXT,
address TEXT,
phone TEXT,
email TEXT);
""")
conn.commit()
for i in range(20):
#idd = g.code.imei()
idd = n
gender = random.choice(gen)
if gender == 'Женский':
first_name = p.first_name(gender=Gender.FEMALE)
last_name = p.last_name(gender=Gender.FEMALE)
#self.patron = rus.patronymic(gender = Gender.FEMALE)
elif gender == 'Мужской':
first_name = p.first_name(gender=Gender.MALE)
last_name = p.last_name(gender=Gender.MALE)
#self.patron = rus.patronymic(gender = Gender.MALE)
age = p.age(minimum=18, maximum=70)
month_ob = dt.month()
day_ob = dt.day_of_month()
year_ob = 2020 - age
city_ob = adr.city()
city = adr.city()
address = adr.address()
phone = p.telephone(mask='+7(###)-###-####')
email = p.email(domains=['mimesis.name'])
class Users:
def __init__(self):
self.person = Person('en')
self.generic = Generic('en')
self.questions = Question.objects.all()
def generate(self):
for i in range(1, 101):
if i % 2 != 0:
user_gender = Gender.MALE
else:
user_gender = Gender.FEMALE
firstname = self.person.first_name(gender=user_gender)
lastname = self.person.last_name(gender=user_gender)
user = User.objects.create_user(self.person.username(),
self.person.email(), 'Qwerty1!')
user.first_name = firstname
user.last_name = lastname
user.is_active = True
self.update_aboutMe(user)
self.update_aboutYou(user)
user.save()
def update_aboutMe(self, user):
user.aboutme.gender = choice(list(commonInfo.GENDER))[0]
randColorHairCount = randrange(1, len(commonInfo.COLOR_HAIR))
user.aboutme.color_hair = sample(
[index[0] for index in list(commonInfo.COLOR_HAIR)],
k=randColorHairCount)
randColorAyeCount = randrange(1, len(commonInfo.COLOR_AYE))
user.aboutme.color_aye = sample(
[index[0] for index in list(commonInfo.COLOR_AYE)],
k=randColorAyeCount)
user.aboutme.activate = True
d1 = datetime.strptime('1/1/1970', '%m/%d/%Y')
d2 = datetime.strptime('1/1/2002', '%m/%d/%Y')
user.aboutme.birthday = self.random_date(d1, d2).strftime('%Y-%m-%d')
user.aboutme.weight = randrange(45, 150)
user.aboutme.growth = randrange(100, 250)
user.aboutme.genres.set(
self.get_random_list_of_multiple_data_from_objects(Genres.objects))
user.aboutme.music_types.set(
self.get_random_list_of_multiple_data_from_objects(
MusicType.objects))
user.aboutme.films.set(
self.get_random_list_of_multiple_data_from_objects(Films.objects))
user.aboutme.books.set(
self.get_random_list_of_multiple_data_from_objects(Books.objects))
user.aboutme.hobbies.set(
self.get_random_list_of_multiple_data_from_objects(
Hobbies.objects))
user.aboutme.foods.set(
self.get_random_list_of_multiple_data_from_objects(Foods.objects))
user.aboutme.country = self.get_random_data_from_objects(
Countries.objects)
user.aboutme.save()
self.save_questionary(user, user.aboutme)
def update_aboutYou(self, user):
user.aboutyou.gender = choice(list(commonInfo.GENDER))[0]
randColorHairCount = randrange(1, len(commonInfo.COLOR_HAIR))
user.aboutyou.color_hair = sample(
[index[0] for index in list(commonInfo.COLOR_HAIR)],
k=randColorHairCount)
randColorAyeCount = randrange(1, len(commonInfo.COLOR_AYE))
user.aboutyou.color_aye = sample(
[index[0] for index in list(commonInfo.COLOR_AYE)],
k=randColorAyeCount)
min_age = randrange(18, 60)
user.aboutyou.min_age = min_age
user.aboutyou.max_age = randrange(min_age, 61)
min_growth = randrange(100, 250)
user.aboutyou.min_growth = min_growth
user.aboutyou.max_growth = randrange(min_growth, 251)
min_weight = randrange(45, 150)
user.aboutyou.min_weight = min_weight
user.aboutyou.max_weight = randrange(min_weight, 151)
user.aboutyou.genres.set(
self.get_random_list_of_multiple_data_from_objects(Genres.objects))
user.aboutyou.music_types.set(
self.get_random_list_of_multiple_data_from_objects(
MusicType.objects))
user.aboutyou.films.set(
self.get_random_list_of_multiple_data_from_objects(Films.objects))
user.aboutyou.books.set(
self.get_random_list_of_multiple_data_from_objects(Books.objects))
user.aboutyou.hobbies.set(
self.get_random_list_of_multiple_data_from_objects(
Hobbies.objects))
user.aboutyou.foods.set(
self.get_random_list_of_multiple_data_from_objects(Foods.objects))
user.aboutyou.countries.set(
self.get_random_list_of_multiple_data_from_objects(
Countries.objects))
user.aboutyou.save()
self.save_questionary(user, user.aboutyou)
def save_questionary(self, user, obj):
for question in self.questions:
question = Question.objects.get(pk=question.pk)
answer = self.get_random_data_from_objects(
Answer.objects.filter(question=question))
obj_type = ContentType.objects.get_for_model(obj)
questionary_obj, is_created = Questionary.objects.get_or_create(
content_type=obj_type,
user=user,
question=question,
defaults={'answer': answer})
questionary_obj.answer = answer
questionary_obj.save()
def random_date(self, start, end):
"""
This function will return a random datetime between two datetime
objects.
"""
delta = end - start
int_delta = (delta.days * 24 * 60 * 60) + delta.seconds
random_second = randrange(int_delta)
return start + timedelta(seconds=random_second)
def get_random_data_from_objects(self, Objects):
modelDataIds = Objects.values_list('id', flat=True)
ChoiceId = choice(list(modelDataIds))
Data = Objects.get(pk=ChoiceId)
return Data
def get_random_list_of_multiple_data_from_objects(self, Objects):
countOfdata = Objects.count()
randDataCount = randrange(1, countOfdata + 1)
modelDataIds = Objects.values_list('id', flat=True)
listOfRandomDataIds = sample(list(modelDataIds), k=randDataCount)
multipleData = Objects.filter(id__in=listOfRandomDataIds)
return multipleData