def migrate_contact(listing_mapper, contact_type_mapper):
logging.debug('migrating contact...')
columns = get_columns('contact')
# ['id', 'version', 'created_by_id', 'created_date', 'edited_by_id', 'edited_date', 'email',
# 'listing_id', 'name', 'organization', 'secure_phone', 'type_id', 'unsecure_phone']
assert columns[0] == 'id'
assert columns[1] == 'version'
assert columns[2] == 'created_by_id'
assert columns[3] == 'created_date'
assert columns[4] == 'edited_by_id'
assert columns[5] == 'edited_date'
assert columns[6] == 'email'
assert columns[7] == 'listing_id'
assert columns[8] == 'name'
assert columns[9] == 'organization'
assert columns[10] == 'secure_phone'
assert columns[11] == 'type_id'
assert columns[12] == 'unsecure_phone'
values = get_values('contact', len(columns))
# logging.debug('category columns: %s' % columns)
logging.info('Contacts to migrate: {0!s}'.format(len(values)))
logging.info('==========================')
for i in values:
try:
email = i[6]
listing_id = i[7]
name = i[8]
organization = i[9]
secure_phone = i[10]
type_id = i[11]
contact_type = models.ContactType.objects.get(
id=contact_type_mapper[type_id])
unsecure_phone = i[12]
listing = models.Listing.objects.get(id=listing_mapper[listing_id])
try:
contact = models.Contact(name=name,
email=email,
secure_phone=secure_phone,
unsecure_phone=unsecure_phone,
contact_type=contact_type)
contact.save()
except Exception:
logging.warning(
'Error: Found duplicate contact entry: {0!s}'.format(name))
contact = models.Tag.objects.get(name=name)
logging.info('Adding contact {0!s} for listing {1!s}'.format(
name, listing.title))
listing.contacts.add(contact)
except Exception as e:
logging.error(
'Error adding contact entry: {0!s}, values: {1!s}'.format(
str(e), i))
def create_contact(self, contact: ContactData) -> models.Contact:
contact_type = self.contact_types_by_name.get(
contact.contact_type_name, None)
assert contact_type is not None, "ContactType with name '{}' not found".format(
contact.contact_type_name)
instance = models.Contact(contact_type=contact_type,
name=contact.name,
email=contact.email,
organization=contact.organization,
secure_phone=contact.secure_phone,
unsecure_phone=contact.unsecure_phone)
instance.save()
self.contacts_by_email[contact.email] = instance
return instance
def run():
"""
Creates basic sample data
"""
total_start_time = time_ms()
db_connection = transaction.get_connection()
db_connection.queries_limit = 100000
db_connection.queries_log = deque(maxlen=db_connection.queries_limit)
############################################################################
# Fast Mode
############################################################################
if FAST_MODE:
section_file_start_time = time_ms()
load_data_from_sql(db_connection, 'dump_sqlite3.sql')
print('-----Took: {} ms'.format(time_ms() - section_file_start_time))
print('---Database Calls: {}'.format(
show_db_calls(db_connection, False)))
return
############################################################################
# Security Markings
############################################################################
unclass = 'UNCLASSIFIED' # noqa: F841
secret = 'SECRET' # noqa: F841
secret_n = 'SECRET//NOVEMBER' # noqa: F841
ts = 'TOP SECRET' # noqa: F841
ts_s = 'TOP SECRET//SIERRA' # noqa: F841
ts_st = 'TOP SECRET//SIERRA//TANGO' # noqa: F841
ts_stgh = 'TOP SECRET//SIERRA//TANGO//GOLF//HOTEL' # noqa: F841
ts_n = 'TOP SECRET//NOVEMBER' # noqa: F841
ts_sn = 'TOP SECRET//SIERRA//NOVEMBER' # noqa: F841
ts_stn = 'TOP SECRET//SIERRA//TANGO//NOVEMBER' # noqa: F841
ts_stghn = 'TOP SECRET//SIERRA//TANGO//GOLF//HOTEL//NOVEMBER' # noqa: F841
############################################################################
# Loading Data Files
############################################################################
object_cache = {}
print('--Loading Files')
section_file_start_time = time_ms()
categories_data = load_yaml_file('categories.yaml')
contact_data = load_yaml_file('contacts.yaml')
profile_data = load_yaml_file('profile.yaml')
listings_data = load_yaml_file('listings.yaml')
listing_types = load_yaml_file('listing_types.yaml')
image_types = load_yaml_file('image_types.yaml')
agency_data = [{
'short_name': 'Minitrue',
'title': 'Ministry of Truth',
'icon.filename': 'ministry_of_truth.jpg'
}, {
'short_name': 'Minipax',
'title': 'Ministry of Peace',
'icon.filename': 'ministry_of_peace.png'
}, {
'short_name': 'Miniluv',
'title': 'Ministry of Love',
'icon.filename': 'ministry_of_love.jpeg'
}, {
'short_name': 'Miniplen',
'title': 'Ministry of Plenty',
'icon.filename': 'ministry_of_plenty.png'
}, {
'short_name': 'Test',
'title': 'Test',
'icon.filename': 'ministry_of_plenty.png'
}, {
'short_name': 'Test 1',
'title': 'Test 1',
'icon.filename': 'ministry_of_plenty.png'
}, {
'short_name': 'Test2',
'title': 'Test 2',
'icon.filename': 'ministry_of_plenty.png'
}, {
'short_name': 'Test 3',
'title': 'Test 3',
'icon.filename': 'ministry_of_plenty.png'
}, {
'short_name': 'Test 4',
'title': 'Test 4',
'icon.filename': 'ministry_of_plenty.png'
}]
print('-----Took: {} ms'.format(time_ms() - section_file_start_time))
############################################################################
# Recreate Index Mapping
############################################################################
print('--Recreate Index Mapping')
section_file_start_time = time_ms()
model_access_es.recreate_index_mapping()
print('-----Took: {} ms'.format(time_ms() - section_file_start_time))
print('--flushing database')
call_command('flush',
'--noinput') # Used to make postgresql work in unittest
############################################################################
# Groups
############################################################################
print('--Creating Groups')
section_file_start_time = time_ms()
models.Profile.create_groups()
print('-----Took: {} ms'.format(time_ms() - section_file_start_time))
show_db_calls(db_connection)
############################################################################
# Categories
############################################################################
print('--Creating Categories')
section_start_time = time_ms()
with transaction.atomic():
for current_category in categories_data['categories']:
current_category_obj = models.Category(
title=current_category['title'],
description=current_category['description'])
current_category_obj.save()
print('-----Took: {} ms'.format(time_ms() - section_start_time))
print('---Database Calls: {}'.format(show_db_calls(db_connection, False)))
############################################################################
# Contact Types and Contacts
############################################################################
print('--Creating Contact Types and Contacts')
section_start_time = time_ms()
with transaction.atomic():
for contact_type in contact_data['contact_types']:
current_contact_type_obj = models.ContactType(name=contact_type)
current_contact_type_obj.save()
object_cache['ContactType.{}'.format(
contact_type)] = current_contact_type_obj
for current_contact in contact_data['contacts']:
if not models.Contact.objects.filter(
email=current_contact['email']).exists():
current_contact_obj = models.Contact(
name=current_contact['name'],
organization=current_contact['organization'],
contact_type=object_cache['ContactType.{}'.format(
current_contact['contact_type'])],
email=current_contact['email'],
unsecure_phone=current_contact['unsecure_phone'],
secure_phone=current_contact['secure_phone'])
current_contact_obj.save()
object_cache['Contact.{}'.format(
current_contact['email'])] = current_contact_obj
print('-----Took: {} ms'.format(time_ms() - section_start_time))
print('---Database Calls: {}'.format(show_db_calls(db_connection, False)))
############################################################################
# Listing Types
############################################################################
print('--Creating Listing Types')
section_start_time = time_ms()
with transaction.atomic():
for listing_type in listing_types['listing_types']:
listing_type_object = models.ListingType(
title=listing_type['title'],
description=listing_type['description'])
listing_type_object.save()
object_cache['ListingType.{}'.format(
listing_type['title'])] = listing_type_object
print('-----Took: {} ms'.format(time_ms() - section_start_time))
print('---Database Calls: {}'.format(show_db_calls(db_connection, False)))
load_custom_field_types(db_connection)
############################################################################
# Image Types
############################################################################
# Note: these image sizes do not represent those that should be used in production
print('--Creating Image Types')
section_start_time = time_ms()
with transaction.atomic():
for image_type in image_types['image_types']:
image_type_obj = models.ImageType(
name=image_type['name'],
max_size_bytes=image_type['max_size_bytes'])
image_type_obj.save()
object_cache['ImageType.{}'.format(
image_type['name'])] = image_type_obj
print('-----Took: {} ms'.format(time_ms() - section_start_time))
print('---Database Calls: {}'.format(show_db_calls(db_connection, False)))
############################################################################
# Intents
############################################################################
# TODO: more realistic data
print('--Creating Intents')
section_start_time = time_ms()
with transaction.atomic():
img = Image.open(TEST_IMG_PATH + 'android.png')
icon = models.Image.create_image(
img,
file_extension='png',
security_marking='UNCLASSIFIED',
image_type=object_cache['ImageType.intent_icon'].name)
i = models.Intent(action='/application/json/view',
media_type='vnd.ozp-intent-v1+json.json',
label='view',
icon=icon)
i.save()
i = models.Intent(action='/application/json/edit',
media_type='vnd.ozp-intent-v1+json.json',
label='edit',
icon=icon)
i.save()
print('-----Took: {} ms'.format(time_ms() - section_start_time))
print('---Database Calls: {}'.format(show_db_calls(db_connection, False)))
############################################################################
# Organizations
############################################################################
print('--Creating Organizations')
section_start_time = time_ms()
with transaction.atomic():
for agency_record in agency_data:
img = Image.open(TEST_IMG_PATH + agency_record['icon.filename'])
icon = models.Image.create_image(
img,
file_extension=agency_record['icon.filename'].split(".")[-1],
security_marking=agency_record.get('icon.security_marking',
'UNCLASSIFIED'),
image_type=object_cache['ImageType.agency_icon'])
agency_object = models.Agency(
title=agency_record['title'],
short_name=agency_record['short_name'],
icon=icon)
agency_object.save()
object_cache['Agency.{}'.format(
agency_record['short_name'])] = agency_object
print('-----Took: {} ms'.format(time_ms() - section_start_time))
print('---Database Calls: {}'.format(show_db_calls(db_connection, False)))
############################################################################
# Tags
############################################################################
print('--Creating Tags')
section_start_time = time_ms()
with transaction.atomic():
tag_names = ['demo', 'example']
for tag_name in tag_names:
tag_object = models.Tag(name=tag_name)
tag_object.save()
object_cache['Tag.{}'.format(tag_name)] = tag_object
print('-----Took: {} ms'.format(time_ms() - section_start_time))
print('---Database Calls: {}'.format(show_db_calls(db_connection, False)))
############################################################################
# Profiles
############################################################################
print('--Creating Profiles')
section_start_time = time_ms()
with transaction.atomic():
for current_profile_data in profile_data:
access_control = json.dumps(current_profile_data['access_control'])
profile_obj = models.Profile.create_user(
current_profile_data['username'], # noqa: F841
email=current_profile_data['email'],
display_name=current_profile_data['display_name'],
bio=current_profile_data['bio'],
access_control=access_control,
organizations=current_profile_data['organizations'],
stewarded_organizations=current_profile_data[
'stewarded_organizations'],
groups=current_profile_data['groups'],
dn=current_profile_data['dn'])
object_cache['Profile.{}'.format(
current_profile_data['username'])] = profile_obj
print('-----Took: {} ms'.format(time_ms() - section_start_time))
print('---Database Calls: {}'.format(show_db_calls(db_connection, False)))
# -*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-
############################################################################
# System Notifications
############################################################################
print('--Creating System Notifications')
section_start_time = time_ms()
with transaction.atomic():
# create some notifications that expire next week
next_week = datetime.datetime.now() + datetime.timedelta(days=7)
eastern = pytz.timezone('US/Eastern')
next_week = eastern.localize(next_week)
n1 = notification_model_access.create_notification(
object_cache['Profile.{}'.format('wsmith')], # noqa: F841
next_week,
'System will be going down for approximately 30 minutes on X/Y at 1100Z'
)
n2 = notification_model_access.create_notification(
object_cache['Profile.{}'.format('julia')], # noqa: F841
next_week,
'System will be functioning in a degredaded state between 1800Z-0400Z on A/B'
)
# create some expired notifications
last_week = datetime.datetime.now() - datetime.timedelta(days=7)
last_week = eastern.localize(last_week)
n1 = notification_model_access.create_notification(
object_cache['Profile.{}'.format('wsmith')], # noqa: F841
last_week,
'System will be going down for approximately 30 minutes on C/D at 1700Z'
)
n2 = notification_model_access.create_notification(
object_cache['Profile.{}'.format('julia')], # noqa: F841
last_week,
'System will be functioning in a degredaded state between 2100Z-0430Z on F/G'
)
print('-----Took: {} ms'.format(time_ms() - section_start_time))
print('---Database Calls: {}'.format(show_db_calls(db_connection, False)))
# -*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-
# ===========================================================================
# Listings Icons
# ===========================================================================
library_entries = []
review_entries = []
print('--Creating Listings Icons')
listing_db_call_total = 0
section_start_time = time_ms()
with transaction.atomic(): # Maybe too large of a transaction
for current_listing_data in listings_data:
listing_obj = create_listing_icons(current_listing_data,
object_cache)
db_calls = show_db_calls(db_connection, False)
listing_db_call_total = listing_db_call_total + db_calls
print('---Took: {} ms'.format(time_ms() - section_start_time))
print('---Total Database Calls: {}'.format(listing_db_call_total))
# ===========================================================================
# Listings
# ===========================================================================
print('--Creating Listings')
listing_db_call_total = 0
temp_counter = 0
section_start_time = time_ms()
with transaction.atomic(): # Maybe too large of a transaction
for current_listing_data in listings_data:
temp_counter = temp_counter + 1
listing_obj = create_listing(current_listing_data, object_cache)
if current_listing_data['listing_review_batch']:
review_entry = {}
review_entry['listing_obj'] = listing_obj
review_entry['listing_review_batch'] = current_listing_data[
'listing_review_batch']
review_entries.append(review_entry)
listing_id = listing_obj.id
listing_library_entries = current_listing_data['library_entries']
if listing_library_entries:
for listing_library_entry in listing_library_entries:
listing_library_entry['listing_obj'] = listing_obj
library_entries.append(listing_library_entry)
db_calls = show_db_calls(db_connection, False)
listing_db_call_total = listing_db_call_total + db_calls
print('----{} \t DB Calls: {}'.format(
current_listing_data['listing']['title'], db_calls))
print('--Creating {} Listings took: {} ms'.format(
temp_counter,
time_ms() - section_start_time))
print('---Total Database Calls: {}'.format(listing_db_call_total))
############################################################################
# Reviews
############################################################################
print('--Creating Reviews')
section_start_time = time_ms()
with transaction.atomic():
for review_entry in review_entries:
create_listing_review_batch(review_entry['listing_obj'],
review_entry['listing_review_batch'],
object_cache)
print('-----Took: {} ms'.format(time_ms() - section_start_time))
print('---Database Calls: {}'.format(show_db_calls(db_connection, False)))
############################################################################
# Library (bookmark listings)
############################################################################
print('--Creating Library')
section_start_time = time_ms()
with transaction.atomic():
create_library_entries(library_entries, object_cache)
for library_entry in library_entries:
current_listing = library_entry['listing_obj']
current_listing_owner = current_listing.owners.first()
# print('={} Creating Notification for {}='.format(current_listing_owner.user.username, current_listing.title))
listing_notification = notification_model_access.create_notification(
current_listing_owner, # noqa: F841
next_week,
'{} update next week'.format(current_listing.title),
listing=current_listing)
print('-----Took: {} ms'.format(time_ms() - section_start_time))
print('---Database Calls: {}'.format(show_db_calls(db_connection, False)))
############################################################################
# Subscription
############################################################################
# Categories
# ['Books and Reference', 'Business', 'Communication', 'Education', 'Entertainment', 'Finance',
# 'Health and Fitness', 'Media and Video', 'Music and Audio', 'News',
# 'Productivity', 'Shopping', 'Sports', 'Tools', 'Weather']
# Tags
# ['demo', 'example', 'tag_0', 'tag_1', 'tag_2', 'tag_3',
# 'tag_4', 'tag_5', 'tag_6', 'tag_7', 'tag_8', 'tag_9']
# Usernames
# ['bigbrother', 'bigbrother2', 'khaleesi', 'wsmith', 'julia', 'obrien', 'aaronson',
# 'pmurt', 'hodor', 'jones', 'tammy', 'rutherford', 'noah', 'syme', 'abe',
# 'tparsons', 'jsnow', 'charrington', 'johnson']
subscriptions = [ # flake8: noqa
['bigbrother', 'category', 'Books and Reference'],
['bigbrother', 'category', 'Business']
]
############################################################################
# Recommendations
############################################################################
print('--Creating Recommendations')
section_start_time = time_ms()
sample_data_recommender = RecommenderDirectory()
sample_data_recommender.recommend('baseline,graph_cf')
print('-----Took: {} ms'.format(time_ms() - section_start_time))
print('---Database Calls: {}'.format(show_db_calls(db_connection, False)))
############################################################################
# End of script
############################################################################
total_end_time = time_ms()
print('Sample Data Generator took: {} ms'.format(total_end_time -
total_start_time))
def run():
"""
Creates basic sample data
"""
total_start_time = time_ms()
# Recreate Index Mapping
model_access_es.recreate_index_mapping()
# Create Groups
models.Profile.create_groups()
############################################################################
# Security Markings
############################################################################
unclass = 'UNCLASSIFIED' # noqa: F841
secret = 'SECRET' # noqa: F841
secret_n = 'SECRET//NOVEMBER' # noqa: F841
ts = 'TOP SECRET' # noqa: F841
ts_s = 'TOP SECRET//SIERRA' # noqa: F841
ts_st = 'TOP SECRET//SIERRA//TANGO' # noqa: F841
ts_stgh = 'TOP SECRET//SIERRA//TANGO//GOLF//HOTEL' # noqa: F841
ts_n = 'TOP SECRET//NOVEMBER' # noqa: F841
ts_sn = 'TOP SECRET//SIERRA//NOVEMBER' # noqa: F841
ts_stn = 'TOP SECRET//SIERRA//TANGO//NOVEMBER' # noqa: F841
ts_stghn = 'TOP SECRET//SIERRA//TANGO//GOLF//HOTEL//NOVEMBER' # noqa: F841
############################################################################
# Categories
############################################################################
with transaction.atomic():
categories_data = None
with open(os.path.join(TEST_DATA_PATH, 'categories.yaml'),
'r') as stream:
try:
categories_data = yaml.load(stream) # TODO: Use Stream API
except yaml.YAMLError as exc:
print(exc)
for current_category in categories_data['categories']:
current_category_obj = models.Category(
title=current_category['title'],
description=current_category['description'])
current_category_obj.save()
############################################################################
# Contact Types and Contacts
############################################################################
with transaction.atomic():
contact_data = None
with open(os.path.join(TEST_DATA_PATH, 'contacts.yaml'),
'r') as stream:
try:
contact_data = yaml.load(stream) # TODO: Use Stream API
except yaml.YAMLError as exc:
print(exc)
for contact_type in contact_data['contact_types']:
current_contact_type_obj = models.ContactType(name=contact_type)
current_contact_type_obj.save()
for current_contact in contact_data['contacts']:
if not models.Contact.objects.filter(
email=current_contact['email']).exists():
current_contact_obj = models.Contact(
name=current_contact['name'],
organization=current_contact['organization'],
contact_type=models.ContactType.objects.get(
name=current_contact['contact_type']),
email=current_contact['email'],
unsecure_phone=current_contact['unsecure_phone'],
secure_phone=current_contact['secure_phone'])
current_contact_obj.save()
############################################################################
# Listing Types
############################################################################
with transaction.atomic():
web_app = models.ListingType(title='Web Application',
description='web applications')
web_app.save()
widget = models.ListingType(title='Widget',
description='widget things')
widget.save()
desktop_app = models.ListingType(title='Desktop App',
description='desktop app')
desktop_app.save()
web_services = models.ListingType(title='Web Services',
description='web services')
web_services.save()
code_library = models.ListingType(title='Code Library',
description='code library')
code_library.save()
############################################################################
# Image Types
############################################################################
# Note: these image sizes do not represent those that should be used in
# production
with transaction.atomic():
small_icon_type = models.ImageType(name='small_icon',
max_size_bytes='4096')
small_icon_type.save()
large_icon_type = models.ImageType(name='large_icon',
max_size_bytes='8192')
large_icon_type.save()
banner_icon_type = models.ImageType(name='banner_icon',
max_size_bytes='2097152')
banner_icon_type.save()
large_banner_icon_type = models.ImageType(name='large_banner_icon',
max_size_bytes='2097152')
large_banner_icon_type.save()
small_screenshot_type = models.ImageType(name='small_screenshot',
max_size_bytes='1048576')
small_screenshot_type.save()
large_screenshot_type = models.ImageType(name='large_screenshot',
max_size_bytes='1048576')
large_screenshot_type.save()
intent_icon_type = models.ImageType(name='intent_icon',
max_size_bytes='2097152')
intent_icon_type.save()
agency_icon_type = models.ImageType(name='agency_icon',
max_size_bytes='2097152')
agency_icon_type.save()
############################################################################
# Intents
############################################################################
# TODO: more realistic data
with transaction.atomic():
img = Image.open(TEST_IMG_PATH + 'android.png')
icon = models.Image.create_image(img,
file_extension='png',
security_marking='UNCLASSIFIED',
image_type=intent_icon_type.name)
i = models.Intent(action='/application/json/view',
media_type='vnd.ozp-intent-v1+json.json',
label='view',
icon=icon)
i.save()
i = models.Intent(action='/application/json/edit',
media_type='vnd.ozp-intent-v1+json.json',
label='edit',
icon=icon)
i.save()
############################################################################
# Organizations
############################################################################
with transaction.atomic():
# Minitrue - Ministry of Truth
img = Image.open(TEST_IMG_PATH + 'ministry_of_truth.jpg')
icon = models.Image.create_image(img,
file_extension='jpg',
security_marking='UNCLASSIFIED',
image_type='agency_icon')
minitrue = models.Agency(title='Ministry of Truth',
short_name='Minitrue',
icon=icon)
minitrue.save()
# Minipax - Ministry of Peace
img = Image.open(TEST_IMG_PATH + 'ministry_of_peace.png')
icon = models.Image.create_image(img,
file_extension='png',
security_marking='UNCLASSIFIED',
image_type='agency_icon')
minipax = models.Agency(title='Ministry of Peace',
short_name='Minipax',
icon=icon)
minipax.save()
# Miniluv - Ministry of Love
img = Image.open(TEST_IMG_PATH + 'ministry_of_love.jpeg')
icon = models.Image.create_image(img,
file_extension='jpeg',
security_marking='UNCLASSIFIED',
image_type='agency_icon')
miniluv = models.Agency(title='Ministry of Love',
short_name='Miniluv',
icon=icon)
miniluv.save()
# Miniplen - Ministry of Plenty
img = Image.open(TEST_IMG_PATH + 'ministry_of_plenty.png')
icon = models.Image.create_image(img,
file_extension='png',
security_marking='UNCLASSIFIED',
image_type='agency_icon')
miniplenty = models.Agency(title='Ministry of Plenty',
short_name='Miniplen',
icon=icon)
miniplenty.save()
img = Image.open(TEST_IMG_PATH + 'ministry_of_plenty.png')
icon = models.Image.create_image(img,
file_extension='png',
security_marking='UNCLASSIFIED',
image_type='agency_icon')
test = models.Agency(title='Test', short_name='Test', icon=icon)
test.save()
img = Image.open(TEST_IMG_PATH + 'ministry_of_plenty.png')
icon = models.Image.create_image(img,
file_extension='png',
security_marking='UNCLASSIFIED',
image_type='agency_icon')
test1 = models.Agency(title='Test 1', short_name='Test 1', icon=icon)
test1.save()
img = Image.open(TEST_IMG_PATH + 'ministry_of_plenty.png')
icon = models.Image.create_image(img,
file_extension='png',
security_marking='UNCLASSIFIED',
image_type='agency_icon')
test2 = models.Agency(title='Test 2', short_name='Test2', icon=icon)
test2.save()
img = Image.open(TEST_IMG_PATH + 'ministry_of_plenty.png')
icon = models.Image.create_image(img,
file_extension='png',
security_marking='UNCLASSIFIED',
image_type='agency_icon')
test3 = models.Agency(title='Test 3', short_name='Test 3', icon=icon)
test3.save()
img = Image.open(TEST_IMG_PATH + 'ministry_of_plenty.png')
icon = models.Image.create_image(img,
file_extension='png',
security_marking='UNCLASSIFIED',
image_type='agency_icon')
test4 = models.Agency(title='Test 4', short_name='Test 4', icon=icon)
test4.save()
############################################################################
# Tags
############################################################################
with transaction.atomic():
demo = models.Tag(name='demo')
demo.save()
example = models.Tag(name='example')
example.save()
############################################################################
# Profiles
############################################################################
with transaction.atomic():
profile_ref = {}
profile_data = None
with open(os.path.join(TEST_DATA_PATH, 'profile.yaml'), 'r') as stream:
try:
profile_data = yaml.load(stream)
except yaml.YAMLError as exc:
print(exc)
for current_profile_data in profile_data:
access_control = json.dumps(current_profile_data['access_control'])
profile_ref[
current_profile_data['username']] = models.Profile.create_user(
current_profile_data['username'], # noqa: F841
email=current_profile_data['email'],
display_name=current_profile_data['display_name'],
bio=current_profile_data['bio'],
access_control=access_control,
organizations=current_profile_data['organizations'],
stewarded_organizations=current_profile_data[
'stewarded_organizations'],
groups=current_profile_data['groups'],
dn=current_profile_data['dn'])
# -*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-
############################################################################
# System Notifications
############################################################################
with transaction.atomic():
# create some notifications that expire next week
next_week = datetime.datetime.now() + datetime.timedelta(days=7)
eastern = pytz.timezone('US/Eastern')
next_week = eastern.localize(next_week)
n1 = notification_model_access.create_notification(
profile_ref['wsmith'], # noqa: F841
next_week,
'System will be going down for approximately 30 minutes on X/Y at 1100Z'
)
n2 = notification_model_access.create_notification(
profile_ref['julia'], # noqa: F841
next_week,
'System will be functioning in a degredaded state between 1800Z-0400Z on A/B'
)
# create some expired notifications
last_week = datetime.datetime.now() - datetime.timedelta(days=7)
last_week = eastern.localize(last_week)
n1 = notification_model_access.create_notification(
profile_ref['wsmith'], # noqa: F841
last_week,
'System will be going down for approximately 30 minutes on C/D at 1700Z'
)
n2 = notification_model_access.create_notification(
profile_ref['julia'], # noqa: F841
last_week,
'System will be functioning in a degredaded state between 2100Z-0430Z on F/G'
)
# -*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-
# ===========================================================================
# Listings
# Listings from File
# ===========================================================================
with transaction.atomic(): # Maybe too large of a transaction
listings_data = None
with open(os.path.join(TEST_DATA_PATH, 'listings.yaml'),
'r') as stream:
try:
listings_data = yaml.load(stream)
except yaml.YAMLError as exc:
print(exc)
library_entries = []
for current_listing_data in listings_data:
listing_obj = create_listing(current_listing_data)
listing_id = listing_obj.id
listing_library_entries = current_listing_data['library_entries']
if listing_library_entries:
for listing_library_entry in listing_library_entries:
listing_library_entry['listing_id'] = listing_id
library_entries.append(listing_library_entry)
############################################################################
# Library (bookmark listings)
############################################################################
with transaction.atomic():
create_library_entries(library_entries)
for current_id in [entry['listing_id'] for entry in library_entries]:
current_listing = models.Listing.objects.get(id=current_id)
current_listing_owner = current_listing.owners.first()
print('={} Creating Notification for {}='.format(
current_listing_owner.user.username, current_listing.title))
listing_notification = notification_model_access.create_notification(
current_listing_owner, # noqa: F841
next_week,
'{} update next week'.format(current_listing.title),
listing=current_listing)
############################################################################
# Subscription
############################################################################
# Categories
# ['Books and Reference', 'Business', 'Communication', 'Education', 'Entertainment', 'Finance',
# 'Health and Fitness', 'Media and Video', 'Music and Audio', 'News',
# 'Productivity', 'Shopping', 'Sports', 'Tools', 'Weather']
# Tags
# ['demo', 'example', 'tag_0', 'tag_1', 'tag_2', 'tag_3',
# 'tag_4', 'tag_5', 'tag_6', 'tag_7', 'tag_8', 'tag_9']
# Usernames
# ['bigbrother', 'bigbrother2', 'khaleesi', 'wsmith', 'julia', 'obrien', 'aaronson',
# 'pmurt', 'hodor', 'jones', 'tammy', 'rutherford', 'noah', 'syme', 'abe',
# 'tparsons', 'jsnow', 'charrington', 'johnson']
subscriptions = [ # flake8: noqa
['bigbrother', 'category', 'Books and Reference'],
['bigbrother', 'category', 'Business']
]
############################################################################
# Recommendations
############################################################################
sample_data_recommender = RecommenderDirectory()
sample_data_recommender.recommend('baseline,graph_cf')
total_end_time = time_ms()
print('Sample Data Generator took: {} ms'.format(total_end_time -
total_start_time))