def test_contact_dto_converter(self):
my_contact = Contact(
first_name="Jordan",
last_name="Degner",
phones=['4029813230'],
email="*****@*****.**",
position="Software Engineer",
)
print 'Converting a contact to a DTO.'
contact_dto = DTOConverter.to_dto(ContactDTO, my_contact)
print 'Testing equality...'
for attr, value in my_contact.__dict__.iteritems():
self.assertEqual(getattr(my_contact, attr),
getattr(contact_dto, attr),
"{0} attribute not equal".format(attr))
print 'Converting a DTO to a contact.'
my_contact = DTOConverter.from_dto(Contact, contact_dto)
print 'Testing equality...'
for attr, value in my_contact.__dict__.iteritems():
self.assertEqual(getattr(my_contact, attr),
getattr(contact_dto, attr),
"{0} attribute not equal".format(attr))
def test_org_dto_converter(self):
my_org = Organization(
name='Yoyodyne, Inc.',
address="1234 Yoyodyne Way, San Narciso, CA",
types=[OrgTypesEnum.GOVERNMENT, OrgTypesEnum.RESEARCH],
phone_numbers=["4026170423"],
emails=["*****@*****.**"],
contacts=[],
organization_url="www.yoyodyne.com",
page_rank_info=PageRankInfo(
total_with_self=10,
total=8,
references=[
PageRankVector(org_domain='yoyodyne.com',
count=2,
pages=[
UrlCountPair(
url='http://www.yoyodyne.com/',
count=2)
]),
PageRankVector(
org_domain='trystero.org',
count=4,
pages=[
UrlCountPair(url='http://www.yoyodyne.com/',
count=3),
UrlCountPair(
url='http://www.yoyodyne.com/contacts.php',
count=1)
]),
PageRankVector(org_domain='thurnandtaxis.info',
count=4,
pages=[
UrlCountPair(
url='http://www.yoyodyne.com/',
count=4)
])
]))
print 'Converting an organization model to a DTO.'
org_dto = DTOConverter.to_dto(OrganizationDTO, my_org)
print 'Testing equality...'
for attr, value in my_org.__dict__.iteritems():
if attr == 'page_rank_info':
self._compare_page_rank_info(my_org, org_dto)
else:
self.assertEqual(getattr(my_org, attr), getattr(org_dto, attr),
"{0} attribute not equal".format(attr))
print 'Converting a DTO to an organization.'
my_org = DTOConverter.from_dto(Organization, org_dto)
print 'Testing equality...'
for attr, value in my_org.__dict__.iteritems():
if attr == 'page_rank_info':
self._compare_page_rank_info(my_org, org_dto)
else:
self.assertEqual(getattr(my_org, attr), getattr(org_dto, attr),
"{0} attribute not equal".format(attr))
def bring_orgs_to_memory(self):
"""
Make a db call to get all organizations and trim it down to a smaller model to conserve space.
Returns:
small_orgs ([SmallOrganization]): A list of all organizations from the database, converted to
SmallOrganization objects to save some space.
"""
# grab all organizations
tmp_org_dtos = self.org_dao.all(
"id",
"page_rank_info.total",
"organization_url",
"page_rank_info.total_with_self",
"page_rank_info.references.count",
"page_rank_info.references.org_domain",
"page_rank_info.references.pages.url",
"page_rank_info.references.pages.count",
)
# convert the dtos to this smaller model
small_orgs = [
SmallOrganization(DTOConverter.from_dto(Organization, o), o.id) for o in tmp_org_dtos if o.organization_url
]
small_orgs = [o for o in small_orgs if o.org_domain]
# At this point, tmp_org_dtos gets gc'd and we're left with the minimum data needed for calculation
return small_orgs
def test_user_dao(self):
user_dto = DTOConverter.to_dto(UserDTO, self.user)
user_dao = self.ctx.get_object("UserDAO")
print 'Testing user creation ...'
user_dao.create_update(user_dto)
assert_user = user_dao.find(id=user_dto.id)
self.assertEqual(assert_user.id, user_dto.id)
print 'Testing user editing ...'
user_dto.first = "Byee"
user_dto.last = "Ybee"
user_dao.create_update(user_dto)
assert_user = user_dao.find(id=user_dto.id)
self.assertEqual(assert_user.id, user_dto.id)
self.assertEqual(assert_user.first_name, user_dto.first_name)
self.assertEqual(assert_user.last_name, user_dto.last_name)
print 'Testing user deletion ...'
user_dao.delete(user_dto)
assert_user = user_dao.find(id=user_dto.id)
self.assertTrue(assert_user is None)
print 'UserDAO tests passed'
def test_urlmetadata_dao(self):
url_dto = DTOConverter.to_dto(URLMetadataDTO, self.urlmetadata)
url_dao = self.ctx.get_object("URLMetadataDAO")
print 'Testing URL creation ...'
url_dao.create_update(url_dto)
assert_url = url_dao.find(id=url_dto.id,
url=url_dto.url)
self.assertEqual(assert_url.url, url_dto.url)
print 'Testing URL editing ...'
url_dto.url = "http://facebook.com"
url_dao.create_update(url_dto)
assert_url = url_dao.find(id=url_dto.id,
url=url_dto.url)
self.assertEqual(assert_url.url, url_dto.url)
print 'Testing URL deletion ...'
url_dao.delete(url_dto)
assert_url = url_dao.find(id=url_dto.id)
self.assertTrue(assert_url is None)
print 'URLMetadataDAO tests passed'
def store_organizations(self, small_orgs):
"""
Store small_orgs to database.
Arguments:
small_orgs ([SmallOrganization]): A list of SmallOrganization objects to be stored to the database.
"""
org_dtos = []
# create list of dtos
for i in range(0, len(small_orgs)):
org_model = self._small_org_to_org_model(small_orgs[i])
org_dto = DTOConverter.to_dto(OrganizationDTO, org_model)
org_dto.id = small_orgs[i].id
org_dtos.append(org_dto)
# store
self.org_dao.page_rank_store(org_dtos)
# set documents without ranks to max
empty_dtos = self.org_dao.findmany(page_rank__exists=False)
for dto in empty_dtos:
dto.page_rank = pow(2, 63) - 1
dto.page_rank_weight = 0.0
self.org_dao.create_update(dto)
def store_organizations(self, small_orgs):
"""
Store small_orgs to database.
Arguments:
small_orgs ([SmallOrganization]): A list of SmallOrganization objects to be stored to the database.
"""
org_dtos = []
# create list of dtos
for i in range(0, len(small_orgs)):
org_model = self._small_org_to_org_model(small_orgs[i])
org_dto = DTOConverter.to_dto(OrganizationDTO, org_model)
org_dto.id = small_orgs[i].id
org_dtos.append(org_dto)
# store
self.org_dao.page_rank_store(org_dtos)
# set documents without ranks to max
empty_dtos = self.org_dao.findmany(page_rank__exists=False)
for dto in empty_dtos:
dto.page_rank = pow(2, 63) - 1
dto.page_rank_weight = 0.0
self.org_dao.create_update(dto)
def test_user_dao(self):
user_dto = DTOConverter.to_dto(UserDTO, self.user)
user_dao = self.ctx.get_object("UserDAO")
print 'Testing user creation ...'
user_dao.create_update(user_dto)
assert_user = user_dao.find(id=user_dto.id)
self.assertEqual(assert_user.id, user_dto.id)
print 'Testing user editing ...'
user_dto.first = "Byee"
user_dto.last = "Ybee"
user_dao.create_update(user_dto)
assert_user = user_dao.find(id=user_dto.id)
self.assertEqual(assert_user.id, user_dto.id)
self.assertEqual(assert_user.first_name, user_dto.first_name)
self.assertEqual(assert_user.last_name, user_dto.last_name)
print 'Testing user deletion ...'
user_dao.delete(user_dto)
assert_user = user_dao.find(id=user_dto.id)
self.assertTrue(assert_user is None)
print 'UserDAO tests passed'
def put_url(self, u):
"""
Puts a new URL in the database.
Arguments:
u (URLMetadata): The URL to be placed in the database.
"""
url_dto = DTOConverter.to_dto(URLMetadataDTO, u)
self.dao().create_update(url_dto)
def test_merge_records(self):
contact_dto = DTOConverter.to_dto(ContactDTO, self.contact)
contact_dao = self.ctx.get_object("ContactDAO")
print 'Saving initial record ...'
contact_dao.create_update(contact_dto)
print 'Creating a duplicate and attempting an insert ...'
new_contact = Contact(email="*****@*****.**",
phones=['4029813230'])
new_contact_dto = DTOConverter.to_dto(ContactDTO, new_contact)
contact_dao.create_update(new_contact_dto)
print 'Asserting that the old contact was updated'
assert_contact = contact_dao.find(id=contact_dto.id)
self.assertEqual(assert_contact.phones, new_contact_dto.phones)
print 'Merge records tests passed'
def test_merge_records(self):
contact_dto = DTOConverter.to_dto(ContactDTO, self.contact)
contact_dao = self.ctx.get_object("ContactDAO")
print 'Saving initial record ...'
contact_dao.create_update(contact_dto)
print 'Creating a duplicate and attempting an insert ...'
new_contact = Contact(email="*****@*****.**",
phones=['4029813230'])
new_contact_dto = DTOConverter.to_dto(ContactDTO, new_contact)
contact_dao.create_update(new_contact_dto)
print 'Asserting that the old contact was updated'
assert_contact = contact_dao.find(id=contact_dto.id)
self.assertEqual(assert_contact.phones, new_contact_dto.phones)
print 'Merge records tests passed'
def test_urlfrontier(self):
for i in range(5):
self.urlfrontier_set_up()
start_time = datetime.utcnow()
url_dao = self.dao_ctx.get_object("URLMetadataDAO")
print 'Creating 2000 URLs'
url_list = []
for x in range(2000):
url_list.append(
URLMetadata(url="http://test" + str(x) + ".com",
last_visited=(start_time - timedelta(days=x))))
print 'Converting these URLs to DTOs'
url_dtos = [
DTOConverter.to_dto(URLMetadataDTO, u) for u in url_list
]
print 'Saving...'
for dto in url_dtos:
url_dao.create_update(dto)
print 'Create the URLFrontier'
frontier = self.frontier_ctx.get_object("URLFrontier")
print 'Start the cache process'
frontier.start_cache_process()
print '"Create" a second frontier and verify that the frontier is a singleton'
frontier2 = self.frontier_ctx.get_object("URLFrontier")
self.assertEqual(frontier, frontier2)
print 'Wait for the cache to populate and get the least recently visited URL'
old_url = frontier.next_url()
print 'Assert that the last visited date matches the last element in urllist'
self.assertEqual(url_list[1999].url, old_url.url)
print 'Push an even older URL'
oldest_url = URLMetadata(url="http://test2001.com",
last_visited=(start_time -
timedelta(days=x + 1)))
frontier.put_url(oldest_url)
print 'Empty the cache'
frontier.empty_cache()
print 'Wait a couple more seconds and verify that the next URL is the oldest one'
next_url = frontier.next_url()
self.assertEqual(oldest_url.url, next_url.url)
print 'Stop the cache process'
frontier.terminate_cache_process()
self.urlfrontier_tear_down()
def test_urlfrontier(self):
for i in range(5):
self.urlfrontier_set_up()
start_time = datetime.utcnow()
url_dao = self.dao_ctx.get_object("URLMetadataDAO")
print 'Creating 2000 URLs'
url_list = []
for x in range(2000):
url_list.append(URLMetadata(url="http://test" + str(x) + ".com",
last_visited=(start_time - timedelta(days=x))))
print 'Converting these URLs to DTOs'
url_dtos = [DTOConverter.to_dto(URLMetadataDTO, u) for u in url_list]
print 'Saving...'
for dto in url_dtos:
url_dao.create_update(dto)
print 'Create the URLFrontier'
frontier = self.frontier_ctx.get_object("URLFrontier")
print 'Start the cache process'
frontier.start_cache_process()
print '"Create" a second frontier and verify that the frontier is a singleton'
frontier2 = self.frontier_ctx.get_object("URLFrontier")
self.assertEqual(frontier, frontier2)
print 'Wait for the cache to populate and get the least recently visited URL'
old_url = frontier.next_url()
print 'Assert that the last visited date matches the last element in urllist'
self.assertEqual(url_list[1999].url, old_url.url)
print 'Push an even older URL'
oldest_url = URLMetadata(url="http://test2001.com",
last_visited=(start_time - timedelta(days=x + 1)))
frontier.put_url(oldest_url)
print 'Empty the cache'
frontier.empty_cache()
print 'Wait a couple more seconds and verify that the next URL is the oldest one'
next_url = frontier.next_url()
self.assertEqual(oldest_url.url, next_url.url)
print 'Stop the cache process'
frontier.terminate_cache_process()
self.urlfrontier_tear_down()
def set_up_url_metadata_scraper_test(self):
"""Set up database for URLMetadataScraper test. Not called automatically"""
urlmetadata = URLMetadata(
url="http://www.google.com",
last_visited=datetime(1, 1, 1),
update_freq=5,
score=0,
checksum=Binary('154916369406075238760605425088915003118'),
)
urlmetadata2 = URLMetadata(
checksum=Binary('199553381546012383114562002951261892300'),
last_visited=datetime(1, 1, 1),
update_freq=1,
url='http://www.halftheskymovement.org/partners'
)
ctx = ApplicationContext(TestableDAOContext())
self.url_dto = DTOConverter.to_dto(URLMetadataDTO, urlmetadata)
self.url_dto2 = DTOConverter.to_dto(URLMetadataDTO, urlmetadata2)
self.url_dao = ctx.get_object("URLMetadataDAO")
self.url_dao.create_update(self.url_dto)
self.url_dao.create_update(self.url_dto2)
def test_organization_dao_field_weights(self):
print "Checking OrganizationDAO._field_weights for consistency with OrganizationDTO"
org_dto = DTOConverter.to_dto(OrganizationDTO, self.organization)
org_dao = self.ctx.get_object("OrganizationDAO")
for key in org_dto:
if key == "id":
continue
self.assertIn(key, org_dao._field_weights.keys(),
"ERROR: a field was added to DTO but not to _field_weights,"
"a member of OrganizationDAO")
total = 0.0
for key, value in org_dao._field_weights.iteritems():
total += value
self.assertEqual(1.0, total, "ERROR: OrganizationDAO._field_weights do not add to 1.0")
def test_contact_dto_converter(self):
my_contact = Contact(first_name="Jordan",
last_name="Degner",
phones=['4029813230'],
email="*****@*****.**",
position="Software Engineer",
)
print 'Converting a contact to a DTO.'
contact_dto = DTOConverter.to_dto(ContactDTO, my_contact)
print 'Testing equality...'
for attr, value in my_contact.__dict__.iteritems():
self.assertEqual(getattr(my_contact, attr), getattr(contact_dto, attr),
"{0} attribute not equal".format(attr))
print 'Converting a DTO to a contact.'
my_contact = DTOConverter.from_dto(Contact, contact_dto)
print 'Testing equality...'
for attr, value in my_contact.__dict__.iteritems():
self.assertEqual(getattr(my_contact, attr), getattr(contact_dto, attr),
"{0} attribute not equal".format(attr))
def test_user_dao_field_weights(self):
print "Checking UserDAO._field_weights for consistency with UserDTO"
user_dto = DTOConverter.to_dto(UserDTO, self.user)
user_dao = self.ctx.get_object("UserDAO")
for key in user_dto:
if key == "id":
continue
self.assertIn(key, user_dao._field_weights.keys(),
"ERROR: a field was added to DTO but not to _field_weights,"
"a member of UserDAO")
total = 0.0
for key, value in user_dao._field_weights.iteritems():
total += value
self.assertEqual(1.0, total, "ERROR: UserDAO._field_weights do not add to 1.0")
def test_contact_dao(self):
contact_dto = DTOConverter.to_dto(ContactDTO, self.contact)
contact_dao = self.ctx.get_object("ContactDAO")
print 'Testing contact creation ...'
contact_dao.create_update(contact_dto)
assert_contact = contact_dao.find(id=contact_dto.id,
first_name=contact_dto.first_name,
last_name=contact_dto.last_name,
email=contact_dto.email)
self.assertEqual(assert_contact.first_name, contact_dto.first_name)
self.assertEqual(assert_contact.last_name, contact_dto.last_name)
self.assertEqual(assert_contact.email, contact_dto.email)
print 'Testing contact text search ...'
assert_contacts = contact_dao.findmany(search='jordan degner',
num_elements=10)
self.assertEqual(len(assert_contacts), 1)
self.assertEqual(assert_contacts[0].first_name, contact_dto.first_name)
assert_contacts = contact_dao.findmany(search='bee yee',
num_elements=10)
self.assertEqual(len(assert_contacts), 0)
assert_contacts = contact_dao.findmany(search='@gmail',
search_fields=[
'email',
],
num_elements=10)
self.assertEqual(len(assert_contacts), 1)
self.assertEqual(assert_contacts[0].first_name, contact_dto.first_name)
print 'Testing contact editing ...'
contact_dto.first_name = "Djordan"
contact_dao.create_update(contact_dto)
assert_contact = contact_dao.find(id=contact_dto.id,
first_name=contact_dto.first_name)
self.assertEqual(assert_contact.first_name, contact_dto.first_name)
print 'Testing contact deletion ...'
contact_dao.delete(contact_dto)
assert_contact = contact_dao.find(id=contact_dto.id)
self.assertTrue(assert_contact is None)
print 'ContactDAO tests passed'
def test_user_dao_field_weights(self):
print "Checking UserDAO._field_weights for consistency with UserDTO"
user_dto = DTOConverter.to_dto(UserDTO, self.user)
user_dao = self.ctx.get_object("UserDAO")
for key in user_dto:
if key == "id":
continue
self.assertIn(
key, user_dao._field_weights.keys(),
"ERROR: a field was added to DTO but not to _field_weights,"
"a member of UserDAO")
total = 0.0
for key, value in user_dao._field_weights.iteritems():
total += value
self.assertEqual(1.0, total,
"ERROR: UserDAO._field_weights do not add to 1.0")
def test_item_converter(self):
ctx = ApplicationContext(TestableDAOContext())
print 'Creating organization and contact item.'
org = ctx.get_object('OrganizationDAO')
org_dto = OrganizationDTO(name="Univerisityee of Nyeebraska-Lincoln")
org.create_update(org_dto)
org_model = DTOConverter.from_dto(Organization, org_dto)
contact_item = ScrapedContact(
first_name='Bee',
last_name='Yee',
organization={'name': "Univerisityee of Nyeebraska-Lincoln"})
print 'Converting contact to model.'
converter = ctx.get_object('ModelConverter')
model_contact = converter.to_model(Contact, contact_item)
self.assertEqual(model_contact.organization.name, org_model.name)
def test_item_converter(self):
ctx = ApplicationContext(TestableDAOContext())
print 'Creating organization and contact item.'
org = ctx.get_object('OrganizationDAO')
org_dto = OrganizationDTO(name="Univerisityee of Nyeebraska-Lincoln")
org.create_update(org_dto)
org_model = DTOConverter.from_dto(Organization, org_dto)
contact_item = ScrapedContact(first_name='Bee',
last_name='Yee',
organization={'name': "Univerisityee of Nyeebraska-Lincoln"}
)
print 'Converting contact to model.'
converter = ctx.get_object('ModelConverter')
model_contact = converter.to_model(Contact, contact_item)
self.assertEqual(model_contact.organization.name, org_model.name)
def test_organization_dao_field_weights(self):
print "Checking OrganizationDAO._field_weights for consistency with OrganizationDTO"
org_dto = DTOConverter.to_dto(OrganizationDTO, self.organization)
org_dao = self.ctx.get_object("OrganizationDAO")
for key in org_dto:
if key == "id":
continue
self.assertIn(
key, org_dao._field_weights.keys(),
"ERROR: a field was added to DTO but not to _field_weights,"
"a member of OrganizationDAO")
total = 0.0
for key, value in org_dao._field_weights.iteritems():
total += value
self.assertEqual(
1.0, total,
"ERROR: OrganizationDAO._field_weights do not add to 1.0")
def test_publication_dao(self):
pub_dto = DTOConverter.to_dto(PublicationDTO, self.publication)
pub_dao = self.ctx.get_object("PublicationDAO")
print 'Testing publication creation ...'
pub_dao.create_update(pub_dto)
assert_pub = pub_dao.find(id=pub_dto.id, title=pub_dto.title)
self.assertEqual(assert_pub.title, pub_dto.title)
self.assertEqual(assert_pub.authors, pub_dto.authors)
self.assertEqual(assert_pub.publisher, pub_dto.publisher)
print 'Testing publication text search ...'
assert_pubs = pub_dao.findmany(search='book of yee degner ADAMS',
num_elements=10)
self.assertEqual(len(assert_pubs), 1)
self.assertEqual(assert_pubs[0].title, pub_dto.title)
assert_pubs = pub_dao.findmany(search='nosuchpublication',
num_elements=10)
self.assertEqual(len(assert_pubs), 0)
assert_pubs = pub_dao.findmany(search='sam adams',
search_fields=[
'authors',
],
num_elements=10)
self.assertEqual(len(assert_pubs), 1)
self.assertEqual(assert_pubs[0].title, pub_dto.title)
print 'Testing publication editing ...'
pub_dto.title = "The Book of Mee"
pub_dao.create_update(pub_dto)
assert_pub = pub_dao.find(id=pub_dto.id, title=pub_dto.title)
self.assertEqual(assert_pub.title, pub_dto.title)
print 'Testing publication deletion ...'
pub_dao.delete(pub_dto)
assert_pub = pub_dao.find(id=pub_dto.id)
self.assertTrue(assert_pub is None)
print 'PublicationDAO tests passed'
def test_contact_dao(self):
contact_dto = DTOConverter.to_dto(ContactDTO, self.contact)
contact_dao = self.ctx.get_object("ContactDAO")
print 'Testing contact creation ...'
contact_dao.create_update(contact_dto)
assert_contact = contact_dao.find(id=contact_dto.id,
first_name=contact_dto.first_name,
last_name=contact_dto.last_name,
email=contact_dto.email)
self.assertEqual(assert_contact.first_name, contact_dto.first_name)
self.assertEqual(assert_contact.last_name, contact_dto.last_name)
self.assertEqual(assert_contact.email, contact_dto.email)
print 'Testing contact text search ...'
assert_contacts = contact_dao.findmany(search='jordan degner',
num_elements=10)
self.assertEqual(len(assert_contacts), 1)
self.assertEqual(assert_contacts[0].first_name, contact_dto.first_name)
assert_contacts = contact_dao.findmany(search='bee yee', num_elements=10)
self.assertEqual(len(assert_contacts), 0)
assert_contacts = contact_dao.findmany(search='@gmail', search_fields=['email', ], num_elements=10)
self.assertEqual(len(assert_contacts), 1)
self.assertEqual(assert_contacts[0].first_name, contact_dto.first_name)
print 'Testing contact editing ...'
contact_dto.first_name = "Djordan"
contact_dao.create_update(contact_dto)
assert_contact = contact_dao.find(id=contact_dto.id,
first_name=contact_dto.first_name)
self.assertEqual(assert_contact.first_name, contact_dto.first_name)
print 'Testing contact deletion ...'
contact_dao.delete(contact_dto)
assert_contact = contact_dao.find(id=contact_dto.id)
self.assertTrue(assert_contact is None)
print 'ContactDAO tests passed'
def request_organization(request):
"""
Sends a request to the Request Organization page if the user is logged in.
Returns:
A rendered page containing the Request Organization form.
"""
if 'user_id' not in request.session:
logger.error('Bad request made for organization seed without login')
return unauthorized(request)
else:
user_id = request.session['user_id']
form = RequestOrgForm(request.POST or None)
error = ''
success = ''
if request.method == 'POST':
if form.is_valid():
url = form.cleaned_data['url']
dao = ctx.get_object('URLMetadataDAO')
try:
metadata = URLMetadata(url=url,
domain=UrlUtility.get_domain(url))
except ValueError:
error = "Oops! We don't recognize that domain. Please try another."
if not error:
try:
dto = DTOConverter.to_dto(URLMetadataDTO, metadata)
dao.create_update(dto)
logger.info(
'Org seed with url={0} requested by user={1}'.format(
url, user_id))
success = 'Your request has been sent successfully!'
except:
error = 'Something went wrong with your request. Please try again later.'
return render(request, 'organization/request_organization.html', {
'form': form,
'success': success,
'error': error
})
def test_publication_dao(self):
pub_dto = DTOConverter.to_dto(PublicationDTO, self.publication)
pub_dao = self.ctx.get_object("PublicationDAO")
print 'Testing publication creation ...'
pub_dao.create_update(pub_dto)
assert_pub = pub_dao.find(id=pub_dto.id,
title=pub_dto.title)
self.assertEqual(assert_pub.title, pub_dto.title)
self.assertEqual(assert_pub.authors, pub_dto.authors)
self.assertEqual(assert_pub.publisher, pub_dto.publisher)
print 'Testing publication text search ...'
assert_pubs = pub_dao.findmany(search='book of yee degner ADAMS',
num_elements=10)
self.assertEqual(len(assert_pubs), 1)
self.assertEqual(assert_pubs[0].title, pub_dto.title)
assert_pubs = pub_dao.findmany(search='nosuchpublication', num_elements=10)
self.assertEqual(len(assert_pubs), 0)
assert_pubs = pub_dao.findmany(search='sam adams', search_fields=['authors', ], num_elements=10)
self.assertEqual(len(assert_pubs), 1)
self.assertEqual(assert_pubs[0].title, pub_dto.title)
print 'Testing publication editing ...'
pub_dto.title = "The Book of Mee"
pub_dao.create_update(pub_dto)
assert_pub = pub_dao.find(id=pub_dto.id,
title=pub_dto.title)
self.assertEqual(assert_pub.title, pub_dto.title)
print 'Testing publication deletion ...'
pub_dao.delete(pub_dto)
assert_pub = pub_dao.find(id=pub_dto.id)
self.assertTrue(assert_pub is None)
print 'PublicationDAO tests passed'
def setUp(self):
with MockDBConnection() as db:
db.dropall()
self.ctx = ApplicationContext(TestablePageRankContext())
with open(os.path.join(RESOURCES_DIRECTORY, 'organizations'), mode='r') as to_read:
org_models = pickle.load(to_read)
# freeze these out b/c we didn't store them
for model in org_models:
del model.contacts
org_dtos = [DTOConverter.to_dto(OrganizationDTO, o) for o in org_models]
org_dao = self.ctx.get_object('OrganizationDAO')
for dto in org_dtos:
org_dao.create_update(dto)
with open(os.path.join(RESOURCES_DIRECTORY, 'ranked_organizations'), mode='r') as to_read:
self.assert_models = pickle.load(to_read)
def _monitor_cache(dao, max_size, cache, job_queue, job_cond, fill_cond, empty_cond,
req_doms, blk_doms, srt_list, logger_lock):
while True:
try:
with job_cond:
next_job = job_queue.get(block=False)
except Empty:
with job_cond:
job_cond.wait(1)
try:
next_job = job_queue.get(block=False)
except Empty:
continue
if next_job == CacheJobs.Fill:
with logger_lock:
logger.info('Filling the cache')
with fill_cond:
urls = dao().findmany_by_domains(max_size - cache.qsize(),
req_doms, blk_doms, srt_list)
for u in urls:
url_obj = DTOConverter.from_dto(URLMetadata, u)
try:
cache.put(url_obj)
except Full:
break
fill_cond.notify_all()
elif next_job == CacheJobs.Empty:
with logger_lock:
logger.info('Emptying the cache')
with empty_cond:
while True:
try:
cache.get(block=False)
except Empty:
break
empty_cond.notify()
def request_organization(request):
"""
Sends a request to the Request Organization page if the user is logged in.
Returns:
A rendered page containing the Request Organization form.
"""
if 'user_id' not in request.session:
logger.error('Bad request made for organization seed without login')
return unauthorized(request)
else:
user_id = request.session['user_id']
form = RequestOrgForm(request.POST or None)
error = ''
success = ''
if request.method == 'POST':
if form.is_valid():
url = form.cleaned_data['url']
dao = ctx.get_object('URLMetadataDAO')
try:
metadata = URLMetadata(url=url, domain=UrlUtility.get_domain(url))
except ValueError:
error = "Oops! We don't recognize that domain. Please try another."
if not error:
try:
dto = DTOConverter.to_dto(URLMetadataDTO, metadata)
dao.create_update(dto)
logger.info('Org seed with url={0} requested by user={1}'.format(url, user_id))
success = 'Your request has been sent successfully!'
except:
error = 'Something went wrong with your request. Please try again later.'
return render(request, 'organization/request_organization.html', {'form': form, 'success': success, 'error': error})
def test_page_rank(self):
print 'Creating PageRankPreprocessor'
prp = self.ctx.get_object('PageRankPreprocessor')
print 'Bring organizations to memory'
orgs = prp.bring_orgs_to_memory()
print 'Cleaning organizations'
orgs = prp.cleanup_data(orgs)
print 'Creating dat matrix'
matrix = prp.create_matrix(orgs)
self.assertIsNotNone(matrix)
print 'Creating the dampened google matrix'
matrix = google_matrix(matrix)
print 'Generating eigenvector'
vector = left_eigenvector(matrix)
print 'Creating PageRankPostprocessor'
post = self.ctx.get_object('PageRankPostprocessor')
print 'Assigning ranks to organizations'
orgs = post.give_orgs_ranks(orgs, vector)
print 'Storing organizations'
post.store_organizations(orgs)
dao = self.ctx.get_object('OrganizationDAO')
new_org_models = [DTOConverter.from_dto(Organization, o) for o in dao.all()]
self.assertEqual(len(new_org_models), len(self.assert_models),
"Error: returned model number different than expected")
for model in self.assert_models:
self._compare_assert_against_test(model, new_org_models)
def test_urlmetadata_dao(self):
url_dto = DTOConverter.to_dto(URLMetadataDTO, self.urlmetadata)
url_dao = self.ctx.get_object("URLMetadataDAO")
print 'Testing URL creation ...'
url_dao.create_update(url_dto)
assert_url = url_dao.find(id=url_dto.id, url=url_dto.url)
self.assertEqual(assert_url.url, url_dto.url)
print 'Testing URL editing ...'
url_dto.url = "http://facebook.com"
url_dao.create_update(url_dto)
assert_url = url_dao.find(id=url_dto.id, url=url_dto.url)
self.assertEqual(assert_url.url, url_dto.url)
print 'Testing URL deletion ...'
url_dao.delete(url_dto)
assert_url = url_dao.find(id=url_dto.id)
self.assertTrue(assert_url is None)
print 'URLMetadataDAO tests passed'
def test_org_dto_converter(self):
my_org = Organization(name='Yoyodyne, Inc.',
address="1234 Yoyodyne Way, San Narciso, CA",
types=[OrgTypesEnum.GOVERNMENT, OrgTypesEnum.RESEARCH],
phone_numbers=["4026170423"],
emails=["*****@*****.**"],
contacts=[],
organization_url="www.yoyodyne.com",
page_rank_info=PageRankInfo(
total_with_self=10,
total=8,
references=[
PageRankVector(
org_domain='yoyodyne.com',
count=2,
pages=[
UrlCountPair(
url='http://www.yoyodyne.com/',
count=2
)
]
),
PageRankVector(
org_domain='trystero.org',
count=4,
pages=[
UrlCountPair(
url='http://www.yoyodyne.com/',
count=3
),
UrlCountPair(
url='http://www.yoyodyne.com/contacts.php',
count=1
)
]
),
PageRankVector(
org_domain='thurnandtaxis.info',
count=4,
pages=[
UrlCountPair(
url='http://www.yoyodyne.com/',
count=4
)
]
)
]
)
)
print 'Converting an organization model to a DTO.'
org_dto = DTOConverter.to_dto(OrganizationDTO, my_org)
print 'Testing equality...'
for attr, value in my_org.__dict__.iteritems():
if attr == 'page_rank_info':
self._compare_page_rank_info(my_org, org_dto)
else:
self.assertEqual(getattr(my_org, attr), getattr(org_dto, attr),
"{0} attribute not equal".format(attr))
print 'Converting a DTO to an organization.'
my_org = DTOConverter.from_dto(Organization, org_dto)
print 'Testing equality...'
for attr, value in my_org.__dict__.iteritems():
if attr == 'page_rank_info':
self._compare_page_rank_info(my_org, org_dto)
else:
self.assertEqual(getattr(my_org, attr), getattr(org_dto, attr),
"{0} attribute not equal".format(attr))
def test_organization_dao(self):
org_dto = DTOConverter.to_dto(OrganizationDTO, self.organization)
org_dao = self.ctx.get_object("OrganizationDAO")
print 'Testing organization creation ...'
org_dao.create_update(org_dto)
assert_org = org_dao.find(id=org_dto.id,
name=org_dto.name,
organization_url=org_dto.organization_url,
email_key=org_dto.email_key,
emails=org_dto.emails,
phone_numbers=org_dto.phone_numbers,
facebook=org_dto.facebook,
twitter=org_dto.twitter,
address=org_dto.address,
keywords=org_dto.keywords,
types=org_dto.types[0],
)
self.assertEqual(assert_org.name, org_dto.name)
self.assertEqual(assert_org.organization_url, org_dto.organization_url)
self.assertEqual(assert_org.contacts, org_dto.contacts)
self.assertEqual(assert_org.email_key, org_dto.email_key)
self.assertEqual(assert_org.emails, org_dto.emails)
self.assertEqual(assert_org.phone_numbers, org_dto.phone_numbers)
self.assertEqual(assert_org.facebook, org_dto.facebook)
self.assertEqual(assert_org.twitter, org_dto.twitter)
self._compare_page_rank_info(assert_org, org_dto)
print 'Testing organization text search ...'
assert_orgs = org_dao.findmany(search='YeE university ers Religious govern secUTION ISFP Yeesy',
num_elements=10)
self.assertEqual(len(assert_orgs), 1)
self.assertEqual(assert_orgs[0].name, org_dto.name)
assert_orgs = org_dao.findmany(search='prevention advocacy', num_elements=10)
self.assertEqual(len(assert_orgs), 0)
assert_orgs = org_dao.findmany(search='religious', search_fields=['types', ], num_elements=10)
self.assertEqual(len(assert_orgs), 1)
self.assertEqual(assert_orgs[0].name, org_dto.name)
assert_orgs = org_dao.findmany(search='ISFP', search_fields=['keywords', ], num_elements=10)
self.assertEqual(len(assert_orgs), 1)
assert_orgs = org_dao.findmany(search='omaha', search_fields=['address', ], num_elements=10)
self.assertEqual(len(assert_orgs), 1)
self.assertEqual(assert_orgs[0].name, org_dto.name)
print 'Testing organization editing ...'
org_dto.name = "Yee Universityee"
org_dto.contacts = []
org_dao.create_update(org_dto)
assert_org = org_dao.find(id=org_dto.id,
name=u'Yee Universityee')
self.assertEqual(assert_org.name, org_dto.name)
self.assertEqual(assert_org.organization_url, org_dto.organization_url)
self.assertEqual(assert_org.contacts, org_dto.contacts)
print 'Testing organization deletion ...'
org_dao.delete(org_dto)
assert_org = org_dao.find(id=org_dto.id)
self.assertTrue(assert_org is None)
print 'OrganizationDAO tests passed'
def test_organization_dao(self):
org_dto = DTOConverter.to_dto(OrganizationDTO, self.organization)
org_dao = self.ctx.get_object("OrganizationDAO")
print 'Testing organization creation ...'
org_dao.create_update(org_dto)
assert_org = org_dao.find(
id=org_dto.id,
name=org_dto.name,
organization_url=org_dto.organization_url,
email_key=org_dto.email_key,
emails=org_dto.emails,
phone_numbers=org_dto.phone_numbers,
facebook=org_dto.facebook,
twitter=org_dto.twitter,
address=org_dto.address,
keywords=org_dto.keywords,
types=org_dto.types[0],
)
self.assertEqual(assert_org.name, org_dto.name)
self.assertEqual(assert_org.organization_url, org_dto.organization_url)
self.assertEqual(assert_org.contacts, org_dto.contacts)
self.assertEqual(assert_org.email_key, org_dto.email_key)
self.assertEqual(assert_org.emails, org_dto.emails)
self.assertEqual(assert_org.phone_numbers, org_dto.phone_numbers)
self.assertEqual(assert_org.facebook, org_dto.facebook)
self.assertEqual(assert_org.twitter, org_dto.twitter)
self._compare_page_rank_info(assert_org, org_dto)
print 'Testing organization text search ...'
assert_orgs = org_dao.findmany(
search='YeE university ers Religious govern secUTION ISFP Yeesy',
num_elements=10)
self.assertEqual(len(assert_orgs), 1)
self.assertEqual(assert_orgs[0].name, org_dto.name)
assert_orgs = org_dao.findmany(search='prevention advocacy',
num_elements=10)
self.assertEqual(len(assert_orgs), 0)
assert_orgs = org_dao.findmany(search='religious',
search_fields=[
'types',
],
num_elements=10)
self.assertEqual(len(assert_orgs), 1)
self.assertEqual(assert_orgs[0].name, org_dto.name)
assert_orgs = org_dao.findmany(search='ISFP',
search_fields=[
'keywords',
],
num_elements=10)
self.assertEqual(len(assert_orgs), 1)
assert_orgs = org_dao.findmany(search='omaha',
search_fields=[
'address',
],
num_elements=10)
self.assertEqual(len(assert_orgs), 1)
self.assertEqual(assert_orgs[0].name, org_dto.name)
print 'Testing organization editing ...'
org_dto.name = "Yee Universityee"
org_dto.contacts = []
org_dao.create_update(org_dto)
assert_org = org_dao.find(id=org_dto.id, name=u'Yee Universityee')
self.assertEqual(assert_org.name, org_dto.name)
self.assertEqual(assert_org.organization_url, org_dto.organization_url)
self.assertEqual(assert_org.contacts, org_dto.contacts)
print 'Testing organization deletion ...'
org_dao.delete(org_dto)
assert_org = org_dao.find(id=org_dto.id)
self.assertTrue(assert_org is None)
print 'OrganizationDAO tests passed'
def test_page_rank_merge(self):
org_dto = DTOConverter.to_dto(OrganizationDTO, self.organization)
org_dao = self.ctx.get_object("OrganizationDAO")
print 'Putting initial data in database'
org_dao.create_update(org_dto)
new_organization = OrganizationDTO(
name="Yee University",
organization_url="www.yee.com",
page_rank_info=PageRankInfoDTO(
total_with_self=18,
total=16,
references=[
PageRankVectorDTO(
org_domain='yee.com',
count=2,
pages=[
UrlCountPairDTO(url='http://www.yee.com/test.php',
count=2)
]),
PageRankVectorDTO(
org_domain='trystero.org',
count=12,
pages=[
UrlCountPairDTO(url='http://www.yee.com/',
count=2),
UrlCountPairDTO(
url='http://www.yee.com/contacts.php',
count=10)
]),
PageRankVectorDTO(
org_domain='philately.com',
count=4,
pages=[
UrlCountPairDTO(url='http://www.yee.com/test.php',
count=4)
])
]))
test_org = Organization(
name="Yee University",
organization_url="www.yee.com",
email_key="*****@*****.**",
emails=["*****@*****.**", "*****@*****.**"],
phone_numbers=[5555555555, "(555)555-5555"],
facebook="http://www.facebook.com/yee",
twitter="http://www.twitter.com/yee",
address="5124 Yeesy Street Omaha, NE 68024",
keywords="intj enfp entp isfp enfj istj",
types=[
OrgTypesEnum.RELIGIOUS,
OrgTypesEnum.GOVERNMENT,
OrgTypesEnum.PROSECUTION,
],
page_rank_info=PageRankInfoDTO(
total_with_self=24,
total=20,
references=[
PageRankVectorDTO(
org_domain='yee.com',
count=4,
pages=[
UrlCountPairDTO(url='http://www.yee.com/',
count=2),
UrlCountPairDTO(url='http://www.yee.com/test.php',
count=2)
]),
PageRankVectorDTO(
org_domain='trystero.org',
count=12,
pages=[
UrlCountPairDTO(url='http://www.yee.com/',
count=2),
UrlCountPairDTO(
url='http://www.yee.com/contacts.php',
count=10)
]),
PageRankVectorDTO(org_domain='thurnandtaxis.info',
count=4,
pages=[
UrlCountPairDTO(
url='http://www.yee.com/',
count=4)
]),
PageRankVectorDTO(
org_domain='philately.com',
count=4,
pages=[
UrlCountPairDTO(url='http://www.yee.com/test.php',
count=4)
])
]))
print 'Merging new data'
org_dao.create_update(new_organization)
assert_org = org_dao.find(id=org_dto.id)
self.assertEqual(assert_org.name, test_org.name)
self.assertEqual(assert_org.organization_url,
test_org.organization_url)
self.assertEqual(assert_org.email_key, test_org.email_key)
self.assertEqual(assert_org.emails, test_org.emails)
self.assertEqual(assert_org.phone_numbers, test_org.phone_numbers)
self.assertEqual(assert_org.facebook, test_org.facebook)
self.assertEqual(assert_org.twitter, test_org.twitter)
self._compare_page_rank_info(assert_org, test_org)
print 'OrganizationDAO tests passed'
def signup(request):
"""
Sends a request to the Signup page.
Returns:
A rendered page of the Signup form if the user is not logged in already.
"""
if 'user_id' in request.session:
user_id = request.session['user_id']
logger.error('Bad request for signup made by user={0}'.format(user_id))
return HttpResponseRedirect('/')
error = ''
form = SignupForm(request.POST or None)
if request.method == 'POST':
user_dao = ctx.get_object('UserDAO')
if form.is_valid():
data = form.cleaned_data
password = make_password(data['password'])
new_user = User(first_name=data['first_name'],
last_name=data['last_name'],
email=data['email'],
password=password,
background=data['background'],
account_type=int(data['account_type']))
org_dao = ctx.get_object('OrganizationDAO')
if 'org_type' in data and data['org_type']:
new_user.org_type = int(data['org_type'])
if 'organization' in data and data['organization']:
existing_org = org_dao.find(name=data['organization'])
if existing_org:
new_user.organization = existing_org
else:
new_org = OrganizationDTO()
new_org.name = data['organization']
if new_user.org_type:
new_org.types.append(new_user.org_type)
else:
new_org.types.append(OrgTypesEnum.UNKNOWN)
new_user.organization = org_dao.create_update(new_org)
try:
user_dto = DTOConverter.to_dto(UserDTO, new_user)
ret_user = user_dao.create_update(user_dto)
request.session['name'] = str(new_user.first_name) + ' ' + str(
new_user.last_name)
request.session['user_id'] = str(ret_user.id)
request.session['account_type'] = str(ret_user.account_type)
request.session.set_expiry(SESSION_TIMEOUT)
return HttpResponseRedirect('/')
except:
logger.error('Error occurred during signup')
error = 'Oops! We had a little trouble signing you up. Please try again later.'
return render(request, 'user/signup.html', {'form': form, 'error': error})