def test_create_object_from_doc(self):
new_object = provider_batch_data.create_objects_from_doc(self.old_doc)
matching = ProviderBatchData.query.filter_by(provider="pmc").first()
assert_equals(matching.provider, "pmc")
assert_equals(matching.aliases, self.old_doc["aliases"])
def test_create_object_from_doc(self):
new_object = provider_batch_data.create_objects_from_doc(self.old_doc)
matching = ProviderBatchData.query.filter_by(provider="pmc").first()
assert_equals(matching.provider, "pmc")
assert_equals(matching.aliases, self.old_doc["aliases"])
def setUp(self):
ProviderTestCase.setUp(self)
self.db = setup_postgres_for_unittests(db, app)
sample_data_dump = open(SAMPLE_EXTRACT_METRICS_PAGE, "r").read()
sample_data_dump_different_month = open(
SAMPLE_EXTRACT_METRICS_PAGE_DIFFERENT_MONTH, "r").read()
test_monthly_data = [{
"_id": "abc",
"type": "provider_data_dump",
"provider": "pmc",
"raw": sample_data_dump,
"provider_raw_version": 1.0,
"created": "2012-11-29T07:34:01.126892",
"aliases": {
"pmid": ["111", "222"]
},
"min_event_date": "2012-10-01T07:34:01.126892",
"max_event_date": "2012-10-31T07:34:01.126892"
}, {
"_id": "def",
"type": "provider_data_dump",
"provider": "pmc",
"raw": sample_data_dump_different_month,
"provider_raw_version": 1.0,
"created": "2012-11-29T08:34:01.126892",
"aliases": {
"pmid": ["111"]
},
"min_event_date": "2012-01-01T07:34:01.126892",
"max_event_date": "2012-01-31T07:34:01.126892"
}, {
"_id": "abc123",
"raw":
"<pmc-web-stat><request year=\"2012\" month=\"10\" jrid=\"elife\" eissn=\"2050-084X\"></request><response status=\"0\" collection=\"eLife\"></response><articles><article id=\"PMC3463246\"><meta-data doi=\"10.7554/eLife.00013\" pmcid=\"PMC3463246\" pubmed-id=\"23066504\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00013\"/><usage unique-ip=\"1368\" full-text=\"1464\" pdf=\"722\" abstract=\"119\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"144\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3463247\"><meta-data doi=\"10.7554/eLife.00240\" pmcid=\"PMC3463247\" pubmed-id=\"23066507\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00240\"/><usage unique-ip=\"514\" full-text=\"606\" pdf=\"230\" abstract=\"0\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"9\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3465569\"><meta-data doi=\"10.7554/eLife.00242\" pmcid=\"PMC3465569\" pubmed-id=\"23066508\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00242\"/><usage unique-ip=\"473\" full-text=\"503\" pdf=\"181\" abstract=\"2\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"13\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3465570\"><meta-data doi=\"10.7554/eLife.00243\" pmcid=\"PMC3465570\" pubmed-id=\"23066509\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00243\"/><usage unique-ip=\"547\" full-text=\"636\" pdf=\"227\" abstract=\"1\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"56\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3466591\"><meta-data doi=\"10.7554/eLife.00065\" pmcid=\"PMC3466591\" pubmed-id=\"23066506\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00065\"/><usage unique-ip=\"2516\" full-text=\"2804\" pdf=\"1583\" abstract=\"195\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"405\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3466783\"><meta-data doi=\"10.7554/eLife.00007\" pmcid=\"PMC3466783\" pubmed-id=\"23066503\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00007\"/><usage unique-ip=\"1331\" full-text=\"1412\" pdf=\"898\" abstract=\"224\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"109\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3467772\"><meta-data doi=\"10.7554/eLife.00270\" pmcid=\"PMC3467772\" pubmed-id=\"23066510\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00270\"/><usage unique-ip=\"1396\" full-text=\"1776\" pdf=\"625\" abstract=\"4\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"0\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3470722\"><meta-data doi=\"10.7554/eLife.00286\" pmcid=\"PMC3470722\" pubmed-id=\"23071903\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00286\"/><usage unique-ip=\"909\" full-text=\"1030\" pdf=\"376\" abstract=\"6\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"0\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3479833\"><meta-data doi=\"10.7554/eLife.00031\" pmcid=\"PMC3479833\" pubmed-id=\"23110253\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00031\"/><usage unique-ip=\"154\" full-text=\"126\" pdf=\"87\" abstract=\"26\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"13\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3470409\"><meta-data doi=\"10.7554/eLife.00048\" pmcid=\"PMC3470409\" pubmed-id=\"23066505\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00048\"/><usage unique-ip=\"1250\" full-text=\"1361\" pdf=\"911\" abstract=\"237\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"317\" supp-data=\"4\" cited-by=\"0\"/></article><article id=\"PMC3482692\"><meta-data doi=\"10.7554/eLife.00102\" pmcid=\"PMC3482692\" pubmed-id=\"23110254\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00102\"/><usage unique-ip=\"259\" full-text=\"232\" pdf=\"133\" abstract=\"36\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"8\" supp-data=\"3\" cited-by=\"0\"/></article><article id=\"PMC3482687\"><meta-data doi=\"10.7554/eLife.00281\" pmcid=\"PMC3482687\" pubmed-id=\"23110255\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00281\"/><usage unique-ip=\"75\" full-text=\"53\" pdf=\"47\" abstract=\"0\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"1\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3482686\"><meta-data doi=\"10.7554/eLife.00005\" pmcid=\"PMC3482686\" pubmed-id=\"23110252\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00005\"/><usage unique-ip=\"324\" full-text=\"249\" pdf=\"263\" abstract=\"71\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"93\" supp-data=\"17\" cited-by=\"0\"/></article></articles></pmc-web-stat>",
"max_event_date": "2012-10-31T07:34:01.126892",
"provider": "pmc",
"aliases": {
"pmid": [
"23066504", "23066507", "23066508", "23066509", "23066506",
"23066503", "23066510", "23071903", "23110253", "23066505",
"23110254", "23110255", "23110252"
]
},
"provider_raw_version": 1,
"type": "provider_data_dump",
"min_event_date": "2012-10-02T07:34:01.126892",
"created": "2012-11-29T09:34:01.126892"
}]
#print test_monthly_data
for doc in test_monthly_data:
new_object = provider_batch_data.create_objects_from_doc(doc)
print new_object
self.provider = pmc.Pmc()
print "after pmc"
def write_batch_dict(batch_dict):
logger.info("connected to postgres at " + os.getenv("POSTGRESQL_URL"))
new_object = provider_batch_data.create_objects_from_doc(batch_dict)
print "added to db if it wasn't already there"
print "current batch data:"
matches = provider_batch_data.ProviderBatchData.query.filter_by(provider="pmc").order_by("min_event_date").all()
for match in matches:
print match
def write_batch_dict(batch_dict):
logger.info("connected to postgres at " + os.getenv("POSTGRESQL_URL"))
new_object = provider_batch_data.create_objects_from_doc(batch_dict)
print "added to db if it wasn't already there"
print "current batch data:"
matches = provider_batch_data.ProviderBatchData.query.filter_by(provider="pmc").order_by("min_event_date").all()
for match in matches:
print match
def setUp(self):
ProviderTestCase.setUp(self)
self.db = setup_postgres_for_unittests(db, app)
sample_data_dump = open(SAMPLE_EXTRACT_METRICS_PAGE, "r").read()
sample_data_dump_different_month = open(SAMPLE_EXTRACT_METRICS_PAGE_DIFFERENT_MONTH, "r").read()
test_monthly_data = [
{"_id": "abc",
"type": "provider_data_dump",
"provider": "pmc",
"raw": sample_data_dump,
"provider_raw_version": 1.0,
"created": "2012-11-29T07:34:01.126892",
"aliases": {"pmid":["111", "222"]},
"min_event_date": "2012-10-01T07:34:01.126892",
"max_event_date": "2012-10-31T07:34:01.126892"
},
{"_id": "def",
"type": "provider_data_dump",
"provider": "pmc",
"raw": sample_data_dump_different_month,
"provider_raw_version": 1.0,
"created": "2012-11-29T08:34:01.126892",
"aliases": {"pmid":["111"]},
"min_event_date": "2012-01-01T07:34:01.126892",
"max_event_date": "2012-01-31T07:34:01.126892"
},
{
"_id": "abc123",
"raw": "<pmc-web-stat><request year=\"2012\" month=\"10\" jrid=\"elife\" eissn=\"2050-084X\"></request><response status=\"0\" collection=\"eLife\"></response><articles><article id=\"PMC3463246\"><meta-data doi=\"10.7554/eLife.00013\" pmcid=\"PMC3463246\" pubmed-id=\"23066504\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00013\"/><usage unique-ip=\"1368\" full-text=\"1464\" pdf=\"722\" abstract=\"119\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"144\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3463247\"><meta-data doi=\"10.7554/eLife.00240\" pmcid=\"PMC3463247\" pubmed-id=\"23066507\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00240\"/><usage unique-ip=\"514\" full-text=\"606\" pdf=\"230\" abstract=\"0\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"9\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3465569\"><meta-data doi=\"10.7554/eLife.00242\" pmcid=\"PMC3465569\" pubmed-id=\"23066508\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00242\"/><usage unique-ip=\"473\" full-text=\"503\" pdf=\"181\" abstract=\"2\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"13\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3465570\"><meta-data doi=\"10.7554/eLife.00243\" pmcid=\"PMC3465570\" pubmed-id=\"23066509\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00243\"/><usage unique-ip=\"547\" full-text=\"636\" pdf=\"227\" abstract=\"1\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"56\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3466591\"><meta-data doi=\"10.7554/eLife.00065\" pmcid=\"PMC3466591\" pubmed-id=\"23066506\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00065\"/><usage unique-ip=\"2516\" full-text=\"2804\" pdf=\"1583\" abstract=\"195\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"405\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3466783\"><meta-data doi=\"10.7554/eLife.00007\" pmcid=\"PMC3466783\" pubmed-id=\"23066503\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00007\"/><usage unique-ip=\"1331\" full-text=\"1412\" pdf=\"898\" abstract=\"224\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"109\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3467772\"><meta-data doi=\"10.7554/eLife.00270\" pmcid=\"PMC3467772\" pubmed-id=\"23066510\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00270\"/><usage unique-ip=\"1396\" full-text=\"1776\" pdf=\"625\" abstract=\"4\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"0\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3470722\"><meta-data doi=\"10.7554/eLife.00286\" pmcid=\"PMC3470722\" pubmed-id=\"23071903\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00286\"/><usage unique-ip=\"909\" full-text=\"1030\" pdf=\"376\" abstract=\"6\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"0\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3479833\"><meta-data doi=\"10.7554/eLife.00031\" pmcid=\"PMC3479833\" pubmed-id=\"23110253\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00031\"/><usage unique-ip=\"154\" full-text=\"126\" pdf=\"87\" abstract=\"26\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"13\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3470409\"><meta-data doi=\"10.7554/eLife.00048\" pmcid=\"PMC3470409\" pubmed-id=\"23066505\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00048\"/><usage unique-ip=\"1250\" full-text=\"1361\" pdf=\"911\" abstract=\"237\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"317\" supp-data=\"4\" cited-by=\"0\"/></article><article id=\"PMC3482692\"><meta-data doi=\"10.7554/eLife.00102\" pmcid=\"PMC3482692\" pubmed-id=\"23110254\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00102\"/><usage unique-ip=\"259\" full-text=\"232\" pdf=\"133\" abstract=\"36\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"8\" supp-data=\"3\" cited-by=\"0\"/></article><article id=\"PMC3482687\"><meta-data doi=\"10.7554/eLife.00281\" pmcid=\"PMC3482687\" pubmed-id=\"23110255\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00281\"/><usage unique-ip=\"75\" full-text=\"53\" pdf=\"47\" abstract=\"0\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"1\" supp-data=\"0\" cited-by=\"0\"/></article><article id=\"PMC3482686\"><meta-data doi=\"10.7554/eLife.00005\" pmcid=\"PMC3482686\" pubmed-id=\"23110252\" pub-year=\"2012\" volume=\"1\" issue=\"\" first-page=\"e00005\"/><usage unique-ip=\"324\" full-text=\"249\" pdf=\"263\" abstract=\"71\" scanned-summary=\"0\" scanned-page-browse=\"0\" figure=\"93\" supp-data=\"17\" cited-by=\"0\"/></article></articles></pmc-web-stat>",
"max_event_date": "2012-10-31T07:34:01.126892",
"provider": "pmc",
"aliases": {
"pmid": [
"23066504",
"23066507",
"23066508",
"23066509",
"23066506",
"23066503",
"23066510",
"23071903",
"23110253",
"23066505",
"23110254",
"23110255",
"23110252"
]
},
"provider_raw_version": 1,
"type": "provider_data_dump",
"min_event_date": "2012-10-02T07:34:01.126892",
"created": "2012-11-29T09:34:01.126892"
}
]
#print test_monthly_data
for doc in test_monthly_data:
new_object = provider_batch_data.create_objects_from_doc(doc)
print new_object
self.provider = pmc.Pmc()
print "after pmc"
