def test_all_with_pipeline_data(self):
sqlite_in_memory = SQLiteConfiguration("")
data_controller = DataController(sqlite_in_memory)
data_controller.run_schema()
crawler = FileCrawler()
crawl_controller = CrawlController(crawler, "d0b7f41f-ad37-3b47-ab70-9feac35557cc")
analysis_controller = AnalysisController()
pipeline_controller = PipelineController(
data_controller=data_controller,
crawl_controller=crawl_controller,
analysis_controller=analysis_controller
)
# Clean run
pipeline_controller.execute()
# Test slim profiles
slim_profiles = data_controller.api_data.get_profiles_slim()
self.assertEqual(len(slim_profiles), 19)
# Test fields
fields = data_controller.api_data.get_fields()
self.assertEqual(len(fields), 14)
def test_all_with_pipeline_data(self):
sqlite_in_memory = SQLiteConfiguration("")
data_controller = DataController(sqlite_in_memory)
data_controller.run_schema()
crawler = FileCrawler()
crawl_controller = CrawlController(
crawler, "d0b7f41f-ad37-3b47-ab70-9feac35557cc")
analysis_controller = AnalysisController()
pipeline_controller = PipelineController(
data_controller=data_controller,
crawl_controller=crawl_controller,
analysis_controller=analysis_controller)
# Clean run
pipeline_controller.execute()
# Test slim profiles
slim_profiles = data_controller.api_data.get_profiles_slim()
self.assertEqual(len(slim_profiles), 19)
# Test fields
fields = data_controller.api_data.get_fields()
self.assertEqual(len(fields), 14)
def __init__(self, *args, **kwargs):
super(MendeleyCache, self).__init__(*args, **kwargs)
# Read configuration
self.configuration = ServiceConfiguration()
self.configuration.load()
log.info("Configuration has been loaded")
# Create service controllers
self.data_controller = DataController(self.configuration.database)
self.data_controller.assert_schema()
log.info("Schema has been checked")
# Create crawler based on configuration
self.crawler = None
""":type : AbstractCrawler"""
if not self.configuration.uses_mendeley:
log.info("Pipeline uses FileCrawler")
self.crawler = FileCrawler()
else:
from mendeleycache.crawler.sdk_crawler import SDKCrawler
log.info("Pipeline uses SDKCrawler".format(
app_id=self.configuration.crawler.app_id,
app_secret=self.configuration.crawler.app_secret))
self.crawler = SDKCrawler(
app_id=self.configuration.crawler.app_id,
app_secret=self.configuration.crawler.app_secret)
# Create the pipeline
self.crawl_controller = CrawlController(
self.crawler, self.configuration.crawler.research_group)
self.analysis_controller = AnalysisController()
self.pipeline_controller = PipelineController(
data_controller=self.data_controller,
crawl_controller=self.crawl_controller,
analysis_controller=self.analysis_controller)
log.info("Pipeline has been initialized")
# Create the routing controllers
self.fields_controller = FieldsController(self, self.data_controller)
self.profiles_controller = ProfilesController(self,
self.data_controller,
self.configuration.cache)
self.publications_controller = DocumentsController(
self, self.data_controller)
self.cache_controller = CacheController(self, self.data_controller,
self.pipeline_controller,
self.configuration)
self.root_controller = RootController(self, self.data_controller,
self.configuration)
# Register the routes
self.register_routes()
log.info("Routes have been registered")
log.info("MendeleyCache has been initialized")
def test_execute(self):
sqlite_in_memory = SQLiteConfiguration("")
data_controller = DataController(sqlite_in_memory)
data_controller.run_schema()
crawler = FileCrawler()
crawl_controller = CrawlController(
crawler, "d0b7f41f-ad37-3b47-ab70-9feac35557cc")
analysis_controller = AnalysisController()
pipeline_controller = PipelineController(
data_controller=data_controller,
crawl_controller=crawl_controller,
analysis_controller=analysis_controller)
# Clean run shall not crash
pipeline_controller.execute()
# Second run shall not crash either
pipeline_controller.execute()
# Third run shall not crash either
pipeline_controller.execute()
def test_execute(self):
sqlite_in_memory = SQLiteConfiguration("")
data_controller = DataController(sqlite_in_memory)
data_controller.run_schema()
crawler = FileCrawler()
crawl_controller = CrawlController(crawler, "d0b7f41f-ad37-3b47-ab70-9feac35557cc")
analysis_controller = AnalysisController()
pipeline_controller = PipelineController(
data_controller=data_controller,
crawl_controller=crawl_controller,
analysis_controller=analysis_controller
)
# Clean run shall not crash
pipeline_controller.execute()
# Second run shall not crash either
pipeline_controller.execute()
# Third run shall not crash either
pipeline_controller.execute()
def sample_pipeline(app_id=None, app_secret=None):
from mendeleycache.crawler.sdk_crawler import SDKCrawler
sqlite_in_memory = SQLiteConfiguration("")
data_controller = DataController(sqlite_in_memory)
data_controller.run_schema()
crawler = None
if app_id is None and app_secret is None:
crawler = FileCrawler()
else:
crawler = SDKCrawler(app_id=app_id, app_secret=app_secret)
crawl_controller = CrawlController(crawler,
"d0b7f41f-ad37-3b47-ab70-9feac35557cc")
analysis_controller = AnalysisController()
pipeline_controller = PipelineController(
data_controller=data_controller,
crawl_controller=crawl_controller,
analysis_controller=analysis_controller)
# Clean run shall not crash
pipeline_controller.execute()
rows = data_controller.engine.execute("SELECT * FROM profile").fetchall()
print()
print("Profiles:")
for row in rows:
print(row)
rows = data_controller.engine.execute(
"SELECT * FROM cache_profile").fetchall()
print()
print("Cache profiles:")
for row in rows:
print(row)
rows = data_controller.engine.execute(
"SELECT id, owner_mendeley_id, title, authors, tags FROM document"
).fetchall()
print()
print("Documents:")
for row in rows:
print(row)
rows = data_controller.engine.execute(
"SELECT * FROM cache_document").fetchall()
print()
print("Cache documents:")
for row in rows:
print(row)
rows = data_controller.engine.execute(
"SELECT * FROM cache_field").fetchall()
print()
print("Cache fields:")
for row in rows:
print(row)
rows = data_controller.engine.execute(
"SELECT * FROM cache_document_has_cache_field").fetchall()
print()
print("LINK: Cache document -> Cache field:")
for row in rows:
print(row)
rows = data_controller.engine.execute(
"SELECT * FROM cache_profile_has_cache_document").fetchall()
print()
print("LINK: Cache profile -> Cache document:")
for row in rows:
print(row)
print()
if not config.uses_mendeley:
log.info("Pipeline uses FileCrawler")
crawler = FileCrawler()
else:
from mendeleycache.crawler.sdk_crawler import SDKCrawler
log.info("Pipeline uses SDKCrawler".format(
app_id=config.crawler.app_id,
app_secret=config.crawler.app_secret))
crawler = SDKCrawler(app_id=config.crawler.app_id,
app_secret=config.crawler.app_secret)
crawl_controller = CrawlController(crawler,
config.crawler.research_group)
analysis_controller = AnalysisController()
pipeline_controller = PipelineController(
data_controller=data_controller,
crawl_controller=crawl_controller,
analysis_controller=analysis_controller)
pipeline_controller.execute()
# Show file-crawler sample data
elif command == "sample-file-pipeline":
sample_pipeline()
# Trigger the pipeline with the mendeley sdk crawler
elif command == "sample-sdk-pipeline":
if not len(sys.argv) >= 4:
log.critical(
"Missing arguments: mendeleycache.runner sample-sdk-pipeline {app-id} {app-secret}"
)
app_id = sys.argv[2]
else:
from mendeleycache.crawler.sdk_crawler import SDKCrawler
log.info("Pipeline uses SDKCrawler".format(
app_id=config.crawler.app_id,
app_secret=config.crawler.app_secret
))
crawler = SDKCrawler(
app_id=config.crawler.app_id,
app_secret=config.crawler.app_secret
)
crawl_controller = CrawlController(crawler, config.crawler.research_group)
analysis_controller = AnalysisController()
pipeline_controller = PipelineController(
data_controller=data_controller,
crawl_controller=crawl_controller,
analysis_controller=analysis_controller
)
pipeline_controller.execute()
# Show file-crawler sample data
elif command == "sample-file-pipeline":
sample_pipeline()
# Trigger the pipeline with the mendeley sdk crawler
elif command == "sample-sdk-pipeline":
if not len(sys.argv) >= 4:
log.critical("Missing arguments: mendeleycache.runner sample-sdk-pipeline {app-id} {app-secret}")
app_id = sys.argv[2]
app_secret = sys.argv[3]
def sample_pipeline(app_id=None, app_secret=None):
from mendeleycache.crawler.sdk_crawler import SDKCrawler
sqlite_in_memory = SQLiteConfiguration("")
data_controller = DataController(sqlite_in_memory)
data_controller.run_schema()
crawler = None
if app_id is None and app_secret is None:
crawler = FileCrawler()
else:
crawler = SDKCrawler(app_id=app_id, app_secret=app_secret)
crawl_controller = CrawlController(crawler, "d0b7f41f-ad37-3b47-ab70-9feac35557cc")
analysis_controller = AnalysisController()
pipeline_controller = PipelineController(
data_controller=data_controller,
crawl_controller=crawl_controller,
analysis_controller=analysis_controller
)
# Clean run shall not crash
pipeline_controller.execute()
rows = data_controller.engine.execute("SELECT * FROM profile").fetchall()
print()
print("Profiles:")
for row in rows:
print(row)
rows = data_controller.engine.execute("SELECT * FROM cache_profile").fetchall()
print()
print("Cache profiles:")
for row in rows:
print(row)
rows = data_controller.engine.execute("SELECT id, owner_mendeley_id, title, authors, tags FROM document").fetchall()
print()
print("Documents:")
for row in rows:
print(row)
rows = data_controller.engine.execute("SELECT * FROM cache_document").fetchall()
print()
print("Cache documents:")
for row in rows:
print(row)
rows = data_controller.engine.execute("SELECT * FROM cache_field").fetchall()
print()
print("Cache fields:")
for row in rows:
print(row)
rows = data_controller.engine.execute("SELECT * FROM cache_document_has_cache_field").fetchall()
print()
print("LINK: Cache document -> Cache field:")
for row in rows:
print(row)
rows = data_controller.engine.execute("SELECT * FROM cache_profile_has_cache_document").fetchall()
print()
print("LINK: Cache profile -> Cache document:")
for row in rows:
print(row)
print()
