class ReportBuilderTest(TestCase):
def setUp(self):
with open("test_report.pkl", 'rb') as f:
self.files = pickle.load(f)
self.report_builder = ReportBuilder(self.files,
"https://server.elscione.com/")
def test_total_size(self):
result = self.report_builder.get_total_size()
self.assertEqual(result, 426737457589)
def test_total_size_formatted(self):
result = self.report_builder.get_total_size_formatted()
self.assertEqual(result, "426.74 GB")
def test_ext_counts(self):
result = self.report_builder.get_ext_counts()
self.assertEqual(result["jpg"], 140)
self.assertEqual(result["zip"], 74)
def test_ext_sizes(self):
result = self.report_builder.get_ext_sizes()
self.assertEqual(result["jpg"], 140972306)
self.assertEqual(result["zip"], 9367400136)
def store_report(self, report_id, title):
report_saver = ReportSaver(self.files,title, ReportBuilder(self.files, self.base_url))
with open("static/reports/" + report_id + "_chart.json", "w") as f:
f.write(report_saver.to_json_chart())
with open("static/reports/" + report_id + ".json", "w") as f:
f.write(report_saver.to_json())
with open("static/reports/" + report_id + ".txt", "w") as f:
f.write(report_saver.to_link_list())
def setUp(self):
with open("test_report.pkl", 'rb') as f:
self.files = pickle.load(f)
self.report_saver = ReportSaver(self.files, "", ReportBuilder(self.files,"https://server.elscione.com/"))
with open("test_report.json", 'r') as f:
self.expected_json = f.read()
with open("test_report_chart.json", 'r') as f:
self.expected_json_chart = f.read()
if __name__ == "__main__":
if len(sys.argv) > 1:
command = sys.argv[1]
if command == "crawl":
if len(sys.argv) > 2:
url = sys.argv[2]
c = Crawler(url, True)
c.crawl()
print("Done")
r = ReportBuilder(c.files, url)
print(r.get_total_size_formatted())
if command == "mkreport":
if len(sys.argv) > 3:
url = sys.argv[2]
report_id = sys.argv[3]
c = Crawler(url, True)
c.crawl()
print("Done")
r = ReportBuilder(c.files, url)
print(r.get_total_size_formatted())
c.store_report(report_id, "")
if links[k]["type"] == "d":
print(links[k]["link"])
self.crawl(links[k]["link"])
else:
self.files.append(dict(link=links[k]["link"], size=links[k]["size"], ext=links[k]["ext"]))
def store_report(self, report_id, title):
report_saver = ReportSaver(self.files,title, ReportBuilder(self.files, self.base_url))
with open("static/reports/" + report_id + "_chart.json", "w") as f:
f.write(report_saver.to_json_chart())
with open("static/reports/" + report_id + ".json", "w") as f:
f.write(report_saver.to_json())
with open("static/reports/" + report_id + ".txt", "w") as f:
f.write(report_saver.to_link_list())
if __name__ == "__main__":
c = Crawler("http://www.downloads.imune.net/medicalbooks/", True)
c.crawl()
r = ReportBuilder(c.files, "http://www.downloads.imune.net/medicalbooks/")
print(r.get_total_size_formatted())
# for f in c.files:
# if f["size"] > 1000000:
# print(f)
c.store_report("000011", "test")
def setUp(self):
with open("test_report.pkl", 'rb') as f:
self.files = pickle.load(f)
self.report_builder = ReportBuilder(self.files,
"https://server.elscione.com/")