def get_urls(self, start_year, end_year):
source = SemiannualUrlSource(
{
"url_template": "https://transparency.twitter.com/content/dam/transparency-twitter/data/download-removal-requests/removal-requests-report-$start_month-$end_month-$report_year.csv",
"start_year": start_year, "end_year": end_year})
return source.get()
def get_urls(self, start_year, end_year):
source = SemiannualUrlSource({
"url_template":
"https://transparency.facebook.com/download/$report_year-$facebook_period/",
"start_year": start_year,
"end_year": end_year
})
return source.get()
def get_urls(self, start_year, end_year):
source = SemiannualUrlSource(
{"url_template": "https://www.snap.com/en-US/privacy/transparency/$date_end",
"start_year": start_year, "end_year": end_year})
# special case for current period - available at https://www.snap.com/en-US/privacy/transparency/
urls = source.get()
current_year = {'url': "https://www.snap.com/en-US/privacy/transparency/",
'report_start': "", 'report_end': ""}
urls.append(current_year)
return urls
def test_read_correctly_handles_date_end(self):
source = SemiannualUrlSource(
{"url_template": "https://transparency.example.com/$date_end",
'start_year': '2016', 'end_year': '2017'})
data = source.get()
expected = [
{"url": "https://transparency.example.com/2016-06-30", "report_start": "2016-01-01 00:00:00", "report_end": "2016-06-30 23:59:59"},
{"url": "https://transparency.example.com/2016-12-31", "report_start": "2016-07-01 00:00:00", "report_end": "2016-12-31 23:59:59"},
{"url": "https://transparency.example.com/2017-06-30", "report_start": "2017-01-01 00:00:00", "report_end": "2017-06-30 23:59:59"},
{"url": "https://transparency.example.com/2017-12-31", "report_start": "2017-07-01 00:00:00", "report_end": "2017-12-31 23:59:59"},
]
self.assertEqual(expected, data)
class TestSemiannualUrlSource(TransparencyTestCase):
def setUp(self):
self.source = SemiannualUrlSource({"url_template": "https://transparency.example.com/$report_year-$facebook_period-$start_month-$end_month", 'start_year': '2016', 'end_year': '2017'})
def test_read_correctly_handles_numbers(self):
data = self.source.get()
expected = [
{"url": "https://transparency.example.com/2016-H1-jan-jun", "report_start": "2016-01-01 00:00:00", "report_end": "2016-06-30 23:59:59"},
{"url": "https://transparency.example.com/2016-H2-jul-dec", "report_start": "2016-07-01 00:00:00", "report_end": "2016-12-31 23:59:59"},
{"url": "https://transparency.example.com/2017-H1-jan-jun", "report_start": "2017-01-01 00:00:00", "report_end": "2017-06-30 23:59:59"},
{"url": "https://transparency.example.com/2017-H2-jul-dec", "report_start": "2017-07-01 00:00:00", "report_end": "2017-12-31 23:59:59"},
]
self.assertEqual(expected, data)
def test_read_correctly_handles_date_end(self):
source = SemiannualUrlSource(
{"url_template": "https://transparency.example.com/$date_end",
'start_year': '2016', 'end_year': '2017'})
data = source.get()
expected = [
{"url": "https://transparency.example.com/2016-06-30", "report_start": "2016-01-01 00:00:00", "report_end": "2016-06-30 23:59:59"},
{"url": "https://transparency.example.com/2016-12-31", "report_start": "2016-07-01 00:00:00", "report_end": "2016-12-31 23:59:59"},
{"url": "https://transparency.example.com/2017-06-30", "report_start": "2017-01-01 00:00:00", "report_end": "2017-06-30 23:59:59"},
{"url": "https://transparency.example.com/2017-12-31", "report_start": "2017-07-01 00:00:00", "report_end": "2017-12-31 23:59:59"},
]
self.assertEqual(expected, data)
def setUp(self):
self.source = SemiannualUrlSource({"url_template": "https://transparency.example.com/$report_year-$facebook_period-$start_month-$end_month", 'start_year': '2016', 'end_year': '2017'})