def main(credentials, data_set_config, query, options, start_at, end_at):
data_type = data_set_config['data-type']
data = Fetcher(credentials, query, start_at, end_at).fetch()
parsed_data = Parser(query, options, data_type).parse(data)
Pusher(data_set_config, options).push(parsed_data)
def main(credentials, data_set_config, query, options, start_at, end_at):
client = create_client(credentials)
documents = query_documents_for(client, query, options,
data_set_config['data-type'], start_at,
end_at)
Pusher(data_set_config, options).push(documents)
def main(credentials, data_set_config, query, options, start_at, end_at):
if start_at or end_at:
print 'Cannot backfill realtime collectors'
sys.exit(1)
data = Fetcher(credentials, query).fetch()
parsed_data = Parser().parse(data)
Pusher(data_set_config, options).push(parsed_data)
def main(credentials, data_set_config, query, options, start_at, end_at):
client = create_client(credentials)
assert "filtersets" in options, "`filtersets` must be specified"
documents = []
original_filters = query.get("filters", [])
def insert_ga(filters):
# Insert ga: on everything but the first.
def gen():
for i, f in enumerate(filters):
if i == 0:
yield f
else:
yield "ga:" + f
return list(gen())
for filters in options["filtersets"]:
all_filters = insert_ga(original_filters + filters)
query["filters"] = [";".join(all_filters)]
options.setdefault("additionalFields", {}).update(
{"department": get_department(filters)})
ds = query_documents_for(
client, query, options, data_set_config['data-type'],
start_at, end_at
)
documents.extend(ds)
Pusher(data_set_config, options).push(documents)
def collect_parse_and_push(self, data_set_config, options):
raw_json_data = self.collect()
parsed_data = self.build_parser(data_set_config,
options).parse(raw_json_data)
Pusher(data_set_config, options).push(parsed_data)
def test_does_not_empty_data_set(self, mock_from_config):
mock_data_set_client = Mock()
mock_from_config.return_value = mock_data_set_client
Pusher(self.data_set_config, {}).push(self.data)
assert_that(mock_data_set_client.empty_data_set.called,
equal_to(False))
def test_empties_data_set(self, mock_from_config):
mock_data_set_client = Mock()
mock_from_config.return_value = mock_data_set_client
Pusher(self.data_set_config, {'empty-data-set': True}).push(self.data)
assert_that(mock_data_set_client.empty_data_set.called, equal_to(True))
def test_pushes_nothing_when_empty_data(self, mock_from_config):
mock_data_set_client = Mock()
mock_from_config.return_value = mock_data_set_client
Pusher(self.data_set_config, {}).push([])
assert_that(mock_data_set_client.post.called, equal_to(False))
def test_pushes_with_chunk_in_options(self, mock_from_config):
mock_data_set_client = Mock()
mock_from_config.return_value = mock_data_set_client
Pusher(self.data_set_config, {'chunk-size': 98}).push(self.data)
mock_data_set_client.post.assert_called_once_with(self.data,
chunk_size=98)
def test_pushes_with_default_chunk_100(self, mock_from_config):
mock_data_set_client = Mock()
mock_from_config.return_value = mock_data_set_client
Pusher(self.data_set_config, {}).push(self.data)
mock_data_set_client.post.assert_called_once_with(self.data,
chunk_size=100)
def test_Pusher_init_sets_up_client_correctly(self, mock_from_config):
Pusher(self.data_set_config, {})
mock_from_config.assert_called_once_with(self.data_set_config)