def test_line_delimited(self):
with open('fixtures/line_delimited.json') as json_file:
outline = make_outline(json_file, True, None)
expected = {
'map': [
('message_Revised', 'message.Revised'),
('message_original', 'message.original'),
('source_author', 'source.author'),
]
}
self.assertEqual(outline, expected)
def test_line_delimited(self):
with open('fixtures/line_delimited.json') as json_file:
outline = make_outline(json_file, True, None)
expected = {
'map': [
('message_Revised', 'message.Revised'),
('message_original', 'message.original'),
('source_author', 'source.author'),
]
}
self.assertEqual(outline, expected)
def test_basic(self):
with open('fixtures/data.json') as json_file:
outline = make_outline(json_file, False, 'nodes')
expected = {
'collection': 'nodes',
'map': [
('message_Revised', 'message.Revised'),
('message_original', 'message.original'),
('source_author', 'source.author'),
]
}
self.assertEqual(outline, expected)
def test_basic(self):
with open('fixtures/data.json') as json_file:
outline = make_outline(json_file, False, 'nodes')
expected = {
'collection': 'nodes',
'map': [
('message_Revised', 'message.Revised'),
('message_original', 'message.original'),
('source_author', 'source.author'),
]
}
self.assertEqual(outline, expected)
def test_deeply_nested(self):
with open('fixtures/deeply_nested.json') as json_file:
outline = make_outline(json_file, False, 'nodes')
expected = {
'collection': 'nodes',
'map': [
('one_0_two_0_three_0', 'one.0.two.0.three.0'),
('one_0_two_0_three_1', 'one.0.two.0.three.1'),
('one_0_two_0_three_2', 'one.0.two.0.three.2'),
('one_0_two_1_three_0', 'one.0.two.1.three.0'),
('one_0_two_1_three_1', 'one.0.two.1.three.1'),
('one_0_two_1_three_2', 'one.0.two.1.three.2'),
]
}
self.assertEqual(outline, expected)
def test_deeply_nested(self):
with open('fixtures/deeply_nested.json') as json_file:
outline = make_outline(json_file, False, 'nodes')
expected = {
'collection': 'nodes',
'map': [
('one_0_two_0_three_0', 'one.0.two.0.three.0'),
('one_0_two_0_three_1', 'one.0.two.0.three.1'),
('one_0_two_0_three_2', 'one.0.two.0.three.2'),
('one_0_two_1_three_0', 'one.0.two.1.three.0'),
('one_0_two_1_three_1', 'one.0.two.1.three.1'),
('one_0_two_1_three_2', 'one.0.two.1.three.2'),
]
}
self.assertEqual(outline, expected)
def test_different_keys_per_row(self):
"Outline should contain the union of the keys."
with open('fixtures/different_keys_per_row.json') as json_file:
outline = make_outline(json_file, False, 'nodes')
expected = {
'collection': 'nodes',
'map': [
('tags_0', 'tags.0'),
('tags_1', 'tags.1'),
('tags_2', 'tags.2'),
('that', 'that'),
('theother', 'theother'),
('this', 'this'),
]
}
self.assertEqual(outline, expected)
def test_different_keys_per_row(self):
"Outline should contain the union of the keys."
with open('fixtures/different_keys_per_row.json') as json_file:
outline = make_outline(json_file, False, 'nodes')
expected = {
'collection': 'nodes',
'map': [
('tags_0', 'tags.0'),
('tags_1', 'tags.1'),
('tags_2', 'tags.2'),
('that', 'that'),
('theother', 'theother'),
('this', 'this'),
]
}
self.assertEqual(outline, expected)
def main():
common.start()
common.CONF['data_to_fetch'] = common.ask('Fetch Tweet Data or User Data? 1/Tweet 2/User',
answer=list, default='2', options=[1, 2])
request_params = {}
if common.CONF['data_to_fetch'] == '2':
print("You requested User Data")
common.CONF['query'] = common.ask('Search terms? ' +
'Found here: https://dev.twitter.com/rest/public/search',
answer=common.str_compat)
request_params['q'] = common.CONF['query']
url = 'https://api.twitter.com/1.1/users/search.json?'
else:
print("You requested Tweet Data")
common.CONF['query'] = common.ask('Search terms? ' +
'Found here: https://dev.twitter.com/rest/public/search',
answer=common.str_compat)
request_params['q'] = common.CONF['query']
result_data_type = common.ask('Type of search results? 1/Popular 2/Recent 3/Mixed',
answer=list, default='1', options=[1, 2, 3])
request_params['result_type'] = common.RESULT_MAP[result_data_type]
location = common.ask('Location? Eg. 1600 Amphitheatre Parkway, Mountain View, CA',
answer=common.str_compat, default=" ")
if location.strip():
encode_location = urllib.urlencode({'address': location})
response_location = requests.get('https://maps.googleapis.com/maps/api/geocode/json?' +
encode_location)
try:
location_json = response_location.json()
location_data = location_json['results'][0]['geometry']['location']
location_array = [str(value) for value in location_data.itervalues()]
if location_array:
radius_mi = common.ask('Distance to search within in miles',
answer=common.str_compat)
location_array.append(radius_mi + u'mi')
common.CONF['geocode'] = ",".join(location_array)
request_params['geocode'] = common.CONF['geocode']
except:
print('Unable to fetch lat and long for location')
# date = common.ask('Include tweets before? eg. 2015-07-19', answer=dateObject, default=" ")
# if date.strip():
# request_params['until'] = date
url = 'https://api.twitter.com/1.1/search/tweets.json?'
output_file_name = common.ask('Output file name',
answer=common.str_compat, default="output")
print('Sending request to API...')
json_search_data = get_json_data(url, request_params, common.CONF['consumer_key'],
common.CONF['consumer_secret'],
common.CONF['api_key'], common.CONF['api_secret'])
if json_search_data['nodes']:
print('API response received.')
with open('json_dump.json', 'w') as outfile:
json.dump(json_search_data, outfile)
outline = gen_outline.make_outline('json_dump.json', False, 'nodes')
print('Generating outline file..')
outfile = 'outline.json'
with open(outfile, 'w') as f:
json.dump(outline, f, indent=2, sort_keys=True)
print('Outline file generation done.')
with open(outfile) as f:
key_map = json.load(f)
loader = json2csv.Json2Csv(key_map)
outfile = output_file_name + '.csv'
if os.path.isfile(outfile):
os.remove(outfile)
print('Writing to %s' % outfile)
with open('json_dump.json') as f:
loader.load(f)
loader.write_csv(filename=outfile, make_strings=True)
print('Output file generated.')
else:
print('Search yield no results')