def search_for_tweets(search_term):
twitter_url = 'https://api.twitter.com/1.1/search/tweets.json'
url = twurl.augment(twitter_url, {'q': search_term, 'count': 10})
connection = urllib2.urlopen(url)
data = connection.read()
js = json.loads(data)
html = ''
for item in js['statuses']:
archives = make_archive_dropdown(item['id'])
html += make_tweet(item, 'myTimeline', archives)
return html
def searchForTweets(searchTerm):
global session
twitter_url = 'https://api.twitter.com/1.1/search/tweets.json'
url = twurl.augment(twitter_url, {'q': searchTerm, 'count': 15}, session)
connection = urllib2.urlopen(url)
data = connection.read()
js = json.loads(data)
html = ''
for item in js['statuses']:
archives = makeArchiveDropdown(item)
html = html + makeTweet(item, 'myTimeline', archives)
return html
def spiderTwitter():
TWITTER_URL = 'https://api.twitter.com/1.1/friends/list.json'
conn = sqlite3.connect('spider.sqlite3')
cur = conn.cursor()
cur.execute('''
CREATE TABLE IF NOT EXISTS Twitter (name TEXT, retrieved INTEGER, friends INTEGER)''')
while True:
acct = raw_input('Enter a Twitter account, or quit: ')
if ( acct == 'quit' ) : break
if ( len(acct) < 1 ) :
cur.execute('SELECT name FROM Twitter WHERE retrieved = 0 LIMIT 1')
try:
acct = cur.fetchone()[0]
except:
print 'No unretrieved Twitter accounts found'
continue
url = twurl.augment(TWITTER_URL, {'screen_name': acct, 'count': '20'} )
print 'Retrieving', url
connection = urllib.urlopen(url)
data = connection.read()
headers = connection.info().dict
# print 'Remaining', headers['x-rate-limit-remaining']
js = json.loads(data)
# print json.dumps(js, indent=4)
cur.execute('UPDATE Twitter SET retrieved=1 WHERE name = ?', (acct, ) )
countnew = 0
countold = 0
for u in js['users'] :
friend = u['screen_name']
print friend
cur.execute('SELECT friends FROM Twitter WHERE name = ? LIMIT 1', (friend, ) )
try:
count = cur.fetchone()[0]
cur.execute('UPDATE Twitter SET friends = ? WHERE name = ?', (count+1, friend) )
countold = countold + 1
except:
cur.execute('''INSERT INTO Twitter (name, retrieved, friends) VALUES ( ?, 0, 1 )''', ( friend, ) )
countnew = countnew + 1
print 'New accounts=',countnew,' revisited=',countold
conn.commit()
cur.close()
import urllib.request, urllib.parse, urllib.error
import twurl
TWITTER_URL = "https://api.twitter.com/1.1/statuses/user_timeline.json"
while True:
print("")
acct = input("Enter Twitter Account:")
if len(acct) < 1:
break
url = twurl.augment(TWITTER_URL, {"screen_name": acct, "count": "2"})
print("Retrieving", url)
connection = urllib.request.urlopen(url)
data = connection.read().decode()
print(data[:250])
headers = dict(connection.getheaders())
# print headers
print("Remaining", headers["x-rate-limit-remaining"])
def call_api(twitter_url, parameters):
url = twurl.augment(twitter_url, parameters)
connection = urllib2.urlopen(url)
return connection.read()
import os
if os.path.exists("~/Desktop/majorProject/Data/search_result.txt"):
os.remove("~/Desktop/majorProject/Data/search_result.txt")
TWITTER_URL = 'https://api.twitter.com/1.1/search/tweets.json?'
word = raw_input('Enter word:')
word_no_retweet = word + ' -RT' #filter out no retweet
f = open("/users/karnageknight/Desktop/majorProject/Data/search_result.txt",
"a+")
count = 0
for x in range(0, 3): #range defines number of pages displayed
if x == 0:
url = twurl.augment(
TWITTER_URL, {
'q': word_no_retweet,
'count': '100',
'lang': 'en',
'include_entities': 'false',
'return_type': 'popular'
}) #for first page
else:
url = twurl.augment(
TWITTER_URL, {
'q': word_no_retweet,
'count': '100',
'lang': 'en',
'include_entities': 'false',
'max_id': next_max_id,
'return_type': 'popular'
}) #for subsequent tweets using their max_id
#print 'Retrieving', url
#print word_no_retweet
import urllib.request, urllib.parse, urllib.error
from twurl import augment
print('* Calling Twitter...')
url = augment('https://api.twitter.com/1.1/statuses/user_timeline.json',
{'screen_name': 'drchuck', 'count': '2'})
print(url)
connection = urllib.request.urlopen(url)
data = connection.read()
print(data)
headers = dict(connection.getheaders())
print(headers)
from pprint import pprint
# https://apps.twitter.com/
# Create App and get the four strings, put them in hidden.py
TWITTER_URL = 'https://api.twitter.com/1.1/friends/list.json'
# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
print("This program will give you a possibility to search something in json")
print('')
acct = input('Enter Twitter Account: ')
url = twurl.augment(TWITTER_URL, {'screen_name': acct, 'count': '5'})
# print('Retrieving', url)
connection = urllib.request.urlopen(url, context=ctx)
data = connection.read().decode()
js = json.loads(data)
print(json.dumps(js, indent=2))
def search_json(json):
"""
dict -> dict
You can serch values by key
"""
nick_pair = []
nicknames = []
import re
import objectpath
import pytz
import sys
non_bmp = dict.fromkeys(range(0x1000, sys.maxunicode + 1), 0xfffd)
TWITTER_URL = 'https://api.twitter.com/1.1/search/tweets.json?'
ctx=ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
while True:
print('')
query = input('Enter the query:')
if len(query)<1:
break
url = twurl.augment(TWITTER_URL,
{'q':query, 'count':'100'})
con = ur.urlopen(url, context = ctx)
jas = json.loads(con.read().decode())
tree_obj = objectpath.Tree(jas)
tweets = list(tree_obj.execute('$..text'))
for x in tweets:
try:
print(x)
except:
print(x.translate(non_bmp))
headers = dict(connection.getheaders())
print('Remaining', headers['x-rate-limit-remaining'])
# https://apps.twitter.com/
# Create App and get the four strings, put them in hidden.py
TWITTER_URL = 'https://api.twitter.com/1.1/friends/list.json'
# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
while True:
print('')
acct = input('Enter Twitter Account:')
if (len(acct) < 1): break
url = twurl.augment(TWITTER_URL,
#Count is number of friends IE.. First five friends
{'screen_name': acct, 'count': '5'})
print('Retrieving', url)
connection = urllib.request.urlopen(url, context=ctx)
data = connection.read().decode()
#add try except
js = json.loads(data)
#Pretty Prints the data
print(json.dumps(js, indent=2))
headers = dict(connection.getheaders())
print('Remaining', headers['x-rate-limit-remaining'])
for u in js['users']:
print(u['screen_name'])
if 'status' not in u:
#this script is authored by Georges BODIONG
import sys
import urllib
import twurl
import json
TWITTER_URL = 'https://api.twitter.com/1.1/search/tweets.json'
while True:
print '-----------------------------------'
q = raw_input('Enter your search query: ')
# if (len(acct) < 1) : break
url = twurl.augment(TWITTER_URL,
{'q' : q})
print 'Retrieving', url
connection = urllib.urlopen(url)
data = connection.read()
headers = connection.info().dict
print '-----------------------------------'
print 'Retrieved', len(data), 'characters'
js = json.loads(data)
# print json.dumps(js, indent=4)
#persist data in the file system
#in a file named followers.json
with open('dBootcamp.json', 'w') as f:
json.dump(js, f)
print 'The operation was successful...'
#!/usr/bin/env python
import urllib
from twurl import augment
print "Connecting..."
request_url = r""
params = {}
url = augment(request_url, params)
print "\n{}\n".format(url)
connection = urllib.urlopen(url)
data = connection.read()
print data
headers = connection.info.dict
print headers
import urllib
import twurl
import json
# Retrieves friend list data from twitter
TWITTER_URL = 'https://api.twitter.com/1.1/friends/list.json'
while True:
print ''
acct = raw_input('Enter Twitter Account:')
if (len(acct) < 1): break
url = twurl.augment(TWITTER_URL, {
'screen_name': acct,
'count': '5'
}) # we only grab the latest 5 users
print 'Retrieving', url
connection = urllib.urlopen(url)
data = connection.read() # Body JSON Data
headers = connection.info(
).dict # JSON Header data. Includes the x rate limit remaining
print 'Remaining', headers['x-rate-limit-remaining']
js = json.loads(data) # json load the body data
print json.dumps(js, indent=4) # Pretty print with indent 4
for u in js['users']:
print u['screen_name']
s = u['status']['text']
print ' ', s[:50] # Print the first 50 characters of each tweet
import urllib.request, urllib.parse, urllib.error
from twurl import augment
import ssl
# https://apps.twitter.com/
# Create App and get the four strings, put them in hidden.py
print('* Calling Twitter...')
url = augment('https://api.twitter.com/1.1/statuses/user_timeline.json', {
'screen_name': 'satvikboorela',
'count': '2'
})
print(url)
# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
connection = urllib.request.urlopen(url, context=ctx)
data = connection.read()
print(data)
print('======================================')
headers = dict(connection.getheaders())
print(headers)
import urllib.request, urllib.parse, urllib.error
import json
from twurl import augment
import ssl
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
while True:
accn = input("Enter Username:- ")
if len(accn) < 1: break
url = augment('https://api.twitter.com/1.1/friends/list.json', {
'screen_name': accn,
'count': '80'
})
print('Retrieving: ', url)
uh = urllib.request.urlopen(url, context=ctx)
data = uh.read().decode()
headers = dict(uh.getheaders())
print('Remaining Request: ', headers['x-rate-limit-remaining'])
js = json.loads(data)
x = open('output.json', 'w')
x.write(json.dumps(js, indent=4))
#print(json.dumps(js, indent = 2))
users = js['users']
for user in users:
print('-------------------------------')
while True:
print('')
keyword = input('Enter Your Search Keyword:')
if (len(keyword) < 1): break
#Represents the 2min timeframe
minutetime = 2
#Describes the number of timeframes you wish to retrieve
for x in range(0, 3):
#max_id is used to set an ending marker of each timeframe
url = twurl.augment(
TWITTER_URL, {
'q': keyword,
'count': '100',
'max_id': identity,
'result_type': 'recent'
})
connection = urllib.request.urlopen(url, context=ctx)
data = connection.read().decode()
jsondata = json.loads(data)
#Count for each timeframe
counter = 0
for id in jsondata['statuses']:
a = id['created_at']
b = list(a)
del b[0:4]
# Create App and get the four strings, put them in hidden.py
TWITTER_URL = 'https://api.twitter.com/1.1/friends/list.json' #pagina para obtener listas de amigos con api de twitter
# Ignore SSL certificate errors
#====instrucciones para ingorar errores de certificado SSL===========================================
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
#====codigo para extraccion de informacion de twitter, requiere token y secrets en el archivo hidden dentro de una carpeta donde este corriendo el programa=========================
while True: #mientras la condicion sea verdadera
print('') #imprime un renglon en blanco
acct = input('Enter Twitter Account:') #solicita al usuiario la entrada de una cuenta de twitter
if (len(acct) < 1): break #en caso de no ingresarse una cuenta de twitter, con break, el programa sale del lazo
url = twurl.augment(TWITTER_URL,
{'screen_name': acct, 'count': '5'}) #biblioteca con tuplas de key:value, en "screen_name" se carga la pagina con la cuenta y en "count" se carga un valor predetermiado de "5"
#esta relacionada con la bibloteca twurl, ya que es un campo de ingreso a la misma
print('Retrieving', url) #envia a impresion la cadena "Retrieving" + la "url"
connection = urllib.request.urlopen(url, context=ctx) # puntero para leer la pagina regresa una cadena de valores, "calla" revisiones de seguridad en errores de certificado con 'context=ctx' para ssl
data = connection.read().decode() #orden de lectura, es la representacion en cadena del "json", viene de internet, se tiene que codificar para cambiar de utf-8 al formato general de pyton: "unicode"
js = json.loads(data) #carga la informacion recuperada en la variable js ya en formato "unicode" y genera un "arbol" de la misma en notacion json
print(json.dumps(js, indent=2)) #envia a impresion las secciones recuparadas en js, dandoles sangria de dos espacios para mayor claridad
headers = dict(connection.getheaders()) #declara "headers" como el diccionario donde se almacenan todos los headers con el comando getheaders() de "urllib", aplicado al puntero de la pagina solicitada
print('Remaining', headers['x-rate-limit-remaining']) #envia a impresion el valor de la key con nombre 'x-rate-limit-remaining', el limite para twitter es muy corto, cerca de 15 unicamente
for u in js['users']: # para cada 'u', abreviatura para usuario, en la informacion que contiene js, ya ordenada, buca lo contenido en la key 'users'
print(u['screen_name']) #envia a impresion el valor de la key dentro de 'users' con el nombre 'screen_name'
if 'status' not in u: #si no hay una key con nombre 'status' dentro de la key 'users'
print(' * No status found') #envia a imporesion : no se encontro un status
def augment(url, parameters): # parameters = dictionary
secrets = hidden.oauth()
consumer = oauth.OAuthConsumer(secrets['consumer_key'], secrets['consumer_secret'])
token = oauth.OAuthToken(secrets['token_key'], secrets['token_secret'])
oauth_request = oauth.OAuthRequest.from_consumer_and_token(consumer, token = token, http_method = 'GET', http_url = url, parameters = parameters)
oauth_request.sign_request(oauth.OAuthSignatureMethod_HMAC_SHA1(), consumer, token)
return oauth_request.to_url()
-- twitter.py
import urllib
from twurl import augment
print '* Calling Twitter... *'
url = augment('https://api.twitter.com/1.1/statuses/user_timeline.json', {'screen_name' : 'drchuck', 'count': 2} ) # show me the user's timeline, for user (screen_name) drchuck and show me the first 2
print url
connection = urllib.urlopen(url)
data = connection.read() # gets the body
print data # will be JSON data
headers = connection.info().dict # gets a dictionary with only the headers
print headers
# python -mjson.tool < json_ugly_file
-- twitter2.py
import urllib
import twurl
import json
import urllib.request, urllib.parse, urllib.error
from twurl import augment
print('* Calling Twitter...')
url = augment('https://api.twitter.com/1.1/statuses/user_timeline.json',
{'screen_name': 'drchuck', 'count': '2'} )
print(url)
connection = urllib.request.urlopen(url)
data = connection.read()
print(data)
headers = dict(connection.getheaders())
print(headers)
import urllib
from twurl import augment
import json
print "reads a Twitter account's timeline"
while True:
try:
cout = int(raw_input('Enter tweet count:'))
break
except ValueError:
print "Oops! That was no valid number. Try again..."
print '* Calling Twitter...'
url = augment('https://api.twitter.com/1.1/statuses/user_timeline.json',
{'screen_name': 'Theidden_one', 'count': cout} )
print url[:100]
connection = urllib.urlopen(url)
data = connection.read()
#print data
headers = connection.info().dict
#print headers
print '\nRemaining', headers['x-rate-limit-remaining']
js = json.loads(data)
#print json.dumps(js, indent=4)
for u in js:
print u['text']
def gettwfriends(acct, count):
APP_ROOT = os.path.dirname(os.path.abspath(__file__))
count = int(count)
TWITTER_URL = 'https://api.twitter.com/1.1/friends/list.json'
USER_URL = 'https://api.twitter.com/1.1/users/lookup.json'
# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
conn = sqlite3.connect(APP_ROOT + '\\friends.sqlite')
cur = conn.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS People
(id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT UNIQUE,
data TEXT DEFAULT NULL,
retrieved INTEGER DEFAULT (0) CHECK (retrieved in (0,1) ) )'''
)
cur.execute('''CREATE TABLE IF NOT EXISTS Follows
(from_id INTEGER,
to_id INTEGER,
UNIQUE (from_id, to_id))''')
conn.commit()
while True:
# acct = input('Enter a Twitter account, or quit: ')
# if (acct == 'quit'):
# break
count = count - 1
if count < 0:
break
if (len(acct) < 1):
cur.execute(
'SELECT id, name FROM People WHERE retrieved=0 LIMIT 1')
try:
(id, acct) = cur.fetchone()
except:
print('No unretrieved Twitter accounts found')
# continue
else:
cur.execute('SELECT id FROM People WHERE name = ? LIMIT 1',
(acct, ))
try:
id = cur.fetchone()[0]
except:
cur.execute(
'''INSERT OR IGNORE INTO People
(name, retrieved) VALUES (?, 0)''', (acct, ))
conn.commit()
if cur.rowcount != 1:
print('Error inserting account:', acct)
continue
id = cur.lastrowid
try:
url2 = twurl.augment(USER_URL, {'screen_name': acct})
try:
connection2 = urllib.request.urlopen(url2, context=ctx)
except Exception as err2:
print('Failed to Retrieve #USER', err2)
break
data2 = connection2.read().decode()
headers2 = dict(connection2.getheaders())
print('Remaining user_lookup',
headers2['x-rate-limit-remaining'])
try:
js2 = json.loads(data2)[0]
save = json.dumps(js2).encode()
except:
print('Unable to parse json')
print(data2)
break
cur.execute('UPDATE People SET data=? WHERE name=?', (
memoryview(save),
acct,
))
conn.commit()
except:
print('Error retrieving User data')
continue
if count > 5:
racct = count
else:
racct = 200
url = twurl.augment(TWITTER_URL, {'screen_name': acct, 'count': racct})
print('Retrieving account', acct)
try:
connection = urllib.request.urlopen(url, context=ctx)
except Exception as err:
print('Failed to Retrieve', err)
break
data = connection.read().decode()
headers = dict(connection.getheaders())
print('Remaining', headers['x-rate-limit-remaining'])
try:
js = json.loads(data)
except:
print('Unable to parse json')
print(data)
break
if 'users' not in js:
print('Incorrect JSON received')
print(json.dumps(js, indent=4))
continue
cur.execute('UPDATE People SET retrieved=1 WHERE name = ?', (acct, ))
countnew = 0
countold = 0
for u in js['users']:
friend = u['screen_name']
print(friend)
cur.execute('SELECT id FROM People WHERE name = ? LIMIT 1',
(friend, ))
try:
friend_id = cur.fetchone()[0]
countold = countold + 1
except:
cur.execute(
'''INSERT OR IGNORE INTO People (name, data,retrieved)
VALUES (?,?, 0)''', (
friend,
memoryview(json.dumps(u).encode()),
))
conn.commit()
if cur.rowcount != 1:
print('Error inserting account:', friend)
continue
friend_id = cur.lastrowid
countnew = countnew + 1
cur.execute(
'''INSERT OR IGNORE INTO Follows (from_id, to_id)
VALUES (?, ?)''', (id, friend_id))
cur.execute(
'''INSERT OR IGNORE INTO Follows (from_id, to_id)
VALUES (?, ?)''', (id, id))
print('New accounts=', countnew, ' revisited=', countold)
print('Remaining', headers['x-rate-limit-remaining'])
conn.commit()
acct = ""
cur.close()
def callAPI(twitter_url, parameters, session):
url = twurl.augment(twitter_url, parameters, session)
connection = urllib2.urlopen(url)
return connection.read()
def callApi(twitter_url, parameters): #
url = twurl.augment(twitter_url, parameters)
connection = urllib2.urlopen(url)
return connection.read()
#enter twitter ID
while True:
twitterID = raw_input("ID:")
if twitterID == "quit": break
if len(twitterID)<1:
twitterID = cur.execute('''SELECT name FROM Twitter WHERE
retrieved = 0 LIMIT 1''')
try:
twitterID = cur.fetchone()[0]
except:
print "No unretrieved twitter ID"
continue #NO NEW Twitter ID
#read the twitter data using API
url = twurl.augment(twitter_url, {'screen_name':twitterID, 'count':'20'})
print "retrieving url", url
connection = urllib.urlopen(url)
data = connection.read()
header = connection.info().dict
print header
js = json.loads(data)
print js.dumps(js,indent = 4)
cur.execute("UPDATE Twitter SET retrieved = 1 WHERE name = ?", (twitterID,))
#find all the friends
countnew = 0
countold = 0
for u in js["users"]:
friendname = u["screen_name"]
cur.execute('''SELECT friends FROM Twitter WHERE name = ?''', friendname)
#-------------------------------------------------------------------------------
# Name: module1
# Purpose:
#
# Author: Trenton J. McKinney
#
# Created: 24/11/2015
# Copyright: (c) Trenton J. McKinney 2015
# Licence: <your licence>
#-------------------------------------------------------------------------------
import urllib
import twurl
print 'Calling Twitter'
url = twurl.augment('https://api.twitter.com/1.1/statuses/user_timeline.json',
{'screen_name': 'Trenton_EE', 'count': '2'})
print url
connection = urllib.urlopen(url)
data = connection.read()
print data
headers = connection.info().dict
print headers
import urllib.request, urllib.parse, urllib.error
import twurl
import json
TWITTER_URL = 'https://api.twitter.com/1.1/friends/list.json'
while True:
print('')
acct = input('Enter Twitter Account:')
if ( len(acct) < 1 ) : break
url = twurl.augment(TWITTER_URL,
{'screen_name': acct, 'count': '5'} )
print('Retrieving', url)
connection = urllib.request.urlopen(url)
data = connection.read().decode()
headers = dict(connection.getheaders())
print('Remaining', headers['x-rate-limit-remaining'])
js = json.loads(data)
print(json.dumps(js, indent=4))
for u in js['users'] :
print(u['screen_name'])
s = u['status']['text']
print(' ',s[:50])
#! python3
import urllib.request, urllib.parse, urllib.error
from twurl import augment
import ssl
# https://apps.twitter.com/
# Create App and get the four strings, put them in hidden.py
print('* Calling Twitter...')
url = augment('https://api.twitter.com/1.1/statuses/user_timeline.json', {
'screen_name': 'Vaibhav_Fubuki',
'count': '2'
})
print(url)
# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
connection = urllib.request.urlopen(url, context=ctx)
data = connection.read()
print(data) # Encoded data
# print(data.decode()) # Decoded data
print('======================================')
headers = dict(connection.getheaders())
print(headers)
