def tweet(command, device, output):
tweets = []
is_trending = False # used to differentiate trending tweets from others
try:
# consumer_key & consumer_secret are application specific and other two
# account specific
api = TwitterAPI(consumer_key, consumer_secret, access_token_key,
access_token_secret)
search_term = command['arguments']['name'] # this is argument name
if command['intent'] == 'trends/place':
is_trending = True
country_codes = api.request(
'trends/available'
) # to find the 'What On Earth ID' an id for a particular place
for code in country_codes:
if code['country'].lower() == search_term[1:]:
woeid = code['woeid']
response = api.request(
command['intent'],
{'id': woeid
}) # you only get links to trending topics in twitter
count = 0 # specifies no of tweets for places 'currently 5'
for item in response:
count += 1
# If tweets starts with a special character then clean up
if not re.match('[a-zA-Z0-9]', item['query'][0]):
topic_name = item['query'][3:]
else:
topic_name = item['query']
tweets.append(
topic_name + '<br /> ' +
str(item['url'])) # change the output format as required
if count == 5:
break
elif command['intent'] == 'statuses/user_timeline':
query = 'screen_name'
else:
query = 'q'
if is_trending == False: # if the request is not on trending topics
response = api.request(command['intent'], {
query: search_term,
'count': 5
})
for item in response:
string = item['text'].replace('\n', '<br />')
tweets.append(string)
output_tweets = []
# Convert links in string format to HTML format
for tweet in tweets:
tweet_length = len(tweet)
first_char = 0
while True:
index = tweet.find('http', first_char, tweet_length)
if index == -1:
break
last_pos_link = tweet.find(' ', index) - 1
if last_pos_link == -2:
last_pos_link = tweet_length - 1
if last_pos_link < tweet_length:
text = tweet[index:last_pos_link + 1]
blank = '_blank'
link = '<a href="' + text + '" target="_blank">' + text + '</a>'
tweet = tweet.replace(text, link)
first_char = index + len(link)
tweet_length = len(tweet)
output_tweets.append(tweet)
output = {
'commands': [],
'error': False,
'final': True,
'parsed': command,
'message': 'Executed command',
'type': None,
'tweet': output_tweets
}
return output
except:
output = {'message': 'Invalid input', 'error': True, 'final': True}
return output
def suck(save_item, handle_error, source):
api = TwitterAPI(settings.TWITTER['consumer_key'],
settings.TWITTER['consumer_secret'],
settings.TWITTER['access_token'],
settings.TWITTER['access_token_secret'])
if 'lastRetrieved' not in source:
source['lastRetrieved'] = {}
def get_and_save(endpoint,
request_filters,
lr_key,
admin1,
admin2=None,
admin3=None,
tags=None):
if lr_key in source['lastRetrieved']:
request_filters['since_id'] = source['lastRetrieved'][lr_key]
request_filters['count'] = 100
#request_filters['max_id'] = '494053583428390914'
r = api.request(endpoint, request_filters)
new_since_id = None
if r.status_code == 200:
for record in r.get_iterator():
if not new_since_id:
new_since_id = record['id_str']
source['lastRetrieved'][lr_key] = new_since_id
item = transform(record, admin1, admin3=admin3, tags=tags)
save_item(item)
else:
logger.error("Twitter API error: " + str(r.status_code))
for l in lists.items:
lr_key = l['owner_screen_name'] + '|' + l['slug']
request_filters = {
'slug': l['slug'],
'owner_screen_name': l['owner_screen_name'],
'per_page': 100
}
tags = None
if 'tags' in l:
tags = l['tags']
get_and_save('lists/statuses',
request_filters,
lr_key,
l['slug'].capitalize(),
tags=tags)
for h in hashtags.items:
lr_key = remove_non_ascii(h['hashtag'])
request_filters = {'q': h['hashtag']}
admin3 = None
tags = None
if 'state' in h:
admin3 = h['state']
if 'tags' in h:
tags = h['tags']
get_and_save('search/tweets',
request_filters,
lr_key,
h['country'],
admin3=admin3,
tags=tags)
return source['lastRetrieved']
print(strTwitterApiTokenSecret)
print(strAwsAccessKeyId)
print(strAwsSecretKey)
print(strAwsRegion)
print(strAwsStreamName)
sys.exit(1)
# Ensure filterText is provided as a command-line argument
if len(sys.argv) < 2:
print(
'Error! Filter keyword for tweets not provided as a command-line argument'
)
sys.exit(1)
# Setting up Twitter and Kinesis objects
api = TwitterAPI(consumer_key, consumer_secret, access_token_key,
access_token_secret)
kinesis = boto3.client('kinesis',
aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_key,
region_name=aws_region)
# Filter and track tweets by specific keyword
filterText = sys.argv[1]
r = api.request('statuses/filter', {'track': filterText})
# Write new tweets into Kinesis
for item in r:
if 'text' in item:
kinesis.put_record(StreamName=aws_stream_name,
Data=json.dumps(item),
PartitionKey=item['user']['screen_name'])
logger.info("Got {0} results".format(c))
except Exception as e:
logger.exception(
"Could not connect to TwitterAPI - are your credentials correct?"
)
if __name__ == '__main__':
#Load config
Config.load('config.json')
#Initialize twitter api
api = TwitterAPI(Config.consumer_key, Config.consumer_secret,
Config.access_token_key, Config.access_token_secret)
#Initialize ignorelist
ignore_list = IgnoreList("ignorelist")
#Initialize scheduler
scheduler = BlockingScheduler()
#First run
RandomTimes()
ClearQueue()
CheckRateLimit()
CheckBlockedUsers()
ScanForContests()
scheduler.add_job(RandomTimes, 'interval', hours=24)
import time
import gzip
import pandas as pd
from TwitterAPI import TwitterAPI
if __name__ == '__main__':
if len(sys.argv) <= 1:
print __doc__.strip()
sys.exit(0)
params = json.load(open(sys.argv[1]))
params.get('users')
api = TwitterAPI(
params['api']['consumer_key'],
params['api']['consumer_secret'],
params['api']['access_token_key'],
params['api']['access_token_secret'])
followers = {}
target = params['followers']['target']
if os.path.exists(target):
followers = json.load(open(target))
src = params['followers']['source']
src = gzip.open(src) if src.endswith('.gz') else open(src)
tweets = pd.Series([l for l in src if l.strip()]).apply(json.loads)
r = None
for tweet in tweets:
if not 'user' in tweet:
def watch_tweet_stream(s3_store,
s3_store_in,
s3_object_expiration_time,
twitter_keys=[],
twitter_request='statuses/filter',
twitter_filter={'track': ''},
check_geo_coords=False,
geo_box_coords_long_lat='',
check_retweeted_status=False,
check_possibly_sensitive=False):
api = TwitterAPI(twitter_keys[0], twitter_keys[1], twitter_keys[2],
twitter_keys[3])
while True:
try:
response = api.request(twitter_request, twitter_filter)
for item in response.get_iterator():
#pprint(item['filter_level'])
if 'text' in item:
if check_retweeted_status and 'retweeted_status' in item and item[
'retweeted_status']:
logging.info("Tweet is a retweet.")
continue
if check_possibly_sensitive and 'possibly_sensitive' in item and item[
'possibly_sensitive']:
logging.info("Tweet is possibly sensitive.")
continue
if check_possibly_sensitive and 'retweet' in item and 'possibly_sensitive' in item[
'retweet'] and item['retweet'][
'possibly_sensitive']:
logging.info(
"Tweet is a retweet and possibly sensitive.")
continue
if check_geo_coords and item['geo'] is None:
logging.info("Geo tag not set.")
continue
if check_geo_coords and not check_coords(
get_coord_array(geo_box_coords_long_lat),
item['geo']['coordinates']):
logging.info("Tweet not in region.")
continue
if 'entities' in item and 'media' in item['entities']:
logging.info('Tweet: ' + '@' +
item['user']['screen_name'] + ':' +
item['text'])
for media_object in item['entities']['media']:
if media_object['type'] == 'photo':
if s3_store:
logging.info(
"Dispatching thread to capture for: " +
media_object['media_url'] +
" with filter: " +
str(twitter_filter) +
" and storing in " +
s3_store_in.lower())
t = Thread(target=capture_photo_to_object,
args=(
s3_object_expiration_time,
media_object['media_url'],
s3_store_in.lower(),
))
t.start()
else:
logging.info(
'Not storing tweet since s3_store is False.'
)
else:
logging.info('Tweet media is not a photo.')
else:
logging.info('Tweet does not have media.')
continue
elif 'message' in item and item['code'] == 88:
print 'SUSPEND, RATE LIMIT EXCEEDED: %s\n' % item['message']
break
except Exception, e:
logging.error(str(e))
logging.debug("Sleeping for 12 seconds")
time.sleep(12)
def get_twitter():
twitter = TwitterAPI(consumer_key, consumer_secret, access_token,
access_token_secret)
return twitter
def __init__(self, consumer_key, consumer_secret, access_token_key,
access_token_secret):
"""Initialize the service."""
from TwitterAPI import TwitterAPI
self.api = TwitterAPI(consumer_key, consumer_secret, access_token_key,
access_token_secret)
import time
import json
import os
import sys
from credentials import credentials
#Variables that contains the user credentials to access Twitter API
# Load credentials from ~/.credentials.json
creds = credentials.require(['access_token',
'access_token_secret',
'consumer_key',
'consumer_secret'])
api = TwitterAPI(creds.consumer_key,
creds.consumer_secret,
creds.access_token,
creds.access_token_secret,
auth_type='oAuth2')
#Grab Draft King data
data_pitchers = pd.read_csv("../Data_Engineering_Project/data/pitchers_2016_06_26.csv")
data_batters = pd.read_csv("../Data_Engineering_Project/data/batters_2016_06_26.csv")
data_DK = pd.read_csv("../Data_Engineering_Project/data/DKSalaries_2016_06_26.csv")
#Grab the players
pitchers_names = data_pitchers["Player Name"]
batters_names = data_batters["Player Name"]
DK_names = data_DK["Name"]
#Concatenate the batters and pitchers and drop dups
frames = [pitchers_names,batters_names]
all_players= pd.concat(frames)
all_players = all_players.drop_duplicates()
import time import pymongo import bson from pymongo.collation import Collation import pytz from TwitterAPI import TwitterAPI from TwitterAPI import TwitterPager from twitter_api_constants import * from process_tweet import process_tweet # Set up Twitter API api = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, auth_type="oAuth2") # Set up Mongo MONGO = pymongo.MongoClient(MONGO_CONNECTION_STRING) twitter_db = MONGO["twitters"] trackers_hashtags = twitter_db["trackersHashtags"] trackers_users = twitter_db["trackersUsers"] update_settings = twitter_db["updated"] tweets = twitter_db["tweets"] users = twitter_db["users"] users_to_search = twitter_db["usersToSearch"] tweets_to_collect = twitter_db["tweetsToCollect"] tweet_tree = twitter_db["tweetTree"] poll_seed = twitter_db["pollSeed"]
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
requests_log = logging.getLogger("requests.packages.urllib3")
requests_log.setLevel(logging.DEBUG)
requests_log.propagate = True
# Connect to Twitter
try:
enable_requests_debugging()
load_configuration()
connect_to_redis()
o = TwitterOAuth.read_file('./credentials.txt')
api = TwitterAPI(o.consumer_key,
o.consumer_secret,
auth_type='oAuth2',
api_version='2')
# ADD STREAM RULES
print("Configuring stream rules.")
delete_all_rules(api)
set_rules(api, [{'value': QUERY}])
get_rules(api)
# START STREAM
print("Starting.")
while True:
try:
stream = api.request('tweets/search/stream')
if stream.status_code != 200:
print(
def send(status_msg):
TWEET_TEXT = "@koonktech " + status_msg
api = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN_KEY,
ACCESS_TOKEN_SECRET)
api.request('statuses/update', {'status': TWEET_TEXT})
def api_init__(self):
self.api = TwitterAPI(consumer_key=creds["vivek"]["consumer_key"],
consumer_secret=creds["vivek"]["consumer_secret"],
access_token_key=creds["vivek"]["access_token_key"],
access_token_secret=creds["vivek"]["access_token_secret"])
import json
import requests
from TwitterAPI import TwitterAPI
import os
api = TwitterAPI(consumer_key=os.environ.get('TWITTER_CONSUMER_KEY'),
consumer_secret=os.environ.get('TWITTER_CONSUMER_SECRET'),
access_token_key=os.environ.get('TWITTER_ACCESS_TOKEN_KEY'),
access_token_secret=os.environ.get('TWITTER_ACCESS_TOKEN_SECRET'))
def hrequest():
url = 'http://hypothes.is/api/search?limit=10'
response = requests.get(url)
response.raise_for_status()
annotations = json.loads(response.text)
return annotations
def make_tweets(annotations):
tweets_list = []
for i in annotations['rows']:
try:
contextlink = i['uri']['incontext']
def make_tweet(username):
""" (str) -> str
Given the twitter username of a twitter user, returns a tweet
recommending user with a new coffee order.
"""
while True:
a, b = random.choice(coffee_types["intro"])
o = u"@{} {} {} {}".format(username, a, order(), b)
if len(o) < 140:
return o
logging.info("Connecting to Twitter API")
api = TwitterAPI(keys.consumer_key, keys.consumer_secret,
keys.access_token_key, keys.access_token_secret)
bot = api.request("account/verify_credentials").json()["screen_name"]
# we keep the last 1000 messages and do not reply to those again
msgs = deque(maxlen=1000)
logging.info("Connected")
try:
for msg in api.request("user", {"replies": "all"}):
logging.info("New event")
logging.debug(msg)
if "text" in msg:
logging.info("Event is Tweet")
msg_id = msg["id"]
other = msg["user"]["screen_name"]
to = msg["in_reply_to_screen_name"]
toid = msg["in_reply_to_status_id"]
from time import sleep
ckey = ''
csecret = ''
atoken = ''
asecret = ''
while True:
from TwitterAPI import TwitterAPI
api = TwitterAPI(ckey, csecret, atoken, asecret)
r = api.request('statuses/filter', {'track': 'could care less'})
print('Connected to stream...')
for item in r.get_iterator():
tweet_id = item['id']
original_tweet_text = item['text']
if not "@" in original_tweet_text and "#" in original_tweet_text and item['user']['screen_name'] != "YouShouldCare":
# new_tweet_text = 'RT @' + item['user']['screen_name'] + ' ' + original_tweet_text
new_tweet_text = original_tweet_text
postresult = api.request('statuses/update', {'status': new_tweet_text})
# friend_result = api.request('friendships/create', {'screen_name': item['user']['screen_name']})
print('Tweeted: ' + new_tweet_text)
else:
# print('Passed on tweet: @' + str(item['user']['screen_name']) + ' ' + str(original_tweet_text))
pass
import sklearn.neighbors
import sklearn.neural_network
!pip install TwitterAPI
from TwitterAPI import TwitterAPI, TwitterOAuth
# Credentials file format:
# consumer_key=YOUR_CONSUMER_KEY
# consumer_secret=YOUR_CONSUMER_SECRET
# access_token_key=YOUR_ACCESS_TOKEN
# access_token_secret=YOUR_ACCESS_TOKEN_SECRET
"""### Connect and authenticate with Twitter REST API."""
o = TwitterOAuth.read_file(os.environ['HOME']+'/.twitterapi_credentials')
twapi = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret)
"""### Convenience function"""
def searchTwitter(q, api, feed="search/tweets", n=100):
return [t for t in api.request(feed, {'q':q,'count':n})]
"""### Get JSON from Twitter"""
cat_tweets = searchTwitter('#cats', twapi, n=100)
dog_tweets = searchTwitter('#dogs', twapi, n=100)
"""### Convert the json returned by Twitter into a dataframe"""
cat_df = pandas.read_json(json.dumps(cat_tweets))
dog_df = pandas.read_json(json.dumps(dog_tweets))
count = 0
cities = get_cities()
api = None
fopen = open("log/output.txt", "w")
for city in cities:
print city
print "-------------"
fopen.write(city + "\n")
max_id = None
finish = False
while not finish:
pre_count = Scraper.count
if api is None:
api = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, Access_token,
Access_token_secret)
json_str = {}
json_str['q'] = "%s %s" % ("jobs", city)
json_str['count'] = "100"
if max_id is not None:
json_str['max_id'] = max_id
try:
r = api.request('search/tweets', json_str)
response = json.loads(r.response.text)
except:
pass
for tweet in response['statuses']:
'access_token_secret': creds[3]
}
def make_pic():
colors = gen_colors()
full_data = get_data_from_colors(colors, 100)
plt.imshow(full_data)
plt.xticks([])
plt.yticks([])
plt.savefig('pic.png')
make_pic()
creds = read_credentials()
api = TwitterAPI(creds['consumer_key'], creds['consumer_secret'],
creds['access_token_key'], creds['access_token_secret'])
#pic_file = open('pic.png','rb')
#pic_data = pic_file.read()
with open('pic.png', 'rb') as image_file:
#encoded_string = base64.b64encode(image_file.read())
data = image_file.read()
#data = data.encode('base64')
image_dict = {
'image': {
'media': data
#'media_data': data.encode('base64')
}
}
from TwitterAPI import TwitterAPI
import json
import pandas as pd
from pandas.io.json import json_normalize
import authentication as auth
api = TwitterAPI(auth.consumer_key, auth.consumer_secret, auth.access_token, auth.token_secret)
request = api.request('tweets/search/30day/:Development', {'query': 'we\'ll donate for every retweet to help'})
tweet_raw = []
for item in request:
tweet_raw.append(json.dumps(item))
tweet_df = pd.DataFrame(json_normalize(tweet_raw), columns=
['text',
'user.screen_name',
'created_at',
'favorite_count',
'in_reply_to_status_id',
'retweeted_status',
'matching_rules'])
tweet_df.to_csv("30DaySandbox.csv", sep='\t')
import pandas as pd
from TwitterAPI import TwitterAPI
import time
from tqdm import tqdm
data = pd.read_csv("https://raw.githubusercontent.com/CivilServiceUSA/us-senate/master/us-senate/data/us-senate.csv")
APP_KEY = 'NBa0kFD92S7aP99xyJU354HlO'
APP_SECRET = 'cOJ4dncbvsBswH79HAZPQBpz57umO5taE4hYsucfOfgMapzEsU'
api = TwitterAPI(APP_KEY,
APP_SECRET,
auth_type='oAuth2')
ids = []
friends = {}
for u in tqdm(data.twitter_handle):
friend_list = []
usercheck = api.request('users/lookup', {'screen_name':u}).json()
if 'errors' in usercheck:
ids.append("")
continue
ids.append(str(usercheck[0]['id']))
import json
import argparse
import time
from TwitterAPI import TwitterAPI
import config
API = 'followers/ids'
api = TwitterAPI(config.API_KEY, config.API_SECRET_KEY, config.ACCESS_TOKEN, config.ACCESS_TOKEN_SECRET)
def parse_arg():
args = argparse.ArgumentParser(description="collect user IDs for every user following the specified user.")
args.add_argument("-f", "--filename", type=str, help="specify output JSON filename.")
args.add_argument("-u", "--user_id", type=int, help="specify user by user_id.")
args.add_argument("-s", "--screen_name", type=str, help="specify user by screen_name.")
args.add_argument("-c", "--count", type=int, default=5000, help="the number of IDs attempt retrieval of.")
return args.parse_args()
def get_friends(user_id=None, screen_name=None, count=1000):
follower_ids = []
params = {}
if user_id is not None:
params['user_id'] = user_id
elif screen_name is not None:
params['screen_name'] = screen_name
else:
print("One of user_id or screen_name must be specified.")
return follower_ids
import asyncio
import random
import re
from collections import defaultdict
from datetime import datetime
from functools import partial, wraps
import pytz
from hoshino import Service, priv, util
from hoshino.config import twitter as cfg
from hoshino.typing import CQEvent
from nonetrip import MessageSegment as ms
from TwitterAPI import TwitterAPI, TwitterResponse
api = TwitterAPI(cfg.consumer_key, cfg.consumer_secret, cfg.access_token_key, cfg.access_token_secret)
sv = Service('twitter-poller', use_priv=priv.SUPERUSER, manage_priv=priv.SUPERUSER, visible=False)
URL_TIMELINE = 'statuses/user_timeline'
subr_dic = {
Service('kc-twitter', enable_on_default=False, help_='艦これ官推转发', bundle='kancolle'): ['KanColle_STAFF', 'C2_STAFF', 'ywwuyi'],
Service('pcr-twitter', enable_on_default=True, help_='日服Twitter转发', bundle='pcr订阅'): ['priconne_redive', 'priconne_anime'],
Service('pripri-twitter', enable_on_default=False, visible=False): ['pripri_anime'],
Service('coffee-favorite-twitter', manage_priv=priv.SUPERUSER,
enable_on_default=False, visible=False): ['shiratamacaron', 'k_yuizaki', 'suzukitoto0323'],
}
latest_info = {} # { account: {last_tweet_id: int, profile_image: str } }
for _, ids in subr_dic.items(): # initialize
for account in ids:
latest_info[account] = {'last_tweet_id': 0, 'profile_image': '', 'media_only': False}
def search(request):
if request.method == "POST":
term = request.POST['term']
data = {
"query": {
"function_score": {
"query": {
"match": {
"title": term
}
},
"functions": [{
"script_score": {
"script":
"_score / (1 + pow(1.5,(-doc['likes'].value)) )"
}
}],
"max_boost":
100,
"score_mode":
"max",
"boost_mode":
"replace",
"min_score":
0
}
}
}
#pro
#command = "curl XGET localhost:9200/blog_index/doc/_search -d '{\"query\":{\"custom_score\":{\"query\":{\"match\":{\"title\":%s}},\"script\":\"_score * (doc['likes'].value)\"}}}' " % term
url = "http://localhost:9200/blog_index/doc/_search"
json_item = json.loads(requests.post(url, json=data).text)
prodocs = [elem['_source'] for elem in json_item['hits']['hits']]
for index, doc in enumerate(prodocs):
prodocs[index]['body'] = prodocs[index]['body'][:250]
prodocs[index]['Id'] = json_item['hits']['hits'][index]['_id']
#avg
avgdocs = []
#consumer_key, consumer_secret, access_token_key, access_token_secret
api = TwitterAPI("tmNbyIDjNn1SkEPJENTUy9XUC",
"raddZenV7CaOiR29akB0DlabmpFUwPVY1V4qZHQwRFGyqacKT3",
"3051564974-ancG9igjFccXFMLe0k8OMTVmW5euR8hES4NFnRM",
"XxU9lcVtnjHeYiHtSxIvqE3ub6cEgI6HBriwW4eYlfVvl")
r = api.request('search/tweets', {
'q': term,
'count': '5',
'lang': 'en'
})
response = r.json()
for post in response["statuses"]:
url = "https://twitter.com/%s/status/%d" % (
post['user']['screen_name'], post['id'])
avgdocs.append({
'created_at': post['created_at'],
'user': post['user']['screen_name'],
'source': url,
'text': post['text']
})
else:
prodocs, avgdocs = [], []
#return HttpResponse("hi")
#context = {'username': '******'}
return render_to_response('result.html', {
'prodocs': prodocs,
'avgdocs': avgdocs
},
context_instance=RequestContext(request))
def __connectAPI(self): # connect to Twitter's API
creds = self._thisAPIInformation.getCredsArray()
try:
self._api = TwitterAPI(creds[0], creds[1], creds[2], creds[3])
except:
print("Unable to connect to TwitterAPI.\n")
def process_item(self, item, spider):
try:
# Insert item into kith table.
if isinstance(item, KithItem):
self.cursor.execute(
"INSERT INTO kith (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into ruvilla table.
elif isinstance(item, RuvillaItem):
self.cursor.execute(
"INSERT INTO ruvilla (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into footlocker table.
elif isinstance(item, FootLockerItem):
self.cursor.execute(
"INSERT INTO footlocker (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into footaction table.
elif isinstance(item, FootActionItem):
self.cursor.execute(
"INSERT INTO footaction (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into champs table.
elif isinstance(item, ChampsItem):
self.cursor.execute(
"INSERT INTO champs (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into eastbay table.
elif isinstance(item, EastBayItem):
self.cursor.execute(
"INSERT INTO eastbay (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into finishline table.
elif isinstance(item, FinishLineItem):
self.cursor.execute(
"INSERT INTO finishline (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into adidas table.
elif isinstance(item, AdidasItem):
self.cursor.execute(
"INSERT INTO adidas (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into nike table.
elif isinstance(item, NikeItem):
self.cursor.execute(
"INSERT INTO nike (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into footshop table.
elif isinstance(item, FootShopItem):
self.cursor.execute(
"INSERT INTO footshop (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into caliroots table.
elif isinstance(item, CalirootsItem):
self.cursor.execute(
"INSERT INTO caliroots (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into afew table.
elif isinstance(item, AfewItem):
self.cursor.execute(
"INSERT INTO afew (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into einhalb table.
elif isinstance(item, EinhalbItem):
self.cursor.execute(
"INSERT INTO einhalb (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into end table.
elif isinstance(item, EndItem):
self.cursor.execute(
"INSERT INTO end (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into sns table.
elif isinstance(item, SNSItem):
self.cursor.execute(
"INSERT INTO sns (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into goodwillout table.
elif isinstance(item, GoodWillOutItem):
self.cursor.execute(
"INSERT INTO goodwillout (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into tint table.
elif isinstance(item, TintItem):
self.cursor.execute(
"INSERT INTO tint (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into overkill table.
elif isinstance(item, OverkillItem):
self.cursor.execute(
"INSERT INTO overkill (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into footdistrict table.
elif isinstance(item, FootDistrictItem):
self.cursor.execute(
"INSERT INTO footdistrict (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into size table.
elif isinstance(item, SizeItem):
self.cursor.execute(
"INSERT INTO size (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into ycmc table.
elif isinstance(item, YCMCItem):
self.cursor.execute(
"INSERT INTO ycmc (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into city table.
elif isinstance(item, CityItem):
self.cursor.execute(
"INSERT INTO city (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into nordstrom table.
elif isinstance(item, NordstromItem):
self.cursor.execute(
"INSERT INTO nordstrom (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into barneys table.
elif isinstance(item, BarneysItem):
self.cursor.execute(
"INSERT INTO barneys (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into jimmyjazz table.
elif isinstance(item, JimmyJazzItem):
self.cursor.execute(
"INSERT INTO jimmyjazz (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into jdsports table.
elif isinstance(item, JDSportsItem):
self.cursor.execute(
"INSERT INTO jdsports (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into footpatrol table.
elif isinstance(item, FootPatrolItem):
self.cursor.execute(
"INSERT INTO footpatrol (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into sneakerbaas table.
elif isinstance(item, SneakerBaasItem):
self.cursor.execute(
"INSERT INTO sneakerbaas (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into sneakerpolitics table.
elif isinstance(item, SneakerPoliticsItem):
self.cursor.execute(
"INSERT INTO sneakerpolitics (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into urbanindustry table.
elif isinstance(item, UrbanIndustryItem):
self.cursor.execute(
"INSERT INTO urbanindustry (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into urbanoutfitters table.
elif isinstance(item, UrbanOutfittersItem):
self.cursor.execute(
"INSERT INTO urbanoutfitters (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into luisa table.
elif isinstance(item, LuisaItem):
self.cursor.execute(
"INSERT INTO luisa (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into slamjam table.
elif isinstance(item, SlamJamItem):
self.cursor.execute(
"INSERT INTO slamjam (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into rise45 table.
elif isinstance(item, Rise45Item):
self.cursor.execute(
"INSERT INTO rise45 (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into undefeated table.
elif isinstance(item, UndefeatedItem):
self.cursor.execute(
"INSERT INTO undefeated (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into zappos table.
elif isinstance(item, ZapposItem):
self.cursor.execute(
"INSERT INTO zappos (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into ubiq table.
elif isinstance(item, UbiqItem):
self.cursor.execute(
"INSERT INTO ubiq (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into pointz table.
elif isinstance(item, PointzItem):
self.cursor.execute(
"INSERT INTO pointz (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into kicks table.
elif isinstance(item, KicksItem):
self.cursor.execute(
"INSERT INTO kicks (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into shoespalace table.
elif isinstance(item, ShoesPalaceItem):
self.cursor.execute(
"INSERT INTO shoespalace (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into stickabush table.
elif isinstance(item, StickABushItem):
self.cursor.execute(
"INSERT INTO stickabush (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into kong table.
elif isinstance(item, KongItem):
self.cursor.execute(
"INSERT INTO kong (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into saveoursole table.
elif isinstance(item, SaveOurSoleItem):
self.cursor.execute(
"INSERT INTO saveoursole (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into inflammable table.
elif isinstance(item, InflammableItem):
self.cursor.execute(
"INSERT INTO inflammable (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into defshop table.
elif isinstance(item, DefShopItem):
self.cursor.execute(
"INSERT INTO defshop (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into offspring table.
elif isinstance(item, OffSpringItem):
self.cursor.execute(
"INSERT INTO offspring (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into solekitchen table.
elif isinstance(item, SoleKitchenItem):
self.cursor.execute(
"INSERT INTO solekitchen (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into drome table.
elif isinstance(item, DromeItem):
self.cursor.execute(
"INSERT INTO drome (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into footasylum table.
elif isinstance(item, FootAsylumItem):
self.cursor.execute(
"INSERT INTO footasylum (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into hhv table.
elif isinstance(item, HHVItem):
self.cursor.execute(
"INSERT INTO hhv (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into concepts table.
elif isinstance(item, ConceptsItem):
self.cursor.execute(
"INSERT INTO concepts (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into socialstatus table.
elif isinstance(item, SocialStatusItem):
self.cursor.execute(
"INSERT INTO socialstatus (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into extrabutter table.
elif isinstance(item, ExtraButterItem):
self.cursor.execute(
"INSERT INTO extrabutter (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into bodega table.
elif isinstance(item, BodegaItem):
self.cursor.execute(
"INSERT INTO bodega (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into saintalfred table.
elif isinstance(item, SaintAlfredItem):
self.cursor.execute(
"INSERT INTO saintalfred (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into lapstonenhammer table.
elif isinstance(item, LapstoneNHammerItem):
self.cursor.execute(
"INSERT INTO lapstonenhammer (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into shelflife table.
elif isinstance(item, ShelfLifeItem):
self.cursor.execute(
"INSERT INTO shelflife (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into asphaltgold table.
elif isinstance(item, AsphaltGoldItem):
self.cursor.execute(
"INSERT INTO asphaltgold (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into hanon table.
elif isinstance(item, HanonItem):
self.cursor.execute(
"INSERT INTO hanon (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into solebox table.
elif isinstance(item, SoleBoxItem):
self.cursor.execute(
"INSERT INTO solebox (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into consortium table.
elif isinstance(item, ConsortiumItem):
self.cursor.execute(
"INSERT INTO consortium (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into haven table.
elif isinstance(item, HavenItem):
self.cursor.execute(
"INSERT INTO haven (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into needsupply table.
elif isinstance(item, NeedSupplyItem):
self.cursor.execute(
"INSERT INTO needsupply (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into loaded table.
elif isinstance(item, LoadedItem):
self.cursor.execute(
"INSERT INTO loaded (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into wellgosh table.
elif isinstance(item, WellGoshItem):
self.cursor.execute(
"INSERT INTO wellgosh (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into capsule table.
elif isinstance(item, CapsuleItem):
self.cursor.execute(
"INSERT INTO capsule (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into yme table.
elif isinstance(item, YMEItem):
self.cursor.execute(
"INSERT INTO yme (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into hypedc table.
elif isinstance(item, HypeDCItem):
self.cursor.execute(
"INSERT INTO hypedc (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into holypop table.
elif isinstance(item, HolyPopItem):
self.cursor.execute(
"INSERT INTO holypop (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into bstn table.
elif isinstance(item, BSTNItem):
self.cursor.execute(
"INSERT INTO bstn (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into trophyroom table.
elif isinstance(item, TrophyRoomItem):
self.cursor.execute(
"INSERT INTO trophyroom (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into sidestep table.
elif isinstance(item, SideStepItem):
self.cursor.execute(
"INSERT INTO sidestep (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into shiekh table.
elif isinstance(item, ShiekhItem):
self.cursor.execute(
"INSERT INTO shiekh (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into rezet table.
elif isinstance(item, RezetItem):
self.cursor.execute(
"INSERT INTO rezet (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into footlockereu table.
elif isinstance(item, FootLockerEUItem):
self.cursor.execute(
"INSERT INTO footlockereu (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into office table.
elif isinstance(item, OfficeItem):
self.cursor.execute(
"INSERT INTO office (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into allike table.
elif isinstance(item, ALLikeItem):
self.cursor.execute(
"INSERT INTO allike (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into sportshoes table.
elif isinstance(item, SportsShoesItem):
self.cursor.execute(
"INSERT INTO sportshoes (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into runnerspoint table.
elif isinstance(item, RunnersPointItem):
self.cursor.execute(
"INSERT INTO runnerspoint (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into graffiti table.
elif isinstance(item, GraffitiItem):
self.cursor.execute(
"INSERT INTO graffiti (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into urbanjungle table.
elif isinstance(item, UrbanJungleItem):
self.cursor.execute(
"INSERT INTO urbanjungle (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into ssense table.
elif isinstance(item, SSenseItem):
self.cursor.execute(
"INSERT INTO ssense (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into backdoor table.
elif isinstance(item, BackDoorItem):
self.cursor.execute(
"INSERT INTO backdoor (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into basket table.
elif isinstance(item, BasketItem):
self.cursor.execute(
"INSERT INTO basket (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into oneblockdown table.
elif isinstance(item, OneBlockDownItem):
self.cursor.execute(
"INSERT INTO oneblockdown (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into dopefactory table.
elif isinstance(item, DopeFactoryItem):
self.cursor.execute(
"INSERT INTO dopefactory (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into nextdoor table.
elif isinstance(item, NextDoorItem):
self.cursor.execute(
"INSERT INTO nextdoor (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into summer table.
elif isinstance(item, SummerItem):
self.cursor.execute(
"INSERT INTO summer (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into mrporter table.
elif isinstance(item, MrPorterItem):
self.cursor.execute(
"INSERT INTO mrporter (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into stormfashion table.
elif isinstance(item, StormFashionItem):
self.cursor.execute(
"INSERT INTO stormfashion (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into tresbien table.
elif isinstance(item, TresBienItem):
self.cursor.execute(
"INSERT INTO tresbien (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into packer table.
elif isinstance(item, PackerItem):
self.cursor.execute(
"INSERT INTO packer (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into addict table.
elif isinstance(item, AddictItem):
self.cursor.execute(
"INSERT INTO addict (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into aphrodite table.
elif isinstance(item, AphroditeItem):
self.cursor.execute(
"INSERT INTO aphrodite (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into bait table.
elif isinstance(item, BaitItem):
self.cursor.execute(
"INSERT INTO bait (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into blends table.
elif isinstance(item, BlendsItem):
self.cursor.execute(
"INSERT INTO blends (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into nicekicks table.
elif isinstance(item, NiceKicksItem):
self.cursor.execute(
"INSERT INTO nicekicks (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into clicks table.
elif isinstance(item, ClicksItem):
self.cursor.execute(
"INSERT INTO clicks (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into feature table.
elif isinstance(item, FeatureItem):
self.cursor.execute(
"INSERT INTO feature (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into hypebeast table.
elif isinstance(item, HypeBeastItem):
self.cursor.execute(
"INSERT INTO hypebeast (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into deadstock table.
elif isinstance(item, DeadStockItem):
self.cursor.execute(
"INSERT INTO deadstock (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into notre table.
elif isinstance(item, NotreItem):
self.cursor.execute(
"INSERT INTO notre (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into nrml table.
elif isinstance(item, NrmlItem):
self.cursor.execute(
"INSERT INTO nrml (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into oneness table.
elif isinstance(item, OnenessItem):
self.cursor.execute(
"INSERT INTO oneness (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into pufferreds table.
elif isinstance(item, PufferRedsItem):
self.cursor.execute(
"INSERT INTO pufferreds (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into renarts table.
elif isinstance(item, RenartsItem):
self.cursor.execute(
"INSERT INTO renarts (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into shoesgallery table.
elif isinstance(item, ShoesGalleryItem):
self.cursor.execute(
"INSERT INTO shoesgallery (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into proper table.
elif isinstance(item, ProperItem):
self.cursor.execute(
"INSERT INTO proper (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into solestop table.
elif isinstance(item, SoleStopItem):
self.cursor.execute(
"INSERT INTO solestop (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into titolo table.
elif isinstance(item, TitoloItem):
self.cursor.execute(
"INSERT INTO titolo (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into uptown table.
elif isinstance(item, UptownItem):
self.cursor.execute(
"INSERT INTO uptown (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into westnyc table.
elif isinstance(item, WestNYCItem):
self.cursor.execute(
"INSERT INTO westnyc (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into wishatl table.
elif isinstance(item, WishATLItem):
self.cursor.execute(
"INSERT INTO wishatl (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into xileclothing table.
elif isinstance(item, XileClothingItem):
self.cursor.execute(
"INSERT INTO xileclothing (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into solefly table.
elif isinstance(item, SoleflyItem):
self.cursor.execute(
"INSERT INTO solefly (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into patta table.
elif isinstance(item, PattaItem):
self.cursor.execute(
"INSERT INTO patta (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into svd table.
elif isinstance(item, SVDItem):
self.cursor.execute(
"INSERT INTO svd (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
# Insert item into dsmny table.
elif isinstance(item, DSMNYItem):
self.cursor.execute(
"INSERT INTO dsmny (link, date) VALUES (%s, %s)",
(item['link'].encode('utf-8'), DATE))
# Insert item into shoesaddictor table.
elif isinstance(item, ShoesAddictorItem):
self.cursor.execute(
"INSERT INTO shoesaddictor (name, link, date) VALUES (%s, %s, %s)",
(item['name'].encode('utf-8'),
item['link'].encode('utf-8'), DATE))
self.conn.commit()
# If item name contain below words. Tweet it.
if ('nmd' in item['name'].encode('utf-8').lower()) or (
'ultra' in item['name'].encode('utf-8').lower()
and 'boost' in item['name'].encode('utf-8').lower()
) or ('jordan' in item['name'].encode('utf-8').lower()
and 'retro' in item['name'].encode('utf-8').lower()
) or ('yeezy' in item['name'].encode('utf-8').lower()) or (
'max' in item['name'].encode('utf-8').lower()
and 'atmos' in item['name'].encode('utf-8').lower()) or (
'max' in item['name'].encode('utf-8').lower()
and 'master' in item['name'].encode('utf-8').lower()
) or ('ronnie' in item['name'].encode('utf-8').lower()
) or ('fieg'
in item['name'].encode('utf-8').lower()):
# Twitter Auth - Tweet the item with date, time, item name, and link.
# To obtain Twitter CONSUMER and ACCESS keys go to https://apps.twitter.com/
CONSUMER_KEY = ' PASTE CONSUMER_KEY HERE '
CONSUMER_SECRET = ' PASTE CONSUMER_SECRET HERE '
ACCESS_TOKEN_KEY = ' PASTE ACCESS_TOKEN_KEY HERE '
ACCESS_TOKEN_SECRET = ' PASTE ACCESS_TOKEN_SECRET HERE '
API = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET,
ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET)
TEXT_TO_TWEET = DATE + " EST " + item['name'] + " " + item[
'link']
TWEET = API.request('statuses/update',
{'status': TEXT_TO_TWEET})
print(Fore.RED + 'TWEET LOG SUCCESS: ' + DATE + ' EST ' +
item['name'] + ' ' + item['link'] +
Style.RESET_ALL if TWEET.status_code ==
200 else Fore.RED +
'TWEET LOG FAILURE: FAILED TO TWEET' + Style.RESET_ALL)
except MySQLdb.Error, e:
# print (Fore.RED + "MYSQL ERROR %d: %s" % (e.args[0], e.args[1] + Style.RESET_ALL))
return item
from threader import Threader
import guizero
# from twython import Twython
import math
# import time
from textwrap import wrap
from apikeys import (consumer_key, consumer_secret, access_token,
access_token_secret)
source_selected = True
keys = dict(consumer_key=consumer_key,
consumer_secret=consumer_secret,
access_token_key=access_token,
access_token_secret=access_token_secret)
api = TwitterAPI(**keys)
def send_tweet():
print("send tweet")
tweet_array = wrap(tweet.value, 240)
th = Threader(tweet_array, api, wait=slider.value)
th.send_tweets()
#
# def insert_selected():
# api_key_line.show()
# choose_key_area.hide()
# global source_selected
# source_selected = True
import datetime
import json
import urllib2
import threading
from lxml import html
import requests
TRACK_TERM = 'Houston'
TRACK_LOC = '-95.788087,29.523624,-95.014496,30.110732'
CONSUMER_KEY = '----'
CONSUMER_SECRET = '----'
ACCESS_TOKEN_KEY = '----'
ACCESS_TOKEN_SECRET = '---'
api = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN_KEY,
ACCESS_TOKEN_SECRET)
r = api.request('statuses/filter', {
'track': TRACK_TERM,
'locations': TRACK_LOC
})
### MongoDB Calling
client = MongoClient("mongodb://127.0.0.1:3001/meteor")
db = client['meteor']
for item in r.get_iterator():
if 'text' in item:
a = item['text']
p = polarity(a)
s = subjectivity(a)
if (p != 0.0):
def get_twitter():
""" Construct an instance of TwitterAPI using the tokens you entered above.
Returns:
An instance of TwitterAPI.
"""
return TwitterAPI(consumer_key, consumer_secret, access_token, access_token_secret)
# get timeline tweet, return up to 3,200 of a user's most recent Tweets.
from TwitterAPI import TwitterAPI, TwitterPager
import pandas as pd
# set twitter api Authorization keys, remember the order for the keys
# api = TwitterAPI("consumer_API_key","consumer_API_secret", "access_token","access_token_secret")
api = TwitterAPI("G9c9PkikFEffLezvmhdkHpWma",
"2hLfuvejl3fhBp6Zxr1NMBT0jZk3LpNLdjTTKbDzWYBbZ3duRC",
"1250078614445608967-vOiELbylTZ0GxGrnhBouPPbLTTzCDG",
"GVDza7InbhVUZeme8g0Okoqsyo4WByTvZkFlZ4iX0jgJP")
def get_all_tweets(screen_name):
pager = TwitterPager(api, 'statuses/user_timeline', {'screen_name': screen_name, 'count': 200})
alltweets = pager.get_iterator(wait=3.5)
outtweets = [
[screen_name, tweet['id_str'], pd.to_datetime(tweet['created_at']), tweet['user']['location'],
tweet['retweet_count'], tweet['favorite_count'], tweet['lang'], tweet['text']] for tweet in alltweets]
df = pd.DataFrame(outtweets, columns=['user_name', 'id', 'create_time', 'geo', 'retweets','favorite_count',
'language', 'text'])
df.to_csv('results.csv', index=False)
print('finish')
if __name__ == '__main__':
get_all_tweets('google')
