def tweet(command, device, output): tweets = [] is_trending = False # used to differentiate trending tweets from others try: # consumer_key & consumer_secret are application specific and other two # account specific api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret) search_term = command['arguments']['name'] # this is argument name if command['intent'] == 'trends/place': is_trending = True country_codes = api.request( 'trends/available' ) # to find the 'What On Earth ID' an id for a particular place for code in country_codes: if code['country'].lower() == search_term[1:]: woeid = code['woeid'] response = api.request( command['intent'], {'id': woeid }) # you only get links to trending topics in twitter count = 0 # specifies no of tweets for places 'currently 5' for item in response: count += 1 # If tweets starts with a special character then clean up if not re.match('[a-zA-Z0-9]', item['query'][0]): topic_name = item['query'][3:] else: topic_name = item['query'] tweets.append( topic_name + '<br /> ' + str(item['url'])) # change the output format as required if count == 5: break elif command['intent'] == 'statuses/user_timeline': query = 'screen_name' else: query = 'q' if is_trending == False: # if the request is not on trending topics response = api.request(command['intent'], { query: search_term, 'count': 5 }) for item in response: string = item['text'].replace('\n', '<br />') tweets.append(string) output_tweets = [] # Convert links in string format to HTML format for tweet in tweets: tweet_length = len(tweet) first_char = 0 while True: index = tweet.find('http', first_char, tweet_length) if index == -1: break last_pos_link = tweet.find(' ', index) - 1 if last_pos_link == -2: last_pos_link = tweet_length - 1 if last_pos_link < tweet_length: text = tweet[index:last_pos_link + 1] blank = '_blank' link = '<a href="' + text + '" target="_blank">' + text + '</a>' tweet = tweet.replace(text, link) first_char = index + len(link) tweet_length = len(tweet) output_tweets.append(tweet) output = { 'commands': [], 'error': False, 'final': True, 'parsed': command, 'message': 'Executed command', 'type': None, 'tweet': output_tweets } return output except: output = {'message': 'Invalid input', 'error': True, 'final': True} return output
def suck(save_item, handle_error, source): api = TwitterAPI(settings.TWITTER['consumer_key'], settings.TWITTER['consumer_secret'], settings.TWITTER['access_token'], settings.TWITTER['access_token_secret']) if 'lastRetrieved' not in source: source['lastRetrieved'] = {} def get_and_save(endpoint, request_filters, lr_key, admin1, admin2=None, admin3=None, tags=None): if lr_key in source['lastRetrieved']: request_filters['since_id'] = source['lastRetrieved'][lr_key] request_filters['count'] = 100 #request_filters['max_id'] = '494053583428390914' r = api.request(endpoint, request_filters) new_since_id = None if r.status_code == 200: for record in r.get_iterator(): if not new_since_id: new_since_id = record['id_str'] source['lastRetrieved'][lr_key] = new_since_id item = transform(record, admin1, admin3=admin3, tags=tags) save_item(item) else: logger.error("Twitter API error: " + str(r.status_code)) for l in lists.items: lr_key = l['owner_screen_name'] + '|' + l['slug'] request_filters = { 'slug': l['slug'], 'owner_screen_name': l['owner_screen_name'], 'per_page': 100 } tags = None if 'tags' in l: tags = l['tags'] get_and_save('lists/statuses', request_filters, lr_key, l['slug'].capitalize(), tags=tags) for h in hashtags.items: lr_key = remove_non_ascii(h['hashtag']) request_filters = {'q': h['hashtag']} admin3 = None tags = None if 'state' in h: admin3 = h['state'] if 'tags' in h: tags = h['tags'] get_and_save('search/tweets', request_filters, lr_key, h['country'], admin3=admin3, tags=tags) return source['lastRetrieved']
print(strTwitterApiTokenSecret) print(strAwsAccessKeyId) print(strAwsSecretKey) print(strAwsRegion) print(strAwsStreamName) sys.exit(1) # Ensure filterText is provided as a command-line argument if len(sys.argv) < 2: print( 'Error! Filter keyword for tweets not provided as a command-line argument' ) sys.exit(1) # Setting up Twitter and Kinesis objects api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret) kinesis = boto3.client('kinesis', aws_access_key_id=aws_access_key, aws_secret_access_key=aws_secret_key, region_name=aws_region) # Filter and track tweets by specific keyword filterText = sys.argv[1] r = api.request('statuses/filter', {'track': filterText}) # Write new tweets into Kinesis for item in r: if 'text' in item: kinesis.put_record(StreamName=aws_stream_name, Data=json.dumps(item), PartitionKey=item['user']['screen_name'])
logger.info("Got {0} results".format(c)) except Exception as e: logger.exception( "Could not connect to TwitterAPI - are your credentials correct?" ) if __name__ == '__main__': #Load config Config.load('config.json') #Initialize twitter api api = TwitterAPI(Config.consumer_key, Config.consumer_secret, Config.access_token_key, Config.access_token_secret) #Initialize ignorelist ignore_list = IgnoreList("ignorelist") #Initialize scheduler scheduler = BlockingScheduler() #First run RandomTimes() ClearQueue() CheckRateLimit() CheckBlockedUsers() ScanForContests() scheduler.add_job(RandomTimes, 'interval', hours=24)
import time import gzip import pandas as pd from TwitterAPI import TwitterAPI if __name__ == '__main__': if len(sys.argv) <= 1: print __doc__.strip() sys.exit(0) params = json.load(open(sys.argv[1])) params.get('users') api = TwitterAPI( params['api']['consumer_key'], params['api']['consumer_secret'], params['api']['access_token_key'], params['api']['access_token_secret']) followers = {} target = params['followers']['target'] if os.path.exists(target): followers = json.load(open(target)) src = params['followers']['source'] src = gzip.open(src) if src.endswith('.gz') else open(src) tweets = pd.Series([l for l in src if l.strip()]).apply(json.loads) r = None for tweet in tweets: if not 'user' in tweet:
def watch_tweet_stream(s3_store, s3_store_in, s3_object_expiration_time, twitter_keys=[], twitter_request='statuses/filter', twitter_filter={'track': ''}, check_geo_coords=False, geo_box_coords_long_lat='', check_retweeted_status=False, check_possibly_sensitive=False): api = TwitterAPI(twitter_keys[0], twitter_keys[1], twitter_keys[2], twitter_keys[3]) while True: try: response = api.request(twitter_request, twitter_filter) for item in response.get_iterator(): #pprint(item['filter_level']) if 'text' in item: if check_retweeted_status and 'retweeted_status' in item and item[ 'retweeted_status']: logging.info("Tweet is a retweet.") continue if check_possibly_sensitive and 'possibly_sensitive' in item and item[ 'possibly_sensitive']: logging.info("Tweet is possibly sensitive.") continue if check_possibly_sensitive and 'retweet' in item and 'possibly_sensitive' in item[ 'retweet'] and item['retweet'][ 'possibly_sensitive']: logging.info( "Tweet is a retweet and possibly sensitive.") continue if check_geo_coords and item['geo'] is None: logging.info("Geo tag not set.") continue if check_geo_coords and not check_coords( get_coord_array(geo_box_coords_long_lat), item['geo']['coordinates']): logging.info("Tweet not in region.") continue if 'entities' in item and 'media' in item['entities']: logging.info('Tweet: ' + '@' + item['user']['screen_name'] + ':' + item['text']) for media_object in item['entities']['media']: if media_object['type'] == 'photo': if s3_store: logging.info( "Dispatching thread to capture for: " + media_object['media_url'] + " with filter: " + str(twitter_filter) + " and storing in " + s3_store_in.lower()) t = Thread(target=capture_photo_to_object, args=( s3_object_expiration_time, media_object['media_url'], s3_store_in.lower(), )) t.start() else: logging.info( 'Not storing tweet since s3_store is False.' ) else: logging.info('Tweet media is not a photo.') else: logging.info('Tweet does not have media.') continue elif 'message' in item and item['code'] == 88: print 'SUSPEND, RATE LIMIT EXCEEDED: %s\n' % item['message'] break except Exception, e: logging.error(str(e)) logging.debug("Sleeping for 12 seconds") time.sleep(12)
def get_twitter(): twitter = TwitterAPI(consumer_key, consumer_secret, access_token, access_token_secret) return twitter
def __init__(self, consumer_key, consumer_secret, access_token_key, access_token_secret): """Initialize the service.""" from TwitterAPI import TwitterAPI self.api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret)
import time import json import os import sys from credentials import credentials #Variables that contains the user credentials to access Twitter API # Load credentials from ~/.credentials.json creds = credentials.require(['access_token', 'access_token_secret', 'consumer_key', 'consumer_secret']) api = TwitterAPI(creds.consumer_key, creds.consumer_secret, creds.access_token, creds.access_token_secret, auth_type='oAuth2') #Grab Draft King data data_pitchers = pd.read_csv("../Data_Engineering_Project/data/pitchers_2016_06_26.csv") data_batters = pd.read_csv("../Data_Engineering_Project/data/batters_2016_06_26.csv") data_DK = pd.read_csv("../Data_Engineering_Project/data/DKSalaries_2016_06_26.csv") #Grab the players pitchers_names = data_pitchers["Player Name"] batters_names = data_batters["Player Name"] DK_names = data_DK["Name"] #Concatenate the batters and pitchers and drop dups frames = [pitchers_names,batters_names] all_players= pd.concat(frames) all_players = all_players.drop_duplicates()
import time import pymongo import bson from pymongo.collation import Collation import pytz from TwitterAPI import TwitterAPI from TwitterAPI import TwitterPager from twitter_api_constants import * from process_tweet import process_tweet # Set up Twitter API api = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, auth_type="oAuth2") # Set up Mongo MONGO = pymongo.MongoClient(MONGO_CONNECTION_STRING) twitter_db = MONGO["twitters"] trackers_hashtags = twitter_db["trackersHashtags"] trackers_users = twitter_db["trackersUsers"] update_settings = twitter_db["updated"] tweets = twitter_db["tweets"] users = twitter_db["users"] users_to_search = twitter_db["usersToSearch"] tweets_to_collect = twitter_db["tweetsToCollect"] tweet_tree = twitter_db["tweetTree"] poll_seed = twitter_db["pollSeed"]
logging.basicConfig() logging.getLogger().setLevel(logging.DEBUG) requests_log = logging.getLogger("requests.packages.urllib3") requests_log.setLevel(logging.DEBUG) requests_log.propagate = True # Connect to Twitter try: enable_requests_debugging() load_configuration() connect_to_redis() o = TwitterOAuth.read_file('./credentials.txt') api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') # ADD STREAM RULES print("Configuring stream rules.") delete_all_rules(api) set_rules(api, [{'value': QUERY}]) get_rules(api) # START STREAM print("Starting.") while True: try: stream = api.request('tweets/search/stream') if stream.status_code != 200: print(
def send(status_msg): TWEET_TEXT = "@koonktech " + status_msg api = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET) api.request('statuses/update', {'status': TWEET_TEXT})
def api_init__(self): self.api = TwitterAPI(consumer_key=creds["vivek"]["consumer_key"], consumer_secret=creds["vivek"]["consumer_secret"], access_token_key=creds["vivek"]["access_token_key"], access_token_secret=creds["vivek"]["access_token_secret"])
import json import requests from TwitterAPI import TwitterAPI import os api = TwitterAPI(consumer_key=os.environ.get('TWITTER_CONSUMER_KEY'), consumer_secret=os.environ.get('TWITTER_CONSUMER_SECRET'), access_token_key=os.environ.get('TWITTER_ACCESS_TOKEN_KEY'), access_token_secret=os.environ.get('TWITTER_ACCESS_TOKEN_SECRET')) def hrequest(): url = 'http://hypothes.is/api/search?limit=10' response = requests.get(url) response.raise_for_status() annotations = json.loads(response.text) return annotations def make_tweets(annotations): tweets_list = [] for i in annotations['rows']: try: contextlink = i['uri']['incontext']
def make_tweet(username): """ (str) -> str Given the twitter username of a twitter user, returns a tweet recommending user with a new coffee order. """ while True: a, b = random.choice(coffee_types["intro"]) o = u"@{} {} {} {}".format(username, a, order(), b) if len(o) < 140: return o logging.info("Connecting to Twitter API") api = TwitterAPI(keys.consumer_key, keys.consumer_secret, keys.access_token_key, keys.access_token_secret) bot = api.request("account/verify_credentials").json()["screen_name"] # we keep the last 1000 messages and do not reply to those again msgs = deque(maxlen=1000) logging.info("Connected") try: for msg in api.request("user", {"replies": "all"}): logging.info("New event") logging.debug(msg) if "text" in msg: logging.info("Event is Tweet") msg_id = msg["id"] other = msg["user"]["screen_name"] to = msg["in_reply_to_screen_name"] toid = msg["in_reply_to_status_id"]
from time import sleep ckey = '' csecret = '' atoken = '' asecret = '' while True: from TwitterAPI import TwitterAPI api = TwitterAPI(ckey, csecret, atoken, asecret) r = api.request('statuses/filter', {'track': 'could care less'}) print('Connected to stream...') for item in r.get_iterator(): tweet_id = item['id'] original_tweet_text = item['text'] if not "@" in original_tweet_text and "#" in original_tweet_text and item['user']['screen_name'] != "YouShouldCare": # new_tweet_text = 'RT @' + item['user']['screen_name'] + ' ' + original_tweet_text new_tweet_text = original_tweet_text postresult = api.request('statuses/update', {'status': new_tweet_text}) # friend_result = api.request('friendships/create', {'screen_name': item['user']['screen_name']}) print('Tweeted: ' + new_tweet_text) else: # print('Passed on tweet: @' + str(item['user']['screen_name']) + ' ' + str(original_tweet_text)) pass
import sklearn.neighbors import sklearn.neural_network !pip install TwitterAPI from TwitterAPI import TwitterAPI, TwitterOAuth # Credentials file format: # consumer_key=YOUR_CONSUMER_KEY # consumer_secret=YOUR_CONSUMER_SECRET # access_token_key=YOUR_ACCESS_TOKEN # access_token_secret=YOUR_ACCESS_TOKEN_SECRET """### Connect and authenticate with Twitter REST API.""" o = TwitterOAuth.read_file(os.environ['HOME']+'/.twitterapi_credentials') twapi = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret) """### Convenience function""" def searchTwitter(q, api, feed="search/tweets", n=100): return [t for t in api.request(feed, {'q':q,'count':n})] """### Get JSON from Twitter""" cat_tweets = searchTwitter('#cats', twapi, n=100) dog_tweets = searchTwitter('#dogs', twapi, n=100) """### Convert the json returned by Twitter into a dataframe""" cat_df = pandas.read_json(json.dumps(cat_tweets)) dog_df = pandas.read_json(json.dumps(dog_tweets))
count = 0 cities = get_cities() api = None fopen = open("log/output.txt", "w") for city in cities: print city print "-------------" fopen.write(city + "\n") max_id = None finish = False while not finish: pre_count = Scraper.count if api is None: api = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, Access_token, Access_token_secret) json_str = {} json_str['q'] = "%s %s" % ("jobs", city) json_str['count'] = "100" if max_id is not None: json_str['max_id'] = max_id try: r = api.request('search/tweets', json_str) response = json.loads(r.response.text) except: pass for tweet in response['statuses']:
'access_token_secret': creds[3] } def make_pic(): colors = gen_colors() full_data = get_data_from_colors(colors, 100) plt.imshow(full_data) plt.xticks([]) plt.yticks([]) plt.savefig('pic.png') make_pic() creds = read_credentials() api = TwitterAPI(creds['consumer_key'], creds['consumer_secret'], creds['access_token_key'], creds['access_token_secret']) #pic_file = open('pic.png','rb') #pic_data = pic_file.read() with open('pic.png', 'rb') as image_file: #encoded_string = base64.b64encode(image_file.read()) data = image_file.read() #data = data.encode('base64') image_dict = { 'image': { 'media': data #'media_data': data.encode('base64') } }
from TwitterAPI import TwitterAPI import json import pandas as pd from pandas.io.json import json_normalize import authentication as auth api = TwitterAPI(auth.consumer_key, auth.consumer_secret, auth.access_token, auth.token_secret) request = api.request('tweets/search/30day/:Development', {'query': 'we\'ll donate for every retweet to help'}) tweet_raw = [] for item in request: tweet_raw.append(json.dumps(item)) tweet_df = pd.DataFrame(json_normalize(tweet_raw), columns= ['text', 'user.screen_name', 'created_at', 'favorite_count', 'in_reply_to_status_id', 'retweeted_status', 'matching_rules']) tweet_df.to_csv("30DaySandbox.csv", sep='\t')
import pandas as pd from TwitterAPI import TwitterAPI import time from tqdm import tqdm data = pd.read_csv("https://raw.githubusercontent.com/CivilServiceUSA/us-senate/master/us-senate/data/us-senate.csv") APP_KEY = 'NBa0kFD92S7aP99xyJU354HlO' APP_SECRET = 'cOJ4dncbvsBswH79HAZPQBpz57umO5taE4hYsucfOfgMapzEsU' api = TwitterAPI(APP_KEY, APP_SECRET, auth_type='oAuth2') ids = [] friends = {} for u in tqdm(data.twitter_handle): friend_list = [] usercheck = api.request('users/lookup', {'screen_name':u}).json() if 'errors' in usercheck: ids.append("") continue ids.append(str(usercheck[0]['id']))
import json import argparse import time from TwitterAPI import TwitterAPI import config API = 'followers/ids' api = TwitterAPI(config.API_KEY, config.API_SECRET_KEY, config.ACCESS_TOKEN, config.ACCESS_TOKEN_SECRET) def parse_arg(): args = argparse.ArgumentParser(description="collect user IDs for every user following the specified user.") args.add_argument("-f", "--filename", type=str, help="specify output JSON filename.") args.add_argument("-u", "--user_id", type=int, help="specify user by user_id.") args.add_argument("-s", "--screen_name", type=str, help="specify user by screen_name.") args.add_argument("-c", "--count", type=int, default=5000, help="the number of IDs attempt retrieval of.") return args.parse_args() def get_friends(user_id=None, screen_name=None, count=1000): follower_ids = [] params = {} if user_id is not None: params['user_id'] = user_id elif screen_name is not None: params['screen_name'] = screen_name else: print("One of user_id or screen_name must be specified.") return follower_ids
import asyncio import random import re from collections import defaultdict from datetime import datetime from functools import partial, wraps import pytz from hoshino import Service, priv, util from hoshino.config import twitter as cfg from hoshino.typing import CQEvent from nonetrip import MessageSegment as ms from TwitterAPI import TwitterAPI, TwitterResponse api = TwitterAPI(cfg.consumer_key, cfg.consumer_secret, cfg.access_token_key, cfg.access_token_secret) sv = Service('twitter-poller', use_priv=priv.SUPERUSER, manage_priv=priv.SUPERUSER, visible=False) URL_TIMELINE = 'statuses/user_timeline' subr_dic = { Service('kc-twitter', enable_on_default=False, help_='艦これ官推转发', bundle='kancolle'): ['KanColle_STAFF', 'C2_STAFF', 'ywwuyi'], Service('pcr-twitter', enable_on_default=True, help_='日服Twitter转发', bundle='pcr订阅'): ['priconne_redive', 'priconne_anime'], Service('pripri-twitter', enable_on_default=False, visible=False): ['pripri_anime'], Service('coffee-favorite-twitter', manage_priv=priv.SUPERUSER, enable_on_default=False, visible=False): ['shiratamacaron', 'k_yuizaki', 'suzukitoto0323'], } latest_info = {} # { account: {last_tweet_id: int, profile_image: str } } for _, ids in subr_dic.items(): # initialize for account in ids: latest_info[account] = {'last_tweet_id': 0, 'profile_image': '', 'media_only': False}
def search(request): if request.method == "POST": term = request.POST['term'] data = { "query": { "function_score": { "query": { "match": { "title": term } }, "functions": [{ "script_score": { "script": "_score / (1 + pow(1.5,(-doc['likes'].value)) )" } }], "max_boost": 100, "score_mode": "max", "boost_mode": "replace", "min_score": 0 } } } #pro #command = "curl XGET localhost:9200/blog_index/doc/_search -d '{\"query\":{\"custom_score\":{\"query\":{\"match\":{\"title\":%s}},\"script\":\"_score * (doc['likes'].value)\"}}}' " % term url = "http://localhost:9200/blog_index/doc/_search" json_item = json.loads(requests.post(url, json=data).text) prodocs = [elem['_source'] for elem in json_item['hits']['hits']] for index, doc in enumerate(prodocs): prodocs[index]['body'] = prodocs[index]['body'][:250] prodocs[index]['Id'] = json_item['hits']['hits'][index]['_id'] #avg avgdocs = [] #consumer_key, consumer_secret, access_token_key, access_token_secret api = TwitterAPI("tmNbyIDjNn1SkEPJENTUy9XUC", "raddZenV7CaOiR29akB0DlabmpFUwPVY1V4qZHQwRFGyqacKT3", "3051564974-ancG9igjFccXFMLe0k8OMTVmW5euR8hES4NFnRM", "XxU9lcVtnjHeYiHtSxIvqE3ub6cEgI6HBriwW4eYlfVvl") r = api.request('search/tweets', { 'q': term, 'count': '5', 'lang': 'en' }) response = r.json() for post in response["statuses"]: url = "https://twitter.com/%s/status/%d" % ( post['user']['screen_name'], post['id']) avgdocs.append({ 'created_at': post['created_at'], 'user': post['user']['screen_name'], 'source': url, 'text': post['text'] }) else: prodocs, avgdocs = [], [] #return HttpResponse("hi") #context = {'username': '******'} return render_to_response('result.html', { 'prodocs': prodocs, 'avgdocs': avgdocs }, context_instance=RequestContext(request))
def __connectAPI(self): # connect to Twitter's API creds = self._thisAPIInformation.getCredsArray() try: self._api = TwitterAPI(creds[0], creds[1], creds[2], creds[3]) except: print("Unable to connect to TwitterAPI.\n")
def process_item(self, item, spider): try: # Insert item into kith table. if isinstance(item, KithItem): self.cursor.execute( "INSERT INTO kith (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into ruvilla table. elif isinstance(item, RuvillaItem): self.cursor.execute( "INSERT INTO ruvilla (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into footlocker table. elif isinstance(item, FootLockerItem): self.cursor.execute( "INSERT INTO footlocker (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into footaction table. elif isinstance(item, FootActionItem): self.cursor.execute( "INSERT INTO footaction (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into champs table. elif isinstance(item, ChampsItem): self.cursor.execute( "INSERT INTO champs (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into eastbay table. elif isinstance(item, EastBayItem): self.cursor.execute( "INSERT INTO eastbay (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into finishline table. elif isinstance(item, FinishLineItem): self.cursor.execute( "INSERT INTO finishline (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into adidas table. elif isinstance(item, AdidasItem): self.cursor.execute( "INSERT INTO adidas (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into nike table. elif isinstance(item, NikeItem): self.cursor.execute( "INSERT INTO nike (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into footshop table. elif isinstance(item, FootShopItem): self.cursor.execute( "INSERT INTO footshop (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into caliroots table. elif isinstance(item, CalirootsItem): self.cursor.execute( "INSERT INTO caliroots (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into afew table. elif isinstance(item, AfewItem): self.cursor.execute( "INSERT INTO afew (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into einhalb table. elif isinstance(item, EinhalbItem): self.cursor.execute( "INSERT INTO einhalb (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into end table. elif isinstance(item, EndItem): self.cursor.execute( "INSERT INTO end (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into sns table. elif isinstance(item, SNSItem): self.cursor.execute( "INSERT INTO sns (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into goodwillout table. elif isinstance(item, GoodWillOutItem): self.cursor.execute( "INSERT INTO goodwillout (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into tint table. elif isinstance(item, TintItem): self.cursor.execute( "INSERT INTO tint (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into overkill table. elif isinstance(item, OverkillItem): self.cursor.execute( "INSERT INTO overkill (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into footdistrict table. elif isinstance(item, FootDistrictItem): self.cursor.execute( "INSERT INTO footdistrict (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into size table. elif isinstance(item, SizeItem): self.cursor.execute( "INSERT INTO size (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into ycmc table. elif isinstance(item, YCMCItem): self.cursor.execute( "INSERT INTO ycmc (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into city table. elif isinstance(item, CityItem): self.cursor.execute( "INSERT INTO city (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into nordstrom table. elif isinstance(item, NordstromItem): self.cursor.execute( "INSERT INTO nordstrom (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into barneys table. elif isinstance(item, BarneysItem): self.cursor.execute( "INSERT INTO barneys (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into jimmyjazz table. elif isinstance(item, JimmyJazzItem): self.cursor.execute( "INSERT INTO jimmyjazz (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into jdsports table. elif isinstance(item, JDSportsItem): self.cursor.execute( "INSERT INTO jdsports (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into footpatrol table. elif isinstance(item, FootPatrolItem): self.cursor.execute( "INSERT INTO footpatrol (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into sneakerbaas table. elif isinstance(item, SneakerBaasItem): self.cursor.execute( "INSERT INTO sneakerbaas (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into sneakerpolitics table. elif isinstance(item, SneakerPoliticsItem): self.cursor.execute( "INSERT INTO sneakerpolitics (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into urbanindustry table. elif isinstance(item, UrbanIndustryItem): self.cursor.execute( "INSERT INTO urbanindustry (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into urbanoutfitters table. elif isinstance(item, UrbanOutfittersItem): self.cursor.execute( "INSERT INTO urbanoutfitters (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into luisa table. elif isinstance(item, LuisaItem): self.cursor.execute( "INSERT INTO luisa (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into slamjam table. elif isinstance(item, SlamJamItem): self.cursor.execute( "INSERT INTO slamjam (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into rise45 table. elif isinstance(item, Rise45Item): self.cursor.execute( "INSERT INTO rise45 (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into undefeated table. elif isinstance(item, UndefeatedItem): self.cursor.execute( "INSERT INTO undefeated (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into zappos table. elif isinstance(item, ZapposItem): self.cursor.execute( "INSERT INTO zappos (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into ubiq table. elif isinstance(item, UbiqItem): self.cursor.execute( "INSERT INTO ubiq (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into pointz table. elif isinstance(item, PointzItem): self.cursor.execute( "INSERT INTO pointz (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into kicks table. elif isinstance(item, KicksItem): self.cursor.execute( "INSERT INTO kicks (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into shoespalace table. elif isinstance(item, ShoesPalaceItem): self.cursor.execute( "INSERT INTO shoespalace (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into stickabush table. elif isinstance(item, StickABushItem): self.cursor.execute( "INSERT INTO stickabush (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into kong table. elif isinstance(item, KongItem): self.cursor.execute( "INSERT INTO kong (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into saveoursole table. elif isinstance(item, SaveOurSoleItem): self.cursor.execute( "INSERT INTO saveoursole (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into inflammable table. elif isinstance(item, InflammableItem): self.cursor.execute( "INSERT INTO inflammable (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into defshop table. elif isinstance(item, DefShopItem): self.cursor.execute( "INSERT INTO defshop (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into offspring table. elif isinstance(item, OffSpringItem): self.cursor.execute( "INSERT INTO offspring (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into solekitchen table. elif isinstance(item, SoleKitchenItem): self.cursor.execute( "INSERT INTO solekitchen (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into drome table. elif isinstance(item, DromeItem): self.cursor.execute( "INSERT INTO drome (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into footasylum table. elif isinstance(item, FootAsylumItem): self.cursor.execute( "INSERT INTO footasylum (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into hhv table. elif isinstance(item, HHVItem): self.cursor.execute( "INSERT INTO hhv (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into concepts table. elif isinstance(item, ConceptsItem): self.cursor.execute( "INSERT INTO concepts (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into socialstatus table. elif isinstance(item, SocialStatusItem): self.cursor.execute( "INSERT INTO socialstatus (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into extrabutter table. elif isinstance(item, ExtraButterItem): self.cursor.execute( "INSERT INTO extrabutter (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into bodega table. elif isinstance(item, BodegaItem): self.cursor.execute( "INSERT INTO bodega (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into saintalfred table. elif isinstance(item, SaintAlfredItem): self.cursor.execute( "INSERT INTO saintalfred (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into lapstonenhammer table. elif isinstance(item, LapstoneNHammerItem): self.cursor.execute( "INSERT INTO lapstonenhammer (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into shelflife table. elif isinstance(item, ShelfLifeItem): self.cursor.execute( "INSERT INTO shelflife (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into asphaltgold table. elif isinstance(item, AsphaltGoldItem): self.cursor.execute( "INSERT INTO asphaltgold (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into hanon table. elif isinstance(item, HanonItem): self.cursor.execute( "INSERT INTO hanon (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into solebox table. elif isinstance(item, SoleBoxItem): self.cursor.execute( "INSERT INTO solebox (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into consortium table. elif isinstance(item, ConsortiumItem): self.cursor.execute( "INSERT INTO consortium (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into haven table. elif isinstance(item, HavenItem): self.cursor.execute( "INSERT INTO haven (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into needsupply table. elif isinstance(item, NeedSupplyItem): self.cursor.execute( "INSERT INTO needsupply (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into loaded table. elif isinstance(item, LoadedItem): self.cursor.execute( "INSERT INTO loaded (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into wellgosh table. elif isinstance(item, WellGoshItem): self.cursor.execute( "INSERT INTO wellgosh (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into capsule table. elif isinstance(item, CapsuleItem): self.cursor.execute( "INSERT INTO capsule (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into yme table. elif isinstance(item, YMEItem): self.cursor.execute( "INSERT INTO yme (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into hypedc table. elif isinstance(item, HypeDCItem): self.cursor.execute( "INSERT INTO hypedc (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into holypop table. elif isinstance(item, HolyPopItem): self.cursor.execute( "INSERT INTO holypop (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into bstn table. elif isinstance(item, BSTNItem): self.cursor.execute( "INSERT INTO bstn (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into trophyroom table. elif isinstance(item, TrophyRoomItem): self.cursor.execute( "INSERT INTO trophyroom (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into sidestep table. elif isinstance(item, SideStepItem): self.cursor.execute( "INSERT INTO sidestep (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into shiekh table. elif isinstance(item, ShiekhItem): self.cursor.execute( "INSERT INTO shiekh (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into rezet table. elif isinstance(item, RezetItem): self.cursor.execute( "INSERT INTO rezet (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into footlockereu table. elif isinstance(item, FootLockerEUItem): self.cursor.execute( "INSERT INTO footlockereu (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into office table. elif isinstance(item, OfficeItem): self.cursor.execute( "INSERT INTO office (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into allike table. elif isinstance(item, ALLikeItem): self.cursor.execute( "INSERT INTO allike (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into sportshoes table. elif isinstance(item, SportsShoesItem): self.cursor.execute( "INSERT INTO sportshoes (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into runnerspoint table. elif isinstance(item, RunnersPointItem): self.cursor.execute( "INSERT INTO runnerspoint (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into graffiti table. elif isinstance(item, GraffitiItem): self.cursor.execute( "INSERT INTO graffiti (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into urbanjungle table. elif isinstance(item, UrbanJungleItem): self.cursor.execute( "INSERT INTO urbanjungle (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into ssense table. elif isinstance(item, SSenseItem): self.cursor.execute( "INSERT INTO ssense (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into backdoor table. elif isinstance(item, BackDoorItem): self.cursor.execute( "INSERT INTO backdoor (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into basket table. elif isinstance(item, BasketItem): self.cursor.execute( "INSERT INTO basket (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into oneblockdown table. elif isinstance(item, OneBlockDownItem): self.cursor.execute( "INSERT INTO oneblockdown (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into dopefactory table. elif isinstance(item, DopeFactoryItem): self.cursor.execute( "INSERT INTO dopefactory (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into nextdoor table. elif isinstance(item, NextDoorItem): self.cursor.execute( "INSERT INTO nextdoor (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into summer table. elif isinstance(item, SummerItem): self.cursor.execute( "INSERT INTO summer (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into mrporter table. elif isinstance(item, MrPorterItem): self.cursor.execute( "INSERT INTO mrporter (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into stormfashion table. elif isinstance(item, StormFashionItem): self.cursor.execute( "INSERT INTO stormfashion (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into tresbien table. elif isinstance(item, TresBienItem): self.cursor.execute( "INSERT INTO tresbien (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into packer table. elif isinstance(item, PackerItem): self.cursor.execute( "INSERT INTO packer (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into addict table. elif isinstance(item, AddictItem): self.cursor.execute( "INSERT INTO addict (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into aphrodite table. elif isinstance(item, AphroditeItem): self.cursor.execute( "INSERT INTO aphrodite (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into bait table. elif isinstance(item, BaitItem): self.cursor.execute( "INSERT INTO bait (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into blends table. elif isinstance(item, BlendsItem): self.cursor.execute( "INSERT INTO blends (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into nicekicks table. elif isinstance(item, NiceKicksItem): self.cursor.execute( "INSERT INTO nicekicks (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into clicks table. elif isinstance(item, ClicksItem): self.cursor.execute( "INSERT INTO clicks (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into feature table. elif isinstance(item, FeatureItem): self.cursor.execute( "INSERT INTO feature (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into hypebeast table. elif isinstance(item, HypeBeastItem): self.cursor.execute( "INSERT INTO hypebeast (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into deadstock table. elif isinstance(item, DeadStockItem): self.cursor.execute( "INSERT INTO deadstock (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into notre table. elif isinstance(item, NotreItem): self.cursor.execute( "INSERT INTO notre (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into nrml table. elif isinstance(item, NrmlItem): self.cursor.execute( "INSERT INTO nrml (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into oneness table. elif isinstance(item, OnenessItem): self.cursor.execute( "INSERT INTO oneness (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into pufferreds table. elif isinstance(item, PufferRedsItem): self.cursor.execute( "INSERT INTO pufferreds (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into renarts table. elif isinstance(item, RenartsItem): self.cursor.execute( "INSERT INTO renarts (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into shoesgallery table. elif isinstance(item, ShoesGalleryItem): self.cursor.execute( "INSERT INTO shoesgallery (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into proper table. elif isinstance(item, ProperItem): self.cursor.execute( "INSERT INTO proper (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into solestop table. elif isinstance(item, SoleStopItem): self.cursor.execute( "INSERT INTO solestop (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into titolo table. elif isinstance(item, TitoloItem): self.cursor.execute( "INSERT INTO titolo (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into uptown table. elif isinstance(item, UptownItem): self.cursor.execute( "INSERT INTO uptown (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into westnyc table. elif isinstance(item, WestNYCItem): self.cursor.execute( "INSERT INTO westnyc (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into wishatl table. elif isinstance(item, WishATLItem): self.cursor.execute( "INSERT INTO wishatl (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into xileclothing table. elif isinstance(item, XileClothingItem): self.cursor.execute( "INSERT INTO xileclothing (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into solefly table. elif isinstance(item, SoleflyItem): self.cursor.execute( "INSERT INTO solefly (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into patta table. elif isinstance(item, PattaItem): self.cursor.execute( "INSERT INTO patta (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into svd table. elif isinstance(item, SVDItem): self.cursor.execute( "INSERT INTO svd (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) # Insert item into dsmny table. elif isinstance(item, DSMNYItem): self.cursor.execute( "INSERT INTO dsmny (link, date) VALUES (%s, %s)", (item['link'].encode('utf-8'), DATE)) # Insert item into shoesaddictor table. elif isinstance(item, ShoesAddictorItem): self.cursor.execute( "INSERT INTO shoesaddictor (name, link, date) VALUES (%s, %s, %s)", (item['name'].encode('utf-8'), item['link'].encode('utf-8'), DATE)) self.conn.commit() # If item name contain below words. Tweet it. if ('nmd' in item['name'].encode('utf-8').lower()) or ( 'ultra' in item['name'].encode('utf-8').lower() and 'boost' in item['name'].encode('utf-8').lower() ) or ('jordan' in item['name'].encode('utf-8').lower() and 'retro' in item['name'].encode('utf-8').lower() ) or ('yeezy' in item['name'].encode('utf-8').lower()) or ( 'max' in item['name'].encode('utf-8').lower() and 'atmos' in item['name'].encode('utf-8').lower()) or ( 'max' in item['name'].encode('utf-8').lower() and 'master' in item['name'].encode('utf-8').lower() ) or ('ronnie' in item['name'].encode('utf-8').lower() ) or ('fieg' in item['name'].encode('utf-8').lower()): # Twitter Auth - Tweet the item with date, time, item name, and link. # To obtain Twitter CONSUMER and ACCESS keys go to https://apps.twitter.com/ CONSUMER_KEY = ' PASTE CONSUMER_KEY HERE ' CONSUMER_SECRET = ' PASTE CONSUMER_SECRET HERE ' ACCESS_TOKEN_KEY = ' PASTE ACCESS_TOKEN_KEY HERE ' ACCESS_TOKEN_SECRET = ' PASTE ACCESS_TOKEN_SECRET HERE ' API = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET) TEXT_TO_TWEET = DATE + " EST " + item['name'] + " " + item[ 'link'] TWEET = API.request('statuses/update', {'status': TEXT_TO_TWEET}) print(Fore.RED + 'TWEET LOG SUCCESS: ' + DATE + ' EST ' + item['name'] + ' ' + item['link'] + Style.RESET_ALL if TWEET.status_code == 200 else Fore.RED + 'TWEET LOG FAILURE: FAILED TO TWEET' + Style.RESET_ALL) except MySQLdb.Error, e: # print (Fore.RED + "MYSQL ERROR %d: %s" % (e.args[0], e.args[1] + Style.RESET_ALL)) return item
from threader import Threader import guizero # from twython import Twython import math # import time from textwrap import wrap from apikeys import (consumer_key, consumer_secret, access_token, access_token_secret) source_selected = True keys = dict(consumer_key=consumer_key, consumer_secret=consumer_secret, access_token_key=access_token, access_token_secret=access_token_secret) api = TwitterAPI(**keys) def send_tweet(): print("send tweet") tweet_array = wrap(tweet.value, 240) th = Threader(tweet_array, api, wait=slider.value) th.send_tweets() # # def insert_selected(): # api_key_line.show() # choose_key_area.hide() # global source_selected # source_selected = True
import datetime import json import urllib2 import threading from lxml import html import requests TRACK_TERM = 'Houston' TRACK_LOC = '-95.788087,29.523624,-95.014496,30.110732' CONSUMER_KEY = '----' CONSUMER_SECRET = '----' ACCESS_TOKEN_KEY = '----' ACCESS_TOKEN_SECRET = '---' api = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET) r = api.request('statuses/filter', { 'track': TRACK_TERM, 'locations': TRACK_LOC }) ### MongoDB Calling client = MongoClient("mongodb://127.0.0.1:3001/meteor") db = client['meteor'] for item in r.get_iterator(): if 'text' in item: a = item['text'] p = polarity(a) s = subjectivity(a) if (p != 0.0):
def get_twitter(): """ Construct an instance of TwitterAPI using the tokens you entered above. Returns: An instance of TwitterAPI. """ return TwitterAPI(consumer_key, consumer_secret, access_token, access_token_secret)
# get timeline tweet, return up to 3,200 of a user's most recent Tweets. from TwitterAPI import TwitterAPI, TwitterPager import pandas as pd # set twitter api Authorization keys, remember the order for the keys # api = TwitterAPI("consumer_API_key","consumer_API_secret", "access_token","access_token_secret") api = TwitterAPI("G9c9PkikFEffLezvmhdkHpWma", "2hLfuvejl3fhBp6Zxr1NMBT0jZk3LpNLdjTTKbDzWYBbZ3duRC", "1250078614445608967-vOiELbylTZ0GxGrnhBouPPbLTTzCDG", "GVDza7InbhVUZeme8g0Okoqsyo4WByTvZkFlZ4iX0jgJP") def get_all_tweets(screen_name): pager = TwitterPager(api, 'statuses/user_timeline', {'screen_name': screen_name, 'count': 200}) alltweets = pager.get_iterator(wait=3.5) outtweets = [ [screen_name, tweet['id_str'], pd.to_datetime(tweet['created_at']), tweet['user']['location'], tweet['retweet_count'], tweet['favorite_count'], tweet['lang'], tweet['text']] for tweet in alltweets] df = pd.DataFrame(outtweets, columns=['user_name', 'id', 'create_time', 'geo', 'retweets','favorite_count', 'language', 'text']) df.to_csv('results.csv', index=False) print('finish') if __name__ == '__main__': get_all_tweets('google')