def open_spider(self, spider):
if not Pipeline.db:
Pipeline.db = models.create_db_session(
getattr(spider, 'dburi', models.DEFAULT_DBURI))
Pipeline.brands = {
i.title.lower(): i
for i in Pipeline.db.query(models.Brand).all()
}
self.item_set = set(
i[0] for i in Pipeline.db.query(models.Product.id).all())
if spider.name == 'RyansComputers':
self.preprocessor = RyansComputersPreProcessor()
elif spider.name == 'StarTech':
self.preprocessor = StarTechPreProcessor()
if len(spider.start_urls) > 1:
spider.log(
'. '.join([
"Should have only one start URL", "Have {} URLs",
"Picking up the <{}> as the main platform URL"
]).format(len(spider.start_urls)), spider.start_urls[0])
# Get or set the Platform object.
platform_url = urlparse(spider.start_urls[0]).netloc
platform_title = spider.platform_title
self.platform = Pipeline.db.query(models.Platform). \
filter(models.Platform.title == platform_title). \
filter(models.Platform.url == platform_url).first()
if not self.platform:
self.platform = models.Platform(title=platform_title,
url=platform_url)
Pipeline.db.add(self.platform)
def get(self, *args, **kwargs):
delete = self.get_argument('delete', None)
if delete:
delete = int(delete)
db_session = models.create_db_session()
try:
server = db_session.query(models.Server).filter(models.Server.id == delete).first()
if server.user_id == int(self.get_current_user_id()):
db_session.delete(server)
db_session.commit()
except Exception as e:
my_log.exception(e)
finally:
db_session.close()
self.redirect('/')
else:
server_id = self.get_argument('server_id', None)
server_name = self.get_argument('server_name', None)
host = self.get_argument('host', None)
port = self.get_argument('port', None)
parameter = {
'server_id': server_id,
'server_name': server_name,
'host': host,
'port': port
}
self.render('server.html', **parameter)
def get(self, *args, **kwargs):
user_id = int(self.get_current_user_id())
db_session = models.create_db_session()
server_list = None
try:
server_list = db_session.query(models.Server).filter(models.Server.user_id == user_id)
except Exception as e:
my_log.exception(e)
finally:
db_session.close()
self.render('index.html', server_list=server_list)
def post(self, *args, **kwargs):
db_session = models.create_db_session()
server_id = self.get_argument('server_id', None)
server_name = self.get_argument('server_name', 'My Server')
host = self.get_argument('host')
port = int(self.get_argument('port', 22))
username = self.get_argument('username')
password = self.get_argument('password')
user_id = int(self.get_current_user_id())
if server_id:
if user_id:
try:
server = db_session.query(models.Server).filter(models.Server.id == int(server_id)).first()
if server.user_id == int(user_id):
server_info = dict(server_name=server_name, host=host, port=port,
username=username, password=password)
server.server_name = server_name
server.host = host
server.port = port
server.username = username
server.password = password
db_session.commit()
else:
pass
except Exception as e:
my_log.exception(e)
finally:
db_session.close()
self.redirect('/')
else:
pass
else:
try:
server_info = dict(server_name=server_name, host=host, port=port,
username=username, password=password, user_id=user_id)
server = models.Server(**server_info)
db_session.add(server)
db_session.commit()
except Exception as e:
my_log.exception(e)
finally:
db_session.close()
self.redirect('/')
def post(self, *args, **kwargs):
email = self.get_argument('email')
password = self.get_argument('password')
db_session = models.create_db_session()
try:
user = db_session.query(models.User).filter(models.User.email == email).first()
if not user:
md5 = hashlib.md5()
md5.update(password)
password = md5.hexdigest()
user = models.User(email=email, password=password)
db_session.add(user)
db_session.commit()
self.render('signup.html', status=1)
else:
self.render('signup.html', status=2)
except Exception as e:
my_log.exception(e)
finally:
db_session.close()
def get(self, *args, **kwargs):
server_id = self.get_argument('server_id')
server_info = {}
if server_id:
db_session = models.create_db_session()
try:
server = db_session.query(models.Server).filter(models.Server.id == int(server_id)).first()
if server and server.user_id == int(self.get_current_user_id()):
server_info = dict(host=server.host,
port=server.port,
username=server.username,
password=server.password)
self.render('ssh.html', **server_info)
else:
self.redirect('/')
except Exception as e:
my_log.exception(e)
finally:
db_session.close()
else:
self.redirect('/')
def post(self, *args, **kwargs):
email = self.get_argument('email')
password = self.get_argument('password')
db_session = models.create_db_session()
try:
user = db_session.query(models.User).filter(models.User.email == email).first()
if user:
md5 = hashlib.md5()
md5.update(password)
password = md5.hexdigest()
if password == user.password:
self.set_secure_cookie('user', email)
self.set_secure_cookie('user_id', str(user.id))
self.redirect('/')
else:
self.render('login.html', status=1)
else:
self.render('login.html', status=2)
except Exception as e:
my_log.exception(e)
finally:
db_session.close()
except:
return url
logger = logging.getLogger('fakenews')
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
db_eng = db_connect()
db_session = create_db_session(db_eng)
# and
tweets = db_session.query(Tweet).filter(
Tweet.UserScreenName == "realDonaldTrump").all()
count = 0
published = 0
for t in tweets:
print(count, t.TweetId)
count += 1
url_list = t.Urls.replace("{", "").replace("}", "").split(",")
print(type(url_list))
for url in url_list:
try:
def application_setup():
flask.g.db_session = models.create_db_session(app.config['SLACKASK_DB_URI'])
from utils import UpdateTweet
from models import Tweet, db_connect, create_db_session
from config import logger, update
import logging
logging.basicConfig(level=logger.get("level"), format=logger.get("format"))
db_session = create_db_session(db_connect())
size = update.get("size")
rows = db_session.query(Tweet).filter(
Tweet.UserScreenName == "realDonaldTrump").order_by(
Tweet.TweetDate.desc()).limit(size)
ids = [r.TweetId for r in rows]
tweets = UpdateTweet(ids)
cnt = 1
for t in tweets:
retweets = t.retweet_count
favorites = t.favorite_count
if hasattr(t, 'retweeted_status'):
favorites = t.retweeted_status.favorite_count
old = db_session.query(Tweet).filter(Tweet.TweetId == t.id).all()[0]
msg = "%s - ID: %s | oR: %6s, oF: %6s | nR: %6s, nF: %6s |" % (
cnt, t.id, old.RetweetCount, old.FavoriteCount, retweets, favorites)
logging.warning(msg)
old.RetweetCount = retweets
old.FavoriteCount = favorites
def application_setup():
flask.g.db_session = models.create_db_session(
app.config['SLACKASK_DB_URI'])
def parse(account):
db_engine = db_connect()
db_session = create_db_session(db_engine)
with open(account + '_long.json', 'r') as content_file:
content = json.loads(content_file.read())
logger.info("%s has %s tweets" % (account, len(content)))
cnt = 0
for doc in content:
TweetId = doc.get('id')
TweetDate = arrow.get(
doc.get('created_at'),
"ddd MMM DD HH:mm:ss Z YYYY").format('YYYY-MM-DD HH:mm:ss ZZ')
Text = doc.get("text")
RetweetCount = doc.get("retweet_count")
FavoriteCount = doc.get("favorite_count")
Retweeted = None
OriginalTweetId = None
OriginalTweetUserId = None
OriginalTweetUserName = None
OriginalTweetUserScreenName = None
OriginalTweetDate = None
if doc.get("retweeted_status") is not None:
Retweeted = True
OriginalTweetId = doc.get("retweeted_status").get('id')
OriginalTweetUserId = doc.get("retweeted_status").get(
'user').get('id')
OriginalTweetUserName = doc.get("retweeted_status").get(
'user').get('name')
OriginalTweetUserScreenName = doc.get("retweeted_status").get(
'user').get('screen_name')
OriginalTweetDate = arrow.get(
doc.get("retweeted_status").get('created_at'),
"ddd MMM DD HH:mm:ss Z YYYY").format(
'YYYY-MM-DD HH:mm:ss ZZ')
Favorited = doc.get("favorited")
TweetLang = doc.get('lang')
Source = doc.get('source',
"").partition('>')[-1].rpartition('<')[0]
Mentions = []
if doc.get('entities').get('user_mentions'):
for m in doc.get('entities').get('user_mentions'):
Mentions.append(m.get('screen_name'))
Hashtags = []
if doc.get('entities').get('hashtags'):
for m in doc.get('entities').get('hashtags'):
Hashtags.append(m.get('text'))
Urls = []
if doc.get('entities').get('urls'):
for m in doc.get('entities').get('urls'):
Urls.append(m.get('expanded_url'))
UserId = doc.get('user').get('id')
UserName = doc.get('user').get('name')
UserScreenName = doc.get('user').get('screen_name')
UserCreatedDate = arrow.get(
doc.get('user').get('created_at'),
"ddd MMM DD HH:mm:ss Z YYYY").format('YYYY-MM-DD HH:mm:ss ZZ')
UserLang = doc.get('user').get('lang')
UserLocation = doc.get('user').get('location')
UserTimeZone = doc.get('user').get('time_zone')
UserUTCOffset = doc.get('user').get('utc_offset')
UserFollowerCount = doc.get('user').get('followers_count')
UserFriendsCount = doc.get('user').get('friends_count')
UserFavoritesCount = doc.get('user').get('statuses_count')
UserStatusesCount = doc.get('user').get('favourites_count')
tweet = Tweet(
TweetId=TweetId,
TweetDate=TweetDate,
Text=Text,
RetweetCount=RetweetCount,
FavoriteCount=FavoriteCount,
Retweeted=Retweeted,
Favorited=Favorited,
TweetLang=TweetLang,
Source=Source,
Mentions=Mentions,
Hashtags=Hashtags,
Urls=Urls,
OriginalTweetId=OriginalTweetId,
OriginalTweetUserId=OriginalTweetUserId,
OriginalTweetUserName=OriginalTweetUserName,
OriginalTweetUserScreenName=OriginalTweetUserScreenName,
OriginalTweetDate=OriginalTweetDate,
UserId=UserId,
UserName=UserName,
UserScreenName=UserScreenName,
UserCreatedDate=UserCreatedDate,
UserLang=UserLang,
UserLocation=UserLocation,
UserTimeZone=UserTimeZone,
UserUTCOffset=UserUTCOffset,
UserFollowerCount=UserFollowerCount,
UserFriendsCount=UserFriendsCount,
UserFavoritesCount=UserFavoritesCount,
UserStatusesCount=UserStatusesCount)
db_session.merge(tweet)
cnt += 1
if not cnt % config.commitnumber:
db_session.commit()
logger.info("Committed %s rows from account %s" %
(cnt, account))
if config.isDev:
break
logger.info("Committed %s rows from account %s" % (cnt, account))
db_session.commit()