def open_spider(self, spider): if not Pipeline.db: Pipeline.db = models.create_db_session( getattr(spider, 'dburi', models.DEFAULT_DBURI)) Pipeline.brands = { i.title.lower(): i for i in Pipeline.db.query(models.Brand).all() } self.item_set = set( i[0] for i in Pipeline.db.query(models.Product.id).all()) if spider.name == 'RyansComputers': self.preprocessor = RyansComputersPreProcessor() elif spider.name == 'StarTech': self.preprocessor = StarTechPreProcessor() if len(spider.start_urls) > 1: spider.log( '. '.join([ "Should have only one start URL", "Have {} URLs", "Picking up the <{}> as the main platform URL" ]).format(len(spider.start_urls)), spider.start_urls[0]) # Get or set the Platform object. platform_url = urlparse(spider.start_urls[0]).netloc platform_title = spider.platform_title self.platform = Pipeline.db.query(models.Platform). \ filter(models.Platform.title == platform_title). \ filter(models.Platform.url == platform_url).first() if not self.platform: self.platform = models.Platform(title=platform_title, url=platform_url) Pipeline.db.add(self.platform)
def get(self, *args, **kwargs): delete = self.get_argument('delete', None) if delete: delete = int(delete) db_session = models.create_db_session() try: server = db_session.query(models.Server).filter(models.Server.id == delete).first() if server.user_id == int(self.get_current_user_id()): db_session.delete(server) db_session.commit() except Exception as e: my_log.exception(e) finally: db_session.close() self.redirect('/') else: server_id = self.get_argument('server_id', None) server_name = self.get_argument('server_name', None) host = self.get_argument('host', None) port = self.get_argument('port', None) parameter = { 'server_id': server_id, 'server_name': server_name, 'host': host, 'port': port } self.render('server.html', **parameter)
def get(self, *args, **kwargs): user_id = int(self.get_current_user_id()) db_session = models.create_db_session() server_list = None try: server_list = db_session.query(models.Server).filter(models.Server.user_id == user_id) except Exception as e: my_log.exception(e) finally: db_session.close() self.render('index.html', server_list=server_list)
def post(self, *args, **kwargs): db_session = models.create_db_session() server_id = self.get_argument('server_id', None) server_name = self.get_argument('server_name', 'My Server') host = self.get_argument('host') port = int(self.get_argument('port', 22)) username = self.get_argument('username') password = self.get_argument('password') user_id = int(self.get_current_user_id()) if server_id: if user_id: try: server = db_session.query(models.Server).filter(models.Server.id == int(server_id)).first() if server.user_id == int(user_id): server_info = dict(server_name=server_name, host=host, port=port, username=username, password=password) server.server_name = server_name server.host = host server.port = port server.username = username server.password = password db_session.commit() else: pass except Exception as e: my_log.exception(e) finally: db_session.close() self.redirect('/') else: pass else: try: server_info = dict(server_name=server_name, host=host, port=port, username=username, password=password, user_id=user_id) server = models.Server(**server_info) db_session.add(server) db_session.commit() except Exception as e: my_log.exception(e) finally: db_session.close() self.redirect('/')
def post(self, *args, **kwargs): email = self.get_argument('email') password = self.get_argument('password') db_session = models.create_db_session() try: user = db_session.query(models.User).filter(models.User.email == email).first() if not user: md5 = hashlib.md5() md5.update(password) password = md5.hexdigest() user = models.User(email=email, password=password) db_session.add(user) db_session.commit() self.render('signup.html', status=1) else: self.render('signup.html', status=2) except Exception as e: my_log.exception(e) finally: db_session.close()
def get(self, *args, **kwargs): server_id = self.get_argument('server_id') server_info = {} if server_id: db_session = models.create_db_session() try: server = db_session.query(models.Server).filter(models.Server.id == int(server_id)).first() if server and server.user_id == int(self.get_current_user_id()): server_info = dict(host=server.host, port=server.port, username=server.username, password=server.password) self.render('ssh.html', **server_info) else: self.redirect('/') except Exception as e: my_log.exception(e) finally: db_session.close() else: self.redirect('/')
def post(self, *args, **kwargs): email = self.get_argument('email') password = self.get_argument('password') db_session = models.create_db_session() try: user = db_session.query(models.User).filter(models.User.email == email).first() if user: md5 = hashlib.md5() md5.update(password) password = md5.hexdigest() if password == user.password: self.set_secure_cookie('user', email) self.set_secure_cookie('user_id', str(user.id)) self.redirect('/') else: self.render('login.html', status=1) else: self.render('login.html', status=2) except Exception as e: my_log.exception(e) finally: db_session.close()
except: return url logger = logging.getLogger('fakenews') logger.setLevel(logging.DEBUG) ch = logging.StreamHandler() formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) logger.addHandler(ch) db_eng = db_connect() db_session = create_db_session(db_eng) # and tweets = db_session.query(Tweet).filter( Tweet.UserScreenName == "realDonaldTrump").all() count = 0 published = 0 for t in tweets: print(count, t.TweetId) count += 1 url_list = t.Urls.replace("{", "").replace("}", "").split(",") print(type(url_list)) for url in url_list: try:
def application_setup(): flask.g.db_session = models.create_db_session(app.config['SLACKASK_DB_URI'])
from utils import UpdateTweet from models import Tweet, db_connect, create_db_session from config import logger, update import logging logging.basicConfig(level=logger.get("level"), format=logger.get("format")) db_session = create_db_session(db_connect()) size = update.get("size") rows = db_session.query(Tweet).filter( Tweet.UserScreenName == "realDonaldTrump").order_by( Tweet.TweetDate.desc()).limit(size) ids = [r.TweetId for r in rows] tweets = UpdateTweet(ids) cnt = 1 for t in tweets: retweets = t.retweet_count favorites = t.favorite_count if hasattr(t, 'retweeted_status'): favorites = t.retweeted_status.favorite_count old = db_session.query(Tweet).filter(Tweet.TweetId == t.id).all()[0] msg = "%s - ID: %s | oR: %6s, oF: %6s | nR: %6s, nF: %6s |" % ( cnt, t.id, old.RetweetCount, old.FavoriteCount, retweets, favorites) logging.warning(msg) old.RetweetCount = retweets old.FavoriteCount = favorites
def application_setup(): flask.g.db_session = models.create_db_session( app.config['SLACKASK_DB_URI'])
def parse(account): db_engine = db_connect() db_session = create_db_session(db_engine) with open(account + '_long.json', 'r') as content_file: content = json.loads(content_file.read()) logger.info("%s has %s tweets" % (account, len(content))) cnt = 0 for doc in content: TweetId = doc.get('id') TweetDate = arrow.get( doc.get('created_at'), "ddd MMM DD HH:mm:ss Z YYYY").format('YYYY-MM-DD HH:mm:ss ZZ') Text = doc.get("text") RetweetCount = doc.get("retweet_count") FavoriteCount = doc.get("favorite_count") Retweeted = None OriginalTweetId = None OriginalTweetUserId = None OriginalTweetUserName = None OriginalTweetUserScreenName = None OriginalTweetDate = None if doc.get("retweeted_status") is not None: Retweeted = True OriginalTweetId = doc.get("retweeted_status").get('id') OriginalTweetUserId = doc.get("retweeted_status").get( 'user').get('id') OriginalTweetUserName = doc.get("retweeted_status").get( 'user').get('name') OriginalTweetUserScreenName = doc.get("retweeted_status").get( 'user').get('screen_name') OriginalTweetDate = arrow.get( doc.get("retweeted_status").get('created_at'), "ddd MMM DD HH:mm:ss Z YYYY").format( 'YYYY-MM-DD HH:mm:ss ZZ') Favorited = doc.get("favorited") TweetLang = doc.get('lang') Source = doc.get('source', "").partition('>')[-1].rpartition('<')[0] Mentions = [] if doc.get('entities').get('user_mentions'): for m in doc.get('entities').get('user_mentions'): Mentions.append(m.get('screen_name')) Hashtags = [] if doc.get('entities').get('hashtags'): for m in doc.get('entities').get('hashtags'): Hashtags.append(m.get('text')) Urls = [] if doc.get('entities').get('urls'): for m in doc.get('entities').get('urls'): Urls.append(m.get('expanded_url')) UserId = doc.get('user').get('id') UserName = doc.get('user').get('name') UserScreenName = doc.get('user').get('screen_name') UserCreatedDate = arrow.get( doc.get('user').get('created_at'), "ddd MMM DD HH:mm:ss Z YYYY").format('YYYY-MM-DD HH:mm:ss ZZ') UserLang = doc.get('user').get('lang') UserLocation = doc.get('user').get('location') UserTimeZone = doc.get('user').get('time_zone') UserUTCOffset = doc.get('user').get('utc_offset') UserFollowerCount = doc.get('user').get('followers_count') UserFriendsCount = doc.get('user').get('friends_count') UserFavoritesCount = doc.get('user').get('statuses_count') UserStatusesCount = doc.get('user').get('favourites_count') tweet = Tweet( TweetId=TweetId, TweetDate=TweetDate, Text=Text, RetweetCount=RetweetCount, FavoriteCount=FavoriteCount, Retweeted=Retweeted, Favorited=Favorited, TweetLang=TweetLang, Source=Source, Mentions=Mentions, Hashtags=Hashtags, Urls=Urls, OriginalTweetId=OriginalTweetId, OriginalTweetUserId=OriginalTweetUserId, OriginalTweetUserName=OriginalTweetUserName, OriginalTweetUserScreenName=OriginalTweetUserScreenName, OriginalTweetDate=OriginalTweetDate, UserId=UserId, UserName=UserName, UserScreenName=UserScreenName, UserCreatedDate=UserCreatedDate, UserLang=UserLang, UserLocation=UserLocation, UserTimeZone=UserTimeZone, UserUTCOffset=UserUTCOffset, UserFollowerCount=UserFollowerCount, UserFriendsCount=UserFriendsCount, UserFavoritesCount=UserFavoritesCount, UserStatusesCount=UserStatusesCount) db_session.merge(tweet) cnt += 1 if not cnt % config.commitnumber: db_session.commit() logger.info("Committed %s rows from account %s" % (cnt, account)) if config.isDev: break logger.info("Committed %s rows from account %s" % (cnt, account)) db_session.commit()