if vaultName == 'LIST':
try:
logging.info('Getting list of vaults...')
vaults = glacier.list_vaults()
except:
printException()
sys.exit(1)
for vault in vaults:
logging.info(vault.name)
exit(0)
try:
logging.info('Getting selected vault...')
vault = glacier.get_vault(vaultName)
except:
printException()
sys.exit(1)
logging.info('Getting jobs list...')
jobList = vault.list_jobs()
jobID = ''
# Check if a job already exists
for job in jobList:
if job.action == 'InventoryRetrieval':
logging.info('Found existing inventory retrieval job...')
jobID = job.id
if jobID == '':
def getProgram(self, which):
if 'info' == which:
clients = []
scrapeservices = []
userservices = []
myuserservice = self.getService('user')
lockservice = self.getService('lock', which)
for i in range(15):
clients.append(self.getTwitterAPI())
userservices.append(self.getService('user'))
scrapeservices.append(self.getService('scrape', 'info'))
scrapeservices.append(self.getService('scrape', 'info')) # append an extra for main thread
scanservice = self.getService('scan', 'info')
return InfoScraper(clients, userservices, myuserservice, lockservice, scrapeservices, scanservice)
elif 'followers_wide' == which:
clients = []
edgeservices = []
scrapeservices = []
userservice = self.getService('user')
lockservice = self.getService('lock', which)
for i in range(32):
clients.append(self.getTwitterAPI())
edgeservices.append(self.getService('edge'))
scrapeservices.append(self.getService('scrape', 'followers_wide'))
scrapeservices.append(self.getService('scrape', 'followers_wide')) # append an extra for main thread
scanservice = self.getService('scan', 'followers_wide')
return WideFollowersScraper(clients, edgeservices, userservice, lockservice, scrapeservices, scanservice)
elif 'followers_bot' == which:
clients = []
edgeservices = []
scrapeservices = []
userservice = self.getService('user')
lockservice = self.getService('lock', which)
for i in range(32):
clients.append(self.getTwitterAPI())
edgeservices.append(self.getService('edge'))
scrapeservices.append(self.getService('scrape', 'followers_bot'))
scrapeservices.append(self.getService('scrape', 'followers_bot')) # append an extra for main thread
scanservice = self.getService('scan', 'followers_bot')
return BotFollowersScraper(clients, edgeservices, userservice, lockservice, scrapeservices, scanservice)
elif 'followers_bot_v2' == which:
clients = []
edgeservices = []
scrapeservices = []
botservices = []
userservice = self.getService('user')
lockservice = self.getService('lock', which)
for i in range(32):
clients.append(self.getTwitterAPI())
edgeservices.append(self.getService('bot_edge'))
botservices.append(self.getService('bot'))
scrapeservices.append(self.getService('scrape', 'followers_bot_v2'))
scrapeservices.append(self.getService('scrape', 'followers_bot_v2')) # append an extra for main thread
scanservice = self.getService('scan', 'followers_bot_v2')
return BotFollowersScraper(clients, edgeservices, userservice, lockservice, scrapeservices, scanservice, botservices)
elif 'followers' == which:
clients = []
edgeservices = []
scrapeservices = []
userservice = self.getService('user')
lockservice = self.getService('lock', which)
for i in range(32):
clients.append(self.getTwitterAPI())
edgeservices.append(self.getService('edge'))
scrapeservices.append(self.getService('scrape', 'followers'))
scrapeservices.append(self.getService('scrape', 'followers')) # append an extra for main thread
scanservice = self.getService('scan', 'followers')
return FollowersScraper(clients, edgeservices, userservice, lockservice, scrapeservices, scanservice)
elif 'competition-tweets' == which:
clients = []
tweetservices = []
scrapeservices = []
userservice = self.getService('user')
lockservice = self.getService('lock', which)
for i in range(32):
clients.append(self.getTwitterAPI())
tweetservices.append(self.getService('tweet'))
scrapeservices.append(self.getService('scrape', 'tweets'))
scrapeservices.append(self.getService('scrape', 'tweets')) # append an extra for main thread
scanservice = self.getService('scan', 'tweets')
stats = self.getStats()
return CompetitionTweetsScraper(clients, tweetservices, userservice, lockservice, scrapeservices, scanservice, stats)
elif 'backup' == which:
glacier = boto.glacier.layer2.Layer2(self.config.get('glacier', 'key'), self.config.get('glacier', 'secret'), region_name=self.config.get('glacier', 'region'))
vault = glacier.get_vault(self.config.get('glacier', 'vault-postgresqlbackups'))
lockservice = self.getService('lock', which)
backupservice = self.getService('backup')
edgeservice = EdgeServiceBig(self.getDatabaseCursor(), self.getDatabaseCursor('backupedge', False))
userservice = self.getService('user')
tweetservice = self.getService('tweet')
scanservices = {
'followers': self.getService('scan', 'followers'),
'info': self.getService('scan', 'info'),
'tweets': self.getService('scan', 'tweets')
}
return Backup(vault, lockservice, backupservice, edgeservice, userservice, tweetservice, scanservices, int(float(self.config.get('smisc', 'backup_page_size'))))
elif 'scoring' == which:
lockservice = self.getService('lock', which)
botservice = self.getService('bot')
teamlinkservice = self.getService('teamlink')
tweetservice = BigTweetService(self.getDatabaseCursor('tweet', False))
scoreservice = self.getService('score')
stats = self.getStats()
return Scoring(lockservice, botservice, teamlinkservice, tweetservice, scoreservice, stats)
elif 'reporter' == which:
stats = self.getStats()
lockservice = self.getService('lock', which)
queues = {
'followers_wide': self.getService('scrape', 'followers_wide'),
'followers_bot_v2': self.getService('scrape', 'followers_bot_v2'),
'info': self.getService('scrape', 'info'),
'tweets': self.getService('scrape', 'tweets')
}
return Reporter(lockservice, stats, queues)