def twapi_search(query, count, *, sleep_time=0, **kwargs):
with open(DEBUG_FILENAME, 'w'):
pass
last_id = None
num_fetched = 0
while count > 0:
next_count = min(count, MAX_COUNT_PER_REQ)
next_id = last_id - 1 if last_id is not None else None
data = list(
twapi_search_page(query,
next_count,
next_id,
sleep_time=sleep_time,
**kwargs))
if not data:
cl.warning('No more data can be retrieved, terminating...')
return
for item in data:
last_id = int(item['id'])
yield item
num_fetched += len(data)
count -= len(data)
cl.info('Current number of records fetched: %d' % num_fetched)
def export_csv(data, outfilename, encoding='utf-8'):
cl.progress('Exporting data to csv file: %s' % outfilename)
it = iter(data)
num_records = 0
try:
first_item = next(it)
except StopIteration:
cl.warning('Empty data. Export aborted.')
return
else:
num_records += 1
with open(outfilename, 'w', newline='', encoding=encoding) as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=first_item.keys())
writer.writeheader()
writer.writerow(first_item)
try:
for item in it:
num_records += 1
writer.writerow(item)
except KeyboardInterrupt:
cl.warning('User hit Ctrl-C, flushing data...')
cl.success('%d record(s) saved to csv file.' % num_records)
def recover_from_csv(csvfilename):
progress = 0
for row in csv_reader(csvfilename):
progress += 1
if progress % 1000 == 0:
cl.progress('%d record(s) have been recovered' % progress)
yield row
if int(row['retweets']):
try:
retweets = twapi.GetRetweets(int(row['id']), count=100)
except Exception:
cl.warning('Error: %s' % get_exc_line())
else:
for tweet in retweets:
yield {
'id': tweet.id_str,
'text': row['text'],
'timestamp': tweet.created_at,
'likes': tweet.favorite_count,
'retweets': tweet.retweet_count,
'replies': None,
'url': None,
'html': None,
'user': merge_whitespaces(tweet.user.screen_name),
'fullname': merge_whitespaces(tweet.user.name)
}
def retry_until_success(func, *args, **kwargs):
MAX_RETRY = 2
retry = 0
ret = None
while True:
try:
try:
ret = func(*args, **kwargs)
except KeyboardInterrupt:
raise
except Exception:
if retry >= MAX_RETRY:
cl.error('Error: %sMax retries exceeded, terminating.' %
traceback.format_exc())
raise
cl.error('Error: %sRetrying...' % traceback.format_exc())
retry += 1
else:
return ret
except KeyboardInterrupt:
cl.warning('User hit Ctrl-C, terminating function %r' %
func.__name__)
raise
def demo2():
cl.section('Demo 2')
username = ''
while not username:
username = cl.input('Username: '******''
while not password:
password = cl.password('Password: '******'Successfully logged in.')
with cl.progress('Checking for update...', mode=cl.PROGRESS_SPIN):
time.sleep(3)
choice = ''
while choice.lower() not in {'y', 'n'}:
choice = cl.question(
'A new version is present, would you like to update? (Y/N)').strip(
)
if choice.lower() == 'y':
with cl.progress('Downloading ', mode=cl.PROGRESS_DETERMINATE) as p:
time.sleep(1)
p.update(0.2, ' 20% (1MB/5MB) ETA 4s')
time.sleep(2)
p.update(0.4, ' 40% (2MB/5MB) ETA 3s')
cl.error('Failed to download package. SSL handshake error.')
else:
cl.warning('Update delayed!')
def overview():
cl.section('Overview of Labels')
cl.success('Good job! All test cases passed!')
cl.warning('Warning! Security update delayed!')
cl.error('Error! Failed to write file!')
cl.info('Server listening on port 8888.')
cl.progress('Downloading package, please wait...')
cl.plain('Nothing interesting.')
cl.question('A new version is present, would you like to update? (Y/N)')
def iterator_aggregate_list(data):
if hasattr(data, '__next__'):
result = []
try:
result.extend(data)
except Exception:
cl.warning('Exception happened while collecting data: %s' %
get_exc_line())
except KeyboardInterrupt:
cl.warning('User hit Ctrl-C. Will not collect more data.')
return result
elif isinstance(data, list):
return data
else:
return list(data)
def main():
try:
while True:
welcome()
option = get_menu_option()
if option != 5:
cl.newline()
if option == 1:
overview()
elif option == 2:
animations()
elif option == 3:
demo1()
elif option == 4:
demo2()
else:
cl.plain('Bye!')
break
cl.newline()
cl.input('Press Enter to continue...')
cl.newline()
except (KeyboardInterrupt, EOFError):
cl.newline()
cl.warning('Ctrl-C or EOF received, quitting.')
def bye():
cl.newline()
cl.warning('Bye~', mark='~')
sys.exit(0)
import itertools
import colorlabels as cl
from decouple import config
from github import Github
from util import merge_whitespaces
try:
gh = Github(config('GITHUB_ACCESS_TOKEN'))
except Exception:
cl.warning('GitHub access token is not correctly configured, '
'cannot retrieve GitHub data.')
gh = None
def get_labels(issue):
for label in issue.labels:
yield label.name
def github_issue_repo_fetch(repo, since):
if isinstance(repo, str):
repo = gh.get_repo(repo)
cl.plain('Fetching issues from repo: %s' % repo.full_name)
for issue in repo.get_issues(state='all', since=since):
yield {
'id': 'issue-%d' % issue.number,
'text': merge_whitespaces('%s %s' % (issue.title, issue.body)),
DEBUG_FILENAME = log_file('twdebug.log')
MAX_COUNT_PER_REQ = 100
credentials = {
'consumer_key': config('TWITTER_CONSUMER_KEY'),
'consumer_secret': config('TWITTER_CONSUMER_SECRET'),
'access_token_key': config('TWITTER_ACCESS_TOKEN_KEY'),
'access_token_secret': config('TWITTER_ACCESS_TOKEN_SECRET')
}
try:
twapi = twitter.Api(**credentials,
application_only_auth=True,
sleep_on_rate_limit=True)
except Exception:
cl.warning('Twitter access token is not correctly configured, '
'cannot retrieve Twitter data in the official way.')
twapi = None
def get_full_text(tweet):
if tweet.retweeted_status:
return tweet.retweeted_status.full_text
else:
return tweet.full_text
def get_hashtags(tweet):
for hashtag in tweet.hashtags:
if hashtag.text:
yield hashtag.text
import sys
import colorlabels as cl
from service.auth import register
from util import validate_password, validate_username
if __name__ == '__main__':
if len(sys.argv) < 3:
cl.warning('Usage: %s username password' % sys.argv[0])
sys.exit(-1)
username = sys.argv[1]
password = sys.argv[2]
r = validate_username(username)
if not r:
cl.error(str(r))
sys.exit(-1)
r = validate_password(password)
if not r:
cl.error(str(r))
sys.exit(-1)
if register(username, password):
cl.success('Successfully registered user %r.' % username)
else:
cl.error('User %r already exists!' % username)