# Grab our tweet list (tweets since last tweet up to 3200), optimized for # fewest requests. try: if last_tweet: tweets = t.getUserTimeline(screen_name=ca.conf['screen_name'], count=200, since_id=last_tweet, include_rts=True) else: tweets = t.getUserTimeline(screen_name=ca.conf['screen_name'], count=200, include_rts=True) except TwythonError, e: ca.log_fail("Error accessing twitter stream. User not found or twitter down.") exit() if len(tweets) == 0: ca.log("No new tweets.") exit() tweet_list.extend(tweets) c = 200 lowest_tweet = tweets[-1]['id'] while tweets[-1]['id'] > last_tweet and c < 3200 and tweets: try: tweets = t.getUserTimeline(screen_name=ca.conf['screen_name'], max_id=tweets[-1]['id'], count=200, include_rts=True) if lowest_tweet == tweets[-1]['id']: break tweet_list.extend(tweets) c += 200 ca.log("Grabbed "+str(len(tweets))+" starting at "+str(tweets[-1]['id'])) lowest_tweet = tweets[-1]['id']
# ca.conf['s3bucket'], # ca.conf['s3accesskey'].encode('ascii'), # ca.conf['s3secretkey'].encode('ascii'))) if not key.name in files: pool.apply_async(worker, args=(region_endpoints['publicURL'], ca.conf['container'], ca.creds['token'], key.name, ca.conf['s3bucket'], ca.conf['s3accesskey'].encode('ascii'), ca.conf['s3secretkey'].encode('ascii'))) pool.join() if ca.conf.get('emailreport') and copied_files: ca.log("Sending email.") ca.email("Copied "+str(len(copied_files))+" files to "+ca.conf['container'],''' Copied the follow files from S3 bucket %s to Swift container %s: %s ''' % (ca.conf['s3bucket'],ca.conf['container'],"\n".join(copied_files))) ca.log("Done.",'',100) def worker(endpoint,container,token,key_name,bucket,accesskey,secretkey):
# key.name, # ca.conf['s3bucket'], # ca.conf['s3accesskey'].encode('ascii'), # ca.conf['s3secretkey'].encode('ascii'))) if not key.name in files: pool.apply_async(worker, args=(region_endpoints['publicURL'], ca.conf['container'], ca.creds['token'], key.name, ca.conf['s3bucket'], ca.conf['s3accesskey'].encode('ascii'), ca.conf['s3secretkey'].encode('ascii'))) pool.join() if ca.conf.get('emailreport') and copied_files: ca.log("Sending email.") ca.email( "Copied " + str(len(copied_files)) + " files to " + ca.conf['container'], ''' Copied the follow files from S3 bucket %s to Swift container %s: %s ''' % (ca.conf['s3bucket'], ca.conf['container'], "\n".join(copied_files))) ca.log("Done.", '', 100) def worker(endpoint, container, token, key_name, bucket, accesskey, secretkey): #ca.log("Starting "+str(key_name)) my_s3 = S3Connection(accesskey, secretkey)