def serviceBuild(authorName, serviceName, incremental, s3Bucket, aws_access_key, aws_secret_key): print("refresh %s serviceEvents for %s" % (authorName,serviceName)) dbSession = SimpleDBSession() # get the service-id for serviceName serviceId, = dbSession.query(Service.id).filter(Service.service_name == serviceName).one() # get author-id for authorName authorId, = dbSession.query(Author.id).filter(Author.author_name == authorName).one() mapping = dbSession.query(AuthorServiceMap).filter_by(service_id=serviceId,author_id=authorId).one() collector = EventCollectorFactory.get_collector_for(serviceName,s3Bucket, aws_access_key, aws_secret_key) if collector: collector.build_one(mapping,dbSession,oAuthConfig.get(serviceName),incremental) dbSession.close()
def main(self): # load the config config = ConfigParser() config.read('config.ini') # load AWS config awsConfig = ConfigParser() awsConfig.read(config.get('Common','aws')) # load the oauth config oauthfd=open(config.get('OAuthConfig','mi.oauthkey_file')) oauthConfig = json.load(oauthfd) oauthfd.close() # read the db url from the config dbUrl = config.get('DBConfig','sqlalchemy.url') # initialize the db engine & session engine = create_engine(dbUrl, encoding='utf-8', echo=False) DBSession.configure(bind=engine) dbSession = DBSession() # for controlling incremental vs. full builds incremental = self.options.incremental # service = "googleplus" # collector = EventCollectorFactory.get_collector_for(service,awsConfig.get('AWS','event_bucket'), awsConfig.get('AWS','aws_access_key'), awsConfig.get('AWS','aws_secret_key')) # if collector: # collector.build_all(dbSession,oauthConfig[service],incremental) for service, oauthConfig in oauthConfig.iteritems(): collector = EventCollectorFactory.get_collector_for(service,awsConfig.get('AWS','event_bucket'), awsConfig.get('AWS','aws_access_key'), awsConfig.get('AWS','aws_secret_key')) if collector: try: collector.build_all(dbSession,oauthConfig,incremental) except Exception, e: self.log.error('Collector error: %s' % e)