def initializeDB(): """ Initialize db and mappers for script and return a session """ pg = getUtility(IDatabase, 'postgres') session = pg.session initialize_declarative_mappers(DeclarativeBase, pg.metadata) initialize_defered_mappers(pg.metadata) return session
def main(): import gites.calendar.scripts parseZCML(gites.calendar.scripts, 'calendar_import.zcml') pg = getUtility(IDatabase, 'postgres') pg.session initialize_declarative_mappers(DeclarativeBase, pg.metadata) initialize_defered_mappers(pg.metadata) WalhebConsumingServer('walhebcalendar', '') connect_all() asyncore.loop()
def setupDatabase(self): configurationContext = self['configurationContext'] xmlconfig.file('testing.zcml', self.package, context=configurationContext) configurationContext.execute_actions() schema_file = os.path.join(CURRENT_DIR, 'tests', 'gites_wallons.sql') self.db = gocept.testdb.PostgreSQL(encoding='UTF8', db_template='gites_wallons_testing', schema_path=schema_file) self.db.create() self.engine = create_engine(self.db.dsn) self.conn = self.engine.connect() self.pg = GitesTestDB() self.pg.dsn = self.db.dsn self.session = self.pg.session initialize_declarative_mappers(DeclarativeBase, self.pg.metadata) initialize_defered_mappers(self.pg.metadata) provideUtility(self.pg, IDatabase, 'postgres')
def main(): import gdw.stats.scripts parseZCML(gdw.stats.scripts, 'scripts.zcml') db = getUtility(IDatabase, 'postgres') session = db.session initialize_declarative_mappers(DeclarativeBase, db.metadata) initialize_defered_mappers(db.metadata) p = apachelog.parser(FORMAT) logfilePath, website = getConfig() maxDate = getMaxDate(website).max_1 if maxDate is None: maxDate = datetime(1970, 1, 1) cpt = 0 for line in open(logfilePath): try: data = p.parse(line) except: continue if data is None: continue date = apachelog.parse_date(data['%t']) date = datetime(*time.strptime(date[0], '%Y%m%d%H%M%S')[:6]) if date <= maxDate: continue code = data['%>s'] if int(code) not in VALID_HTTP_CODE: continue path = re.match('(.*) (.*) (.*)', data['%r']).group(2) path = urlparse.urlparse(path)[2] # path : '/hebergement/logement/beauraing/hebid' if len(path.strip('/').split('/')) != 4: continue if path.lstrip('/').split('/')[0] != 'hebergement': continue if path.endswith('/view'): continue if path.endswith('/gallery'): continue hebid = path.rstrip('/').split('/')[-1] if os.path.splitext(hebid)[1] in SKIP_TYPES: continue if hebid.lower() in ['robots.txt', '/misc_/ExternalEditor/edit_icon']: continue if 'manage_' in hebid.lower(): continue if '/p_/' in path.lower(): continue if '/misc_/' in path.lower(): continue agent = data['%{User-Agent}i'] stop = False agent_lower = agent.lower() for robot in ROBOTS: if robot in agent_lower: stop = True break if stop: continue host = data['%h'] if host in SKIP_HOSTS: continue cpt += 1 if cpt % 10000 == 0: session.flush() session.commit() print cpt heb_pk = getHebPkFromId(hebid) logline = LogItem() logline.log_date = date logline.log_path = path logline.log_hebid = hebid logline.log_hebpk = heb_pk logline.log_host = host logline.log_agent = agent logline.log_website = website session.add(logline) maxDate = date session.flush() session.commit()
def set_mysql_mappers(metadata, event=None): utils.initialize_declarative_mappers(DeclarativeBase, metadata) utils.initialize_defered_mappers(metadata)
def connect(self): initialize_declarative_mappers(DeclarativeBase, self.pg.metadata) initialize_defered_mappers(self.pg.metadata) self.session = session()
def set_mappers(metadata, event=None): utils.initialize_declarative_mappers(DeclarativeBase, metadata) utils.initialize_defered_mappers(metadata)
def getModel(self, metadata=None): """ The model is described as an ordered dictionary. The entries are (tablename, some_dict) where 'some_dict' is a dictionary containing a key 'table' referencing a Table() instance and an optional key 'relationships' referencing a sequence of related table names. An optional mapper class can be specified through the 'class' key (otherwise a default mapper class will be autogenerated). """ utils.initialize_declarative_mappers(DeclarativeBase, metadata) model = Model() model.metadata = metadata model.add('reservation_proprio', table=ReservationProprio.__table__, mapper_class=ReservationProprio) model.add('commune', table=Commune.__table__, mapper_class=Commune) model.add('localite', table=Localite.__table__, mapper_class=Localite) model.add('proprio', table=Proprio.__table__, mapper_class=Proprio) model.add('proprio_maj', table=ProprioMaj.__table__, mapper_class=ProprioMaj) model.add('province', table=Province.__table__, mapper_class=Province) model.add('charge', table=Charge.__table__, mapper_class=Charge) model.add('hebergement', table=Hebergement.__table__, mapper_class=Hebergement) model.add('hebergement_app', table=HebergementApp.__table__, mapper_class=HebergementApp) model.add('hebergement_maj', table=HebergementMaj.__table__, mapper_class=HebergementMaj) model.add('hebergement_video', table=HebergementVideo.__table__, mapper_class=HebergementVideo) model.add('maison_tourisme', table=MaisonTourisme.__table__, mapper_class=MaisonTourisme) model.add('type_heb', TypeHebergement.__table__, mapper_class=TypeHebergement) model.add('info_touristique', table=InfoTouristique.__table__, mapper_class=InfoTouristique) model.add('type_info_touristique', table=TypeInfoTouristique.__table__, mapper_class=TypeInfoTouristique) model.add('info_pratique', table=InfoPratique.__table__, mapper_class=InfoPratique) model.add('type_info_pratique', table=TypeInfoPratique.__table__, mapper_class=TypeInfoPratique) model.add('table_hote', table=TableHote.__table__, mapper_class=TableHote) model.add('heb_tab_hote_maj', table=TypeTableHoteOfHebergementMaj.__table__, mapper_class=TypeTableHoteOfHebergementMaj) model.add('log_item', table=LogItem.__table__, mapper_class=LogItem) model.add('map_provider', table=MapProvider.__table__, mapper_class=MapProvider) model.add('map_blacklist', table=MapBlacklist.__table__, mapper_class=MapBlacklist) model.add('map_external_data', table=MapExternalData.__table__, mapper_class=MapExternalData) model.add('link_hebergement_metadata', table=LinkHebergementMetadata.__table__, mapper_class=LinkHebergementMetadata) model.add('metadata_type', table=MetadataType.__table__, mapper_class=MetadataType) model.add('metadata', table=Metadata.__table__, mapper_class=Metadata) model.add('civilite', table=Civilite.__table__, mapper_class=Civilite) model.add('blockinghistory', table=BlockingHistory.__table__, mapper_class=BlockingHistory) model.add('hebergementblockinghistory', table=HebergementBlockingHistory.__table__, mapper_class=HebergementBlockingHistory) model.add('linkhebergementepis', table=LinkHebergementEpis.__table__, mapper_class=LinkHebergementEpis) model.add('notification_origin', table=NotificationOrigin.__table__, mapper_class=NotificationOrigin) model.add('notification', table=Notification.__table__, mapper_class=Notification) model.add('cron', table=Cron.__table__, mapper_class=Cron) utils.initialize_defered_mappers(model.metadata) return model