def initializeDB():
"""
Initialize db and mappers for script and return a session
"""
pg = getUtility(IDatabase, 'postgres')
session = pg.session
initialize_declarative_mappers(DeclarativeBase, pg.metadata)
initialize_defered_mappers(pg.metadata)
return session
def initializeDB():
"""
Initialize db and mappers for script and return a session
"""
pg = getUtility(IDatabase, 'postgres')
session = pg.session
initialize_declarative_mappers(DeclarativeBase, pg.metadata)
initialize_defered_mappers(pg.metadata)
return session
def main():
import gites.calendar.scripts
parseZCML(gites.calendar.scripts, 'calendar_import.zcml')
pg = getUtility(IDatabase, 'postgres')
pg.session
initialize_declarative_mappers(DeclarativeBase, pg.metadata)
initialize_defered_mappers(pg.metadata)
WalhebConsumingServer('walhebcalendar', '')
connect_all()
asyncore.loop()
def setupDatabase(self):
configurationContext = self['configurationContext']
xmlconfig.file('testing.zcml', self.package,
context=configurationContext)
configurationContext.execute_actions()
schema_file = os.path.join(CURRENT_DIR, 'tests', 'gites_wallons.sql')
self.db = gocept.testdb.PostgreSQL(encoding='UTF8',
db_template='gites_wallons_testing',
schema_path=schema_file)
self.db.create()
self.engine = create_engine(self.db.dsn)
self.conn = self.engine.connect()
self.pg = GitesTestDB()
self.pg.dsn = self.db.dsn
self.session = self.pg.session
initialize_declarative_mappers(DeclarativeBase, self.pg.metadata)
initialize_defered_mappers(self.pg.metadata)
provideUtility(self.pg, IDatabase, 'postgres')
def main():
import gdw.stats.scripts
parseZCML(gdw.stats.scripts, 'scripts.zcml')
db = getUtility(IDatabase, 'postgres')
session = db.session
initialize_declarative_mappers(DeclarativeBase, db.metadata)
initialize_defered_mappers(db.metadata)
p = apachelog.parser(FORMAT)
logfilePath, website = getConfig()
maxDate = getMaxDate(website).max_1
if maxDate is None:
maxDate = datetime(1970, 1, 1)
cpt = 0
for line in open(logfilePath):
try:
data = p.parse(line)
except:
continue
if data is None:
continue
date = apachelog.parse_date(data['%t'])
date = datetime(*time.strptime(date[0], '%Y%m%d%H%M%S')[:6])
if date <= maxDate:
continue
code = data['%>s']
if int(code) not in VALID_HTTP_CODE:
continue
path = re.match('(.*) (.*) (.*)', data['%r']).group(2)
path = urlparse.urlparse(path)[2]
# path : '/hebergement/logement/beauraing/hebid'
if len(path.strip('/').split('/')) != 4:
continue
if path.lstrip('/').split('/')[0] != 'hebergement':
continue
if path.endswith('/view'):
continue
if path.endswith('/gallery'):
continue
hebid = path.rstrip('/').split('/')[-1]
if os.path.splitext(hebid)[1] in SKIP_TYPES:
continue
if hebid.lower() in ['robots.txt',
'/misc_/ExternalEditor/edit_icon']:
continue
if 'manage_' in hebid.lower():
continue
if '/p_/' in path.lower():
continue
if '/misc_/' in path.lower():
continue
agent = data['%{User-Agent}i']
stop = False
agent_lower = agent.lower()
for robot in ROBOTS:
if robot in agent_lower:
stop = True
break
if stop:
continue
host = data['%h']
if host in SKIP_HOSTS:
continue
cpt += 1
if cpt % 10000 == 0:
session.flush()
session.commit()
print cpt
heb_pk = getHebPkFromId(hebid)
logline = LogItem()
logline.log_date = date
logline.log_path = path
logline.log_hebid = hebid
logline.log_hebpk = heb_pk
logline.log_host = host
logline.log_agent = agent
logline.log_website = website
session.add(logline)
maxDate = date
session.flush()
session.commit()
def set_mysql_mappers(metadata, event=None):
utils.initialize_declarative_mappers(DeclarativeBase, metadata)
utils.initialize_defered_mappers(metadata)
def connect(self):
initialize_declarative_mappers(DeclarativeBase, self.pg.metadata)
initialize_defered_mappers(self.pg.metadata)
self.session = session()
def set_mappers(metadata, event=None):
utils.initialize_declarative_mappers(DeclarativeBase, metadata)
utils.initialize_defered_mappers(metadata)
def getModel(self, metadata=None):
"""
The model is described as an ordered dictionary. The entries are
(tablename, some_dict) where 'some_dict' is a dictionary containing a
key 'table' referencing a Table() instance and an optional key
'relationships' referencing a sequence of related table names. An
optional mapper class can be specified through the 'class' key
(otherwise a default mapper class will be autogenerated).
"""
utils.initialize_declarative_mappers(DeclarativeBase, metadata)
model = Model()
model.metadata = metadata
model.add('reservation_proprio',
table=ReservationProprio.__table__,
mapper_class=ReservationProprio)
model.add('commune', table=Commune.__table__,
mapper_class=Commune)
model.add('localite', table=Localite.__table__,
mapper_class=Localite)
model.add('proprio', table=Proprio.__table__,
mapper_class=Proprio)
model.add('proprio_maj', table=ProprioMaj.__table__,
mapper_class=ProprioMaj)
model.add('province', table=Province.__table__,
mapper_class=Province)
model.add('charge', table=Charge.__table__,
mapper_class=Charge)
model.add('hebergement', table=Hebergement.__table__,
mapper_class=Hebergement)
model.add('hebergement_app', table=HebergementApp.__table__,
mapper_class=HebergementApp)
model.add('hebergement_maj', table=HebergementMaj.__table__,
mapper_class=HebergementMaj)
model.add('hebergement_video', table=HebergementVideo.__table__,
mapper_class=HebergementVideo)
model.add('maison_tourisme', table=MaisonTourisme.__table__,
mapper_class=MaisonTourisme)
model.add('type_heb', TypeHebergement.__table__,
mapper_class=TypeHebergement)
model.add('info_touristique', table=InfoTouristique.__table__,
mapper_class=InfoTouristique)
model.add('type_info_touristique', table=TypeInfoTouristique.__table__,
mapper_class=TypeInfoTouristique)
model.add('info_pratique', table=InfoPratique.__table__,
mapper_class=InfoPratique)
model.add('type_info_pratique', table=TypeInfoPratique.__table__,
mapper_class=TypeInfoPratique)
model.add('table_hote', table=TableHote.__table__,
mapper_class=TableHote)
model.add('heb_tab_hote_maj', table=TypeTableHoteOfHebergementMaj.__table__,
mapper_class=TypeTableHoteOfHebergementMaj)
model.add('log_item', table=LogItem.__table__,
mapper_class=LogItem)
model.add('map_provider',
table=MapProvider.__table__,
mapper_class=MapProvider)
model.add('map_blacklist',
table=MapBlacklist.__table__,
mapper_class=MapBlacklist)
model.add('map_external_data',
table=MapExternalData.__table__,
mapper_class=MapExternalData)
model.add('link_hebergement_metadata',
table=LinkHebergementMetadata.__table__,
mapper_class=LinkHebergementMetadata)
model.add('metadata_type',
table=MetadataType.__table__,
mapper_class=MetadataType)
model.add('metadata',
table=Metadata.__table__,
mapper_class=Metadata)
model.add('civilite',
table=Civilite.__table__,
mapper_class=Civilite)
model.add('blockinghistory',
table=BlockingHistory.__table__,
mapper_class=BlockingHistory)
model.add('hebergementblockinghistory',
table=HebergementBlockingHistory.__table__,
mapper_class=HebergementBlockingHistory)
model.add('linkhebergementepis',
table=LinkHebergementEpis.__table__,
mapper_class=LinkHebergementEpis)
model.add('notification_origin', table=NotificationOrigin.__table__,
mapper_class=NotificationOrigin)
model.add('notification', table=Notification.__table__,
mapper_class=Notification)
model.add('cron', table=Cron.__table__, mapper_class=Cron)
utils.initialize_defered_mappers(model.metadata)
return model