def __init__(self):
super(Service, self).__init__()
self._queue = AgentRxQueue(Service.PLUGIN_NAME)
self._queue.purge()
self._table_size = LogMessage.objects.count()
self._parser = LogMessageParser()
dse.patch_models()
def handle(self, *args, **options):
if not args or not os.path.exists(args[0]) or not os.path.isdir(args[0]):
print "Path supplied does not exist or is not a directory."
return
#Necessary call to allow dse to work
dse.patch_models()
#Files must be loaded in particular order because of foreign key relationships.
self._import_calendar(os.path.join(args[0], 'calendar.txt'))
self._import_calendar_dates(os.path.join(args[0], 'calendar_dates.txt'))
self._import_routes(os.path.join(args[0], 'routes.txt'))
self._import_trips(os.path.join(args[0], 'trips.txt'))
self._import_stops(os.path.join(args[0], 'stops.txt'))
self._import_stop_times(os.path.join(args[0], 'stop_times.txt'))
#Preform post-processing on database
print "Performing post-processing"
self._postprocess()
print "Script completed."
#raise dse.InsertManyException(e, self.tablename, self.insert_sql, fieldvalues)
#dse.DSE.execute_insert_statements = dse_execute_insert_statements_patch
# 3.1.0 version of execte_insert_statements:
#def execute_insert_statements(self):
#"Executes all bulk insert statements."
#fieldvalues = []
#for items in self.insert_items:
#m = []
#for fieldname in self.fields:
#if fieldname in items:
#m.append(items[fieldname])
#elif fieldname != self.pk or not self.pk_is_autofield:
#m.append(None)
#fieldvalues.append(m)
#self.records_processed += 1
#if self.debug:
#logging.debug("Executing insert: %s" % self.insert_sql)
#for f in fieldvalues:
#logging.debug(str(f))
#try:
#self._execute(self.insert_sql, fieldvalues, many=True)
#except Exception, e:
#raise InsertManyException(e, self.tablename, self.insert_sql, fieldvalues)
databases_hash = settings.DATABASES
settings.DATABASES = {'default': databases_hash['default']}
dse.patch_models(specific_models=[StagedWebexEvent,StagedGTWEvent,StagedWebexRegistrant,StagedGTWRegistrant,StagedHubSpotRegistrant, CmsForm, LandingPage, EventSync, EventSyncShard, AccountSyncShard, WebexRegistrantSnapshot, GTWRegistrantSnapshot, HubSpotRegistrantSnapshot, Registrant])
settings.DATABASES = databases_hash
import sys
import time
import traceback
import cStringIO
import datetime
from django.contrib.contenttypes.models import ContentType
from models import *
import dse
dse.patch_models()
def singleton(cls, context_text):
instances = {}
def getinstance():
if cls not in instances:
instances[cls] = cls(context_text)
return instances[cls]
return getinstance
class DelayedLoggingContext(object):
def __init__(self, context_text):
self.context_text = context_text
self.items = []
def __enter__(self):
return self
from django.core.management.base import NoArgsCommand
from django.db import transaction
import sys
import os
import languagemodelserver.settings as settings
from languagemodelserver.languagemodel.models import Ngram, getNgrams
import dse
dse.patch_models(specific_models=[Ngram])
dse.ITEM_LIMIT = 10000
class Command(NoArgsCommand):
def handle_noargs(self, **options):
# now do the things that you want with your models here
lmFilepath = os.path.join(settings.BASE_DIR, 'corpus', 'nltk-combined.lm')
orders = range(1, settings.NGRAM_ORDER+1)
for order in orders:
ngramArgs = getNgrams(lmFilepath, order)
counter = 0
# commit_on_success() is deprecated in Django 1.7
# with transaction.commit_on_success():
with transaction.atomic():
with Ngram.delayed as d:
for n in ngramArgs:
counter += 1
d.insert(n)
if counter % 1000 == 0:
sys.stdout.write("\rNumber of %s-grams committed to database: %s"% (str(order), str(counter)))
sys.stdout.flush()