def parse( out, infile, modulestore ): logfile = open( out + '/transform.log', 'w') outfile = OutputFile( out + '/data', format , options='wb') parser = JSONToRelation( InURI( infile ) , outfile, mainTableName='EdxTrackEvent' ) parser.setParser( EdXTrackLogJSONParser( parser, 'EdxTrackEvent', dbName='Edx', moduleStore = modulestore ) ) parser.convert()
def parse(out, infile, modulestore):
logfile = open(out + '/transform.log', 'w')
outfile = OutputFile(out + '/data', format, options='wb')
parser = JSONToRelation(InURI(infile),
outfile,
mainTableName='EdxTrackEvent')
parser.setParser(
EdXTrackLogJSONParser(parser,
'EdxTrackEvent',
dbName='Edx',
moduleStore=modulestore))
parser.convert()
outSQLFile,
mainTableName='EdxTrackEvent',
logFile=logFile)
try:
# Setting useDisplayNameCache to True prevents guaranteed
# pulling of Modulestore from the backup---and expensive
# operation. Note that cronRefreshModulestore.sh will
# cause the cache to be refreshed:
jsonConverter.setParser(
EdXTrackLogJSONParser(jsonConverter,
'EdxTrackEvent',
replaceTables=args.dropTables,
dbName='Edx',
useDisplayNameCache=True))
except Exception as e:
with open(logFile, 'w') as fd:
fd.write(
"In json2sql: could not create EdXTrackLogJSONParser; infile: %s; outfile: %s; logfile: %s (%s)"
% (InURI(args.inFilePath), outSQLFile, logFile, ` e `))
# Try to delete the .sql file that was created when
# the OutputFile instance was made in the JSONToRelation
# instantiation statement above:
try:
outSQLFile.remove()
except Exception as e:
pass
sys.exit(1)
jsonConverter.convert()
outputFormat = OutputDisposition.OutputFormat.SQL_INSERT_STATEMENTS
else:
outputFormat = OutputDisposition.OutputFormat.SQL_INSERTS_AND_CSV
outSQLFile = OutputFile(outFullPath, outputFormat, options='wb') # overwrite any sql file that's there
jsonConverter = JSONToRelation(InURI(args.inFilePath),
outSQLFile,
mainTableName='EdxTrackEvent',
logFile=logFile
)
try:
jsonConverter.setParser(EdXTrackLogJSONParser(jsonConverter,
'EdxTrackEvent',
replaceTables=args.dropTables,
dbName='Edx'
))
except Exception as e:
with open(logFile, 'w') as fd:
fd.write("In json2sql: could not create EdXTrackLogJSONParser: %s" % `e`)
# Try to delete the .sql file that was created when
# the OutputFile instance was made in the JSONToRelation
# instantiation statement above:
try:
outSQLFile.remove();
except Exception as e:
pass
sys.exit(1)
jsonConverter.convert()
