def __init__(self):
self.modelName = "IBM1WithPOSTagAndAlignmentType"
self.version = "0.2b"
self.logger = logging.getLogger('IBM1')
self.evaluate = evaluate
self.fe = ()
self.s = defaultdict(list)
self.sTag = defaultdict(list)
self.index = 0
self.typeList = []
self.typeIndex = {}
self.typeDist = []
self.lambd = 1 - 1e-20
self.lambda1 = 0.9999999999
self.lambda2 = 9.999900827395436E-11
self.lambda3 = 1.000000082740371E-15
self.loadTypeDist = {"SEM": .401, "FUN": .264, "PDE": .004,
"CDE": .004, "MDE": .012, "GIS": .205,
"GIF": .031, "COI": .008, "TIN": .003,
"NTR": .086, "MTA": .002}
self.modelComponents = ["t", "s", "sTag",
"typeList", "typeIndex", "typeDist",
"lambd", "lambda1", "lambda2", "lambda3"]
IBM1Base.__init__(self)
return
def __init__(self):
self.t = defaultdict(float)
if "logger" not in vars(self):
self.logger = logging.getLogger('IBM1BASE')
if "modelComponents" not in vars(self):
self.modelComponents = ["t"]
Base.__init__(self)
return
def __init__(self):
self.t = []
if "logger" not in vars(self):
self.logger = logging.getLogger('IBM1BASE')
if "modelComponents" not in vars(self):
self.modelComponents = ["t", "fLex", "eLex", "fIndex", "eIndex"]
Base.__init__(self)
return
def __init__(self):
self.modelName = "IBM1"
self.version = "0.1b"
self.logger = logging.getLogger('IBM1')
self.evaluate = evaluate
IBM1Base.__init__(self)
return
def __init__(self):
self.modelName = "IBM1"
self.version = "0.4b"
self.logger = logging.getLogger('IBM1')
self.evaluate = evaluate
self.fLex = self.eLex = self.fIndex = self.eIndex = None
IBM1Base.__init__(self)
return
def __init__(self):
self.modelName = "HMM"
self.version = "0.1b"
self.logger = logging.getLogger('HMM')
self.p0H = 0.3
self.nullEmissionProb = 0.000005
self.smoothFactor = 0.1
self.task = None
self.evaluate = evaluate
self.modelComponents = ["t", "pi", "a", "eLengthSet"]
Base.__init__(self)
return
def __init__(self):
self.modelName = "HMMWithAlignmentType"
self.version = "0.1b"
self.logger = logging.getLogger('HMM')
self.p0H = 0.3
self.nullEmissionProb = 0.000005
self.smoothFactor = 0.1
self.task = None
self.evaluate = evaluate
self.fe = ()
self.s = defaultdict(list)
self.sTag = defaultdict(list)
self.index = 0
self.typeList = []
self.typeIndex = {}
self.typeDist = []
self.lambd = 1 - 1e-20
self.lambda1 = 0.9999999999
self.lambda2 = 9.999900827395436E-11
self.lambda3 = 1.000000082740371E-15
self.loadTypeDist = {
"SEM": .401,
"FUN": .264,
"PDE": .004,
"CDE": .004,
"MDE": .012,
"GIS": .205,
"GIF": .031,
"COI": .008,
"TIN": .003,
"NTR": .086,
"MTA": .002
}
self.modelComponents = [
"t", "pi", "a", "eLengthSet", "s", "sTag", "typeList", "typeIndex",
"typeDist", "lambd", "lambda1", "lambda2", "lambda3"
]
Base.__init__(self)
return
def __init__(self):
if "nullEmissionProb" not in vars(self):
self.nullEmissionProb = 0.000005
if "task" not in vars(self):
self.task = None
if "t" not in vars(self):
self.t = defaultdict(float)
if "eLengthSet" not in vars(self):
self.eLengthSet = defaultdict(int)
if "a" not in vars(self):
self.a = [[[]]]
if "pi" not in vars(self):
self.pi = []
if "logger" not in vars(self):
self.logger = logging.getLogger('HMMBASE')
if "modelComponents" not in vars(self):
self.modelComponents = ["t", "pi", "a", "eLengthSet"]
Base.__init__(self)
return
def __init__(self):
if "nullEmissionProb" not in vars(self):
self.nullEmissionProb = 0.000005
if "t" not in vars(self):
self.t = []
if "eLengthSet" not in vars(self):
self.eLengthSet = {}
if "a" not in vars(self):
self.a = [[[]]]
if "pi" not in vars(self):
self.pi = []
if "logger" not in vars(self):
self.logger = logging.getLogger('HMMBASE')
if "modelComponents" not in vars(self):
self.modelComponents = [
"t", "pi", "a", "eLengthSet", "fLex", "eLex", "fIndex",
"eIndex"
]
Base.__init__(self)
return
def __init__(self):
'''
self.modelComponents contains all of the names of the variables that
own wishes to save in the model. It is vital that one calles __init__
here at the end of their own __init__ as it would try to automatically
load the model specified in self._savedModelFile.
One should always include a logger in ones own model. Otherwise a blank
one will be provided here.
One should also, always come up with a unique name for their model. It
will be marked in the saved model files to prevent accidentally loading
the wrong model. It should be saved in self.modelName.
Optionally, when there is a self.supportedVersion list and self.version
str, the loader will only load the files with supported versions.
'''
if "logger" not in vars(self):
self.logger = logging.getLogger('MODEL')
if "modelComponents" not in vars(self):
self.modelComponents = []
if "_savedModelFile" not in vars(self):
self._savedModelFile = ""
return
__author__ = 'samyvilar'
import sys
import inspect
from itertools import izip, imap, ifilter, repeat, starmap, takewhile
from utils.sequences import exhaust
from loggers import logging
from front_end.loader.locations import loc, LocationNotSet, Location
from front_end.tokenizer.tokens import TOKENS
from utils import get_attribute_func
logger = logging.getLogger('parser')
current_module = sys.modules[__name__]
class StrictlySigned(object):
@property
def unsigned(self):
return False
class StrictlyUnsigned(object):
@property
def unsigned(self):
return True
configFileTrainSection = {
'TextFilePrefix': 'trainData',
'TagFilePrefix': 'trainDataTag',
'AlignmentFileSuffix': 'trainAlignment'
}
configFileTestSection = {
'TextFilePrefix': 'testData',
'TagFilePrefix': 'testDataTag',
'Reference': 'reference'
}
# Initialise logger
init_logger('aligner.log')
__logger = logging.getLogger('MAIN')
# Dealing with arguments here
if True: # Adding arguments
# Parsing the options
ap = argparse.ArgumentParser(
description="""SFU HMM Aligner %s""" % __version__)
ap.add_argument(
"-d", "--datadir", dest="dataDir",
help="data directory")
ap.add_argument(
"--train", dest="trainData",
help="prefix of training data file")
ap.add_argument(
"--test", dest="testData",
help="prefix of testing data file")
from itertools import imap, izip, chain, repeat, ifilter, izip_longest
from ctypes import c_ulonglong, c_uint, Structure, POINTER, CDLL, CFUNCTYPE, byref, c_int, c_char_p, c_void_p
from ctypes import c_float, c_double, cast, sizeof, pointer, c_ushort, c_ubyte, c_longlong, c_short, c_byte
from ctypes import pythonapi, py_object
import struct
from struct import pack, unpack
import back_end.virtual_machine.instructions.architecture as architecture
from back_end.virtual_machine.instructions.architecture import Address, RealOperand
from back_end.virtual_machine.instructions.architecture import Word, Half, Quarter, OneEighth, DoubleHalf, Double
from loggers import logging
logger = logging.getLogger('virtual_machine')
current_module = sys.modules[__name__]
word_names = tuple((p + 'word') for p in ('', 'half_', 'quarter_', 'one_eighth_'))
signed_word_names = tuple(imap('signed_'.__add__, word_names))
float_names = 'float', 'half_float'
word_factories = c_ulonglong, c_uint, c_ushort, c_ubyte
signed_word_factories = c_longlong, c_int, c_short, c_byte
float_factories = c_double, c_float
word_formats = 'Q', 'I', 'H', 'B'
signed_word_formats = tuple(imap(str.lower, word_formats))
float_formats = 'd', 'f'
def checkAlignmentModel(modelClass, logger=True):
'''
This function will examine the model class.
If the return value is False, then the model is unrecognisable by this
function.
@param modelClass: class, a model class
@param logger: use loggers or not
@return: bool. False for unrecognisable
'''
if logger:
from loggers import logging, init_logger
error = logging.getLogger('CheckModel').error
else:
def error(msg):
print "[ERROR]:", msg
if not inspect.isclass(modelClass):
error(
"Specified Model needs to be a class named AlignmentModel under " +
"models/ModelName.py")
return False
try:
from models.cModelBase import AlignmentModelBase as cBase
if issubclass(modelClass, cBase):
logging.getLogger('CheckModel').info(
"Loading Cython model, unable to check further")
return True
except ImportError:
pass
if not issubclass(modelClass, Base):
error("Specified Model needs to be a subclass of " +
"models.modelBase.AlignmentModelBase ")
return False
try:
model = modelClass()
except all:
error("Specified Model instance cannot be created by calling " +
"AlignmentModel()")
return False
for methodName in requiredMethods:
method = getattr(modelClass, methodName, None)
if not callable(method):
error(
"Specified Model class needs to have a method called '" +
methodName + "', " +
"containing at least the following arguments(without Tag): " +
str(requiredMethods1[methodName]) + " or the following" +
"(with Tag) " + str(requiredMethods2[methodName]))
return False
args, _, _, _ = inspect.getargspec(method)
if [a for a in requiredMethods[methodName] if a not in args]:
error("Specified Model class's '" + methodName +
"' method should " +
"contain the following arguments(with exact same names): " +
str(requiredMethods[methodName]))
return False
if [a for a in args if a not in requiredMethods[methodName]]:
error("Specified Model class's '" + methodName +
"' method should " +
"contain only the following arguments: " +
str(requiredMethods[methodName]))
return False
if not [a for a in requiredMethods[methodName] if a not in args]:
return True
else:
error("Unrecognisable model type.")
return False
return mode
__author__ = 'samyvilar'
from loggers import logging
logger = logging.getLogger('tokenizer')
import migrator_kit.table_creator as table_creator
from migrator_kit.checksum_table import ChecksumTable
from util.table_process_helper import TableProcessQueue
from connection_coordinator import ConnectionCoordinator
from os.path import join, splitext
from loggers import logging
from os import rename
from os.path import exists
import sys
import json
import datetime
import functools
import argparse
from typing import *
logger = logging.getLogger("sql2sqlite.main")
logger.setLevel(logging.INFO)
file_handler = logging.FileHandler("main.log")
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(logging.Formatter(fmt='%(asctime)s %(message)s'))
logger.addHandler(file_handler)
# One huge problem for the design of this program is its verbose and hard to remember.
# Even though I wrote it, I still have to acknowledge that I don't how to find a way to
# make the code intuitive and less error prone. More effort should be put on the thought
# of the design but I don't have time, and it probably needs a major change in thinking.
#
# JJ 9/29/2017
db_directory = "" # type: str
db_name = "" # type: str
__author__ = 'samyvilar'
from loggers import logging
logger = logging.getLogger('preprocessor')
__author__ = 'samyvilar'
from loggers import logging
logger = logging.getLogger('loader')
import os
import sys
import inspect
import optparse
currentdir = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0, parentdir)
from fileIO import exportToFile, loadAlignment
from loggers import logging, init_logger
if __name__ == '__main__':
init_logger('evaluator.log')
logger = logging.getLogger('EVALUATOR')
__version__ = "0.4a"
def evaluate(result, reference, showFigure=0):
if showFigure > 0:
from models.plot import addAlignmentToFigure
totalAlign = 0
totalCertain = 0
totalCertainAlignment = 0
totalProbableAlignment = 0
for i in range(min(len(result), len(reference))):
testAlign = []
for entry in result[i]:
if (len(entry)) < 3:
logger.warning("Result missing element." +
from front_end.parser.expressions.cast import cast_expression
from front_end.parser.expressions.initializer import initializer
from front_end.parser.expressions.postfix import postfix_expression
from front_end.parser.expressions.primary import primary_expression, compound_literal
from front_end.parser.expressions.unary import unary_expression
from front_end.parser.statements.labels import labeled_statement
from front_end.parser.expressions.constant import constant_expression as _constant_expression_func
from front_end.parser.expressions.expression import expression as _expression_func
from front_end.parser.statements.compound import statement as _statement_func
from front_end.parser.declarations.declarations import translation_unit as _translation_unit_func
from loggers import logging
logger = logging.getLogger('parser')
current_module = sys.modules[__name__]
class SymbolTable(utils.symbol_table.SymbolTable):
def __setitem__(self, key, value):
# C allows multiple declarations, so long as long they are all consistent, with previous declarations
# AND a single definition.
# possible scenarios
# 1) Giving a declaration, check its consistent with previous declaration or definition if any.
# 2) Giving a definition, check its consistent with previous declaration and its consistent with previous
# declaration if any.
if isinstance(value, Declaration) and key in self: # check declarations/definitions ...
# either function definition, definition or declaration or constant_expression(for enums) ...
