class Transaction(object):
Side = enum(BUY='buy', SELL='sell')
def __init__(self, symbol, dt, px, qty):
if qty == 0:
raise Exception('Cannot transact in 0-value quantity.')
self.symbol = symbol
self.dt = dt
self.px = px
self.qty = qty
@staticmethod
def sort(txns):
# TODO: sort by multiple keys (dt - symbol - px - qty)
sorted_txns = sorted(txns, key=lambda txn: txn.dt)
for i in range(0, len(sorted_txns)):
yield sorted_txns[i]
def cost(self):
return self.px * self.qty
def side(self):
return Transaction.Side.SELL if self.qty < 0 else Transaction.Side.BUY
def __repr__(self):
return 'Transaction({}, {}, {}, {})'.format(self.symbol, self.dt,
self.px, self.qty)
def __str__(self):
return '[{}] {} {} of {} @ {}'.format(
datetime.strftime(self.dt, '%Y-%m-%d'), self.side(), abs(self.qty),
self.symbol, self.px)
def __init__(self, agent):
super().__init__(agent)
self.squads = {
Role.OFFENSE: [],
Role.DEFENSE: []
} #id => unit (maybe squad => {id => unit})
self.assigned = set() #id's
self.unit = util.enum(
MARINE=UnitType(UNIT_TYPEID.TERRAN_MARINE, agent),
SIEGETANK=UnitType(UNIT_TYPEID.TERRAN_SIEGETANK, agent),
SIEGETANKSIEGED=UnitType(UNIT_TYPEID.TERRAN_SIEGETANKSIEGED,
agent),
)
self.chokepoints = []
def __init__(self, agent):
super().__init__(agent)
self.buildings = {
} # key = building enum (library.UnitType), value = set of buildings (library.Unit)
self.construction = {} # key = UnitType, value = buildings (list)
self.unit_types = {}
self.bases = []
self.building = util.enum(
SUPPLYDEPOT=UnitType(UNIT_TYPEID.TERRAN_SUPPLYDEPOT, agent),
SUPPLYDEPOTLOWERED=UnitType(UNIT_TYPEID.TERRAN_SUPPLYDEPOTLOWERED,
agent),
REFINERY=UnitType(UNIT_TYPEID.TERRAN_REFINERY, agent),
BARRACKS=UnitType(UNIT_TYPEID.TERRAN_BARRACKS, agent),
COMMANDCENTER=UnitType(UNIT_TYPEID.TERRAN_COMMANDCENTER, agent),
FACTORY=UnitType(UNIT_TYPEID.TERRAN_FACTORY, agent),
FACTORYTECHLAB=UnitType(UNIT_TYPEID.TERRAN_FACTORYTECHLAB, agent))
self.buildings[self.building.REFINERY] = set()
self.buildings[self.building.BARRACKS] = set()
self.building_commandcenter = False
self.previous_expansion = None
def getLowercaseCountProfilerFactory():
lowercaseAlphabet = ['a','c','g','t','n']
uppercaseAlphabet = ['A','C','G','T']
def letterToKey(x):
if (x in lowercaseAlphabet):
return 'acgt';
if (x in uppercaseAlphabet):
return 'ACGT';
if (x == 'N'):
return 'N';
raise Exception("Unexpected dna input: "+x);
return LetterByLetterCountProfilerFactory(letterToKey, 'LowercaseCount');
#!!!if you change this, PLEASE remember to update getGCcontent in perSequence_profile.py
GCkeys = util.enum(GC='GC', AT='AT', N='N');
def gcLetterToKey(x):
if (x in ['c','g','C','G']):
return GCkeys.GC;
if (x in ['a','t','A','T']):
return GCkeys.AT;
if (x == 'N' or x=='n'):
return GCkeys.N;
raise Exception("Unexpected dna input: "+x);
def getGcCountProfilerFactory():
return LetterByLetterCountProfilerFactory(gcLetterToKey, 'GC-content');
def getBaseCountProfilerFactory():
return LetterByLetterCountProfilerFactory(lambda x: x.upper(), 'BaseCount');
def motifGrammarSimulation(options):
pc = 0.001;
bestHit = options.bestHit;
bestHitMode = options.bestHitMode;
pathToMotifs = options.pathToMotifs;
loadedMotifs = synthetic.LoadedEncodeMotifs(pathToMotifs, pseudocountProb=pc)
motifName1 = options.motifName1;
motifName2 = options.motifName2;
seqLength = options.seqLength;
numSeq = options.numSeq;
generationSetting = options.generationSetting;
outputFileName = "motifGrammarSimulation_"+generationSetting+("_bestHit_mode-"+bestHitMode if bestHit else "");
if (generationSetting is not generationSettings.singleMotif2):
outputFileName+="_motif1-"+motifName1;
if (generationSetting is not generationSettings.singleMotif1):
outputFileName+="_motif2-"+motifName2;
outputFileName+="_seqLength"+str(seqLength)+"_numSeq"+str(numSeq)+".simdata";
kwargs={'loadedMotifs':loadedMotifs}
if (bestHit):
theClass=synthetic.BestHitPwmFromLoadedMotifs;
kwargs['bestHitMode']=bestHitMode
else:
theClass=synthetic.PwmSamplerFromLoadedMotifs;
motif1Generator=theClass(motifName=motifName1,**kwargs)
motif2Generator=theClass(motifName=motifName2,**kwargs)
motif1Embedder=synthetic.SubstringEmbedder(substringGenerator=motif1Generator)
motif2Embedder=synthetic.SubstringEmbedder(substringGenerator=motif2Generator)
embedders = [];
if (generationSetting == generationSettings.allBackground or
generationSetting == generationSettings.twoMotifs):
namePrefix="synthNeg";
else:
namePrefix="synthPos"
if (generationSetting == generationSettings.allBackground):
pass;
elif (generationSetting in [generationSettings.singleMotif1, generationSettings.twoMotifs, generationSettings.singleMotif2]):
if (generationSetting == generationSettings.singleMotif1):
embedders.append(motif1Embedder);
elif (generationSetting == generationSettings.singleMotif2):
embedders.append(motif2Embedder);
elif (generationSetting == generationSettings.twoMotifs):
embedders.append(motif1Embedder);
embedders.append(motif2Embedder);
else:
raise RuntimeError("Unsupported generation setting: "+generationSetting);
elif (generationSetting in [generationSettings.twoMotifsFixedSpacing, generationSettings.twoMotifsVariableSpacing]):
if (generationSetting==generationSettings.twoMotifsFixedSpacing):
separationGenerator=synthetic.FixedQuantityGenerator(options.fixedSpacingOrMinSpacing);
elif (generationSetting==generationSettings.twoMotifsVariableSpacing):
separationGenerator=synthetic.UniformIntegerGenerator(minVal=options.fixedSpacingOrMinSpacing
,maxVal=options.maxSpacing);
else:
raise RuntimeError("unsupported generationSetting:"+generationSetting);
embedders.append(synthetic.EmbeddableEmbedder(
embeddableGenerator=synthetic.PairEmbeddableGenerator(
substringGenerator1=motif1Generator
,substringGenerator2=motif2Generator
,separationGenerator=separationGenerator
)
));
else:
raise RuntimeError("unsupported generationSetting:"+generationSetting);
embedInBackground = synthetic.EmbedInABackground(
backgroundGenerator=synthetic.ZeroOrderBackgroundGenerator(seqLength)
, embedders=embedders
, namePrefix=namePrefix
);
sequenceSet = synthetic.GenerateSequenceNTimes(embedInBackground, numSeq)
synthetic.printSequences(outputFileName, sequenceSet);
#also call scoreSeq on outputFileName
from pwm import pwm;
options = util.enum(motifsFile=pathToMotifs
,pwmName=motifName1
,pseudocountProb=pc
,fileToScore=outputFileName
,scoreSeqMode=pwm.SCORE_SEQ_MODE.continuous
,reverseComplementToo=True
,seqCol=1
,auxillaryCols=[0,1]
,topN=None
,greedyTopN=False);
from pwm import scoreSeq;
scoreSeq.scoreSeqs(options,returnScores=False);
options.pwmName=motifName2;
scoreSeq.scoreSeqs(options,returnScores=False);
metadataAttrName=perfToTrackOptions.perfAttrName
,recordAttrName=perfToTrackOptions.perfAttrName
,updateFunc=getBestUpdateFunc(
isLargerBetter=perfToTrackOptions.isLargerBetter
,metadataCallbacks=metadataCallbacks)
,initVal=None)
,jsondb.NumRecordsMetadataUpdateInfo]
,[RunTrackerMetadataFields.bestPerfSavedFiles]);
jsonDbFactory = jsondb.JsonDb.getFactory(JsonableRecordClass=JsonableRecordClass
,JsonableRecordsHolderClass=JsonableRecordsHolderClass
,MetadataClass=MetadataClass
,callbacks_beforeAdd=callbacks_beforeAdd
,callbacks_afterAdd=callbacks_afterAdd);
return jsonDbFactory;
RunTrackerRecordFields = util.enum(savedFiles="savedFiles");
RunTrackerMetadataFields = util.enum(bestPerfSavedFiles="bestPerfSavedFiles");
EmailModes = util.enum(noEmails="noEmails", onlyErrorEmails="onlyErrorEmails", errorsAndNewBest="errorsAndNewBest", allEmails="allEmails");
def addRunTrackerArgumentsToParser(parser):
parser.add_argument("--emails", nargs="+", required=True, help="Provide a dummy val if don't want emails");
parser.add_argument("--emailMode", choices=EmailModes.vals, default=EmailModes.errorsAndNewBest);
parser.add_argument("--jobName", help="Used to create email subjects and log files");
parser.add_argument("--logFile");
parser.add_argument("--jsonDbFile", required=True, help="Used to save the records");
parser.add_argument("--thresholdPerfToEmailAt", type=float, help="New Best emails only sent above this threshold")
parser.add_argument("--minThresholdPerfToSaveFiles", type=float, help="Only files above this threshold are saved");
parser.add_argument("--topNtoSave", default=100, type=int, help="Keep top N performing models");
def runTrackerArguments_fillDefaults(options):
coreJsonDb = fp.getCoreFileName(options.jsonDbFile);
conditionCheck.Any([conditionCheck.ValueAmongOptions(options.quantMotifMode, 'quantMotifMode', minAndMaxQuantMotifModes)
,conditionCheck.Notter(conditionCheck.ValueIsSetInOptions(options, 'quantMotifMax'))]
,description="quantMotifMax should only be specified if certain quantMotif modes are chosen").enforce();
#POSITIONAL_MODE.embedInCentralBp
if (options.positionalMode == POSITIONAL_MODE.embedInCentralBp):
if (options.centralBp > options.seqLength):
raise RuntimeError("centralBp must be <= seqLength; "+str(options.centralBp)+" and "+str(options.seqLength)+" respectively");
if (options.centralBp < options.pwm.pwmSize):
raise RuntimeError("if mode is embedInCentralBp, then centralBp must be at least as large as the pwmSize; "+str(options.centralBp)+" and "+str(options.pwm.pwmSize));
#POSITIONAL_MODE.embedOutsideCentralBp
if (options.positionalMode == POSITIONAL_MODE.embedOutsideCentralBp):
if ((options.seqLength-options.centralBp)/2 < options.pwm.pwmSize):
raise RuntimeError("(options.seqLength-options.centralBp)/2 should be >= options.pwm.pwmSize; got len ",str(options.seqLength)+", centralBp "+str(options.centralBp)+" and pwmSize "+str(options.pwm.pwmSize));
POSITIONAL_MODE = util.enum(uniform='unif', embedInCentralBp='embedInCent', embedOutsideCentralBp='embedOutCent', gaussian='gauss');
QUANTITY_OF_MOTIFS_MODE = util.enum(poisson='poisson', fixed='fixed');
positionalModeOptionsAssociatedWithCentralBp = [POSITIONAL_MODE.embedInCentralBp, POSITIONAL_MODE.embedOutsideCentralBp];
if __name__ == "__main__":
parser = argparse.ArgumentParser(parents=[makePwmSamples.getParentArgparse(),synthetic.getParentArgparse()]);
parser.add_argument("--numSamples", type=int, required=True);
parser.add_argument("--quantMotifMode", choices=QUANTITY_OF_MOTIFS_MODE.vals, required=True);
parser.add_argument("--quantMotifMean", type=float, help="Parameter associated with quantity of pwm sampling conditions");
parser.add_argument("--quantMotifMin", type=int, help="Minimum number of pwms in a given sequence");
parser.add_argument("--quantMotifMax", type=int, help="Max number of pwms in a given sequence");
parser.add_argument("--positionalMode", choices=POSITIONAL_MODE.vals, default=POSITIONAL_MODE.uniform);
parser.add_argument("--centralBp", type=int, help="Associated with some positional mode options.");
options = parser.parse_args();
makePwmSamples.processOptions(options);
makePwmSamples.performChecksOnOptions(options);
from util import enum Genders = enum(M=u'Male', F=u'Female') Roles = enum(USER=u'User', ADMIN=u'Administrator', APP=u'Application Owner')
from __future__ import print_function;
from __future__ import absolute_import;
import os;
import sys;
scriptsDir = os.environ.get("UTIL_SCRIPTS_DIR");
if (scriptsDir is None):
raise Exception("Please set environment variable UTIL_SCRIPTS_DIR");
sys.path.insert(0,scriptsDir);
sys.path.insert(0,scriptsDir+"/synthetic/APIscripts/");
import pathSetter
from synthetic import synthetic;
import util;
from pwm import pwm;
from apiHelperFunctions import getMotifGenerator;
GrammarYamlKeys = util.enum(motif1="motif1",motif2="motif2",spacingSetting="spacingSetting",grammarName="grammarName");
SpacingSettings = util.enum(fixedSpacing="fixedSpacing", variableSpacing="variableSpacing");
GrammarYamlKeys_fixedSpacing = util.enum(fixedSpacingValues="fixedSpacingValues"); #keys applicable to the case of fixedSpacing
GrammarYamlKeys_variableSpacing = util.enum(minimum="minimum", maximum="maximum"); #keys applicable to the case of variable spacing
def coinGrammarName(aGrammarYamlDict):
motif1 = aGrammarYamlDict[GrammarYamlKeys.motif1];
motif2 = aGrammarYamlDict[GrammarYamlKeys.motif2];
spacingSetting = aGrammarYamlDict[GrammarYamlKeys.spacingSetting];
if (spacingSetting == SpacingSettings.fixedSpacing):
return motif1+"-"+motif2+"-fixedSpacing-"+"-".join(str(x) for x in aGrammarYamlDict[GrammarYamlKeys_fixedSpacing.fixedSpacingValues]);
elif (spacingSetting == SpacingSettings.variableSpacing):
return motif1+"-"+motif2+"-variableSpacing-min"+str(aGrammarYamlDict[GrammarYamlKeys_variableSpacing.minimum])+"-max"+str(aGrammarYamlDict[GrammarYamlKeys_variableSpacing.maximum]);
else:
raise RuntimeError("Unsupported value for spacing setting: "+str(spacingSetting));
# OS stack thinks it's still alive for some reason (e.g. program is in an
# infinite loop so socket is maintained but nothing can really happen now).
# SERIES means the rule should be merged into the master grammar
# so it can be chained together into a larger utterance.
# TERMINAL means the rule should be merged into the master grammar,
# but only allow it to appear at the end of utterances, typically
# because it contains a dictation element that would get confused
# by commands in series occurring after.
# INDEPENDENT means to not merge this rule into the terminator or
# terminal master rules. Typically this means you will only be
# using the rule by reference. You may also want to put infrequent
# commands in this category to improve recognition since it avoids
# the recognizer from needing to discern when these rules are used
# in combination with others, since they can only be used alone.
RuleType = enum(SERIES=0, TERMINAL=1, INDEPENDENT=2)
dataTypes = set()
def _newDataType(name, members):
newType = namedtuple(name, members)
global dataTypes
dataTypes.add(newType)
return newType
# type is rule type
# seriesMergeGroup lets you have mutually exclusive series rules, to
# avoid for example having window commands mixed with editing.
# mapping, extras, default have their normal dragonfly MappingRule meanings
###Done checkers
import os;
import sys;
scriptsDir = os.environ.get("UTIL_SCRIPTS_DIR");
if (scriptsDir is None):
raise Exception("Please set environment variable UTIL_SCRIPTS_DIR");
sys.path.insert(0,scriptsDir);
import pathSetter;
import util;
DONE_STATE = util.enum(success="success", error="error");
class DoneInfo:
def __init__(self, doneStatus, message):
"""
doneStatus: instance of DONE_STATE
message: string
"""
self.doneStatus = doneStatus;
self.message = message;
class DoneChecker:
def isDone(self):
"""
return: boolean indicating whether the job is done
"""
raise NotImplementedError();
def getStatus(self):
"""
returns a DONE_STATE; checks that isDone() is true
"""
#!/usr/bin/env python
import sys, os;
scriptsDir = os.environ.get("UTIL_SCRIPTS_DIR");
if (scriptsDir is None):
raise Exception("Please set environment variable UTIL_SCRIPTS_DIR");
sys.path.insert(0,scriptsDir);
import pathSetter;
import util;
import fileProcessing as fp;
import argparse;
NORMALISATION_MODE = util.enum(meanVariance = "meanVariance");
NORMALISATION_STATS_NAME = util.enum(Mean = "Mean", StandardDeviation = "StandardDeviation", featureName="featureName");
class StatsForFeature(object):
def __init__(self, featureName):
self.featureName = featureName;
def getFeatureToInfoMap(options):
transformation = util.chainFunctions(fp.trimNewline, fp.splitByTabs);
featureToInfoMap = {};
def actionFromTitle(title):
titleArr = transformation(title);
colNameToIdxMap = util.valToIndexMap(titleArr);
def action(inp, lineNumber):
featureName = inp[colNameToIdxMap[NORMALISATION_STATS_NAME.featureName]];
statsForFeature = StatsForFeature(featureName);
featureToInfoMap[featureName] = statsForFeature;
if (options.normalisationMode == NORMALISATION_MODE.meanVariance):
statsForFeature.mean = float(inp[colNameToIdxMap[NORMALISATION_STATS_NAME.Mean]]);
#!/usr/bin/env python
import sys
import yaml
import zlib
from util import enum
# Global enums.
_VERSION = 1
_NOTIFICATION_TYPES = enum('UPLOAD_PROGRESS', 'FS_EVENT')
_FS_EVENT_TYPES = enum('MODIFIED', 'CREATED', 'MOVED', 'DELETED')
class Notification(object):
def __init__(self, user, timestamp, notification_type):
# TODO(tierney): Verify notification_type.
assert isinstance(timestamp, float)
self.__dict__ = { 'version' : _VERSION,
'user' : user,
'timestamp' : timestamp,
'notification_type' : notification_type }
class ProgressNotification(Notification):
def __init__(self, user, timestamp, notification_type, uploaded_bytes,
total_bytes, elapsed_time):
super(ProgressNotification, self).__init__(user, timestamp, notification_type)
assert isinstance(uploaded_bytes, float)
assert isinstance(total_bytes, float)
assert isinstance(elapsed_time, float)
sys.path.insert(0,scriptsDir);
import pathSetter;
import util;
import argparse;
from pwm import scoreSeq;
from pwm import pwm;
from itertools import izip;
import numpy as np;
from sklearn.cross_validation import train_test_split;
from sklearn.tree import DecisionTreeClassifier;
from sklearn.ensemble import RandomForestClassifier;
from sklearn.grid_search import GridSearchCV;
from sklearn.metrics import accuracy_score;
from sklearn.metrics import confusion_matrix;
CLASSIFIER_TYPE = util.enum(decisionTree="decisionTree", randomForest="randomForest");
SCORING = util.enum(roc_auc='roc_auc', accuracy='accuracy', recall='recall');
def runDecisionTree(scoringResultList, scoringResultListTrainValid, scoringResultListTest, labelsTrainValid, labelsTest, options):
# Run a decision tree classifier on the top PWMs
if (len(labelsTrainValid == 0) == 0) or (len(labelsTrainValid == 1) == 0):
# There are no examples in the training/validation set from one of the classes
raise RuntimeError("Only one class is present in the training/validation set")
ind_0 = labelsTest == 0
ind_1 = labelsTest == 1
if (len(ind_0) == 0) or (len(ind_1) == 0):
# There are no examples in the test set from one of the classes
raise RuntimeError("Only one class is present in the test set")
tuned_parameters = [{'max_depth': range(1, options.topN + 1), 'max_features': range(options.topN, options.topN + 2)}]
if options.usePositions == True:
# Allow for larger trees because the positional information is being included
#!/usr/bin/env python
from __future__ import division;
from __future__ import print_function;
import os, sys;
scriptsDir = os.environ.get("UTIL_SCRIPTS_DIR");
if (scriptsDir is None):
raise Exception("Please set environment variable UTIL_SCRIPTS_DIR");
sys.path.insert(0,scriptsDir);
import pathSetter;
import util;
import argparse;
import pwm;
import fileProcessing as fp;
import random;
PWM_SAMPLING_MODE = util.enum(bestHit="bestHit", default="default");
def getFileNamePieceFromOptions(options):
return (pwm.getFileNamePieceFromOptions(options)
+"_pwmSampMode-"+options.pwmSamplingMode
+"_revCmpPrb"+str(options.reverseComplementProb));
def getParentArgparse():
parser = pwm.getLoadPwmArgparse();
parser.add_argument("--pwmSamplingMode", default=PWM_SAMPLING_MODE.default, choices=PWM_SAMPLING_MODE.vals);
parser.add_argument("--reverseComplementProb", default=0.5, type=float, help="Optional: probability of reverse complementing");
return parser;
def performChecksOnOptions(options):
if (options.reverseComplementProb < 0.0 or options.reverseComplementProb > 1.0):
raise RuntimeError("Reverse complement prob should be >= 0.0 and <= 1.0; was "+str(options.reverseComplementProb));
#TODO: more error checking here
logging.info("misc field %s %s %s" % (field, op, value))
q.filter('%s %s' % (field, op), value)
else:
logging.info("field not found '%s'" % (field, ))
#TODO: error check the sort fields
for s in sorting:
q.order(s)
logging.info("limit: %s, offset: %s" % (limit, offset))
resultCount = q.count(1000)
return (resultCount, q.fetch(limit, offset))
GamePhases = util.enum('GamePhases', 'join', 'buildFirst', 'buildSecond',
'main')
TurnPhases = util.enum('TurnPhases', 'buildInitialSettlement',
'buildInitialRoad', 'playKnightCard', 'mainTurn')
class TurnPhase(db.Model):
phase = db.StringProperty()
order = db.IntegerProperty()
class GamePhase(db.Model):
phase = db.StringProperty()
order = db.IntegerProperty()
def getTurnPhases(self):
return db.Query(TurnPhase).ancestor(self).order("order").fetch(100)
#Next:
#Extract the values from the gene dictionary
#set up an enum for the names of the esc attribute and stuff.
#processing pipeline:
#filter out values where score is zero for esc score
#sort by d7 score
#
CONFIG_ATTR = util.enum(
INPUT_FILE = 'inputFile'
, VALUE_COLUMNS_TO_STORE = 'valueColumnsToStore'
, GENE_ID_COLUMN = 'geneIdColumn'
, FILE_TYPE = 'fileType'
, SAMPLE1COL = 'sample_1'
, SAMPLE2COL = 'sample_2'
, RENAMINGS = 'renamings'
);
METASCORES_ATTR = util.enum(
SIGNED_LOG_PVAL_METASCORES = 'signedLogPvalMetascores'
);
VALUE_COLUMNS_TO_STORE_ATTR = util.enum(
VALUE_NAME = 'valueName'
, COLUMN_NAME = 'columnName'
, TYPE = 'type'
, FLIP_SIGN_ON_INVERT = 'flipSignOnInvert'
);
SIGNED_LOG_PVAL_METASCORES_ATTR = util.enum(
SCORE_NAME = 'scoreName'
from util import fail, enum using = enum(camera = True, watchdog = False, victor = False)
from __future__ import print_function;
import os, sys;
scriptsDir = os.environ.get("UTIL_SCRIPTS_DIR");
if (scriptsDir is None):
raise Exception("Please set environment variable UTIL_SCRIPTS_DIR");
sys.path.insert(0,scriptsDir);
import pathSetter;
import util;
import numpy as np;
import fileProcessing as fp;
import math;
import random;
import argparse;
from collections import OrderedDict;
PWM_FORMAT = util.enum(encodeMotifsFile="encodeMotifsFile", singlePwm="singlePwm");
DEFAULT_LETTER_TO_INDEX={'A':0,'C':1,'G':2,'T':3};
SCORE_SEQ_MODE = util.enum(maxScore="maxScore", bestMatch="bestMatch", continuous="continuous", topN="topN");
def getLoadPwmArgparse():
parser = argparse.ArgumentParser(add_help=False);
parser.add_argument("--motifsFile", required=True);
parser.add_argument("--pwmName", required=True);
parser.add_argument("--pseudocountProb", type=float, default=0.0);
return parser;
def processOptions(options):
thePwm = getSpecfiedPwmFromPwmFile(options);
options.pwm = thePwm;
from twisted.python import log from twisted.internet.threads import deferToThread from notification import mailer import gps, cdb, blam from util import enum #----------------------------------- Globals ---------------------------------# CDB = None MAILER = None LOWBAT = None LASTPOS = None BLAM = None BATTERY_THRESHOLD = None GPS_STATUS = enum(NONE=0, CHARGING=1, BATTERY=2, EMPTY=3, SILENCE=4) def load_configuration(config_file): ''' I read the configfile and populate the config dict ''' file = open(config_file) config = json.load(file) file.close() return config #----------------------------------- Background checker------------------------# def monitor(): silent_vehicles = [k for (k,v) in LASTPOS.items() if v < (datetime.now()-timedelta(minutes=5))] for v in silent_vehicles:
rowName = inp[0] if rowNamesPresent else None;
arr = [contentType(x) for x in inp[contentStartIndex:contentEndIndexWrapper.var]]; #ignore the column denoting the name of the row
titled2DMatrix.addRow(arr, rowName=rowName);
performActionOnEachLineOfFile(
fileHandle=fileHandle
,action=action
,transformation=defaultTabSeppd
,ignoreInputTitle=False
,progressUpdate=progressUpdate
);
if (numpify):
import numpy as np;
titled2DMatrix.rows = np.array(titled2DMatrix.rows);
return titled2DMatrix;
SubsetOfColumnsToUseMode = util.enum(setOfColumnNames="setOfColumnNames", topN="topN");
class SubsetOfColumnsToUseOptions(object):
def __init__(self, mode=SubsetOfColumnsToUseMode.setOfColumnNames, columnNames=None, N=None):
import errorMessages;
self.mode = mode;
self.columnNames = columnNames;
self.N = N;
self.integrityChecks();
def integrityChecks(self):
if (self.mode == SubsetOfColumnsToUseMode.setOfColumnNames):
errorMessages.assertParameterIrrelevantForMode("N", self.N, "subsetOfColumnsToUseMode", self.mode);
errorMessages.assertParameterNecessaryForMode("columnNames", self.columnNames, "subsetOfColumnsToUseMode", self.mode);
elif (self.mode == SubsetOfColumnsToUseMode.topN):
errorMessages.assertParameterIrrelevantForMode("columnNames", self.columnNames, "subsetOfColumnsToUseMode", self.mode);
errorMessages.assertParameterNecessaryForMode("N", self.N, "subsetOfColumnsToUseMode", self.mode);
else:
"""
self.queue = multiprocessing.Queue() if collectReturnValues else None;
self.parallelProcessKickerOffer = ppko.ParallelProcessKickerOffer_Multiprocessing(functionToExecute, returnQueue=self.queue);
self.waitInterval = waitInterval;
class ParalleliserFactory: #wow this class has minimal functionality why do I even use it?
def __init__(self, paralleliserInfo):
self.paralleliserInfo = paralleliserInfo;
def getParalleliser(self, inputs):
"""
inputs: an array of FunctionInputs
"""
return Paralleliser(inputs, self.paralleliserInfo);
ParalleliserState = util.enum(
NOT_STARTED = "NOT_STARTED"
, STARTED = "STARTED"
, DONE = "DONE");
class FunctionInputs(object):
"""
Stores the inputs that will be used to call some function
"""
def __init__(self, args=[], kwargs={}):
self.args = args;
self.kwargs = kwargs;
class Paralleliser(object):
"""
takes an instance of paralleliserInfo (which contains info on how to kick off the jobs) and
a series of inputs, and executes the jobs in parallel.
"""
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import time
import util
from cloud.serialization import serialize, deserialize
MSG_TYPES = util.enum(unknown=0, getattr=1, value=2, method=3, call=4, exception=5)
class Message(object):
""" Abstract class representing a message.
A message contains a checksum (TODO) a timestamp and
methods for serializing/deserializing it.
"""
def __init__(self, checksum=None, timestamp=None):
self.checksum = checksum
self.timestamp = timestamp or time.time()
def serialize(self):
return serialize(self)
#!/usr/bin/env python
import os;
import sys;
scriptsDir = os.environ.get("UTIL_SCRIPTS_DIR");
if (scriptsDir is None):
raise Exception("Please set environment variable UTIL_SCRIPTS_DIR");
sys.path.insert(0,scriptsDir);
import pathSetter
import util;
from synthetic import synthetic;
from pwm import pwm;
generationSettings = util.enum(
allBackground="allBackground"
,singleMotif1="singleMotif1" #embeds first motif
,singleMotif2="singleMotif2" #embeds second motif
,twoMotifs="twoMotifs" #embeds one of both motifs
,twoMotifsFixedSpacing="twoMotifsFixedSpacing" #embeds both motifs with a fixed spacing
,twoMotifsVariableSpacing="twoMotifsVariableSpacing" #embeds both motifs with a variable spacing
);
def motifGrammarSimulation(options):
pc = 0.001;
bestHit = options.bestHit;
bestHitMode = options.bestHitMode;
pathToMotifs = options.pathToMotifs;
loadedMotifs = synthetic.LoadedEncodeMotifs(pathToMotifs, pseudocountProb=pc)
motifName1 = options.motifName1;
motifName2 = options.motifName2;
seqLength = options.seqLength;
numSeq = options.numSeq;
generationSetting = options.generationSetting;
if internalNamesOfKeysToFillDefaultsFor is None:
internalNamesOfKeysToFillDefaultsFor = self.keys.getKeys();
for aKey in internalNamesOfKeysToFillDefaultsFor:
if aKey not in aDict:
if (self.keysDefaults.hasKey(aKey)==False):
raise RuntimeError("Default for "+str(aKey)+" not present, and a value was not provided");
aDict[aKey] = self.keysDefaults.getKey(aKey);
return aDict;
def checkForUnsupportedKeysAndFillInDefaults(self, aDict):
self.checkForUnsupportedKeys(aDict)
self.fillInDefaultsForKeys(aDict);
#something like pytable is sufficiently different that
#we can assume all this loading is for in-memory situations
ContentType=namedtuple('ContentType',['name','castingFunction']);
ContentTypes=util.enum(integer=ContentType("int",int),floating=ContentType("float",float),string=ContentType("str",str));
ContentTypesLookup = dict((x.name,x.castingFunction) for x in ContentTypes.vals);
RootKeys=Keys(Key("features"), Key("labels"), Key("splits"), Key("weights"));
FeaturesFormat=util.enum(rowsAndColumns='rowsAndColumns'
, fasta='fasta'
, fastaInCol="fastaInCol"
, signalFromBigWig="signalFromBigWig");
DefaultModeNames = util.enum(labels="defaultOutputModeName", features="defaultInputModeName");
FeaturesKeys = Keys(Key("featuresFormat")
, Key("opts")
, Key("inputModeName", default=DefaultModeNames.features));
FeatureSetYamlKeys_RowsAndCols = Keys(
Key("fileNames")
,Key("contentType",default=ContentTypes.floating.name)
,Key("contentStartIndex",default=1)
,Key("subsetOfColumnsToUseOptions",default=None)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab
from util import enum
ResponseTypes = enum(ERROR='error', SUCCESS='success', WARNING='warning')
def make_json_data(item=None, messages=None):
if not messages:
messages = make_json_message()
if isinstance(item, list):
total = len(item)
return {
'response': {
'total': total,
'data': item
},
'messages': messages
}
else:
return {'response': item, 'messages': messages}
def make_json_message(type=ResponseTypes.SUCCESS, text=None):
return {'type': type, 'text': text}
## @package exception
# Exceptions and error codes.
import util
## Enumeration indicating error types.
ErrorCode = util.enum(SUCCESS=0,
INTERNAL_FAILURE=1,
STREAM_EXCEEDS_MAXIMUM=2,
INVALID_IMAGE_FORMAT=3,
METHOD_NOT_SUPPORTED=4,
AUTHENTICATION_FAILED=100,
NOT_AUTHORIZED=101,
MISSING_PARAMETER=102,
INVALID_PARAMETER=200,
NOT_FOUND=201,
CONFLICT=202,
INVALID_REQUEST_CODE=203,
USER_NOT_FOUND=300,
USER_IS_BLOCKED=301,
WRONG_PASSWORD=302,
WRONG_EMAIL_ADDRESS=303,
NO_FRIENDSHIP=304,
OBJECT_NOT_FOUND=400,
OBJECT_IS_LOCKED=401,
HTTP_FAILURE=500)
## Exception base class.
class BaseException:
## The constructor.
# @param code error code
#!/usr/bin/env python
from __future__ import division;
from __future__ import print_function;
from __future__ import absolute_import;
import sys;
import os;
scriptsDir = os.environ.get("UTIL_SCRIPTS_DIR");
if (scriptsDir is None):
raise Exception("Please set environment variable UTIL_SCRIPTS_DIR");
sys.path.insert(0,scriptsDir);
import pathSetter;
import fileProcessing as fp;
import util;
import numpy as np;
DistanceMetrics = util.enum(euclidean="euclidean");
def getDistanceMetric(distanceMetricName):
if (distanceMetricName==DistanceMetrics.euclidean):
return euclideanDistance;
else:
raise RuntimeError("Unsupported distance metric: "+distanceMetricName);
def euclideanDistance(rowVals, matrixVals):
assert len(matrixVals.shape)==2;
assert rowVals.shape[1]==matrixVals.shape[1];
assert len(rowVals.shape)==2;
assert rowVals.shape[0]==1;
return np.sqrt(np.sum(np.square(matrixVals-rowVals),axis=1))
return np.sqrt(np.sum(np.square(arr1-arr2)));
from twisted.python import log
from twisted.internet.threads import deferToThread
from notification import mailer
import gps, cdb, blam
from util import enum
#----------------------------------- Globals ---------------------------------#
CDB = None
MAILER = None
LOWBAT = None
LASTPOS = None
BLAM = None
BATTERY_THRESHOLD = None
GPS_STATUS = enum(NONE=0, CHARGING=1, BATTERY=2, EMPTY=3, SILENCE=4)
def load_configuration(config_file):
'''
I read the configfile and populate the config dict
'''
file = open(config_file)
config = json.load(file)
file.close()
return config
#----------------------------------- Background checker------------------------#
def monitor():
elif not props.get(field, None) is None:
#TODO: more error checking here
logging.info("misc field %s %s %s" % (field, op, value))
q.filter('%s %s' % (field, op), value)
else:
logging.info("field not found '%s'" % (field,))
#TODO: error check the sort fields
for s in sorting:
q.order(s)
logging.info("limit: %s, offset: %s" %(limit, offset))
resultCount = q.count(1000);
return (resultCount, q.fetch(limit, offset));
GamePhases = util.enum('GamePhases', 'join', 'buildFirst', 'buildSecond', 'main')
TurnPhases = util.enum('TurnPhases', 'buildInitialSettlement', 'buildInitialRoad', 'playKnightCard', 'mainTurn')
class TurnPhase(db.Model):
phase = db.StringProperty()
order = db.IntegerProperty()
class GamePhase(db.Model):
phase = db.StringProperty()
order = db.IntegerProperty()
def getTurnPhases(self):
return db.Query(TurnPhase).ancestor(self).order("order").fetch(100)
def getTurnPhaseByName(self, phase):
class Cacheable(db.Model):
"""
Model caching helper (sub)class
Usage:
class MyModel(Cacheable):
...
these are over-ridden methods of Model:
get_by_key_name(key_name, parent)
retreives model from cache if possible
put()
write-through cache and put to store
get_or_insert(key_name, **kwds)
these are additional methods:
set_dirty() - marks model as dirty or critical - the later forces
a write to store on exit, the later throttles to one write per second
for this model
deferred_put() - writes the model to store if dirty
ensure_cached() - return a cached instance of the current model
flush_cache() - put the model, and remove all cached copies
Deriving from this class provides:
- Saving models to local storage and memcache where they can be retrieved quickly.
- Throttled write-through to storage for high-volume, but delay-able writes.
Issues:
Cacheable looks for Model instances in:
- in request-local storage (for fast local access for same-request accesses)
- in memcache (key'ed on app instance version, model name, and key name
- in the App Engine data store
All Cacheable Models must use unique key names (not id's) for their instances.
Queries of this model class will NOT return the cached instance of this model. You should
call ensure_cached() to get the cached version.
"""
cache_state = util.enum('clean', 'dirty', 'critical')
def __init__(self, *args, **kwargs):
self._cache_state = self.cache_state.clean
self._is_memcached = False
self._secs_put_last = 0
self._write_rate = timescore.RateLimit(
30) # Peak writes to store - once each 2 seconds
super(Cacheable, self).__init__(*args, **kwargs)
@classmethod
def get_by_key_name(cls, key_name, parent=None):
"""
override the Model.get_by_key_name method, to look in local or memcache
storage first.
All key_name(s) must be strings.
TODO: Support list of key_names - will need to call super are partial list of those
keys that are not yet cached!
"""
model = cls._model_from_cache(key_name)
if model is not None:
return model
# Go to storage
model = super(Cacheable, cls).get_by_key_name(key_name, parent)
if model is not None:
if DEBUG:
logging.info("Reading from storage: %s" %
cls._cache_key(key_name))
model.ensure_cached()
return model
@classmethod
def get_or_insert(cls, key_name, **kwargs):
# Look in cache first
model = cls._model_from_cache(key_name)
if model is not None:
return model
# Go to storage
model = super(Cacheable, cls).get_or_insert(key_name, **kwargs)
if model is not None:
model.ensure_cached()
return model
def put(self):
key = super(Cacheable, self).put()
if DEBUG:
logging.info("Writing to storage: %s" % self._model_cache_key())
self._cache_state = self.cache_state.clean
self.ensure_cached()
return key
def set_dirty(self, state=cache_state.dirty):
"""
Mark the model as having changes to write to the store before the request is over.
"""
self._is_memcached = False
if state > self._cache_state:
self._cache_state = state
self.ensure_cached()
def deferred_put(self):
"""
If the model has been marked as dirty, try to write it out.
"""
if self._cache_state == self.cache_state.clean:
return
self.ensure_cached()
try:
# Write to storage if critical or dirty AND old
if self._cache_state == self.cache_state.critical or \
not self._write_rate.is_exceeded(reqfilter.get_request().secsNow):
self.put()
except Exception, e:
logging.info("Failed to write deferred-write cache: %s (%s)" %
(self._model_cache_key(), e.message))
pass
from __future__ import division;
from __future__ import print_function;
from __future__ import absolute_import;
import sys;
import os;
scriptsDir = os.environ.get("UTIL_SCRIPTS_DIR");
if (scriptsDir is None):
raise Exception("Please set environment variable UTIL_SCRIPTS_DIR");
sys.path.insert(0,scriptsDir);
import pathSetter;
import fileProcessing as fp;
import util;
from collections import OrderedDict;
#ad-hoc script
metadataCols = util.enum(fileAccession='File accession', fileFormat='File format', experimentAccession='Experiment accession', biosampleTermName='Biosample term name', experimentTarget='Experiment target', biosampleAge='Biosample Age', outputType='Output type');
metadataColsForExperiment = [metadataCols.biosampleTermName, metadataCols.biosampleAge];
def readMetadataFile(metadataFile, relevantColumns=metadataCols.vals):
#returns an instance of util.titledMapping
titledMapping = fp.readTitledMapping(fp.getFileHandle(metadataFile), contentType=str, subsetOfColumnsToUseOptions=fp.SubsetOfColumnsToUseOptions(columnNames=relevantColumns));
return titledMapping;
def getMappingFromExperimentAccessionToValuableFields(metadataTitledMapping):
experimentToDetailsTitledMapping = util.TitledMapping(titleArr=metadataColsForExperiment, flagIfInconsistent=True);
for metadataTitledArr in metadataTitledMapping:
dataFilledIn = True;
for metadataColForExperiment in metadataColsForExperiment:
if metadataTitledArr.getCol(metadataColForExperiment) == "":
dataFilledIn = False;
break;
## @package exception
# Exceptions and error codes.
import util
## Enumeration indicating error types.
ErrorCode = util.enum(SUCCESS = 0,
INTERNAL_FAILURE = 1,
STREAM_EXCEEDS_MAXIMUM = 2,
INVALID_IMAGE_FORMAT = 3,
METHOD_NOT_SUPPORTED = 4,
AUTHENTICATION_FAILED = 100,
NOT_AUTHORIZED = 101,
MISSING_PARAMETER = 102,
INVALID_PARAMETER = 200,
NOT_FOUND = 201,
CONFLICT = 202,
INVALID_REQUEST_CODE = 203,
USER_NOT_FOUND = 300,
USER_IS_BLOCKED = 301,
WRONG_PASSWORD = 302,
WRONG_EMAIL_ADDRESS = 303,
NO_FRIENDSHIP = 304,
OBJECT_NOT_FOUND = 400,
OBJECT_IS_LOCKED = 401,
HTTP_FAILURE = 500)
## Exception base class.
class BaseException:
## The constructor.
# @param code error code
# @param http_status a mapped HTTP status
