def __init__(self,
dataRootPath,
singleChainfeatsToInclude,
pairfeatsToInclude=None,
verbose=False):
'''
@param dataRootPath: str. A path to computedFeatures directory that contains needed features. Example:
computedFeatures/
common/
contactMaps/
seqStep/
conservation/
...
structStep/
PSAIA/
VORONOI/
...
@param singleChainfeatsToInclude: dict. Dictionary that contains the paths where features needed for complex codification
that describre single chain properties are
located. Must have the following format:
{"featName":(relativePath_from_dataRootPath, listOfColumnNumbers, dictForNamedColums)}
dictForNamedColums= {"myFeatName":colNumber}
@param pairfeatsToInclude: dict. Dictionary that contains the paths where features needed for complex codification
that describre single chain properties are
located. Must have the following format:
{"featName":(relativePath_from_dataRootPath, listOfColumnNumbers, dictForNamedColums)}
dictForNamedColums= {"myFeatName":colNumber}
@param verbose: bool.
'''
Configuration.__init__(self)
self.dataRootPath = dataRootPath
self.verbose = verbose
self.singleChainfeatsToInclude = singleChainfeatsToInclude
self.pairfeatsToInclude = None if pairfeatsToInclude is None else pairfeatsToInclude[:]
def setUp(self, is_unit_test):
self.ExceptionHandler = CustomException.CustomException.getInstance()
self.Configuration = Configuration()
self.Validation = Validation
self.Setup = Setup.Setup()
self.Setup.setupApp()
self.logger = Debugger.Debugger.getInstance()
self.Database = Database.Database.getInstance(is_unit_test)
self.Database.connect(is_unit_test)
self.ProjectGUI = ProjectGUI
self.Setup.createTables()
self.ProjectRepo = ProjectRepository.ProjectRepository()
email_configuration = self.Database.getConfiguration()
try:
self.Configuration.setEmailConfiguration(email_configuration[0])
except:
self.Configuration.setEmailConfiguration(email_configuration)
pass
self.ProjectsList = {}
self.queue = {}
self.loadAllProjects()
def login(self, request):
try:
data = json.loads(request.data.decode())
self.ip = Configuration().GetData()['PrivateIp']
self.port = Configuration().GetData()['MongoPort']
self.db = Configuration().GetData()['MongoDB']
obj = MongoDB()
obj.ConnectMongo(self.ip, self.port, self.db)
record = obj.ReadValue("users", data["email"])
if (record != None):
record = ast.literal_eval(record['Data'])
if (record['password'] == data["password"]):
ret = {
'access_token':
create_access_token(identity=data["email"]),
'refresh_token':
create_refresh_token(identity=data["email"]),
'status':
"True"
}
return jsonify(ret), 200
else:
return jsonify({"status":
"Invalid username or password"}), 401
else:
return jsonify({"status": "Invalid username or password"}), 401
except Exception as e:
generate_log('auth', str(e), str(request))
def send_email():
sender = Configuration().GetData()['EmailID']
gmail_password = Configuration().GetData()['Password']
COMMASPACE = ', '
recipients = ['*****@*****.**']
# Create the enclosing (outer) message
outer = MIMEMultipart()
outer['Subject'] = 'DataFeed @ ' + str(datetime.datetime.now().date())
outer['To'] = COMMASPACE.join(recipients)
outer['From'] = sender
outer.preamble = 'You will not see this in a MIME-aware mail reader.\n'
msg = MIMEText('Data Feeding Start in Mongodb' + str(datetime.datetime.now()))
outer.attach(msg)
composed = outer.as_string()
# Send the email
try:
with smtplib.SMTP('smtp.gmail.com', 587) as s:
s.ehlo()
s.starttls()
s.ehlo()
s.login(sender, gmail_password)
s.sendmail(sender, recipients, composed)
s.close()
print("Email sent!")
except:
print("Unable to send the email. Error: ", sys.exc_info()[0])
raise
def __init__(self,
rFname,
lFname,
computedFeatsRootDir=None,
statusManager=None):
'''
@param rFname: str. path to receptor pdb file
@param lFname: str. path to ligand pdb file
@param computedFeatsRootDir: str. path where features will be stored
@param statusManager: class that implements .setStatus(msg) to communicate
'''
Configuration.__init__(
self) # Load configuration parameters such as path to programs
self.statusManager = statusManager
if computedFeatsRootDir != None:
self.computedFeatsRootDir = computedFeatsRootDir
self.computedFeatsRootDir = os.path.expanduser(
self.computedFeatsRootDir
) #Creates root path where features will be saved
myMakeDir(self.computedFeatsRootDir)
self.rFname = rFname
self.lFname = lFname
if not os.path.isdir(self.computedFeatsRootDir):
os.mkdir(self.computedFeatsRootDir)
def __init__(self, name):
# does not try loading a formation if name is not passed
if not name:
return
path = os.path.join("data", "actors", "formations")
formationini = Configuration(os.path.join("..", path, name + ".ini")).formation
# formation's name can come from it's ini or just the filename
self.name = formationini.__getattr__("name", default=name)
self.enemies = []
names = formationini.enemies.split("|")
while len(names) > 0:
self.enemies.append(Enemy(names[0]))
if names.count(names[0]) > 0:
self.enemies[-1].name += " " + chr(64 + names.count(names[0]))
names.pop(0)
self.enemies.reverse()
# assigns the positions and scales to the enemies
for i, enemy in enumerate(self.enemies):
n = formationini.__getattr__("coord").split("|")[i]
x = int(n.split(",")[0])
y = int(n.split(",")[1])
enemy.position = (x, y)
n = formationini.__getattr__("scale").split("|")[i]
enemy.scale = float(n)
self.terrain = formationini.__getattr__("terrain")
def __init__(self,
stepName,
isHomoComplex,
savedModelsPath=None,
averageLRscores=False):
'''
:param stepName: str. Must startswith seq_train or struct or mixed (seq_train, mixed_2, structX, seq_train1... are also valid)
:param isHomoComplex: boolean. Is the target complex h**o or hetero
:param savedModelsPath: str. A path to the directory where models have been saved. If None,
it will used the path indicated in Config
:param averageLRscores: True if Ligand and receptor are the same protein and thus, binding site prediction should be averaged
'''
Configuration.__init__(self)
self.isHomoComplex = isHomoComplex
self.stepName = stepName
self.averageLRscores = averageLRscores
if not savedModelsPath is None:
self.savedModelsPath = savedModelsPath
self.model = None
print(stepName)
self.savedModelsPath = os.path.join(
self.savedModelsPath, "h**o" if self.isHomoComplex else "hetero")
for fname in os.listdir(self.savedModelsPath):
if fname.endswith(stepName):
print("Loading model %s %s" %
("h**o" if isHomoComplex else "hetero", fname))
self.model = joblib_load(
os.path.join(self.savedModelsPath, fname))
assert not self.model is None, "Error, there is no valid model in %s for step %s" % (
self.savedModelsPath, self.stepName)
def __init__(self, family):
self.family = family #save data for the bestiary
#gathers all the different enemy types
path = os.path.join("..", "data", "actors", "enemies")
enemies = os.listdir(path)
enemies.sort()
path = os.path.join("..", "data", "actors", "families", family.name, "bestiary.ini")
if not os.path.exists(path):
Configuration(path).save()
self.catalog = Configuration(path)
self.catalog.bestiary.__setattr__(enemies[0], 0)
self.catalog.save()
self.catalog = Configuration(path)
self.beasts = {}
for beast in enemies:
try:
self.beasts[beast] = self.catalog.bestiary.__getattr__(beast, int)
except:
self.catalog.bestiary.__setattr__(beast, 0)
self.catalog.save()
self.catalog = Configuration(path)
self.beasts[beast] = self.catalog.bestiary.__getattr__(beast, int)
def __init__(self, consDbSqlite=None, consDbFilesPath=None):
Configuration.__init__(self)
self.isReady = True
if consDbFilesPath:
self.consDbFilesPath = consDbFilesPath
if consDbSqlite:
self.consDbSqlite = consDbSqlite
self.unirefType = None
if not os.path.isfile(
self.consDbSqlite) or not self.checkIfDbFilesAvailable():
self.isReady = False
else:
self.unirefType = os.path.basename(self.consDbFilesPath)
assert self.unirefType in ["uniref90", "uniref100"], ("Error, consDbFilesPath %s "+\
":must be path/to/data/[uniref90|uniref100]")%(self.consDbFilesPath)
self.sqliteConn = sqlite3.connect(self.consDbSqlite)
self.sqliteCursor = self.sqliteConn.cursor()
try: #check if sqlite was correctly opened
self.sqliteCursor.execute(
"SELECT seqId FROM sequencesTable where sequence== 0"
).fetchone()
except sqlite3.OperationalError:
self.isReady = False
def __init__(self,
trainDataPath,
testPath,
outputPath=None,
nFolds=None,
isLastStep=False,
saveModelFname=None,
verbose=True,
numProc=1):
'''
builder
:param trainDataPath: str. Path to a dir where training data files are stored
:param testPath: str. Path to a dir where testing data files are stored
:param outputPath: str. Path to a dir where predictions will be stored. If None, results will not be saved
and just performance evaluation will be carried out
:param nFolds: int. Number of folds for k-fold cross-validation. If -1, leave-one-out will be performed.
If 0, testing set will be used as if it where independent. Cannot be 1
:param isLastStep: bool. True if this train is the second step of a two steps workflow or the first one in one step workflow
:param saveModelFname: str. A path where the final model, trained with all data will be saved. If None,
model won't be saved
:param verbose: boolean. Whether or not print to stdout info
:param numProc: int. Number of processes to use in parallel
'''
Configuration.__init__(
self) # Load configuration parameters such as path to programs
parentPath, __ = os.path.split(trainDataPath)
parentPath, stepName = os.path.split(parentPath)
parentPath, __ = os.path.split(parentPath)
self.outputPath = outputPath
self.saveModelFname = saveModelFname
self.verbose = verbose
self.numProc = numProc
self.nFolds = nFolds
self.isLastStep = isLastStep
self.trainPath = trainDataPath
trainFilesNames = sorted(os.listdir(self.trainPath))
self.trainPrefixes = sorted(
set([fname.split(".")[0] for fname in trainFilesNames]))
self.testPath = testPath
if not self.testPath is None:
testFilesNames = sorted(os.listdir(self.testPath))
self.testPrefixes = sorted(
set([fname.split(".")[0] for fname in testFilesNames]))
else:
self.testPrefixes = []
self.data, self.prefixesUsedInModel = None, None # self.loadTrainingData(sharedMemoryPath=None) will be executed latter
if self.verbose:
print("%d train complexes loaded." % (len(self.trainPrefixes)))
self.numTestComplexes = 0 if self.testPrefixes == None else len(
self.testPrefixes)
def create(self, name, difficulty, gold=100):
path = os.path.join("..", "data", "actors", "families", name)
if not os.path.exists(path):
os.mkdir(path)
Configuration(os.path.join(path, "family.ini")).save()
familyini = Configuration(os.path.join(path, "family.ini"))
familyini.family.__setattr__("difficulty", difficulty)
familyini.family.__setattr__("gold", int(gold))
familyini.save()
def __init__(self, computedFeatsRootDir, statusManager=None):
'''
:param computedFeatsRootDir: str. root path where results will be saved
:param statusManager: class that implements .setStatus(msg) to communicate
'''
Configuration.__init__(self)
self.computedFeatsRootDir = computedFeatsRootDir
self.filterOutLabels = FILTER_OUT_LABELS
self.filterOutNoStandard = IGNORE_NO_STANDARD
self.statusManager = statusManager
def __init__(self, data_path= None):
'''
@param data_path: str: Path where AAIndex files are located
'''
Configuration.__init__(self) # Load configuration parameters such as path to programs
self.protein_proteinIndexes=["KESO980101","KESO980102","MOOG990101"]
if data_path is None:
self.data_path= self.AAindexPath
else:
self.data_path=data_path
self.data=self.load()
def __init__(self, seqsManager, outPath, winSize):
'''
@param seqsManager: ..manageSeqs.seqsManager.SeqsManager
@param outPath: str. root path where psiblast and al2co scores will be saved
@param winSize: int. The size of sliding window
'''
Configuration.__init__(self)
self.seqsManager = seqsManager
self.seqsWorkingDir = self.seqsManager.getSeqsOutDir()
self.outPath = outPath
self.winSize = winSize
self.filterOutLabels = FILTER_OUT_LABELS
def __init__(self, prefix, computedFeatsRootDir=None, statusManager=None):
'''
@prefix. An id for a complex. Example: 1A2K
:param computedFeatsRootDir: str. path where features will be stored. If None, read from Confinguration
:param statusManager: class that implements .setStatus(msg) to communicate
'''
Configuration.__init__(
self) # Load configuration parameters such as path to programs
self.prefix = prefix
self.statusManager = statusManager
if computedFeatsRootDir != None:
self.computedFeatsRootDir = computedFeatsRootDir
self.computedFeatsRootDir = self.computedFeatsRootDir #Creates root path where features will be saved
def start(self,port=None,systems=None):
connect = Configuration.getMongoConnection()
self.db = connect['cves']
self.systems=systems
app = Flask(__name__, static_folder='static', static_url_path='/static')
app.config['MONGO_DBNAME'] = Configuration.getMongoDB()
# functions
def getEnhance():
for system in self.systems:
cpe=system['cpe']
if cpe:
system['cves']=list(self.db.find({'vulnerable_configuration': system['cpe']}))
#get possible dpe info and store in dpe
for service in system['services']:
if 'cpe' in service:
service['cves']=list(self.db.find({'vulnerable_configuration':service['cpe']}))
#get dpe info for service
return systems
# routes
@app.route('/')
def index():
systems=getEnhance()
return render_template('index.html', systems=systems)
@app.route('/cve/<cveid>')
def cve(cveid):
cvesp = cves.last(rankinglookup = True, namelookup = True, vfeedlookup = True)
cve = cvesp.getcve(cveid=cveid)
if cve is None:
return page_not_found(404)
cve = markCPEs(cve)
return render_template('cve.html', cve=cve)
# filters
@app.template_filter('product')
def product(banner):
if banner:
p=re.search('(product:).([^\s]+)', banner)
return p.group(2)
else:
return "unknown"
# debug filter
@app.template_filter('type')
def isType(var):
return type(var)
#start webserver
app.run(host='127.0.0.1', port=5000, debug=True)
def get_user(self):
try:
email = get_jwt_identity()
self.ip = Configuration().GetData()['PrivateIp']
self.port = Configuration().GetData()['MongoPort']
self.db = Configuration().GetData()['MongoDB']
obj = MongoDB()
obj.ConnectMongo(self.ip, self.port, self.db)
record = obj.ReadValue("users", email)
record = ast.literal_eval(record['Data'])
record.pop('password', None)
return jsonify(record)
except Exception as e:
generate_log('get_user', str(e), 'get_user method')
def email_logfile():
sender = Configuration().GetData()['EmailID']
gmail_password = Configuration().GetData()['Password']
dr_tariq_team = Configuration().GetData()['team_emails']
COMMASPACE = ', '
recipients = dr_tariq_team
# Create the enclosing (outer) message
outer = MIMEMultipart()
outer['Subject'] = 'dpdmlog @ ' + str(datetime.datetime.now().date())
outer['To'] = COMMASPACE.join(recipients)
outer['From'] = sender
outer.preamble = 'You will not see this in a MIME-aware mail reader.\n'
# List of attachments
attachments = ['sp_error.log']
# Add the attachments to the message
for file in attachments:
try:
with open(file, 'rb') as fp:
msg = MIMEBase('application', "octet-stream")
msg.set_payload(fp.read())
encoders.encode_base64(msg)
msg.add_header('Content-Disposition',
'attachment',
filename=os.path.basename(file))
outer.attach(msg)
except:
print("Unable to open one of the attachments. Error: ",
sys.exc_info()[0])
raise
composed = outer.as_string()
# Send the email
try:
with smtplib.SMTP('smtp.gmail.com', 587) as s:
s.ehlo()
s.starttls()
s.ehlo()
s.login(sender, gmail_password)
s.sendmail(sender, recipients, composed)
s.close()
# print("Email sent!")
except:
print("Unable to send the email. Error: ", sys.exc_info()[0])
raise
def setUp(self, is_unit_test):
self.ExceptionHandler = CustomException.CustomException.getInstance()
self.Configuration = Configuration()
self.Validation = Validation
self.Setup = Setup.Setup()
self.Setup.setupApp()
self.logger = Debugger.Debugger.getInstance()
self.Database = Database.Database.getInstance(is_unit_test)
self.Database.connect(is_unit_test)
self.ProjectGUI = ProjectGUI
self.Setup.createTables()
self.ProjectRepo = ProjectRepository.ProjectRepository()
email_configuration = self.Database.getConfiguration()
try:
self.Configuration.setEmailConfiguration(email_configuration[0])
except:
self.Configuration.setEmailConfiguration(email_configuration)
pass
self.ProjectsList = {}
self.queue = {}
self.loadAllProjects()
def data_feed():
app = TestApp("0.0.0.0", 4001, 9)
for com in Configuration().GetData()['CompanyList']:
ibcontract = IBcontract()
ibcontract.secType = "STK"
ibcontract.lastTradeDateOrContractMonth = "201809"
ibcontract.symbol = com
ibcontract.exchange = "SMART"
resolved_ibcontract = app.resolve_ib_contract(ibcontract)
durationstr = "1 D"
for bar in barSize:
historic_data = app.get_IB_historical_data(resolved_ibcontract,
durationstr, bar)
signal = 1
dataset = pd.DataFrame(historic_data)
dataset['signal'] = signal
for index, row in dataset.iterrows():
if dataset['open'][index] > dataset['close'][index]:
signal = 1
else:
signal = 0
print(signal)
dataset['signal'][index] = signal
print(com)
print(bar)
Feed_IntraDay(com, bar, dataset)
print(dataset)
def getdata(self):
app = TestApp("0.0.0.0", 4001, 10)
for com in Configuration().GetData()['CompanyList']:
ibcontract = IBcontract()
ibcontract.secType = "STK"
ibcontract.lastTradeDateOrContractMonth = "201809"
ibcontract.symbol = com
ibcontract.exchange = "SMART"
resolved_ibcontract = app.resolve_ib_contract(ibcontract)
dataset1 = {
0: ['20190502 13:30:00', '20190502 16:00:00'],
1: [209.95, 208.65],
2: [212.65, 210.29],
3: [208.13, 208.41],
4: [208.63, 209.17],
5: [149612, 100915]
}
durationstr = "3600 S"
# historic_data = app.get_IB_historical_data(resolved_ibcontract, durationstr, bar)
df = pd.DataFrame(dataset1)
df.rename(columns={
0: "date",
1: "open",
2: "high",
3: "low",
4: "close",
5: "volume"
},
inplace=True)
for bar in barSize:
dataset = self.strategy(df)
print(com)
print(bar)
# MongoStore().Feed_IntraDay(com, bar, dataset)
print(df)
def loadAccesibility(pdbId, chainType="l", rasaThr=10.0):
'''
Loads psaia files for por a given pdbId and returns a set of accesible
resIds and non-accesible resIds.
:param pdbId: str. The identifier for pdb file
:param chainType: str. "l" for ligan and "r" for receptor
:param rasaThr: float. A threshold of relative asa to decide whether or not a residue is accesible or not
:return (accesibleSet, nonAccesibleSet)
accesibleSet: set(str[]). Set of resIds of residues that are accesible according to PSAIA and the threshold
nonAccesibleSet: set(str[]). Set of resIds of residues that are non-accesible according to PSAIA and the threshold
'''
PSAIA_PATH = os.path.join(Configuration().computedFeatsRootDir, "structStep/PSAIA/procPSAIA")
accesibleSet=set([])
nonAccesibleSet=set([])
for fname in os.listdir(PSAIA_PATH):
if fname.startswith(pdbId+"_"+chainType) and fname.endswith(".psaia.tab"):
with open(os.path.join(PSAIA_PATH, fname)) as f_:
f_.readline()
for line in f_:
lineArray= line.split()
chainId= lineArray[0]
resId= lineArray[1]
res_full_id= chainId+"_"+resId
if float(lineArray[8])> rasaThr:
accesibleSet.add(res_full_id)
else:
nonAccesibleSet.add(res_full_id)
return accesibleSet, nonAccesibleSet
def __init__(self):
self.name = "user"
self.ip = Configuration().GetData()['PrivateIp']
self.port = Configuration().GetData()['MongoPort']
self.db = Configuration().GetData()['MongoDB']
self.email = Configuration().GetData()['EmailID']
self.password = Configuration().GetData()['Password']
self.reg_emailLink = Configuration().GetData()['RegisterEmail']
self.forgot_passwordLink = Configuration().GetData()['ForgotPassword']
self.email_ids = Configuration().GetData()['Email']
self.sp_link = Configuration().GetData()['SP']
self.mongoObj = MongoDB()
self.mongoObj.ConnectMongo(self.ip, self.port, self.db)
def computeFeaturesAllComplexes(OneFeaturesComputerClass,
pdbsIndir,
computedFeatsRootDir,
classArgs={},
ncpu=1):
'''
Computes one type of feature over all complexes that are inside pdbsIndir.
@param OneFeaturesComputerClass: FeaturesComputer. class to use for compute one kind of features
@param pdbsIndir: str. path where pdb files to be computed are located. There must be 2 pdb files per complex
To distinguish them _l_ or _r_ infixed are used. P.e: "1A2K_l_u.pdb" and "1A2K_r_u.pdb".
pdb files ended with "b.pdb" will be skipped.
@param computedFeatsRootDir: str. path where features will be stored
@param classArgs: Dict. The arguments that will be passed to OneFeaturesComputerClass()
@param ncpu: the number of subprocess to use in parallel (parallelism at complex level)
'''
pdbsIndir = os.path.expanduser(pdbsIndir)
computedFeatsRootDir = os.path.expanduser(computedFeatsRootDir)
ConfigObject = Configuration()
if pdbsIndir == None:
pdbsIndir = ConfigObject.pdbsIndir
if computedFeatsRootDir == None:
computedFeatsRootDir = ConfigObject.computedFeatsRootDir
fnames = {}
for fname in sorted(os.listdir(pdbsIndir)):
if not fname.endswith(".pdb"): continue #skip no pdb files
if not fname.endswith("b.pdb"):
prefix = fname.split("_")[0]
if "_r_" in fname or "_l_" in fname:
if prefix not in fnames:
fnames[prefix] = [None, None]
if "_r_" in fname:
fnames[prefix][0] = os.path.join(pdbsIndir, fname)
if "_l_" in fname:
fnames[prefix][1] = os.path.join(pdbsIndir, fname)
else:
fnames[prefix].append(os.path.join(pdbsIndir, fname))
if len(fnames) < 1:
raise ValueError("There are not files to be processed")
for prefix in fnames: # check for errors
if len(fnames[prefix]) > 2 or sum(
[1 for elem in fnames[prefix] if elem is None]):
raise ValueError(
"There must be just 2 pdb files for each complex to be predicted"
)
else:
fnames[prefix] = tuple(fnames[prefix])
Parallel(n_jobs=ncpu, backend="multiprocessing",
batch_size=2)(delayed(computeFunction)(
OneFeaturesComputerClass, fnames[prefix][0],
fnames[prefix][1], computedFeatsRootDir, classArgs)
for prefix in sorted(fnames))
def __init__(self, name):
# does not try loading a family if name is not passed
# ONLY do this during family creation
if not name:
return
path = os.path.join("data", "actors", "families", name)
familyini = Configuration(os.path.join("..", path, "family.ini")).family
# family's last name, all members of the family will have this last name
self.name = name
# all the members in your party
path = os.path.join("..", path)
members = [n.split("/")[-1] for n in glob.glob(os.path.join(path, "*.ini"))]
exclude = ["family.ini", "bestiary.ini", "inventory.ini"]
for item in exclude:
if item in members:
members.remove(item)
self.members = [Character(self, n.replace(".ini", "")) for n in members]
# the party used in battle is the first 3 members you have ordered
if len(self.members) >= 3:
self.party = Party(self.members[0:3])
# party should only be 0 during character creation
elif len(self.members) == 0:
self.party = None
else:
self.party = Party(self.members[0 : len(self.members)])
# the items your family has
self.inventory = Inventory(self)
print self.inventory
# amount of money your family possesses
self.gold = familyini.__getattr__("gold", int)
# gameplay difficulty (0 - easy, 1- normal, 2 - hard)
self.difficulty = familyini.__getattr__("difficulty", int)
# family bestiary
self.bestiary = Bestiary(self)
def get_history(self):
try:
email = get_jwt_identity()
self.ip = Configuration().GetData()['PrivateIp']
self.port = Configuration().GetData()['MongoPort']
self.db = Configuration().GetData()['MongoDB']
obj = MongoDB()
obj.ConnectMongo(self.ip, self.port, self.db)
record = obj.ReadValue("history", email)
if record != None:
record = ast.literal_eval(record["Data"])
toreturn = {"status": "True", "record": record}
else:
# record = ast.literal_eval(record["Data"])
toreturn = {"status": "False"}
return jsonify(toreturn)
except Exception as e:
generate_log('get_history', str(e))
def __init__(self, dataRootPath, verbose=False):
'''
:param dataRootPath: str. A path to computedFeatures directory that contains needed features. Example:
computedFeatures/
common/
contactMaps/
seqStep/
conservation/
...
structStep/
PSAIA/
...
:param verbose: bool.
'''
Configuration.__init__(self)
if dataRootPath is None:
dataRootPath = self.computedFeatsRootDir
self.dataRootPath = dataRootPath
self.verbose = verbose
def admin():
status=["default","none"]
if Configuration.loginRequired():
if not current_user.is_authenticated():
return render_template('login.html', status=status)
else:
return render_template('admin.html', status=status)
else:
person = User.get("_dummy_")
login_user(person)
return render_template('admin.html',status=status)
def __init__(self, family):
self.family = family #save data for the inventory
#gathers all the different enemy types
path = os.path.join("..", "data", "items")
items = os.listdir(path)
items.sort()
path = os.path.join("..", "data", "actors", "families", family.name, "inventory.ini")
if not os.path.exists(path):
Configuration(path).save()
self.catalog = Configuration(path)
self.catalog.inventory.__setattr__(items[0], 0)
self.catalog.save()
self.catalog = Configuration(path)
self.items = {}
def loadList():
changeMade = False
for i in items:
if self.catalog.parser.has_option("inventory", i):
ini = Configuration(os.path.join("..", "data", "items", i, "item.ini"))
#detecting what type of item it is
if ini.parser.has_section("weapon"):
item = Weapon(i)
elif ini.parser.has_section("armor"):
item = Armor(i)
elif ini.parser.has_section("loot"):
item = Loot(i)
elif ini.parser.has_section("usable"):
item = Usable(i)
else:
item = Item(i)
self.items[i] = [item, self.catalog.inventory.__getattr__(i, int)]
else:
self.catalog.inventory.__setattr__(i, 0)
self.catalog.save()
loadList()
def checkIfAllCodified(self, inputPdbs=None):
if inputPdbs is None:
# Default parameters
conf = Configuration()
inputPdbs = conf.pdbsIndir
check0 = self.checkIfAllCmapsComputed(inputPdbs)
check1 = self.checkIfAllCodifiedWorker(self.trainingDataPath)
check2 = self.checkIfAllCodifiedWorker(self.testingDataPath)
return check0 and check1 and check2
def set_default_params(self):
cfg = Configuration()
self.learning_rate_value.setText(str(cfg.learning_rate))
self.learning_steps_value.setText("200 400")
self.decay_value.setText(str(cfg.decay))
self.momentum_value.setText(str(cfg.momentum))
self.epoch_value.setText(str(cfg.number_of_epochs))
self.iterations_value.setText(str(cfg.number_of_iterations))
self.path_content.setText('Dataset')
self.dir_path = 'Dataset'
self.class_value.setText(str(cfg.number_of_classes))
self.seed_value.setText(str(cfg.seed))
self.warm_up_value.setText(str(cfg.iterations_to_warmup))
def __init__(self, stepName, savedModelsPath=None):
'''
@param stepName: str. Must startswith seq_train or struct or mixed (seq_train, mixed_2, structX, seq_train1... are also valid)
@param savedModelsPath: str. A path to the directory where models have been saved. If None,
it will used the path indicated in Config
'''
Configuration.__init__(self)
self.stepName = stepName
if not savedModelsPath is None:
self.savedModelsPath = savedModelsPath
self.model = None
print(stepName)
for fname in os.listdir(self.savedModelsPath):
if fname.endswith(stepName):
print("Loading model %s" % (fname))
self.model = joblib_load(
os.path.join(self.savedModelsPath, fname))
assert not self.model is None, "Error, there is no valid model in %s for step %s" % (
self.savedModelsPath, self.stepName)
def data_feed():
app = TestApp("0.0.0.0", 4001, 10)
for com in Configuration().GetData()['CompanyList']:
ibcontract = IBcontract()
ibcontract.secType = "STK"
ibcontract.lastTradeDateOrContractMonth="201809"
ibcontract.symbol = com
ibcontract.exchange = "SMART"
resolved_ibcontract = app.resolve_ib_contract(ibcontract)
bar='1 sec'
durationstr = "120 sec"
historic_data = app.get_IB_historical_data(resolved_ibcontract,durationstr,bar)
print(com)
print(bar)
Feed_IntraDay(com,bar,historic_data)
print(historic_data)0
def __init__(self, dataRootPath=None, featuresToInclude=None):
if dataRootPath is None:
dataRootPath = Configuration().computedFeatsRootDir
if featuresToInclude is None:
featuresToInclude = FEATURES_TO_INCLUDE
else:
if "psaia" not in zip(*featuresToInclude)[0]:
featuresToInclude.insert(0,
("psaia",
("structStep/PSAIA/procPSAIA", [8], {
"total_RASA": 8
})))
self.psaiaIndex = zip(*featuresToInclude)[0].index("psaia")
DataLoader.__init__(self, dataRootPath, featuresToInclude)
self.rasaLDict = {}
self.rasaRDict = {}
def moveAndWriteAsPDBIfMmcif(fnameIn, fnameOut, removeInput=False):
from Config import Configuration
conf = Configuration()
minNumResidues, maxNumResidues = conf.minNumResiduesPartner, conf.maxNumResiduesPartner
try:
struct, __ = loadPdbIfIsPath(fnameIn)
totalNumRes = 0
for chain in struct[0]:
nResInChain = len(chain.get_list())
totalNumRes += nResInChain
if not (minNumResidues < totalNumRes < maxNumResidues):
raise BadNumberOfResidues(totalNumRes)
else:
writter = PDBIO()
writter.set_structure(struct)
writter.save(fnameOut)
if removeInput: os.remove(fnameIn)
return True
except Exception as e:
print("Error in moveAndWriteAsPDBIfMmcif !!!", e)
return False
class User(UserMixin):
'''Simple User class'''
if not Configuration.loginRequired():
# dummy account for when logon is not required.
USERS = {"_dummy_": "_dummy_"}
else:
USERS = {}
for user in collection.find({}):
USERS[user['username']] = user['password']
def __init__(self, id):
if not id in self.USERS:
raise UserNotFoundError()
self.id = id
self.password = self.USERS[id]
@classmethod
def get(self_class, id):
'''Return user instance of id, return None if not exist'''
try:
return self_class(id)
except UserNotFoundError:
return None
class Bestiary:
def __init__(self, family):
self.family = family #save data for the bestiary
#gathers all the different enemy types
path = os.path.join("..", "data", "actors", "enemies")
enemies = os.listdir(path)
enemies.sort()
path = os.path.join("..", "data", "actors", "families", family.name, "bestiary.ini")
if not os.path.exists(path):
Configuration(path).save()
self.catalog = Configuration(path)
self.catalog.bestiary.__setattr__(enemies[0], 0)
self.catalog.save()
self.catalog = Configuration(path)
self.beasts = {}
for beast in enemies:
try:
self.beasts[beast] = self.catalog.bestiary.__getattr__(beast, int)
except:
self.catalog.bestiary.__setattr__(beast, 0)
self.catalog.save()
self.catalog = Configuration(path)
self.beasts[beast] = self.catalog.bestiary.__getattr__(beast, int)
def update(self):
for beast in self.beasts:
self.catalog.bestiary.__setattr__(beast[0], beast[1])
self.catalog.save()
self.catalog = Configuration(path)
def __str__(self):
str = ""
for i, beast in enumerate(self.beasts):
str += "%03i: %-16s%02i\n" % (i+1, beast[0], beast[1])
return str
import signal
from User import User
from Config import Configuration
import cves
from db_whitelist import *
from db_blacklist import *
# parse command line arguments
argparser = argparse.ArgumentParser(description='populate/update the whitelist used in webviews')
argparser.add_argument('-v', action='store_true', help='verbose output')
args = argparser.parse_args()
# variables
app = Flask(__name__, static_folder='static', static_url_path='/static')
app.config['MONGO_DBNAME'] = Configuration.getMongoDB()
app.config['SECRET_KEY'] = str(random.getrandbits(256))
pageLength = Configuration.getPageLength()
# login manager
login_manager = LoginManager()
login_manager.init_app(app)
# db connectors
mongo = PyMongo(app)
db = Configuration.getMongoConnection()
redisdb = Configuration.getRedisVendorConnection()
# functions
def matchFilePath(path):
pattern = re.compile('^([a-zA-Z/ 0-9._-])+$')
if pattern.match(path):
self.cpe[-1]['references'].append(self.href)
def characters(self, ch):
if self.titletag:
self.title += ch
def endElement(self, name):
if name == 'cpe-item':
self.titletag = False
self.cpe[-1]['title'].append(self.title.rstrip())
elif name == 'references':
self.referencestag = False
elif name == 'reference':
self.referencetag = False
self.href = None
# dict
cpedict = Configuration.getCPEDict()
# connect to db
db = Configuration.getMongoConnection()
cpe = db.cpe
info = db.info
parser = make_parser()
ch = CPEHandler()
parser.setContentHandler(ch)
f = urlopen(cpedict)
i = info.find_one({'db': 'cpe'})
if i is not None:
if f.headers['last-modified'] == i['last-modified']:
sys.exit("Not modified")
class App(object):
_instance = None
def __init__(self):
self.setUp()
def selfDestruct(self):
del self
@staticmethod
def getInstance(is_unit_test):
if not isinstance(App._instance, App):
App._instance = object.__new__(App)
SharedApp.SharedApp.App = App._instance
App._instance.setUp(is_unit_test)
return App._instance
def setUp(self, is_unit_test):
self.ExceptionHandler = CustomException.CustomException.getInstance()
self.Configuration = Configuration()
self.Validation = Validation
self.Setup = Setup.Setup()
self.Setup.setupApp()
self.logger = Debugger.Debugger.getInstance()
self.Database = Database.Database.getInstance(is_unit_test)
self.Database.connect(is_unit_test)
self.ProjectGUI = ProjectGUI
self.Setup.createTables()
self.ProjectRepo = ProjectRepository.ProjectRepository()
email_configuration = self.Database.getConfiguration()
try:
self.Configuration.setEmailConfiguration(email_configuration[0])
except:
self.Configuration.setEmailConfiguration(email_configuration)
pass
self.ProjectsList = {}
self.queue = {}
self.loadAllProjects()
def loadAllProjects(self):
all_projects = self.ProjectRepo.getAll()
if all_projects is not None and all_projects is not False:
if len(all_projects) > 0:
for single_project in all_projects:
project_logic = ProjectCore.ProjectCore()
project_logic.setProjectInfo(all_projects[single_project])
self.ProjectsList[all_projects[single_project]["title"]] = project_logic
def getSingleThreadToQueue(self, thread_id):
return self.queue.get(thread_id)
def addSingleThreadToQueue(self, thread_id, thread_object):
return self.queue.put(thread_object)
def getProjectList(self):
information = []
if self.ProjectsList is not None and self.ProjectsList is not False:
if len(self.ProjectsList) > 0:
for project in self.ProjectsList:
information.append(str(self.ProjectsList[project].getTitle()))
return information
def getSingleProject(self, project_name):
try:
selected_project_object = self.ProjectsList[project_name]
return selected_project_object
except:
return False
def removeProject(self, project_name):
self.ProjectsList.__delitem__(project_name)
def create(
self,
family,
name,
job,
stats,
points=0,
equipment=None,
proficiency=None,
gender="male",
sprite="01",
level=1,
exp=0,
):
Configuration(os.path.join("..", "data", "actors", "families", family, name + ".ini")).save()
ini = Configuration(os.path.join("..", "data", "actors", "families", family, name + ".ini"))
# nothing like short-handing
base = ini.character
dist = ini.distribution
eqp = ini.equipment
prof = ini.proficiency
if proficiency == None:
proficiency = eval(job + "()").proficiencies
if equipment == None:
equipment = [None for i in range(10)]
base.__setattr__("gender", gender)
base.__setattr__("spriteset", sprite)
base.__setattr__("job", job)
base.__setattr__("level", level)
base.__setattr__("exp", exp)
base.__setattr__("points", points)
dist.__setattr__("hpDist", stats[0])
dist.__setattr__("strDist", stats[1])
dist.__setattr__("defDist", stats[2])
dist.__setattr__("spdDist", stats[3])
dist.__setattr__("evdDist", stats[4])
dist.__setattr__("magDist", stats[5])
dist.__setattr__("resDist", stats[6])
eqp.__setattr__("right hand", equipment[0])
eqp.__setattr__("left hand", equipment[1])
eqp.__setattr__("accessory1", equipment[2])
eqp.__setattr__("accessory2", equipment[3])
eqp.__setattr__("accessory3", equipment[4])
eqp.__setattr__("helmet", equipment[5])
eqp.__setattr__("armor", equipment[6])
eqp.__setattr__("legs", equipment[7])
eqp.__setattr__("feet", equipment[8])
eqp.__setattr__("gloves", equipment[9])
prof.__setattr__("sword", proficiency[0])
prof.__setattr__("dagger", proficiency[1])
prof.__setattr__("spear", proficiency[2])
prof.__setattr__("staff", proficiency[3])
prof.__setattr__("guns", proficiency[4])
prof.__setattr__("fist", proficiency[5])
ini.save()
# Copyright (c) 2014 Alexandre Dulaunoy - [email protected] # Copyright (c) 2014-2015 Pieter-Jan Moreels - [email protected] # Imports import os import sys runPath = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(runPath, "./lib/")) import pymongo import redis from Config import Configuration # connect to db db = Configuration.getMongoConnection() cpe = db.cpe try: r = Configuration.getRedisVendorConnection() except: sys.exit(1) for e in cpe.find({}): try: value = e['id'] if e['id'].count(':') > 4: value = ":".join(str(x) for x in (value.split(':')[:5])) (prefix, cpetype, vendor, product, version) = value.split(':') except: pass
def nbelement(db = Configuration.getMongoDB(), collection = None):
if collection is None:
collection = "cves"
c = Configuration.getMongoConnection()
return c[collection].count()
from Config import Configuration
from list import CPEList
# parse command line arguments
argparser = argparse.ArgumentParser(description='populate/update the whitelist used in webviews')
argparser.add_argument('-a', action='append', help='add one or more CPE to whitelist')
argparser.add_argument('-r', action='append', help='remove one or more CPE from whitelist')
argparser.add_argument('-i', help='filename of the whitelist to import')
argparser.add_argument('-e', help='filename of the whitelist to export')
argparser.add_argument('-d', action='store_true', help='drop the whitelist')
argparser.add_argument('-f', action='store_true', help='force an action')
argparser.add_argument('-v', action='store_true', help='verbose output')
args = argparser.parse_args()
# connect to db
db = Configuration.getMongoConnection()
collection = db.mgmt_whitelist
def importWhitelist(importFile):
oList = CPEList(collection, args)
oList.importList(importFile)
def exportWhitelist(exportFile):
oList = CPEList(collection, args)
oList.exportList(exportFile)
def dropWhitelist():
oList = CPEList(collection, args)
oList.dropCollection()
def countWhitelist():
def main():
# Imports
import os
import sys
_runPath = os.path.dirname(os.path.realpath(__file__))
sys.path.append(os.path.join(_runPath, "./lib/"))
import re
import argparse
import csv
from urllib.parse import urlparse
import json
from bson import json_util
import cves
from Config import Configuration
# connect to DB
db = Configuration.getMongoConnection()
collection = db.cves
# init control variables
csvOutput = 0
htmlOutput = 0
jsonOutput = 0
xmlOutput = 0
# init various variables :-)
vSearch = ""
vOutput = ""
vFreeSearch = ""
# parse command-line arguments
argParser = argparse.ArgumentParser(description='Search for vulnerabilities in the National Vulnerability DB. Data from http://nvd.nist.org.')
argParser.add_argument('-p', type=str, help='S = search product, e.g. o:microsoft:windows_7 or o:cisco:ios:12.1')
argParser.add_argument('-f', type=str, help='F = free text search in vulnerability summary')
argParser.add_argument('-c', action='append', help='search one or more CVE-ID')
argParser.add_argument('-o', type=str, help='O = output format [csv|html|json|xml|cveid]')
argParser.add_argument('-l', action='store_true', help='sort in descending mode')
argParser.add_argument('-n', action='store_true', help='lookup complete cpe (Common Platform Enumeration) name for vulnerable configuration')
argParser.add_argument('-r', action='store_true', help='lookup ranking of vulnerable configuration')
argParser.add_argument('-v', type=str, help='vendor name to lookup in reference URLs')
args = argParser.parse_args()
vSearch = args.p
cveSearch = args.c
vOutput = args.o
vFreeSearch = args.f
sLatest = args.l
namelookup = args.n
rankinglookup = args.r
cves = cves.last(rankinglookup = rankinglookup, namelookup = namelookup)
# replace special characters in vSearch with encoded version.
# Basically cuz I'm to lazy to handle conversion on DB creation ...
if vSearch:
vSearch = re.sub(r'\(','%28', vSearch)
vSearch = re.sub(r'\)','%29', vSearch)
# define which output to generate.
if vOutput == "csv":
csvOutput = 1
elif vOutput == "html":
htmlOutput = 1
elif vOutput == "xml":
xmlOutput = 1
from xml.etree.ElementTree import Element, SubElement, tostring
from xml.sax.saxutils import escape as SaxEscape
r = Element('cve-search')
elif vOutput == "json":
jsonOutput = 1
elif vOutput == "cveid":
cveidOutput = 1
else:
cveidOutput = False
# Print first line of html output
if htmlOutput and args.p is not None:
print("<html><body><h1>CVE search "+ args.p +" </h1>")
elif htmlOutput and args.c is not None:
print("<html><body><h1>CVE-ID "+ str(args.c) +" </h1>")
# search default is ascending mode
sorttype=1
if sLatest:sorttype=-1
def printCVE(item):
if not namelookup and not rankinglookup:
print(json.dumps(item, sort_keys=True, default=json_util.default))
else:
if "vulnerable_configuration" in item:
vulconf = []
ranking = []
for conf in item['vulnerable_configuration']:
if namelookup:
vulconf.append(cves.getcpe(cpeid=conf))
if rankinglookup:
rank = cves.getranking(cpeid=conf)
if rank and rank not in ranking:
ranking.append(rank)
if namelookup:
item['vulnerable_configuration'] = vulconf
if rankinglookup:
item['ranking'] = ranking
print(json.dumps(item, sort_keys=True, default=json_util.default))
if cveSearch:
for cveid in cveSearch:
for item in collection.find({'id': cveid}).sort("Modified",sorttype):
printCVE(item)
# Basic freetext search (in vulnerability summary).
# Full-text indexing is more efficient to search across all CVEs.
if vFreeSearch:
for item in collection.find({'summary': {'$regex' : re.compile(vFreeSearch, re.IGNORECASE)}}).sort("Modified",sorttype):
print(item)
# Search Product (best to use CPE notation, e.g. cisco:ios:12.2
if vSearch:
for item in collection.find({"vulnerable_configuration": {'$regex' : vSearch}}).sort("Modified",sorttype):
if csvOutput:
# We assume that the vendor name is usually in the hostame of the
# URL to avoid any match on the resource part
refs=[]
for entry in item['references']:
if args.v is not None:
url = urlparse(entry)
hostname = url.netloc
if re.search(args.v, hostname):
refs.append(entry)
if not refs:
refs = "[no vendor link found]"
if namelookup:
nl = " ".join(item['vulnerable_configuration'])
csvoutput = csv.writer(sys.stdout, delimiter='|', quotechar='|', quoting=csv.QUOTE_MINIMAL)
if not namelookup:
csvoutput.writerow([item['id'],item['Published'],item['cvss'],item['summary'],refs])
else:
csvoutput.writerow([item['id'],item['Published'],item['cvss'],item['summary'],refs,nl])
elif htmlOutput:
print("<h2>"+item['id']+"<br></h2>CVSS score: "+item['cvss']+"<br>"+"<b>"+item['Published']+"<b><br>"+item['summary']+"<br>")
print("References:<br>")
for entry in item['references']:
print(entry+"<br>")
print("<hr><hr>")
# bson straight from the MongoDB db - converted to JSON default
# representation
elif jsonOutput:
printCVE(item)
elif xmlOutput:
c = SubElement(r,'id')
c.text = item['id']
c = SubElement(r,'Published')
c.text = item['Published']
c = SubElement(r,'cvss')
c.text = item['cvss']
c = SubElement(r,'summary')
c.text = SaxEscape(item['summary'])
for e in item['references']:
c = SubElement(r,'references')
c.text = SaxEscape(e)
for e in item['vulnerable_configuration']:
c = SubElement(r,'vulnerable_configuration')
c.text = SaxEscape(e)
elif cveidOutput:
print(item['id'])
else:
print("CVE\t: " + item['id'])
print("DATE\t: " + item['Published'])
print("CVSS\t: " + str(item['cvss']))
print(item['summary'])
print("\nReferences:")
print("-----------")
for entry in item['references']:
print(entry)
print("\nVulnerable Configs:")
print("-------------------")
for entry in item['vulnerable_configuration']:
if not namelookup:
print(entry)
else:
print(cves.getcpe(cpeid=entry))
print("\n\n")
if htmlOutput:
print("</body></html>")
if xmlOutput:
# default encoding is UTF-8. Should this be detected on the terminal?
s = tostring(r).decode("utf-8")
print(s)
self.contributor = attrs.get('contributor')
self.vendor.append({'organization': self.organization, 'lastmodified': self.lastmodified, 'id': self.cvename, 'contributor': self.contributor, 'statement': self.statement})
self.statementtag = True
def characters(self, ch):
if self.statementtag:
self.statement += ch
def endElement(self, name):
if name == 'nvd:statement':
self.statementtag = False
self.statement = self.statement + self.statement.rstrip()
self.vendor[-1]['statement'] = self.statement
# dictionary
vendordict = Configuration.getVendorDict()
# connect to db
db= Configuration.getMongoConnection()
vendor = db.vendor
info = db.info
parser = make_parser()
ch = VendorHandler()
parser.setContentHandler(ch)
f = urlopen(vendordict)
i = info.find_one({'db': 'vendor'})
if i is not None:
if f.headers['last-modified'] == i['last-modified']:
sys.exit("Not modified")
def configure_app(flask_app):
flask_app.config['SWAGGER_UI_DOC_EXPANSION'] = Configuration.get("RESTPLUS_SWAGGER_UI_DOC_EXPANSION")
flask_app.config['RESTPLUS_VALIDATE'] = Configuration.get("RESTPLUS_VALIDATE")
flask_app.config['RESTPLUS_MASK_SWAGGER'] = Configuration.get("RESTPLUS_MASK_SWAGGER")
flask_app.config['ERROR_404_HELP'] = Configuration.get("RESTPLUS_ERROR_404_HELP")
# Main
if __name__=='__main__':
description='''Queries the database for cookies matching criteria'''
parser = argparse.ArgumentParser(description=description)
parser.add_argument('-s', action='store_true', help='Display stats')
parser.add_argument('-d', metavar='domain', help='Domain to query cookies from (e.g github.com)')
parser.add_argument('-b', metavar='browser', help='Browser the cookie was grabbed from (e.g firefox)')
parser.add_argument('-n', metavar='name', help='Cookie name')
parser.add_argument('-v', metavar='value', help='Cookie value')
parser.add_argument('-u', metavar='user', help='User:profile the cookies originate from (e.g NorthernSec:12345ua0.default)')
parser.add_argument('-id', metavar='id', help='ID of stored cookie')
parser.add_argument('db', metavar='database', nargs='?', help='Database')
args = parser.parse_args()
where=[]
if args.d: where.append("domain='%s'"%args.d)
if args.b: where.append("browser='%s'"%args.b)
if args.n: where.append("name='%s'"%args.n)
if args.v: where.append("value='%s'"%args.v)
if args.u: where.append("user='******'"%args.u)
if args.id: where.append("id='%s'"%args.id)
db=args.db if args.db else conf.getCookieJar()
print(db)
cookies = selectAllFrom(db, 'CookieJar', where)
print("id | domain | host | name | value | browser | user | lastUsed | creationTime | timeJarred | notes")
for c in cookies:
print("%s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s"%(c['id'],c['domain'],c['host'],c['name'],c['value'],c['browser'],c['user'],c['lastused'],c['creationtime'],c['timejarred'],c['notes']))
if args.s:
print("Total of %s cookie(s)"%len(cookies))
def update(self):
for item in self.items.keys():
self.catalog.inventory.__setattr__(item, self.items[item][1])
self.catalog.save()
self.catalog = Configuration(path)
del y[i]
del y[i-1]
elif z == ".":
del y[i]
return "/".join(y)
# Main
if __name__=='__main__':
# Argparse
description='''CookieJar web interface'''
parser = argparse.ArgumentParser(description=description)
parser.add_argument('db', metavar='database', nargs='?', help='Database')
args = parser.parse_args()
grabbers={"Mozilla Firefox": MozillaGrabber(args), "Chromium": ChromiumGrabber(args), "Google Chrome", ChromeGrabber(args)}
db=args.db if args.db else conf.getCookieJar()
grabJar(db)
host=conf.getHost()
port=conf.getPort()
global info
info={'db':db,
'supported':['Mozilla Firefox', 'Chromium', 'Google Chrome']}
global supported
supported=[MozillaGrabber(args), ChromiumGrabber(args)]
if conf.getDebug():
app.run(host=host, port=port, debug=True)
else:
print("[*] Starting server...")
def update(self):
for beast in self.beasts:
self.catalog.bestiary.__setattr__(beast[0], beast[1])
self.catalog.save()
self.catalog = Configuration(path)
#for list files
import glob
#command line parser, if running python 2.7 use argparse, else fall-back to optparse
try:
import argparse
except ImportError:
import optparse as argparse
FPS = 60
caption = 'UlDunAd - Ultimate Dungeon Adventure [FPS: %i]'
if not os.path.exists(os.path.join("..", "uldunad.ini")):
Configuration(os.path.join("..", "uldunad.ini")).save()
runini = Configuration(os.path.join("..", "uldunad.ini"))
runini.video.__setattr__("resolution", str(800) + "x" + str(600) + "x" + "W")
runini.audio.__setattr__("volume", str(10))
runini.save()
Input.create(runini)
Input.load(runini)
else:
runini = Configuration(os.path.join("..", "uldunad.ini"))
Input.load(runini)
w, h, fullscreen = runini.video.__getattr__("resolution").split("x")
w, h = float(w), float(h)
resolution = (int(w), int(h))
volume = int(runini.audio.__getattr__("volume"))
if fullscreen == "F":
args = argparser.parse_args()
# init parts of the file names to enable looped file download
file_prefix = "nvdcve-2.0-"
file_suffix = ".xml"
file_mod = "modified"
file_rec = "recent"
# get the current year. This enables us to download all CVE's up to this year :-)
date = datetime.datetime.now()
year = date.year+1
# default config
defaultvalue = {}
defaultvalue['cvss'] = Configuration.getDefaultCVSS()
defaultvalue['cwe'] = "Unknown"
cveStartYear = Configuration.getCVEStartYear()
# define the CVE parser. Thanks to Meredith Patterson (@maradydd) for help on this one.
class CVEHandler(ContentHandler):
def __init__(self):
self.cves = []
self.inCVSSElem = 0
self.inSUMMElem = 0
self.inDTElem = 0
self.inPUBElem = 0
def startElement(self, name, attrs):
if name == 'entry':
class Inventory:
def __init__(self, family):
self.family = family #save data for the inventory
#gathers all the different enemy types
path = os.path.join("..", "data", "items")
items = os.listdir(path)
items.sort()
path = os.path.join("..", "data", "actors", "families", family.name, "inventory.ini")
if not os.path.exists(path):
Configuration(path).save()
self.catalog = Configuration(path)
self.catalog.inventory.__setattr__(items[0], 0)
self.catalog.save()
self.catalog = Configuration(path)
self.items = {}
def loadList():
changeMade = False
for i in items:
if self.catalog.parser.has_option("inventory", i):
ini = Configuration(os.path.join("..", "data", "items", i, "item.ini"))
#detecting what type of item it is
if ini.parser.has_section("weapon"):
item = Weapon(i)
elif ini.parser.has_section("armor"):
item = Armor(i)
elif ini.parser.has_section("loot"):
item = Loot(i)
elif ini.parser.has_section("usable"):
item = Usable(i)
else:
item = Item(i)
self.items[i] = [item, self.catalog.inventory.__getattr__(i, int)]
else:
self.catalog.inventory.__setattr__(i, 0)
self.catalog.save()
loadList()
#adds the amount of said item to the inventory
# do not use negative numbers to remove item, use removeItem method instead
def addItem(self, name, amount):
self.items[name][1] = math.max(_itemMax, self.items[name][1]+amount)
#removes the amount of said item from the inventory
def removeItem(self, name, amount):
self.items[name][1] = math.max(_itemMax, self.items[name][1]-amount)
#creates string rep of the inventory
def __str__(self):
str = ""
for i, item in enumerate(self.items.keys()):
str += "%03i: %-16s%02i\n" % (i+1, self.items[item][0].name, self.items[item][1])
return str
#saves the updated inventory
def update(self):
for item in self.items.keys():
self.catalog.inventory.__setattr__(item, self.items[item][1])
self.catalog.save()
self.catalog = Configuration(path)
#all items by type
def getByType(self, type = None):
available = {}
items = self.available()
if type == None:
return items
else:
for item in items:
if isinstance(items[item][0], type):
available[item] = items[item]
return available
#items that the party possesses
def available(self):
available = {}
for item in self.items.keys():
if self.items[item][1] > 0:
available[item] = self.items[item]
return available
#number of different items the party possesses
def __len__(self):
return len(self.available())
def main():
initialize_app(app)
app.run(debug=Configuration.get("FLASK_DEBUG"), port = 5000, host = "0.0.0.0")
import os
import sys
runPath = os.path.dirname(os.path.realpath(__file__))
sys.path.append(os.path.join(runPath, "./lib/"))
import pymongo
from urllib.request import urlopen
import tarfile
import shutil
import sqlite3
from Config import Configuration
vFeedurl = Configuration.getvFeedURL()
vFeedstatus = Configuration.getvFeedStatus()
# connect to db
db = Configuration.getMongoConnection()
info = db.info
u = urlopen(vFeedurl)
i = info.find_one({'db': 'vfeed'})
if i is not None:
if u.headers['last-modified'] == i['last-modified']:
sys.exit("Not modified")
info.update({'db': 'vfeed'}, {"$set":{'last-modified': u.headers['last-modified']}}, upsert=True)
if not os.path.exists('./tmp'):
os.mkdir('./tmp')
argParser.add_argument('-l', action='store_true', default=False, help='List notification entries')
args = argParser.parse_args()
def checkreq():
if args.c is None:
print ("You need at least one cpe or partial cpe entry (-c) \n")
argParser.print_help()
exit(1)
if args.g is None:
print ("Organization is missing (-g) \n")
argParser.print_help()
exit(1)
# connect to db
db = Configuration.getMongoConnection()
cves = db.cves
def searchcve(cpe=None):
if cpe is None:
return False
cve=cves.find({"vulnerable_configuration":{'$regex':cpe}})
return cve
def updatestate(org=None, cve=None):
if cve is None or org is None:
return False
if name == 'name':
self.nametag = False
if name == 'url':
self.urltag = False
if name == 'ref':
self.reftag = False
if name == 'exploit':
for refl in self.refl:
self.d2sec.append({'name': self.name, 'url': self.url, 'id': refl})
self.exploittag = False
self.refl = []
if name == 'elliot':
self.elliottag = False
# dictionary
d2securl = Configuration.getd2secDict()
# connect to db
db = Configuration.getMongoConnection()
d2sec = db.d2sec
info = db.info
parser = make_parser()
ch = ExploitHandler()
parser.setContentHandler(ch)
f = urlopen(d2securl)
i = info.find_one({'db': 'd2sec'})
if i is not None:
if f.headers['last-modified'] == i['last-modified']:
sys.exit("Not modified")
parser.parse(f)
