def build(cls, data_handler, matrix_creation_parameters):
"""
Generates a matrix with the method used in the handler creation.
@param trajectory_handler:
@param matrix_creation_parameters:
@return: The created matrix.
"""
calculator_type = matrix_creation_parameters.get_value("calculator_type",
default_value = "QTRFIT_OMP_CALCULATOR")
calculator_options = matrix_creation_parameters.get_value("calculator_options",
default_value = ProtocolParameters({"number_of_threads":8,
"blocks_per_grid":8,
"threads_per_block":32}))
calculator_options = ProtocolParameters(calculator_options)
structure = data_handler.get_data()
fit_selection_coordsets = structure.getFittingCoordinates()
calc_selection_coordsets = structure.getCalculationCoordinates()
if calc_selection_coordsets is None:
calculator = RMSDCalculator(calculatorType = calculator_type,
fittingCoordsets = fit_selection_coordsets)
else:
symm_groups = []
if "symmetries" in matrix_creation_parameters:
# Then prepare it to handle calculation symmetries
# Description of equivalences must have the same number of atoms
symm_groups = cls.process_symm_groups(matrix_creation_parameters,
structure,
calc_selection_coordsets)
print "Using symmetries", symm_groups
calculator = RMSDCalculator(calculatorType = calculator_type,
fittingCoordsets = fit_selection_coordsets,
calculationCoordsets = calc_selection_coordsets,
calcSymmetryGroups = symm_groups)
try:
calculator.setNumberOfOpenMPThreads(calculator_options.get_value("number_of_threads",
default_value = 8))
except KeyError:
pass
try:
calculator.setCUDAKernelThreadsPerBlock(calculator_options.get_value("threads_per_block",
default_value = 32),
calculator_options.get_value("blocks_per_grid",
default_value = 8))
except KeyError:
pass
rmsds = calculator.pairwiseRMSDMatrix()
return CondensedMatrix(rmsds)
def __init__(self, workspace_parameters, observer):
super(WorkspaceHandler,self).__init__(observer)
self.parameters = ProtocolParameters(workspace_parameters.get_value("parameters", default_value = ProtocolParameters({
"overwrite":True,
"clear_after_exec":["tmp"]
})))
self.data = {
"results": os.path.join(workspace_parameters["base"], workspace_parameters.get_value("results", default_value="results")),
"tmp" : os.path.join(workspace_parameters["base"], workspace_parameters.get_value("tmp", default_value= "tmp")),
"clusters" : os.path.join(workspace_parameters["base"], workspace_parameters.get_value("clusters", default_value= "clusters")),
"matrix" : os.path.join(workspace_parameters["base"], workspace_parameters.get_value("matrix", default_value= "matrix"))
}
def setUpClass(cls):
cls.parameters = ProtocolParameters.get_default_params("data/params.json")
distances = [94, 6, 43, 14, 96,
18, 59, 54, 69,
56, 96, 69,
54, 50,
8]
cls.matrix_1 = CondensedMatrix(distances)
class WorkspaceHandler(Observable):
def __init__(self, workspace_parameters, observer):
super(WorkspaceHandler,self).__init__(observer)
self.parameters = ProtocolParameters(workspace_parameters.get_value("parameters", default_value = ProtocolParameters({
"overwrite":True,
"clear_after_exec":["tmp"]
})))
self.data = {
"results": os.path.join(workspace_parameters["base"], workspace_parameters.get_value("results", default_value="results")),
"tmp" : os.path.join(workspace_parameters["base"], workspace_parameters.get_value("tmp", default_value= "tmp")),
"clusters" : os.path.join(workspace_parameters["base"], workspace_parameters.get_value("clusters", default_value= "clusters")),
"matrix" : os.path.join(workspace_parameters["base"], workspace_parameters.get_value("matrix", default_value= "matrix"))
}
def __getitem__(self,key):
return self.data[key]
def __str__(self):
return json.dumps(self.data, sort_keys=False, indent=4, separators=(',', ': '))
def create_directories(self):
"""
Recreates the workspace structure. Removes the old location if necessary.
"""
self.notify("MSG","Creating workspace...")
if self.parameters.get_value("overwrite", default_value = True) :
self.clear_directories(self.data.keys())
for folder_key in self.data:
scripts_common.create_directory(self.data[folder_key])
def clear_directories(self, directory_keys):
"""
Removes the directories given as parameters.
@param directory_keys: The keys of the 'data' object which defined paths will be erased.
"""
for folder_key in directory_keys:
folder_path = self.data[folder_key]
if os.path.exists(folder_path):
shutil.rmtree(folder_path)
self.notify("MSG","Removing %s ..."%folder_path)
# print "Removing %s ..."%folder_path
def __enter__(self):
self.create_directories()
return self
def __exit__(self, exception_type, exception_val, trace):
self.clear_directories(self.parameters["clear_after_exec"])
def setup_loader(self):
"""
Builds a loader with 2 preloaded pdbs from the data folder.
"""
loader = ProteinEnsembleDataLoader(ProtocolParameters({"matrix":{}}))
source1 = DataSource(os.path.join(test_data.__path__[0], "pdb1.pdb"))
loader.load(source1)
source2 = DataSource({
"source":os.path.join(test_data.__path__[0], "pdb2.dcd"),
"atoms_source": os.path.join(test_data.__path__[0], "pdb2.pdb")
})
loader.load(source2)
return loader
def test_to_dic(self):
input = ProtocolParameters({
"ciao":
ProtocolParameters({
"come": "ti",
"chiami": 4
}),
"io_mi": {
"chiamo": 87,
"Victor": [1, 2, 3, 4]
}
})
expected = {
'ciao': {
'come': 'ti',
'chiami': 4
},
'io_mi': {
'chiamo': 87,
'Victor': [1, 2, 3, 4]
}
}
self.assertDictEqual(expected, ProtocolParameters.to_dict(input))
def test_to_dic(self):
input = ProtocolParameters({
"ciao":ProtocolParameters({
"come": "ti",
"chiami": 4}),
"io_mi":{
"chiamo":87,
"Victor":[1,2,3,4]
}
})
expected = {
'ciao': {
'come': 'ti',
'chiami': 4
},
'io_mi': {
'chiamo': 87,
'Victor': [1, 2, 3, 4]
}
}
self.assertDictEqual(expected, ProtocolParameters.to_dict(input))
def __init__(self, workspace_parameters, observer):
super(WorkspaceHandler, self).__init__(observer)
self.parameters = ProtocolParameters(
workspace_parameters.get_value(
"parameters", default_value=ProtocolParameters({"overwrite": True, "clear_after_exec": ["tmp"]})
)
)
self.data = {
"results": os.path.join(
workspace_parameters["base"], workspace_parameters.get_value("results", default_value="results")
),
"tmp": os.path.join(
workspace_parameters["base"], workspace_parameters.get_value("tmp", default_value="tmp")
),
"clusters": os.path.join(
workspace_parameters["base"], workspace_parameters.get_value("clusters", default_value="clusters")
),
"matrix": os.path.join(
workspace_parameters["base"], workspace_parameters.get_value("matrix", default_value="matrix")
),
}
def __init__(self, parameters, observer):
"""
Class creator. It parses the needed files and extracts info and coordinates.
"""
super(TrajectoryHandler, self).__init__(observer)
print "Reading conformations..."
prody.confProDy(verbosity="none")
self.parameters = parameters
matrix_parameters = parameters.get_value(
"data.matrix.parameters", default_value=ProtocolParameters.empty())
parameters["data"]["files"] = self.expand_file_lists(
parameters["data"]["files"])
self.files = parameters["data"]["files"]
self.pdbs = []
if len(self.files) == 0:
common.print_and_flush("[ERROR] no pdbs. Exiting...\n")
self.notify("SHUTDOWN", "No pdbs defined in script.")
exit()
self.notify("Loading", "Loading Trajectories")
# Bookmarking structure
self.bookmarking = {"pdb": None, "selections": {}}
merged_structure = self.getMergedStructure()
self.coordsets = merged_structure.getCoordsets()
self.number_of_conformations = self.coordsets.shape[0]
self.number_of_atoms = self.coordsets.shape[1]
self.handle_selection_parameters(matrix_parameters)
print "%d conformations of %d atoms were read." % (
merged_structure.numCoordsets(), merged_structure.numAtoms())
def build(cls, data_handler, matrix_creation_parameters):
"""
Generates a matrix with the method used in the handler creation.
@param trajectory_handler:
@param matrix_creation_parameters:
@return: The created matrix.
"""
calculator_type = matrix_creation_parameters.get_value(
"calculator_type", default_value="QTRFIT_OMP_CALCULATOR")
calculator_options = matrix_creation_parameters.get_value(
"calculator_options",
default_value=ProtocolParameters({
"number_of_threads": 8,
"blocks_per_grid": 8,
"threads_per_block": 32
}))
calculator_options = ProtocolParameters(calculator_options)
structure = data_handler.get_data()
fit_selection_coordsets = structure.getFittingCoordinates()
calc_selection_coordsets = structure.getCalculationCoordinates()
if calc_selection_coordsets is None:
calculator = RMSDCalculator(
calculatorType=calculator_type,
fittingCoordsets=fit_selection_coordsets)
else:
symm_groups = []
if "symmetries" in matrix_creation_parameters:
# Then prepare it to handle calculation symmetries
# Description of equivalences must have the same number of atoms
symm_groups = cls.process_symm_groups(
matrix_creation_parameters, structure,
calc_selection_coordsets)
print "Using symmetries", symm_groups
calculator = RMSDCalculator(
calculatorType=calculator_type,
fittingCoordsets=fit_selection_coordsets,
calculationCoordsets=calc_selection_coordsets,
calcSymmetryGroups=symm_groups)
try:
calculator.setNumberOfOpenMPThreads(
calculator_options.get_value("number_of_threads",
default_value=8))
except KeyError:
pass
try:
calculator.setCUDAKernelThreadsPerBlock(
calculator_options.get_value("threads_per_block",
default_value=32),
calculator_options.get_value("blocks_per_grid",
default_value=8))
except KeyError:
pass
rmsds = calculator.pairwiseRMSDMatrix()
return CondensedMatrix(rmsds)
class WorkspaceHandler(Observable):
def __init__(self, workspace_parameters, observer):
super(WorkspaceHandler, self).__init__(observer)
self.parameters = ProtocolParameters(
workspace_parameters.get_value(
"parameters", default_value=ProtocolParameters({"overwrite": True, "clear_after_exec": ["tmp"]})
)
)
self.data = {
"results": os.path.join(
workspace_parameters["base"], workspace_parameters.get_value("results", default_value="results")
),
"tmp": os.path.join(
workspace_parameters["base"], workspace_parameters.get_value("tmp", default_value="tmp")
),
"clusters": os.path.join(
workspace_parameters["base"], workspace_parameters.get_value("clusters", default_value="clusters")
),
"matrix": os.path.join(
workspace_parameters["base"], workspace_parameters.get_value("matrix", default_value="matrix")
),
}
def __getitem__(self, key):
return self.data[key]
def __str__(self):
return json.dumps(self.data, sort_keys=False, indent=4, separators=(",", ": "))
def create_directories(self):
"""
Recreates the workspace structure. Removes the old location if necessary.
"""
self.notify("MSG", "Creating workspace...")
if self.parameters.get_value("overwrite", default_value=True):
self.clear_directories(self.data.keys())
for folder_key in self.data:
scripts_common.create_directory(self.data[folder_key])
def clear_directories(self, directory_keys):
"""
Removes the directories given as parameters.
@param directory_keys: The keys of the 'data' object which defined paths will be erased.
"""
for folder_key in directory_keys:
folder_path = self.data[folder_key]
if os.path.exists(folder_path):
shutil.rmtree(folder_path)
self.notify("MSG", "Removing %s ..." % folder_path)
# print "Removing %s ..."%folder_path
def __enter__(self):
self.create_directories()
return self
def __exit__(self, exception_type, exception_val, trace):
self.clear_directories(self.parameters["clear_after_exec"])
# Saving matrices
for dataset_name in data.all_datasets:
handler = MatrixHandler(condensed_matrices[dataset_name],
{"method": "load"})
handler.save_matrix("./matrices/%s" % dataset_name)
# Run pyProCT for each of them
base_script = "".join(open("base_script.json", "r").readlines())
for dataset_name in ['concentric_circles'
]: #data.all_datasets: #["spaeth_06"]:#
print dataset_name
# Change placeholders
script_str = base_script % (os.path.abspath(
"./tmp/%s" % dataset_name), "./matrices/%s" % dataset_name)
parameters = ProtocolParameters.get_params_from_json(script_str)
# And change another hypothesis stuff
parameters["clustering"]["evaluation"]["maximum_noise"] = data.noise[
dataset_name]
parameters["clustering"]["evaluation"][
"minimum_cluster_size"] = data.minsize[dataset_name]
parameters["clustering"]["evaluation"][
"minimum_clusters"] = data.num_cluster_ranges[dataset_name][0]
parameters["clustering"]["evaluation"][
"maximum_clusters"] = data.num_cluster_ranges[dataset_name][1]
print parameters["clustering"]["evaluation"][
"minimum_clusters"], parameters["clustering"]["evaluation"][
"maximum_clusters"]
if dataset_name in data.criteria:
parameters["clustering"]["evaluation"][
"evaluation_criteria"] = data.criteria[dataset_name]
@author: victor
"""
from pyproct.driver.parameters import ProtocolParameters
from pyproct.driver.driver import Driver
from pyproct.driver.observer.observer import Observer
import sys
from pyproct.driver.handlers.matrix.matrixHandler import MatrixHandler
from pyproct.tools.matrixTools import get_submatrix
from pyproct.tools.plotTools import matrixToImage
from itertools import product
import numpy
from pyproct.tools.pdbTools import get_number_of_frames
if __name__ == '__main__':
base_script = "".join(open("base_script.json", "r").readlines())
parameters = ProtocolParameters.get_params_from_json(base_script)
parameters["global"]["workspace"]["base"] = sys.argv[3]
parameters["data"]["files"] = [sys.argv[1], sys.argv[2]]
frames_ini = get_number_of_frames(sys.argv[1])
frames_proto = get_number_of_frames(sys.argv[2])
print sys.argv[1], "->", frames_ini
print sys.argv[2], "->", frames_proto
try:
Driver(Observer()).run(parameters)
except SystemExit:
# Expected improductive search
# Load again the matrix
handler = MatrixHandler({
"method": "load",
if __name__ == '__main__':
parser = optparse.OptionParser(usage='%prog [--mpi] [--print] script', version=pyproct.__version__)
parser.add_option('--mpi', action="store_true", dest = "use_mpi", help="Add this flag if you want to use MPI-based scheduling.")
parser.add_option('--print', action="store_true", dest = "print_messages", help="Add this flag to print observed messages to stdout.")
options, args = parser.parse_args()
if(len(args)==0):
parser.error("You need to specify the script to be executed.")
json_script = args[0]
parameters = None
try:
parameters = ProtocolParameters.get_params_from_json(tools.remove_comments(open(json_script).read()))
parameters["global"]["workspace"]["base"] = os.path.abspath(parameters["global"]["workspace"]["base"])
except ValueError, e:
print "Malformed json script."
print e.message
exit()
observer = None
cmd_thread = None
if options.use_mpi:
from pyproct.driver.mpidriver import MPIDriver
from pyproct.driver.observer.MPIObserver import MPIObserver
observer = MPIObserver()
if options.print_messages:
cmd_thread = CmdLinePrinter(observer)
cmd_thread.start()
create_directory("./matrices")
create_directory("./tmp")
condensed_matrices, all_observations = vtools.create_matrices(data)
# Saving matrices
for dataset_name in data.all_datasets:
handler = MatrixHandler(condensed_matrices[dataset_name], {"method":"load"})
handler.save_matrix("./matrices/%s"%dataset_name)
# Run pyProCT for each of them
base_script = "".join(open("base_script.json","r").readlines())
for dataset_name in ['concentric_circles']: #data.all_datasets: #["spaeth_06"]:#
print dataset_name
# Change placeholders
script_str = base_script%(os.path.abspath("./tmp/%s"%dataset_name),"./matrices/%s"%dataset_name)
parameters = ProtocolParameters.get_params_from_json(script_str)
# And change another hypothesis stuff
parameters["clustering"]["evaluation"]["maximum_noise"] = data.noise[dataset_name]
parameters["clustering"]["evaluation"]["minimum_cluster_size"] = data.minsize[dataset_name]
parameters["clustering"]["evaluation"]["minimum_clusters"] = data.num_cluster_ranges[dataset_name][0]
parameters["clustering"]["evaluation"]["maximum_clusters"] = data.num_cluster_ranges[dataset_name][1]
print parameters["clustering"]["evaluation"]["minimum_clusters"], parameters["clustering"]["evaluation"]["maximum_clusters"]
if dataset_name in data.criteria:
parameters["clustering"]["evaluation"]["evaluation_criteria"] = data.criteria[dataset_name]
else:
parameters["clustering"]["evaluation"]["evaluation_criteria"] = data.criteria["default"]
Driver(Observer()).run(parameters)
for dataset_name in ['concentric_circles']: #data.all_datasets:
results_file = os.path.join(os.path.abspath("./tmp/%s"%dataset_name),"results/results.json")
results = convert_to_utf8(json.loads(open(results_file).read()))
parser.add_option(
'--print',
action="store_true",
dest="print_messages",
help="Add this flag to print observed messages to stdout.")
options, args = parser.parse_args()
if (len(args) == 0):
parser.error("You need to specify the script to be executed.")
json_script = args[0]
parameters = None
try:
parameters = ProtocolParameters.get_params_from_json(
tools.remove_comments(open(json_script).read()))
parameters["global"]["workspace"]["base"] = os.path.abspath(
parameters["global"]["workspace"]["base"])
except ValueError, e:
print "Malformed json script."
print e.message
exit()
observer = None
cmd_thread = None
if options.use_mpi:
from pyproct.driver.mpidriver import MPIDriver
from pyproct.driver.observer.MPIObserver import MPIObserver
observer = MPIObserver()
if options.print_messages:
cmd_thread = CmdLinePrinter(observer)
if __name__ == '__main__':
parser = optparse.OptionParser(usage='%prog [--mpi] [--print] script', version=pyproct.__version__)
parser.add_option('--mpi', action="store_true", dest = "use_mpi", help="Add this flag if you want to use MPI-based scheduling.")
parser.add_option('--print', action="store_true", dest = "print_messages", help="Add this flag to print observed messages to stdout.")
options, args = parser.parse_args()
if(len(args)==0):
parser.error("You need to specify the script to be executed.")
json_script = args[0]
parameters = None
try:
parameters = ProtocolParameters.get_params_from_json(open(json_script).read())
parameters["global"]["workspace"]["base"] = os.path.abspath(parameters["global"]["workspace"]["base"])
except ValueError, e:
print "Malformed json script."
print e.message
exit()
observer = None
cmd_thread = None
if options.use_mpi:
from pyproct.driver.mpidriver import MPIDriver
from pyproct.driver.observer.MPIObserver import MPIObserver
observer = MPIObserver()
if options.print_messages:
cmd_thread = CmdLinePrinter(observer)
cmd_thread.start()
@author: victor
"""
from pyproct.driver.parameters import ProtocolParameters
from pyproct.driver.driver import Driver
from pyproct.driver.observer.observer import Observer
import sys
from pyproct.driver.handlers.matrix.matrixHandler import MatrixHandler
from pyproct.tools.matrixTools import get_submatrix
from pyproct.tools.plotTools import matrixToImage
from itertools import product
import numpy
from pyproct.tools.pdbTools import get_number_of_frames
if __name__ == '__main__':
base_script = "".join(open("base_script.json","r").readlines())
parameters = ProtocolParameters.get_params_from_json(base_script)
parameters["global"]["workspace"]["base"] = sys.argv[3]
parameters["data"]["files"] = [sys.argv[1], sys.argv[2]]
frames_ini = get_number_of_frames(sys.argv[1])
frames_proto = get_number_of_frames(sys.argv[2])
print sys.argv[1],"->",frames_ini
print sys.argv[2],"->",frames_proto
try:
Driver(Observer()).run(parameters)
except SystemExit:
# Expected improductive search
# Load again the matrix
handler = MatrixHandler({
"method": "load",
