def __init__(self, **kwargs):
"""Construct a tree -- typically there is one tree per user
"""
self._branchTree = C_snode.C_stree() # This is the WHOLE tree
self.branch = C_snode.C_stree() # This is a single branch
self.debug = message.Message(logTo='./debug.log')
self.debug._b_syslog = True
self._log = message.Message()
self._log._b_syslog = True
self.__name = "BranchTree"
self.within = None # The feed or plugin this branch is within
for key, val in kwargs.iteritems():
if key == 'within': self.within = val
def __init__(self, **kwargs):
"""Construct a tree -- typically there is one tree per user
"""
self._feedTree = C_snode.C_stree()
self.feed = C_snode.C_stree()
self.BR = branch.BranchTree(within=self)
self.plugin = plugin.Plugin_homePage()
self.debug = message.Message(logTo='./debug.log')
self.debug._b_syslog = True
self._log = message.Message()
self._log._b_syslog = True
self.__name = "FeedTree"
def __init__(self, **kwargs):
"""
constructor
"""
self._stree = C_snode.C_stree()
self.id = ""
self.request = ""
self.saveTo = ""
self.loadFrom = ""
self.b_stdout = ""
self.b_dataReady = False
for key, val in kwargs.iteritems():
if key == 'id': self.id = val
if key == 'request': self.request = val
if key == 'saveTo': self.saveTo = val
if key == 'loadFrom': self.loadFrom = val
if key == 'stdout': self.b_stdout = val
self.client = ChRIS_DB_client(id=self.id,
request=self.request,
saveTo=self.saveTo,
loadFrom=self.loadFrom,
stdout=self.b_stdout,
within=self)
def __init__(self, **kwargs):
# This class contains a reference back to the chris parent object that
# contains this Core
self.chris = None
self.str_DBpath = ""
self.b_ignorePersistentDB = False
self.b_createNewDB = 0
self.str_VERB = ""
self.str_API = ""
for key, value in kwargs.iteritems():
if key == "chris": self.chris = value
if key == "DB": self.str_DBpath = value
if key == 'ignorePersistentDB': self.b_ignorePersistentDB = value
if key == 'createNewDB': self.b_createNewDB = value
if key == 'VERB': self.str_VERB = value
if key == 'APIcall': self.str_API = value
self.s_tree = C_snode.C_stree()
self._userDB = ChRIS_SMUserDB(
chris=self.chris,
DB=self.str_DBpath,
ignorePersistentDB=self.b_ignorePersistentDB,
createNewDB=self.b_createNewDB,
VERB=self.str_VERB,
APIcall=self.str_API)
def __init__(self, **kwargs):
self.identity = "0"
self.request = "NULL"
self.b_saveResult = False
self.saveTo = ""
self.loadFrom = ""
self.b_loadFrom = False
self.b_stdout = False
self.within = None
self.b_dataReady = False
self.json_comms = {}
self.json_data = {}
self.b_json_data = False
for key, val in kwargs.iteritems():
if key == 'id': self.identity = val
if key == 'request': self.request = val
if key == 'saveTo': self.saveTo = val
if key == 'loadFrom': self.loadFrom = val
if key == 'stdout': self.b_stdout = val
if key == 'within': self.within = val
if key == 'data':
self.b_json_data = True
self.json_data = val
self.b_saveResult = len(self.saveTo)
self.b_loadFrom = len(self.loadFrom)
threading.Thread.__init__(self)
self.zmq_context = zmq.Context()
self.stree = C_snode.C_stree()
def __init__(self, **kwargs):
"""Example of docstring on the __init__ method.
The __init__ method may be documented in either the class level
docstring, or as a docstring on the __init__ method itself.
Either form is acceptable, but the two should not be mixed. Choose one
convention to document the __init__ method and be consistent with it.
Note:
Do not include the `self` parameter in the ``Args`` section.
Args:
param1 (str): Description of `param1`.
param2 (list of str): Description of `param2`. Multiple
lines are supported.
param3 (int, optional): Description of `param3`, defaults to 0.
"""
self._stree = C_snode.C_stree()
self.debug = message.Message(logTo='./debug.log')
self.debug._b_syslog = True
self._log = message.Message()
self._log._b_syslog = True
self.__name = "Feed"
self._name = ""
self.within = None # The feed or plugin this branch is within
for key, val in kwargs.iteritems():
if key == 'within': self.within = val
def __init__(self, **kwargs):
"""
"""
self._stree = C_snode.C_stree()
self.debug = message.Message(logTo = './debug.log')
self.debug._b_syslog = True
self._log = message.Message()
self._log._b_syslog = True
self.__name = "ChRIS_DB"
self._name = ""
self.within = None # An encapsulating object
# DB
self.str_DBpath = '/tmp/ChRIS_DB'
self._stree = C_snode
# Comms
self.str_protocol = "tcp"
self.str_IP = "127.0.0.1"
self.str_port = "5001"
self.DB_read()
for key, val in kwargs.iteritems():
if key == 'within': self.within = val
self.zmq_context = zmq.Context()
def branch_singleBranch_process(self, **kwargs):
"""
Assuming a branch has been grafted from the tree space, process this
branch's components.
:return:
"""
d_ret = {
'branch': None,
'debug': 'branch_singleBranch_process(): ',
'path': '',
'status': False,
'payload': {},
'URL_get': []
}
for key, val in kwargs.iteritems():
if key == 'VERB': str_VERB = val
if key == 'pathInBranch': str_pathInBranch = val
if key == 'd_ret': d_ret = val
str_ROOT = d_ret['ROOT']
d_ret['b_returnTree'] = True
d_ret['debug'] += 'str_pathInBranch = %s ' % str_pathInBranch
s = d_ret['branch']
d_cd = s.cd(str_pathInBranch)
if d_cd['status']:
# We are retrieving a directory
d_ret['status'] = d_cd['status']
d_ret['pathInBranch'] = d_cd['path']
subTree = C_snode.C_stree()
if subTree.cd('/')['status']:
subTree.graft(s, str_pathInBranch)
d_ret['branch'] = subTree
else:
d_ret['b_returnTree'] = False
# Check if we are in fact processing a "file"
l_p = str_pathInBranch.split('/')
str_dirUp = '/'.join(l_p[0:-1])
d_cd = s.cd(str_dirUp)
if d_cd['status']:
d_ret['debug'] += '../ = %s, file = %s' % (str_dirUp, l_p[-1])
d_ret['status'] = d_cd['status']
d_ret['pathInBranch'] = d_cd['path']
str_fileName = l_p[-1]
if str_VERB != "GET":
self.branch_singleBranch_VERBprocess(**kwargs)
self.debug('%s\n' % l_p)
contents = s.cat(l_p[-1])
if not contents:
d_ret['branch'] = {str_fileName: ''}
else:
d_ret['branch'] = {str_fileName: contents}
self.debug('Returning file contents in payload: "%s"\n' %
d_ret)
d_ret['payload'] = d_ret['branch']
return d_ret
def __init__(self, **kwargs):
self.fake = Faker()
self.contents = ""
self.FS = C_snode.C_stree()
self.numFiles = 10
for key,val in kwargs.iteritems():
if key == 'numFiles': self.numFiles = val
def __init__(self, **kwargs):
'''
Basic constructor. Checks on named input args, checks that files
exist and creates directories.
'''
base.FNNDSC.__init__(self, **kwargs)
self._lw = 120
self._rw = 20
self._l_ROI = []
self._l_subsample = []
self._l_subsampleDir = []
self._l_annotation = []
# Tracking dictionaries
self._d_subsampleDir = {}
self._d_subsampleSet = {}
self._l_pval = []
self._l_group = []
self._l_surface = []
self._l_statFunc = []
self._l_curvFunc = []
self._l_hemi = []
self._l_ctype = ['thickness', 'curv']
self._outDir = ''
self._stageslist = '12'
# Internal tracking vars
self._str_pval = ''
self._str_group = ''
self._str_roi = ''
self._str_hemi = ''
self._str_surface = ''
self._str_statFunc = ''
self._str_subsample = ''
self._str_annotation = ''
self._str_ctype = ''
self._topDir = ''
self._dtree = C_snode.C_stree(['/'])
for key, value in kwargs.iteritems():
if key == 'rsampleList': self._l_subsample = value
if key == 'annot': self._l_annotation = value.split(',')
if key == 'outDir': self._outDir = value
if key == 'stages': self._stageslist = value
if key == 'pval': self._l_pval = value.split(',')
if key == 'group': self._l_group = value.split(',')
if key == 'surface': self._l_surface = value.split(',')
if key == 'statFunc': self._l_statFunc = value.split(',')
if key == 'curvFunc': self._l_curvFunc = value.split(',')
if key == 'hemi': self._l_hemi = value.split(',')
def container_singleFeed_fromTreeGet(self, **kwargs):
'''
Graft a single target container from the container tree to internal storage.
This basically just "links" the container to be processed to a convenience
variable.
:return:
d_ret: dictionary various values
'''
# Initialize the d_ret return dictionary
d_ret = {
'container': None,
'status': False,
'payload': {},
'URL_get': []
}
for key, val in kwargs.iteritems():
if key == 'searchType': str_searchType = val
if key == 'searchTarget': str_searchTarget = val
if key == 'schema': str_schema = val
if key == 'd_ret': d_ret = val
d_ret['debug'] = 'container_fromFeedTree_get():'
F = self._containerTree
F.cd('/containers')
self.container = C_snode.C_stree()
s = self.container
if str_searchType.lower() == 'name':
if F.cd(str_searchTarget)['status']:
Froot = F.cwd()
# s.graft(F, '%s/' % F.cwd())
s.graft(F, '%s/title' % Froot)
s.graft(F, '%s/note' % Froot)
s.graft(F, '%s/data' % Froot)
s.graft(F, '%s/comment' % Froot)
d_ret['status'] = True
d_ret['container'] = s
s.tree_metaData_print(False)
if str_searchType.lower() == 'id':
for containerNode in f.lstr_lsnode('/'):
if F.cd('/containers/%s' % containerNode)['status']:
if str_searchTarget == F.cat('ID'):
d_ret['status'] = True
# s.graft(F, '%s/' % F.cwd())
s.graft(F, '%s/title' % Froot)
s.graft(F, '%s/note' % Froot)
s.graft(F, '%s/data' % Froot)
s.graft(F, '%s/comment' % Froot)
d_ret['container'] = s
break
return d_ret
def __init__(self, **kwargs):
self.contents = C_snode.C_stree()
self.d_REST = {'PUSH': {'body': 'file', 'timestamp': 'file'}}
self.debug = message.Message(logTo='./debug.log')
self.debug._b_syslog = True
self._log = message.Message()
self._log._b_syslog = True
self.__name = "Feed"
self.contents.cd('/')
def branch_branchList_fromTreeGet(self, **kwargs):
"""
Process the main branchTree (i.e. the tree that has all the Feeds.
:param kwargs: schema='name'|'id' -- how to return the list of Feeds.
:return: a list of "hits" in URI format
"""
str_searchType = 'name'
d_ret = {
'debug': 'branch_branchList_fromTreeGet(): ',
'status': False,
'payload': {},
'URL_get': []
}
for key, val in kwargs.iteritems():
if key == 'searchType': str_searchType = val
if key == 'd_ret': d_ret = val
str_ROOT = d_ret['ROOT']
d_ret['debug'] = 'branch_branchList_fromTreeGet(): '
l_URI = []
l_keys = []
s = C_snode.C_stree()
F = self._branchTree
if F.cd('/%s' % str_ROOT.lower())['status']:
if str_searchType.lower() == "name":
# Generate a list of branch elements
l_keys = F.lstr_lsnode()
l_URI = [str_ROOT + '/NAME__' + name for name in l_keys]
d_ret['status'] = True
if str_searchType.lower() == 'id':
for branchNode in F.lstr_lsnode():
F.cd('/%s/%s' % (str_ROOT.lower(), branchNode))
str_ID = F.cat('ID')
l_keys.append(str_ID)
l_URI.append(str_ROOT + '/ID__' + str_ID)
d_ret['status'] = True
s.cd('/')
for feed in l_keys:
s.graft(F, '/%s/%s' % (str_ROOT.lower(), feed))
# d_ret['payload'] = l_keys
d_ret['payload'] = dict(s.snode_root)
d_ret['URL_get'] = l_URI
return d_ret
def __init__(self, **kwargs):
self._pluginTree = C_snode.C_stree()
self.BR = branch.BranchTree(within=self)
self._l_plugin = []
self.pluginName = ""
self.plugin = None
self.debug = message.Message(logTo="./debug.log")
self.debug._b_syslog = True
self._log = message.Message()
self._log._b_syslog = True
self.__name = "Plugin"
self.within = None # The feed or plugin this branch is within
for key, val in kwargs.iteritems():
if key == 'within': self.within = val
def __init__(self, **kwargs):
"""The CHRIS_SM constructor.
"""
self.__name = "ChRIS_SM"
self.str_DBpath = ''
self.b_ignorePersistentDB = False
self.b_createNewDB = 0
self.str_VERB = ""
self.str_API = ""
for key, val in kwargs.iteritems():
if key == 'DB': self.str_DBpath = val
if key == 'ignorePersistentDB': self.b_ignorePersistentDB = val
if key == 'createNewDB': self.b_createNewDB = val
if key == 'APIcall': self.str_API = val
if key == 'VERB': self.str_VERB = val
self._userTree = C_snode.C_stree()
self._SMCore = ChRIS_SMCore(
chris=self,
DB=self.str_DBpath,
ignorePersistentDB=self.b_ignorePersistentDB,
createNewDB=self.b_createNewDB,
VERB=self.str_VERB,
APIcall=self.str_API)
self._name = ""
self._log = message.Message()
self._log.tee(True)
self._log.syslog(True)
# Convenience members
self.DB = self._SMCore._userDB
# The "homePage" is essentially the user's userTree object, created
# in the underlying DB module.
self.homePage = None
def __init__(self, **kwargs):
# This class contains a reference back to the chris parent object that
# contains this DB
self._str_apiCall = ""
self.debug = message.Message(logTo='./debug.log')
self.debug._b_syslog = True
self._log = message.Message()
self._log._b_syslog = True
self.__name = "ChRIS_SMUserDB"
self.str_DBpath = '/tmp'
self.str_VERB = ""
self.str_APIcall = ""
self.chris = None
self.b_createNewDB = 0
self.b_ignorePersistentDB = False
self.b_readDB = False
self.DB = C_snode.C_stree()
self.userTree = None
for key, value in kwargs.iteritems():
if key == "chris": self.chris = value
if key == 'ignorePersistentDB': self.b_ignorePersistentDB = value
if key == 'DB': self.str_DBpath = value
if key == 'createNewDB': self.b_createNewDB = value
if key == 'VERB': self.str_VERB = value
if key == 'APIcall': self.str_APIcall = value
self.debug = message.Message(logTo="./debug.log")
self.debug._b_syslog = True
self._md5 = hashlib
# The entire DataBase is represented here as a tree
self.DB_build()
def branch_singleBranch_fromTreeGet(self, **kwargs):
"""
Graft a single target branch from the branch tree to internal storage.
This basically just "links" the branch to be processed to a convenience
variable.
:return:
d_ret: dictionary various values
"""
# Initialize the d_ret return dictionary
d_ret = {'branch': None, 'status': False, 'payload': {}, 'URL_get': []}
for key, val in kwargs.iteritems():
if key == 'searchType': str_searchType = val
if key == 'searchTarget': str_searchTarget = val
if key == 'schema': str_schema = val
if key == 'd_ret': d_ret = val
str_ROOT = d_ret['ROOT']
d_ret['debug'] = 'branch_singleBranch_fromTreeGet():'
F = self._branchTree
F.cd('/%s' % str_ROOT.lower())
self.branch = C_snode.C_stree()
s = self.branch
if d_ret['ROOT'] == 'Feeds':
if str_searchType.lower() == 'name':
if F.cd(str_searchTarget)['status']:
Froot = F.cwd()
s.graft(F, '%s/' % Froot)
# s.graft(F, '%s/title' % Froot)
# s.graft(F, '%s/note' % Froot)
# s.graft(F, '%s/data' % Froot)
# s.graft(F, '%s/comment' % Froot)
d_ret['status'] = True
d_ret['branch'] = s
s.tree_metaData_print(False)
if str_searchType.lower() == 'id':
for branchNode in f.lstr_lsnode('/'):
if F.cd('/feeds/%s' % branchNode)['status']:
if str_searchTarget == F.cat('ID'):
d_ret['status'] = True
# s.graft(F, '%s/' % F.cwd())
s.graft(F, '%s/title' % Froot)
s.graft(F, '%s/note' % Froot)
s.graft(F, '%s/data' % Froot)
s.graft(F, '%s/comment' % Froot)
s.graft(F, '%s/detail' % Froot)
d_ret['branch'] = s
break
if d_ret['ROOT'] == 'Plugins':
if str_searchType.lower() == 'name':
if F.cd(str_searchTarget)['status']:
Froot = F.cwd()
# s.graft(F, '%s/' % F.cwd())
s.graft(F, '%s/' % Froot)
d_ret['status'] = True
d_ret['branch'] = s
s.tree_metaData_print(False)
return d_ret
def __init__(self, **kwargs):
'''
Basic constructor. Checks on named input args, checks that files
exist and creates directories.
'''
base.FNNDSC.__init__(self, **kwargs)
self._lw = 120
self._rw = 20
self._l_ROI = []
self._l_subj = []
self._l_subsample = []
self._l_subsampleDir = []
self._l_annotation = []
# Tracking dictionaries
self._d_subsampleDir = {}
self._d_subsampleSet = {}
self._l_pval = []
self._l_group = []
self._l_surface = []
self._l_statFunc = []
self._l_curvFunc = []
self._l_hemi = []
self._l_ctype = ['thickness', 'curv']
self._outDir = ''
self._workingDir = ''
self._stageslist = '12'
self._str_threshold = "0.5"
# Internal tracking vars
self._str_pval = ''
self._str_group = ''
self._str_roi = ''
self._str_hemi = ''
self._str_surface = ''
self._str_statFunc = ''
self._str_subsample = ''
self._str_subj = ''
self._str_annotation = ''
self._str_ctype = ''
self._topDir = ''
# Scheduler std out/err dirs
self._str_schedulerStdOutDir = '~/scratch'
self._str_schedulerStdErrDir = '~/scratch'
self._dtree = C_snode.C_stree(['/'])
for key, value in kwargs.iteritems():
if key == 'subjectList': self._l_subj = value
if key == 'sampleList': self._l_subsample = value.split(',')
if key == 'annot': self._l_annotation = value.split(',')
if key == 'outDir': self._outDir = value
if key == 'workingDir': self._workingDir = value
if key == 'stages': self._stageslist = value
if key == 'pval': self._l_pval = value.split(',')
if key == 'group': self._l_group = value.split(',')
if key == 'surface': self._l_surface = value.split(',')
if key == 'statFunc': self._l_statFunc = value.split(',')
if key == 'curvFunc': self._l_curvFunc = value.split(',')
if key == 'hemi': self._l_hemi = value.split(',')
if key == 'threshold': self._str_threshold = value
if key == 'schedulerStdOutDir':
self._str_schedulerStdOutDir = value
if key == 'schedulerStdErrDir':
self._str_schedulerStdErrDir = value
if not os.path.isdir(self._workingDir):
errorFatal(self, 'workingDirNotExist')
self._topDir = self._workingDir
self._l_stages = list(self._stageslist)
def process(self, request):
""" Process the call return result.
Since the client only ever processes a "single" feed, there is no need to
ever send the whole data structure back and forth. Only the feed that
is requested needs to be processed.
This process() method in fact implements a mini-REST API of its own.
"""
s = C_snode.C_stree()
b_returnFeedList = False
s.cd('/')
s.mkcd('users')
s.mkcd('chris')
s.graft(self._stree, '/users/chris/plugins')
s.graft(self._stree, '/users/chris/login')
print("Worker ID - %s: process() - request = '%s'" % (self.worker_id, request))
d_request = json.loads(request)
str_verb = d_request['VERB']
str_URL = d_request['URL']
d_data = d_request['data']
o_URL = urlparse(str_URL)
str_path = o_URL.path
l_path = str_path.split('/')[2:]
if str_verb == 'QUIT':
print('Shutting down server...')
os._exit(1)
# 'paths' are of form:
# /v1/Feeds/NAME__Feed-1
# -- or --
# /v1/Feeds/NAME
str_feed = l_path[1]
# Strip out the NAME<__>
str_feed = str_feed.replace('NAME', '')
str_feed = str_feed.replace('__', '')
if str_verb == 'PULL' or str_verb == 'GET':
if len(l_path) >= 2 and len(str_feed):
s.mkcd('feeds')
s.graft(self._stree, '/users/chris/feeds/%s' % str_feed)
if len(l_path) == 1:
b_returnFeedList = True
if len(l_path) == 2 and not len(str_feed):
b_returnFeedList = True
if b_returnFeedList:
s.mkcd('feeds')
self._stree.cd('/users/chris/feeds')
l_feeds = self._stree.lstr_lsnode()
s.mknode(l_feeds)
for f in l_feeds:
s.cd(f)
s.graft(self._stree, '/users/chris/feeds/%s/detail' % f)
s.cd('../')
if str_verb == 'PUSH':
if len(l_path) >= 2 and len(str_feed):
self._stree.cd('/users/chris/feeds')
# self._stree.rm(str_feed)
s.initFromDict(d_data)
self._stree.graft(s, '/users/chris/feeds/%s' % str_feed)
else:
s.mkcd('feeds')
self._stree.cd('/users/chris/feeds')
l_feeds = self._stree.lstr_lsnode()
s.mknode(l_feeds)
# print(s)
# print(dict(s.snode_root))
return json.dumps(dict(s.snode_root))
def __init__(self, **kwargs):
self.contents = C_snode.C_stree()
self.contents.cd('/')
self.fake = faker.Faker()