def setup_container(container, cname, device,
root_name, my_cpu_parent, my_io_parent):
"""Create a new os container for constraining and isolating the cpus, mem,
and disk IO of one set of io workers, from other workers.
An os container is a pairing of a cpuset cgroup with an io cgroup,
generally from separate cgroup hierarchies. (They can also be
separate fields within a combined cgroup in a single hierarchy.)
my_cpu_parent and my_io_parent describe the existing cpu and io
cgroups of the new container's parent container.
"""
# Create a new cpus+mem cgroup, below my_cpu_parent:
mbytes = plan_container_size(container)
weight = container['weight']
path = cpuset.create_container_cpuset(
cname, 'cpuset', root=my_cpu_parent.name, mbytes=mbytes)
blk_path = cpuset.create_container_blkio(
device, cname, 'io',
root=my_io_parent.name, weight=weight,
priority=container['priority'],
shared_sync_queues=container['shared_sync_queues'])
logging.info( "path: " + path + " blk_path: " + blk_path)
# Setup a view.
cpu_cgroup = cgroup.cgroup('cpuset', path)
blkio_cgroup = cgroup.cgroup('io', blk_path)
container['cpu_cgroup'] = cpu_cgroup
container['blkio_cgroup'] = blkio_cgroup
name = cpu_cgroup.name # eg default/g0/g1
if root_name: # remove default/
name = name[len(root_name)+1:]
container['name'] = name # eg g0/g1
def main():
if len(sys.argv)< 3:
print "Usage: proc_N-mer.py tip_file cluster_file n_mer dist_cutoff >> n_mer_outfile"
return
infile = sys.argv[1]
infile2 = sys.argv[2]
n_mer = int(sys.argv[3])
cutoff = float(sys.argv[4])
p = protein(infile, center='TIP')
p.initCGResiMap()
with open(infile2) as fp:
for line in fp:
cg = cgroup(line.strip())
if cg.getSize() == n_mer:
if p.cgResiGroupFilter(cg, cutoff) == True:
print cg.getString()
elif cg.getSize() < n_mer:
continue
else:
# generate combinations
for idx in list(itertools.combinations(range(cg.getSize()),n_mer)):
sub_cg = cgroup()
sub_cg.pdb = cg.pdb
sub_cg.chain = cg.chain
for i in idx: # iterate all the tuples
sub_cg.AAgroup = sub_cg.AAgroup + cg.AAgroup[i]
sub_cg.resi.append(cg.resi[i])
if p.cgResiGroupFilter(sub_cg, cutoff) == True:
print sub_cg.getString()
fp.close()
def remove_account_from_group(group_id=None, account_id=None):
session_account = get_account()
storage = get_storage(namespace='object', account=session_account)
if not group_id or not account_id:
return HTTPError(400, 'Bad request, must specified group and account')
#get group && account
if group_id.find('group.') == -1:
group_id = 'group.%s' % group_id
if account_id.find('account.') == -1:
account_id = 'account.%s' % account_id
logger.debug('Try to get %s and %s' % (account_id, group_id))
try:
account_record = storage.get(account_id, account=session_account)
account = caccount(account_record)
group_record = storage.get(group_id, account=session_account)
group = cgroup(group_record)
except Exception, err:
logger.error('error while fetching %s and %s : %s' %
(account_id, group_id, err))
return HTTPError(403, 'Record not found or insufficient rights')
def remove_account_from_group(group_id=None,account_id=None):
session_account = get_account()
storage = get_storage(namespace='object',account=session_account)
if not group_id or not account_id:
return HTTPError(400, 'Bad request, must specified group and account')
#get group && account
if group_id.find('group.') == -1:
group_id = 'group.%s' % group_id
if account_id.find('account.') == -1:
account_id = 'account.%s' % account_id
logger.debug('Try to get %s and %s' % (account_id,group_id))
try:
account_record = storage.get(account_id,account=session_account)
account = caccount(account_record)
group_record = storage.get(group_id,account=session_account)
group = cgroup(group_record)
except Exception,err:
logger.error('error while fetching %s and %s : %s' % (account_id,group_id,err))
return HTTPError(403, 'Record not found or insufficient rights')
def test_05_CheckGet(self):
record = STORAGE.get('group.group_name')
record.cat()
GROUP = cgroup(record)
GROUP.cat()
if ACCOUNT._id not in GROUP.account_ids:
raise Exception('group.account_ids: Corruption in load...')
def test_11_check_group_func_autosav(self):
account = caccount(user='******', lastname='testify', storage=STORAGE)
group = cgroup(name='Mgroup')
STORAGE.put(account)
STORAGE.put(group)
account.add_in_groups(group._id)
bdd_account = caccount(STORAGE.get(account._id))
bdd_group = cgroup(STORAGE.get(group._id))
if group._id not in bdd_account.groups:
raise Exception('Group corruption while stock in bdd after add in group')
if account._id not in bdd_group.account_ids:
raise Exception('Group corruption while stock in bdd after add in group')
'''
def test_08_cgroup_with_storage(self): global GROUP GROUP = cgroup(name='group_name', storage=STORAGE) STORAGE.put(GROUP) global ACCOUNT ACCOUNT = caccount(user="******", lastname="Pain", firstname="William", mail="*****@*****.**", group="capensis") STORAGE.put(ACCOUNT)
def test_11_check_group_func_autosav(self):
account = caccount(user='******', lastname='testify', storage=STORAGE)
group = cgroup(name='Mgroup')
STORAGE.put(account)
STORAGE.put(group)
account.add_in_groups(group._id)
bdd_account = caccount(STORAGE.get(account._id))
bdd_group = cgroup(STORAGE.get(group._id))
if group._id not in bdd_account.groups:
raise Exception(
'Group corruption while stock in bdd after add in group')
if account._id not in bdd_group.account_ids:
raise Exception(
'Group corruption while stock in bdd after add in group')
'''
def add_description_to_group():
storage = get_storage(account=root, namespace='object')
for name in groups:
try:
record = storage.get('group.%s' % name)
group_record = cgroup(record)
if not group_record.description:
group_record.description = groups[name]
storage.put(group_record)
except:
pass
def add_description_to_group():
storage = get_storage(account=root, namespace='object')
for name in groups:
try:
record = storage.get('group.%s' % name)
group_record = cgroup(record)
if not group_record.description:
group_record.description = groups[name]
storage.put(group_record)
except:
pass
def test_09_check_admin_rights(self):
account = caccount(user='******')
group = cgroup(name='administrator')
group.add_accounts(account)
record = crecord(admin_group=group._id,group='nothing',owner='refrigerator')
check = record.check_write(account)
if not check:
raise Exception('Admin group are not handle ...')
def test_09_check_admin_rights(self):
account = caccount(user='******')
group = cgroup(name='administrator')
group.add_accounts(account)
record = crecord(admin_group=group._id,
group='nothing',
owner='refrigerator')
check = record.check_write(account)
if not check:
raise Exception('Admin group are not handle ...')
def setup_container(container, cname, device, root_name, my_cpu_parent,
my_io_parent):
"""Create a new os container for constraining and isolating the cpus, mem,
and disk IO of one set of io workers, from other workers.
An os container is a pairing of a cpuset cgroup with an io cgroup,
generally from separate cgroup hierarchies. (They can also be
separate fields within a combined cgroup in a single hierarchy.)
my_cpu_parent and my_io_parent describe the existing cpu and io
cgroups of the new container's parent container.
"""
# Create a new cpus+mem cgroup, below my_cpu_parent:
mbytes = plan_container_size(container)
weight = container['weight']
path = cpuset.create_container_cpuset(cname,
'cpuset',
root=my_cpu_parent.name,
mbytes=mbytes)
blk_path = cpuset.create_container_blkio(
device,
cname,
'io',
root=my_io_parent.name,
weight=weight,
priority=container['priority'],
shared_sync_queues=container['shared_sync_queues'])
logging.info("path: " + path + " blk_path: " + blk_path)
# Setup a view.
cpu_cgroup = cgroup.cgroup('cpuset', path)
blkio_cgroup = cgroup.cgroup('io', blk_path)
container['cpu_cgroup'] = cpu_cgroup
container['blkio_cgroup'] = blkio_cgroup
name = cpu_cgroup.name # eg default/g0/g1
if root_name: # remove default/
name = name[len(root_name) + 1:]
container['name'] = name # eg g0/g1
def main(): if len(sys.argv) < 2: print "Usage python proc_scoreboard.py cluster_file" return fin = open(sys.argv[1], 'r') fo = open(sys.argv[1]+'.score', 'w') for line in fin.readlines(): cg = cgroup(line.strip(), []) fo.write(cg.scoreboard2str()+'\n') fin.close() fo.close() '''
def test_10_check_addgroup_removegroup(self):
GROUP = cgroup(name='mygroup')
ACCOUNT.add_in_groups(GROUP)
if GROUP._id not in ACCOUNT.groups:
raise Exception('Error while add_in_groups, group not added')
if ACCOUNT._id not in GROUP.account_ids:
raise Exception('Error while add_in_groups, account not added to group')
ACCOUNT.remove_from_groups(GROUP)
if GROUP._id in ACCOUNT.groups:
raise Exception('Error while remove_from_groups, group not removed')
if ACCOUNT._id in GROUP.account_ids:
raise Exception('Error while remove_from_groups, group not removed from account')
def test_22_admin_group_access(self):
root_account = caccount(user="******", group="root")
storage = STORAGE
group = cgroup(name="administrator")
record = crecord(_id="test_record", admin_group="group.administrator")
account = caccount(user="******", group="user")
storage.put(record, account=root_account)
group.add_accounts(account)
storage.put(account, account=root_account)
try:
output = storage.get(record._id, account=account)
except:
raise Exception("admin group can't access all the ressources of his group")
def test_22_admin_group_access(self):
root_account = caccount(user="******", group="root")
storage = STORAGE
group = cgroup(name='administrator')
record = crecord(_id='test_record',admin_group='group.administrator')
account = caccount(user='******',group='user')
storage.put(record,account=root_account)
group.add_accounts(account)
storage.put(account,account=root_account)
try:
output= storage.get(record._id,account=account)
except:
raise Exception('admin group can\'t access all the ressources of his group')
def test_22_admin_group_access(self):
root_account = caccount(user="******", group="root")
storage = STORAGE
group = cgroup(name='administrator')
record = crecord(_id='test_record', admin_group='group.administrator')
account = caccount(user='******', group='user')
storage.put(record, account=root_account)
group.add_accounts(account)
storage.put(account, account=root_account)
try:
output = storage.get(record._id, account=account)
except:
raise Exception(
'admin group can\'t access all the ressources of his group')
def add_in_groups(self, groups, storage=None):
if not storage:
storage = self.storage
if not isinstance(groups,list):
groups = [groups]
# String _id to cgroup
group_list = []
for group in groups:
if isinstance(group,cgroup):
group_list.append(group)
elif isinstance(group, basestring):
if storage:
try:
record = storage.get(group)
group_list.append(cgroup(record,storage=storage))
except Exception,err:
raise Exception('Group not found: %s', err)
def test_10_check_addgroup_removegroup(self):
GROUP = cgroup(name='mygroup')
ACCOUNT.add_in_groups(GROUP)
if GROUP._id not in ACCOUNT.groups:
raise Exception('Error while add_in_groups, group not added')
if ACCOUNT._id not in GROUP.account_ids:
raise Exception(
'Error while add_in_groups, account not added to group')
ACCOUNT.remove_from_groups(GROUP)
if GROUP._id in ACCOUNT.groups:
raise Exception(
'Error while remove_from_groups, group not removed')
if ACCOUNT._id in GROUP.account_ids:
raise Exception(
'Error while remove_from_groups, group not removed from account'
)
def remove_from_groups(self, groups, storage=None):
if not storage:
storage = self.storage
if not isinstance(groups,list):
groups = [groups]
# String _id to cgroup
group_list = []
for group in groups:
if isinstance(group,crecord):
group_list.append(group)
elif isinstance(group, str) and isinstance(group, unicode):
if storage:
try:
record = storage.get(group)
group_list.append(cgroup(record, storage=storage))
except Exception,err:
raise Exception('Group not found: %s', err)
def main():
if len(sys.argv) < 3:
print 'Procedure for adding additional properties on contact groups'
print 'Usage: python proc_cgAugment.py AAtips.def cgfile'
print '\tAAtips.def: Amino Acid property description file'
print '\tcgfile: contact group file, each line is a contact group'
print '\tcontact group example: 2nrt.tip,LEY,380 364 417'
return
AADesFile = sys.argv[1]
print AADesFile
CGFile = sys.argv[2]
print CGFile
# split in lines without '\n'
print 'loading AA description ...'
with open(AADesFile) as f:
AAlines = f.read().splitlines()
# compose type map from Amino Acid property file
# AAtips.def
typeMap={}
for line in AAlines:
strArr = line.split(',')
typeMap[strArr[1]]=strArr[3]
fout = open(CGFile+'.aug', 'w')
# load contact groups
with open(CGFile) as f:
CGlines = f.read().splitlines()
# output augmented contact group strings
print 'adding properties ...'
for line in CGlines:
cg = cgroup(line)
cg.getType(typeMap)
fout.write(cg.getString())
fout.close()
print 'done. output: ' + CGFile+'.aug'
'''
def account_post():
#get the session (security)
account = get_account()
if not check_group_rights(account,group_managing_access):
return HTTPError(403, 'Insufficient rights')
root_account = caccount(user="******", group="root")
storage = get_storage(namespace='object',account=account)
logger.debug("POST:")
data = request.body.readline()
if not data:
return HTTPError(400, "No data received")
data = json.loads(data)
## Clean data
try:
del data['_id']
del data['id']
del data['crecord_type']
except:
pass
if data['user']:
#check if already exist
update = False
_id = "account." + str(data['user'])
try:
record = storage.get(_id ,account=account)
logger.debug('Update account %s' % _id)
update = True
except:
logger.debug('Create account %s' % _id)
#-----------------------UPDATE----------------------
if update:
#Get password
if data['passwd']:
passwd = str(data['passwd'])
else:
passwd = None
#Get group
group = str(data['aaa_group'])
if group:
if group.find('group.') == -1:
group = 'group.%s' % group
#get secondary groups
groups = data['groups']
secondary_groups = []
if groups:
if not isinstance(groups,list):
groups = [groups]
for one_group in groups:
if one_group.find('group.') == -1:
one_group = 'group.%s' % one_group
try :
secondary_groups.append(cgroup(storage.get(one_group,account=account)))
except Exception,err:
logger.error('Error while searching secondary group: %s',err)
#clean secondary groups
for one_record in record.data['groups']:
if unicode(one_record) not in secondary_groups:
remove_account_from_group(one_record,record._id)
#get clean account
record = storage.get(_id ,account=account)
#clean
del data['passwd']
del data['aaa_group']
del data['groups']
#new record
for key in dict(data).keys():
record.data[key] = data[key]
update_account = caccount(record)
#updating
if passwd:
logger.debug(' + Update password ...')
update_account.passwd(passwd)
if group:
logger.debug(' + Update group ...')
update_account.chgrp(group)
if secondary_groups:
logger.debug(' + Update groups ...')
update_account.add_in_groups(secondary_groups)
storage.put(update_account, account=account)
storage.put(secondary_groups, account=account)
reload_account(update_account._id)
else:
#----------------------------CREATION--------------------------
logger.debug(' + New account')
new_account = caccount(user=data['user'], group=data['aaa_group'], lastname=data['lastname'], firstname=data['firstname'], mail=data['mail'])
#passwd
passwd = data['passwd']
new_account.passwd(passwd)
logger.debug(" + Passwd: '%s'" % passwd)
#secondary groups
groups = data['groups']
secondary_groups = []
if groups:
if not isinstance(groups,list):
groups = [groups]
for one_group in groups:
if one_group.find('group.') == -1:
one_group = 'group.%s' % one_group
try :
secondary_groups.append(cgroup(storage.get(one_group,account=account)))
except Exception,err:
logger.error('Error while searching secondary group: %s',err)
new_account.add_in_groups(secondary_groups)
storage.put(secondary_groups)
#put record
logger.debug(' + Save new account')
new_account.chown(new_account._id)
storage.put(new_account, account=account)
#get rootdir
logger.debug(' + Create view directory')
rootdir = storage.get('directory.root', account=root_account)
if rootdir:
userdir = crecord({'_id': 'directory.root.%s' % new_account.user,'id': 'directory.root.%s' % new_account.user ,'expanded':'true'}, type='view_directory', name=new_account.user)
userdir.chown(new_account._id)
userdir.chgrp(new_account.group)
userdir.chmod('g-w')
userdir.chmod('g-r')
storage.put(userdir, account=account)
rootdir.add_children(userdir)
storage.put(rootdir, account=root_account)
storage.put(userdir, account=account)
else:
logger.error('Impossible to get rootdir')
def test_01_Init(self): global ACCOUNT ACCOUNT = caccount(user="******", lastname="Pain", firstname="William", mail="*****@*****.**", group="capensis") global GROUP GROUP = cgroup(name='group_name')