def repo_build(f, t):
try:
f_repo_fn = os.path.join(f, 'repo.js')
repo_js = utils.json_load(f_repo_fn)
except FileNotFoundError:
print(
'No repo.js found in the source directory. '
'Creating a new repository.'
)
repo_js = {}
enter_missing(repo_js, 'id', 'Enter a repository ID: ')
enter_missing(repo_js, 'title', 'Enter a nice repository title: ')
enter_missing(repo_js, 'contact', 'Enter a contact e-mail address: ')
utils.json_store('repo.js', repo_js, dirs=[f, t])
while not 'servers' in repo_js or not repo_js['servers'][0].strip():
repo_js['servers'] = [input(
'Enter the public URL of your repository '
'(the path that contains repo.js): '
)]
repo_js['patches'] = {}
for root, dirs, files in os.walk(f):
del(dirs)
if 'patch.js' in files:
patch_id = os.path.basename(root)
repo_js['patches'][patch_id] = patch_build(
patch_id, repo_js['servers'], f, t
)
print('Done.')
utils.json_store('repo.js', repo_js, dirs=[f, t])
def repo_build(f, t):
try:
f_repo_fn = os.path.join(f, 'repo.js')
repo_js = utils.json_load(f_repo_fn)
except FileNotFoundError:
print('No repo.js found in the source directory. '
'Creating a new repository.')
repo_js = {}
enter_missing(repo_js, 'id', 'Enter a repository ID: ')
enter_missing(repo_js, 'title', 'Enter a nice repository title: ')
enter_missing(repo_js, 'contact', 'Enter a contact e-mail address: ')
utils.json_store('repo.js', repo_js, dirs=[f, t])
while 'servers' not in repo_js or not repo_js['servers'][0].strip():
repo_js['servers'] = [
input('Enter the public URL of your repository '
'(the path that contains repo.js): ')
]
repo_js['patches'] = {}
ignored = set(IGNORED_BY_DEFAULT).union(thcrap_ignore_get(f))
for root, dirs, files in os.walk(f):
del (dirs)
if 'patch.js' in files:
patch_id = os.path.basename(root)
repo_js['patches'][patch_id] = patch_build(patch_id,
repo_js['servers'], f,
t, ignored)
print('Done.')
utils.json_store('repo.js', repo_js, dirs=[f, t])
def add(self, args):
#check args
update = 1
count = 0
data = {}
group_list = []
if os.path.exists(self.json_file):
data = json_load(self.json_file)
if len(data) != 0:
group_list = [(k, data[k]) for k in sorted(data.keys())]
gname, v = group_list[len(group_list) - 1]
count = filter(str.isdigit, json.dumps(gname))
if args:
gname = "group" + str(int(count) + 1)
data[gname] = {}
group = data[gname]
#generate vipconf json
group['nodes'] = args.host
group['type'] = args.type
group['vip'] = args.vip
else:
update = 0
#data = sorted(data.items(), key=lambda d: d[0])
#modify vipconf about vip or nodes
if update:
json_store(data, self.config.vip_conf)
dmsg('add vipconf sucessfully')
else:
dwarn('vipconf was updated.')
def vip_confdel_group(args):
data = {}
config = Config()
json_file = config.vip_conf
if os.path.exists(json_file):
data = json_load(json_file)
if len(data) == 0:
dmsg('vipconf was null')
return
if not data.has_key(args.group):
derror("vipconfdel: %s not exist." % (args.group))
sys.exit(2)
else:
del_vips = data[args.group]["vips"]
mask = data[args.group]["mask"]
hosts = data[args.group]["nodes"].split(",")
del data[args.group]
json_store(data, json_file)
for host in hosts:
try:
_del_vip_from_host(config, host, del_vips, mask)
except Exp, e:
print e
def vip_add(args):
#check args
if not is_valid_args(args):
derror('please check args')
sys.exit(1)
update = 1
count = 0
data = {}
group_list = []
config = Config()
json_file = config.vip_conf
if os.path.exists(json_file):
data = json_load(json_file)
args.host = list_distinct(args.host)
args.vip = list_distinct(args.vip)
if len(data) != 0:
group_list = [(k, data[k]) for k in sorted(data.keys())]
for k,v in group_list:
if v['nodes'] == args.host and v['vips'] == args.vip \
and v['type'] == args.type and v['mask'] == args.mask:
update = 0
break
gname, v = group_list[len(group_list) - 1]
count = filter(str.isdigit, json.dumps(gname))
if args:
gname = "group" + str(int(count)+1)
data[gname] = {}
group = data[gname]
#generate vipconf json
group['nodes'] = args.host
group['type'] = args.type
group['vips'] = args.vip
group['mask'] = args.mask
else:
update = 0
#data = sorted(data.items(), key=lambda d: d[0])
#modify vipconf about vip or nodes
if update:
json_store(data, json_file)
dmsg('add vipconf sucessfully')
else:
dwarn('vipconf was updated.')
def vip_confdel(args):
#check args
if not is_valid_ip(args):
derror('please check args')
sys.exit(1)
data = {}
new_host = []
new_vip = []
#load vipconf
config = Config()
json_file = config.vip_conf
if os.path.exists(json_file):
data = json_load(json_file)
if len(data) == 0:
print 'vipconf was null'
return
if not data.has_key(args.group):
derror("vipconfdel: %s not exist." % (args.group))
sys.exit(2)
#reload new host and vip
old_host = json.dumps(data[args.group]['nodes']).strip('"').split(',')
old_vip = json.dumps(data[args.group]['vips']).strip('"').split(',')
if args.host:
new_host = [json.dumps(h.strip('"')) for h in old_host if h.strip('"') not in args.host.split(',') and h != '']
if args.vip:
new_vip = [json.dumps(v.strip('"')) for v in old_vip if v.strip('"') not in args.vip.split(',') and v != '']
#print old_host
#print new_host
if len(new_host) != 0:
data[args.group]['nodes'] = _list2str(new_host)
if len(new_vip) != 0:
data[args.group]['vips'] = _list2str(new_vip)
if len(new_host) == 0 and len(new_vip) == 0:
del data[args.group]
json_store(data, json_file)
def vip_confadd(args):
#check args
if not is_valid_ip(args):
derror('please check args')
sys.exit(1)
data = {}
new_host = []
new_vip = []
#load vipconf
config = Config()
json_file = config.vip_conf
if os.path.exists(json_file):
data = json_load(json_file)
if len(data) == 0:
print 'vipconf was null'
return
#reload new host and vip
old_host = json.dumps(data[args.group]['nodes']).strip('"').split(',')
old_vip = json.dumps(data[args.group]['vips']).strip('"').split(',')
if args.host:
new_host = [json.dumps(h.strip('"')) for h in args.host.split(',') if h.strip('"') not in old_host]
if args.vip:
new_vip = [json.dumps(v.strip('"')) for v in args.vip.split(',') if v.strip('"') not in old_vip]
#print old_host
#print new_host
if len(new_host) != 0:
new_host.extend(old_host)
data[args.group]['nodes'] = _list2str(new_host)
if len(new_vip) != 0:
new_vip.extend(old_vip)
data[args.group]['vips'] = _list2str(new_vip)
if len(new_host) != 0 or len(new_vip) != 0:
json_store(data, json_file)
dmsg('update vipconf sucessfully')
else:
dwarn('vipconf is already updated.')
def tenant_add(self, name, password, port):
if self._check_exists(name, port):
raise Exp(errno.EEXIST, "name or port has been used, please check it!")
#create tenant config directory
dirname = "%s_%s" % (name, port)
abs_path = "%s/%s" % (MINIO_CONF_PATH, dirname)
os.mkdir(abs_path)
#generate config.json
cmd = "cp /opt/minio/config.json %s" % (abs_path)
exec_shell(cmd, p=False)
#modify config.json about accessKey and secretKey
json_file = "%s/config.json" % (abs_path)
data = json_load(json_file)
data["credential"]["accessKey"] = name
data["credential"]["secretKey"] = password
json_store(data, json_file)
def patch_build(patch_id, servers, f, t):
"""Updates the patch in the [f]/[patch_id] directory.
Ensures that patch.js contains all necessary keys and values, then updates
the checksums in files.js and, if [t] differs from [f], copies all patch
files from [f] to [t].
Returns the contents of the patch ID key in repo.js."""
f_path, t_path = [os.path.join(i, patch_id) for i in [f, t]]
# Prepare patch.js.
f_patch_fn = os.path.join(f_path, 'patch.js')
patch_js = utils.json_load(f_patch_fn)
enter_missing(
patch_js, 'title', 'Enter a nice title for "{}": '.format(patch_id)
)
patch_js['id'] = patch_id
patch_js['servers'] = []
# Delete obsolete keys.
if 'files' in patch_js:
del(patch_js['files'])
for i in servers:
url = os.path.join(i, patch_id) + '/'
patch_js['servers'].append(str_slash_normalize(url))
utils.json_store(f_patch_fn, patch_js)
# Reset all old entries to a JSON null. This will delete any files on the
# client side that no longer exist in the patch.
try:
files_js = utils.json_load(os.path.join(f_path, 'files.js'))
for i in files_js:
files_js[i] = None
except FileNotFoundError:
files_js = {}
patch_size = 0
print(patch_id, end='')
for root, dirs, files in os.walk(f_path):
for fn in utils.patch_files_filter(files):
print('.', end='')
f_fn = os.path.join(root, fn)
patch_fn = f_fn[len(f_path) + 1:]
t_fn = os.path.join(t_path, patch_fn)
with open(f_fn, 'rb') as f_file:
f_file_data = f_file.read()
# Ensure Unix line endings for JSON input
if fn.endswith(('.js', '.jdiff')) and b'\r\n' in f_file_data:
f_file_data = f_file_data.replace(b'\r\n', b'\n')
with open(f_fn, 'wb') as f_file:
f_file.write(f_file_data)
f_sum = zlib.crc32(f_file_data) & 0xffffffff
files_js[str_slash_normalize(patch_fn)] = f_sum
patch_size += len(f_file_data)
del(f_file_data)
os.makedirs(os.path.dirname(t_fn), exist_ok=True)
if f != t:
shutil.copy2(f_fn, t_fn)
utils.json_store('files.js', files_js, dirs=[f_path, t_path])
print(
'{num} files, {size}'.format(
num=len(files_js), size=sizeof_fmt(patch_size)
)
)
return patch_js['title']
def patch_build(patch_id, servers, f, t, ignored):
"""Updates the patch in the [f]/[patch_id] directory, ignoring the files
that match [ignored].
Ensures that patch.js contains all necessary keys and values, then updates
the checksums in files.js and, if [t] differs from [f], copies all patch
files from [f] to [t].
Returns the contents of the patch ID key in repo.js."""
f_path, t_path = [os.path.join(i, patch_id) for i in [f, t]]
# Prepare patch.js.
f_patch_fn = os.path.join(f_path, 'patch.js')
patch_js = utils.json_load(f_patch_fn)
enter_missing(patch_js, 'title',
'Enter a nice title for "{}": '.format(patch_id))
patch_js['id'] = patch_id
patch_js['servers'] = []
# Delete obsolete keys.
if 'files' in patch_js:
del (patch_js['files'])
for i in servers:
url = os.path.join(i, patch_id) + '/'
patch_js['servers'].append(str_slash_normalize(url))
utils.json_store(f_patch_fn, patch_js)
# Reset all old entries to a JSON null. This will delete any files on the
# client side that no longer exist in the patch.
try:
files_js = utils.json_load(os.path.join(f_path, 'files.js'))
for i in files_js:
files_js[i] = None
except FileNotFoundError:
files_js = {}
patch_size = 0
print(patch_id, end='')
for f_fn in patch_files_walk(f, f_path, ignored):
print('.', end='')
patch_fn = f_fn[len(f_path) + 1:]
t_fn = os.path.join(t_path, patch_fn)
with open(f_fn, 'rb') as f_file:
f_file_data = f_file.read()
# Ensure Unix line endings for JSON input
if f_fn.endswith(('.js', '.jdiff')) and b'\r\n' in f_file_data:
f_file_data = f_file_data.replace(b'\r\n', b'\n')
with open(f_fn, 'wb') as f_file:
f_file.write(f_file_data)
f_sum = zlib.crc32(f_file_data) & 0xffffffff
files_js[str_slash_normalize(patch_fn)] = f_sum
patch_size += len(f_file_data)
del (f_file_data)
os.makedirs(os.path.dirname(t_fn), exist_ok=True)
if f != t:
shutil.copy2(f_fn, t_fn)
utils.json_store('files.js', files_js, dirs=[f_path, t_path])
print('{num} files, {size}'.format(num=len(
{k: v
for k, v in files_js.items() if v is not None}),
size=sizeof_fmt(patch_size)))
return patch_js['title']
def tenant_mod(self, name, password):
fname = _get_full_name(name)
json_file = "%s/%s/config.json" % (MINIO_CONF_PATH, fname)
data = json_load(json_file)
data["credential"]["secretKey"] = password
json_store(data, json_file)
choices=thcrap_hashes,
default=thcrap_hashes[0])
parser.add_argument('-k', '--key', help='Private key file.', required=True)
def alg_mod_from_str(alg):
"""Returns the PyCrypto hash module that corresponds to [alg]."""
if alg is 'SHA1':
alg = 'SHA'
return importlib.import_module('Crypto.Hash.' + alg)
def sign(arc_fn, alg, key_fn):
alg_mod = alg_mod_from_str(alg)
with open(key_fn, 'r') as key_file:
key = RSA.importKey(key_file.read())
signer = PKCS1_v1_5.new(key)
ret = {'alg': alg}
with open(arc_fn, 'rb') as arc_file:
m = alg_mod.new(arc_file.read())
signature = signer.sign(m)
ret['sig'] = base64.b64encode(signature).decode('ascii')
return ret
if __name__ == '__main__':
arg = parser.parse_args()
sig = sign(arg.arc, arg.alg, arg.key)
utils.json_store(arg.arc + '.sig', sig)
'-k', '--key',
help='Private key file.',
required=True
)
def alg_mod_from_str(alg):
"""Returns the PyCrypto hash module that corresponds to [alg]."""
if alg is 'SHA1':
alg = 'SHA'
return importlib.import_module('Crypto.Hash.' + alg)
def sign(arc_fn, alg, key_fn):
alg_mod = alg_mod_from_str(alg)
with open(key_fn, 'r') as key_file:
key = RSA.importKey(key_file.read())
signer = PKCS1_v1_5.new(key)
ret = {'alg': alg}
with open(arc_fn, 'rb') as arc_file:
m = alg_mod.new(arc_file.read())
signature = signer.sign(m)
ret['sig'] = base64.b64encode(signature).decode('ascii')
return ret
if __name__ == '__main__':
arg = parser.parse_args()
sig = sign(arg.arc, arg.alg, arg.key)
utils.json_store(arg.arc + '.sig', sig)
