def patch_instance(inst='instance-debug'):
idp = 'parts/{}/bin/interpreter'.format(inst)
if not os.path.exists(idp):
error("'{}' doesn't exist: cannot patch it".format(idp))
return False
lines = read_file(idp)
if 'ploneCustom.css' not in ''.join(lines):
sp = 0
for (i, line) in enumerate(lines):
if 'exec(_val)' in line:
nl = line.lstrip()
sp = len(line) - len(nl)
break
lines.insert(
i, "{}{}".format(
' ' * sp,
'_val = _val.replace("\'); from AccessControl.SpecialUsers import system '
'as user;", "/ploneCustom.css\'); from AccessControl.SpecialUsers import '
'system as user;")'))
verbose("=> Patching: '{}'".format(idp))
fh = open(idp, 'w')
fh.write('\n'.join(lines))
fh.close()
else:
verbose("=> Already patched: '{}'".format(idp))
def generate(output_dir, domain):
""" Generate taskjuggler report """
verbose("Begin of taskjuggler report")
records_nb = 0
output_dir = output_dir.rstrip('/')
DAY_DIR = os.path.join(output_dir, date.strftime(date.today(), "%Y-%m-%d"))
rep_cmd = base_rep_cmd.replace('DAY_DIR', DAY_DIR)
if not os.path.exists(DAY_DIR):
os.makedirs(os.path.join(DAY_DIR, 'css'))
os.symlink('%s/custom.css' % BUILD_PATH, '%s/css/custom.css' % DAY_DIR)
outfiles['index']['file'] = os.path.join(output_dir,
outfiles['index']['filename'])
outfiles['error']['file'] = os.path.join(output_dir,
outfiles['error']['filename'])
report_err = [outfiles['error']['filename'], 0]
verbose("Running command: %s" % rep_cmd)
(cmd_out, cmd_err) = runCommand(rep_cmd)
errors = [err for err in cmd_err if 'Error: ' in err]
if errors:
errors_str = '\n'.join(errors)
error("error running command %s : %s" % (rep_cmd, errors_str))
write_to(outfiles, 'error', errors_str)
report_err[1] = len(errors)
gen_err = ['generation_errors.html', 0]
if os.path.exists(os.path.join(output_dir, gen_err[0])):
lines = read_file(os.path.join(output_dir, gen_err[0]),
skip_empty=True)
# Records number:
for line in lines:
# mo = records_pat.match(line)
mo = tickets_pat.match(line)
if mo:
records_nb = mo.group(1)
break
if lines:
gen_err[1] = len(lines) - 1
olds = read_dir(output_dir, only_folders=True)
template = env.get_template('index.html')
rendered = template.render(report_err=report_err,
gen_err=gen_err,
olds=olds,
records_nb=records_nb,
domain=domain)
write_to(outfiles, 'index', rendered.encode('utf8'))
close_outfiles(outfiles)
# delete wrong generated folder
if report_err[1] and os.path.exists(DAY_DIR):
shutil.rmtree(DAY_DIR, ignore_errors=True)
verbose("End of taskjuggler report")
def generate(output_dir, domain):
""" Generate taskjuggler report """
verbose("Begin of taskjuggler report")
records_nb = 0
output_dir = output_dir.rstrip('/')
DAY_DIR = os.path.join(output_dir, date.strftime(date.today(), "%Y-%m-%d"))
rep_cmd = base_rep_cmd.replace('DAY_DIR', DAY_DIR)
if not os.path.exists(DAY_DIR):
os.makedirs(os.path.join(DAY_DIR, 'css'))
os.symlink('%s/custom.css' % BUILD_PATH, '%s/css/custom.css' % DAY_DIR)
outfiles['index']['file'] = os.path.join(output_dir, outfiles['index']['filename'])
outfiles['error']['file'] = os.path.join(output_dir, outfiles['error']['filename'])
report_err = [outfiles['error']['filename'], 0]
verbose("Running command: %s" % rep_cmd)
(cmd_out, cmd_err) = runCommand(rep_cmd)
errors = [err for err in cmd_err if 'Error: ' in err]
if errors:
errors_str = '\n'.join(errors)
error("error running command %s : %s" % (rep_cmd, errors_str))
write_to(outfiles, 'error', errors_str)
report_err[1] = len(errors)
gen_err = ['generation_errors.html', 0]
if os.path.exists(os.path.join(output_dir, gen_err[0])):
lines = read_file(os.path.join(output_dir, gen_err[0]), skip_empty=True)
# Records number:
for line in lines:
# mo = records_pat.match(line)
mo = tickets_pat.match(line)
if mo:
records_nb = mo.group(1)
break
if lines:
gen_err[1] = len(lines)-1
olds = read_dir(output_dir, only_folders=True)
template = env.get_template('index.html')
rendered = template.render(report_err=report_err, gen_err=gen_err, olds=olds, records_nb=records_nb, domain=domain)
write_to(outfiles, 'index', rendered.encode('utf8'))
close_outfiles(outfiles)
# delete wrong generated folder
if report_err[1] and os.path.exists(DAY_DIR):
shutil.rmtree(DAY_DIR, ignore_errors=True)
verbose("End of taskjuggler report")
def _stats_database(self):
# soft dependency
from imio.pyutils.system import error
from Products.CPUtils.Extensions.utils import tobytes
# zope
database = {"fs_sz": 0, "bl_sz": 0}
app = self.context.restrictedTraverse("/")
dbs = app["Control_Panel"]["Database"]
for db in dbs.getDatabaseNames():
readable_size = dbs[db].db_size()
size = int(
tobytes(readable_size[:-1] + " " + readable_size[-1:] + "B"))
# keep only largest
if size > database["fs_sz"]:
database["fs_sz"] = size
database["fs_sz_readable"] = sizeof_fmt(size)
# blobstorage
instdir = os.getenv("PWD")
if instdir is None:
# try with INSTANCE_HOME, it is like
# /srv/instances/instance_name/parts/instance1
instdir = os.getenv("INSTANCE_HOME").split("/parts/")[0]
if os.path.exists(instdir):
# .sizes.json
sizefile = os.path.join(instdir, '.sizes.json')
try:
fh = open(sizefile)
res = json.load(fh)
fh.close()
size = int(res.get(u'local_size', 0))
if size > database['fs_sz']:
size -= database['fs_sz']
database['bl_sz'] = size
database["bl_sz_readable"] = sizeof_fmt(size)
except Exception, msg:
error(u".sizes.json not valid in '{}': '{}'".format(
instdir, msg))
def herror(msg):
error('%s<br />' % msg)
# -*- coding: utf-8 -*-
import sys
from imio.pyutils.system import verbose, error
import transaction
from plone import api
# Parameters check
if len(sys.argv) < 3 or sys.argv[2] != 'run-scripts.py':
error("Inconsistent or unexpected args len: %s" % sys.argv)
sys.exit(0)
def script2():
verbose('Updating ports on %s' % obj.absolute_url_path())
from collective.documentgenerator.utils import update_oo_config
#from imio.dms.mail.utils import update_solr_config
#update_solr_config()
update_oo_config()
transaction.commit()
def script1():
verbose('Pst budget correction on %s' % obj.absolute_url_path())
catalog = obj.portal_catalog
from imio.project.core.events import onModifyProject
for brain in catalog(portal_type='projectspace'):
ps = brain.getObject()
verbose(ps.absolute_url())
ret = ps.restrictedTraverse('clean_budget/display')()
verbose("Before: {}".format(ret.split('<br />\n')[0]))
def add_archived():
"""Add archived columns to inforius exported file"""
parser = argparse.ArgumentParser(
description='From mixed archived column, add 2 new columns')
parser.add_argument('-p',
'--parts',
dest='parts',
help='Run parts: 1 (add archived), 2 (write)',
default='12')
parser.add_argument('tree_file', help='Tree file (csv format)')
parser.add_argument(
'-c',
'--config',
dest='tree_conf',
required=True,
help=
'Tree file configuration: "separator|archived col|sf id col|sf tit col" (starting at 0). '
'Like: ;|1||')
ns = parser.parse_args()
verbose("Start of %s" % sys.argv[0])
verbose("Reading tree file '{}'".format(ns.tree_file))
tree_confs = ns.tree_conf.split('|')
if len(tree_confs) != 4:
error("config parameter not well formated: {}".format(ns.tree_conf))
parser.print_help()
sys.exit(1)
sep, arc_col, id_col, tit_col = tree_confs[0], int(tree_confs[1]), int(
tree_confs[2]), int(tree_confs[3])
lines = read_csv(ns.tree_file, strip_chars=' ', delimiter=sep)
titles = lines.pop(0)
titles.extend(['Archivé farde', 'Archivé chemise'])
new_lines = [titles]
if '1' in ns.parts:
for i, line in enumerate(lines, start=1):
ln_nb = i + 1
archived = line[arc_col] # '0', '1', 'VRAI', 'FAUX'
f_arc = sf_arc = ''
if archived == 'VRAI':
archived = '1'
elif archived == 'FAUX':
archived = '0'
if archived not in ('0', '1'):
error("{}, bad archived value '{}'".format(ln_nb, archived))
else:
if line[id_col] or line[tit_col]: # is subfolder
if archived == '1':
sf_arc = '1'
else: # is folder
if archived == '1':
f_arc = '1'
line.extend([f_arc, sf_arc])
new_lines.append(line)
if '2' in ns.parts:
new_file = ns.tree_file.replace('.csv', '_archived.csv')
with open(new_file, 'wb') as csvfile:
csvwriter = csv.writer(
csvfile, delimiter=sep,
quoting=csv.QUOTE_NONNUMERIC) # csv.QUOTE_ALL
for line in new_lines:
csvwriter.writerow(line)
verbose("End of %s" % sys.argv[0])
def compare_tree_files():
"""Compares a tree file with a reference tree file"""
parser = argparse.ArgumentParser(
description='Compare a tree file with a reference one')
parser.add_argument(
'-p',
'--parts',
dest='parts',
help='Run parts: 1 (load ref), 2 (load tree), 3 (compare)',
default='123')
parser.add_argument('-r',
'--reference',
dest='ref_file',
help='Reference file (csv format)')
parser.add_argument(
'-rc',
'--reference_config',
dest='ref_conf',
help=
'Reference file configuration: "skip lines|separator|id col|title col" (starting at 0). '
'Like: 1|;|0|1')
parser.add_argument('-f',
'--file',
dest='tree_file',
help='Tree file (csv format)',
required=True)
parser.add_argument(
'-fc',
'--file_config',
dest='tree_conf',
required=True,
help=
'Tree file configuration: "skip lines|separator|id cols|title col" (starting at 0). '
'Like: 1|;|0,4|1')
parser.add_argument('-u',
'--unicity',
action='store_true',
dest='check_unicity',
help='Check code unicity')
ns = parser.parse_args()
verbose("Start of %s" % sys.argv[0])
if '1' in ns.parts:
if not ns.ref_file or not ns.ref_conf:
error("Missing -r or -rc parameters for part 1 !")
parser.print_help()
sys.exit(1)
verbose("Reading ref file '{}'".format(ns.ref_file))
ref_confs = ns.ref_conf.split('|')
if len(ref_confs) != 4:
error("rc parameter not well formated: {}".format(ns.ref_conf))
parser.print_help()
sys.exit(1)
skip_lines, ref_id_col, ref_tit_col = int(ref_confs[0]), int(
ref_confs[2]), int(ref_confs[3])
lines = read_csv(ns.ref_file,
skip_lines=skip_lines,
delimiter=ref_confs[1])
ref_dic = {}
for i, line in enumerate(lines, start=skip_lines + 1):
k = line[ref_id_col]
if k in ref_dic:
error("Ref id already exists: {} : {} <=> {}".format(
k, ref_dic[k]['t'], line[ref_tit_col]))
else:
if not re.match(decimal_identifier, k):
error("{}, bad ref identifier value '{}', '{}'".format(
i, k, line[ref_tit_col]))
ref_dic[k] = {'u': '', 't': line[ref_tit_col]}
if '2' in ns.parts:
verbose("Reading tree file '{}'".format(ns.tree_file))
tree_confs = ns.tree_conf.split('|')
if len(tree_confs) != 4:
error("fc parameter not well formated: {}".format(ns.tree_conf))
parser.print_help()
sys.exit(1)
skip_lines, tree_tit_col = int(tree_confs[0]), tree_confs[3]
tree_id_cols = [int(c) for c in tree_confs[2].split(',')]
lines = read_csv(ns.tree_file,
skip_lines=skip_lines,
delimiter=tree_confs[1])
tree_dic = OrderedDict()
for i, line in enumerate(lines, start=skip_lines + 1):
for j, id_col in enumerate(tree_id_cols):
code = line[id_col]
if not code:
continue
for k in code.split(','):
v = tree_tit_col and j == 0 and line[int(
tree_tit_col)] or ''
if k not in tree_dic:
# A. = specific to comblain
if not k.startswith('A.') and not re.match(
decimal_identifier, k):
error(
"{},{}, bad tree identifier value '{}', '{}'".
format(i, id_col, k, v))
tree_dic[k] = {'l': i, 'c': id_col, 't': v}
elif ns.check_unicity:
error("{}, id '{}' already found line {}".format(
i, k, tree_dic[k]['l']))
if '123' == ns.parts:
verbose("Comparing...")
for k in sorted(tree_dic):
tdk = tree_dic[k]
o_k = k
if k[0:1] in ('1', '2', '3', '4'): # must begin with '-'
k = '-{}'.format(k)
if k in ref_dic:
ref_dic[k]['u'] = 'd' # direct usage
if tdk['t'] and tdk['t'] != ref_dic[k]['t']:
print("{},{}, id '{}', different titles: '{}' <=> '{}'".
format(tdk['l'], tdk['c'], k, tdk['t'],
ref_dic[k]['t']))
elif not k.startswith('A.'): # specific to comblain
com = "{},{}, id '{}', not found in ref".format(
tdk['l'], tdk['c'], o_k)
if tdk['t']:
com += " (tit='{}')".format(tdk['t'])
print(com)
verbose("End of %s" % sys.argv[0])
def add_parent():
"""Add parent column"""
parser = argparse.ArgumentParser(description='Analyse code to find parent')
parser.add_argument(
'-p',
'--parts',
dest='parts',
help='Run parts: 1 (load codes), 2 (get parent), 3 (write)',
default='123')
parser.add_argument('tree_file', help='Tree file (csv format)')
parser.add_argument(
'-c',
'--config',
dest='tree_conf',
required=True,
help=
'Tree file configuration: "separator|code col|id col|id parent" (starting at 0). '
'Like: ;|1||')
parser.add_argument('-u',
'--unicity',
action='store_true',
dest='check_unicity',
help='Check code unicity')
ns = parser.parse_args()
verbose("Start of %s" % sys.argv[0])
verbose("Reading tree file '{}'".format(ns.tree_file))
tree_confs = ns.tree_conf.split('|')
if len(tree_confs) != 4:
error("config parameter not well formated: {}".format(ns.tree_conf))
parser.print_help()
sys.exit(1)
sep, code_col, id_col, id_parent = tree_confs[0], int(
tree_confs[1]), tree_confs[2], tree_confs[3]
has_id = id_col != ''
has_parent = id_parent != ''
lines = read_csv(ns.tree_file, strip_chars=' ', delimiter=sep)
code_ids = {} # store code and id
all_ids = {} # store id and code
titles = lines.pop(0)
titles.append('Parent')
cols_nb = len(titles)
new_lines = [titles]
if '1' in ns.parts or '2' in ns.parts or '3' in ns.parts:
for i, line in enumerate(lines, start=1):
ln_nb = i + 1
code = line[code_col]
if not code:
continue
if code in code_ids:
if ns.check_unicity:
error("{}, code already found '{}'".format(ln_nb, code))
else:
cid = has_id and int(line[int(id_col)]) or i
code_ids[code] = cid
all_ids[cid] = code
next_id = 1
def get_next_id(nid):
while nid in all_ids:
nid += 1
all_ids.append(nid)
return nid
if '2' in ns.parts or '3' in ns.parts:
for i, line in enumerate(lines, start=1):
ln_nb = i + 1
code = line[code_col]
if not code:
parent_code = ''
elif has_parent:
cid = code_ids[code]
parent_code = all_ids[cid]
elif re.match(decimal_identifier, code):
parent_code = get_decimal_parent(code)
if parent_code is None:
parent_code = ''
elif False:
if parent_code in code_ids:
parent_code = code_ids[parent_code]
else:
parents = get_parents(parent_code)
prev_parent_id = ''
for level in parents:
if level not in code_ids:
next_id = get_next_id(next_id)
code_ids[level] = next_id
new_line = [''] * cols_nb
new_line[code_col] = level
nid_col = not has_id and -2 or int(id_col)
new_line[nid_col] = next_id
new_line[-1] = prev_parent_id
new_lines.append(new_line)
verbose("{}, added decimal level '{}'".format(
ln_nb, level))
else:
prev_parent_id = code_ids[level]
parent_code = code_ids[parent_code]
else:
error("{}, bad ref identifier value '{}'".format(ln_nb, code))
parent_code = '!'
line.append(str(parent_code))
new_lines.append(line)
if '3' in ns.parts:
new_file = ns.tree_file.replace('.csv', '_parent.csv')
with open(new_file, 'wb') as csvfile:
csvwriter = csv.writer(csvfile,
delimiter=sep,
quoting=csv.QUOTE_NONNUMERIC)
for line in new_lines:
csvwriter.writerow(line)
verbose("End of %s" % sys.argv[0])