def create_user(dbuser, dbpwd=None):
try:
print(
"Création d'un nouvel utilisateur en cours ...\n"
"Un nouvel utilisateur doit-être créé pour pouvoir lui"
" associer la base de données de Open Food Facts.\nVeuillez"
" entrer le mot de passe de l'utilisateur root sous"
" mysql : ",
end='')
pwd = getpass("")
tmpf = NamedTemporaryFile(mode='w', encoding='utf8')
tmpf.file.write("[client]\n" "user=root\n" "password="******"CREATE USER IF NOT EXISTS '{}'@'localhost'{};".format(
dbuser, "" if dbpwd is None else " IDENTIFY BY " + dbpwd)
req += "GRANT ALL PRIVILEGES ON * . * TO '{}'@'localhost';" \
.format(dbuser)
req += "FLUSH PRIVILEGES;"
ret = process(
["mysql", "--defaults-file={}".format(tmpf.name), "-e", req],
stderr=PIPE)
tmpf.close()
except Exception:
if tmpf:
tmpf.close()
print("Exception dans fonction :", __name__)
return False
if ret.returncode:
print("Erreur process -> code de retour :", ret.returncode)
print(ret.stderr)
return False
print("{:5}L'utilisateur a été créé correctement.".format(''))
return True
def mergebase(self, commit_or_commits, db=None):
if db and isinstance(commit_or_commits, Commit):
cursor = db.cursor()
cursor.execute("SELECT mergebase FROM mergebases WHERE commit=%s",
(commit_or_commits.getId(db), ))
try:
return cursor.fetchone()[0]
except:
result = self.mergebase(commit_or_commits)
cursor.execute(
"INSERT INTO mergebases (commit, mergebase) VALUES (%s, %s)",
(commit_or_commits.getId(db), result))
return result
try:
sha1s = commit_or_commits.parents
except:
sha1s = map(str, commit_or_commits)
assert len(sha1s) >= 2
git = process([configuration.executables.GIT, 'merge-base'] + sha1s,
stdout=PIPE,
stderr=PIPE,
cwd=self.path)
stdout, stderr = git.communicate()
if git.returncode == 0: return stdout.strip()
else: raise Exception, "'git merge-base' failed: %s" % stderr.strip()
def runCustom(self, cwd, command, *arguments, **kwargs):
argv = [configuration.executables.GIT, command]
argv.extend(arguments)
stdin_data = kwargs.get("input")
if stdin_data is None: stdin = None
else: stdin = PIPE
env = {}
env.update(environ)
env.update(kwargs.get("env", {}))
if "GIT_DIR" in env: del env["GIT_DIR"]
git = process(argv,
stdin=stdin,
stdout=PIPE,
stderr=PIPE,
cwd=cwd,
env=env)
stdout, stderr = git.communicate(stdin_data)
if kwargs.get("check_errors", True):
if git.returncode == 0:
if kwargs.get("include_stderr", False):
return stdout + stderr
else:
return stdout
else:
raise Exception, "'%s' failed: %s (in %s)" % (
" ".join(argv), stderr.strip(), cwd)
else:
return git.returncode, stdout, stderr
def execute(*command, **kwargs):
kwargs.update(universal_newlines=True)
proc = process(command, **kwargs)
_, _ = proc.communicate()
code = proc.returncode
if not code: return code
sys.exit(code)
def run(self):
try:
batch = process([configuration.executables.GIT, 'cat-file', '--batch'], stdin=PIPE, stdout=PIPE, stderr=STDOUT, cwd=self.repository.path)
stdout, stderr = batch.communicate("\n".join(self.sha1s.keys()) + "\n")
gitobjects = []
for sha1, commit_id in self.sha1s.items():
line, stdout = stdout.split("\n", 1)
try: object_sha1, object_type, object_size = line.split(" ")
except: raise Exception, "unexpected header line: %r" % line
assert object_sha1 == sha1, "%s != %s (%s)" % (object_sha1, sha1)
assert object_type == "commit"
object_size = int(object_size)
object_data = stdout[:object_size]
stdout = stdout[object_size + 1:]
gitobjects.append((GitObject(object_sha1, object_type, object_size, object_data), commit_id))
self.gitobjects = gitobjects
except:
self.error = format_exc()
def execute(*command, **kwargs):
kwargs.update(universal_newlines=True)
proc = process(command, **kwargs)
_, _ = proc.communicate()
code = proc.returncode
if not code: return code
sys.exit(code)
def iscommit(self, name):
git = process([configuration.executables.GIT, 'cat-file', '-t', name],
stdout=PIPE,
stderr=PIPE,
cwd=self.path)
stdout, stderr = git.communicate()
if git.returncode == 0: return stdout.strip() == "commit"
else: return False
def branch(self, name, startpoint):
git = process(
[configuration.executables.GIT, 'branch', name, startpoint],
stdout=PIPE,
stderr=PIPE,
cwd=self.path)
stdout, stderr = git.communicate()
if git.returncode != 0: raise Exception, stderr
def __startBatchCheck(self):
if self.__batchCheck is None:
self.__batchCheck = process(
[configuration.executables.GIT, 'cat-file', '--batch-check'],
stdin=PIPE,
stdout=PIPE,
stderr=STDOUT,
cwd=self.path)
def check_input(args, stdin, **kwargs):
assert isinstance(stdin, str)
child = process(args, stdin=PIPE, **kwargs)
stdout, stderr = child.communicate(stdin)
if child.returncode != 0:
raise CalledProcessError(child.returncode, args, None)
def lstree(remote, regexp=None):
if regexp: name_check = lambda item: bool(regexp.match(item[1]))
else: name_check = lambda item: True
git = process([configuration.executables.GIT, 'ls-remote', remote], stdout=PIPE, stderr=PIPE, cwd="/tmp")
stdout, stderr = git.communicate()
if git.returncode == 0: return filter(name_check, (line.split() for line in stdout.splitlines()))
else: raise Exception, "'git ls-remote' failed: %s" % stderr.strip()
def readObject(repository_path, object_type, object_sha1):
git = process([
configuration.executables.GIT, 'cat-file', object_type, object_sha1
],
stdout=PIPE,
stderr=PIPE,
cwd=repository_path)
stdout, stderr = git.communicate()
if git.returncode == 0: return stdout
else: raise Exception, "'git cat-file' failed: %s" % stderr.strip()
def __compact(self):
import syntaxhighlight
now = time.time()
max_age_uncompressed = 7 * 24 * 60 * 60
max_age_compressed = 90 * 24 * 60 * 60
uncompressed_count = 0
compressed_count = 0
purged_paths = []
db = dbutils.Database()
cursor = db.cursor()
cursor.execute("CREATE TEMPORARY TABLE purged (sha1 CHAR(40) PRIMARY KEY)")
cache_path = configuration.services.HIGHLIGHT["cache_dir"]
for section in sorted(os.listdir(cache_path)):
if len(section) == 2:
for filename in os.listdir("%s/%s" % (cache_path, section)):
fullname = "%s/%s/%s" % (cache_path, section, filename)
age = now - os.stat(fullname).st_mtime
if len(filename) > 38 and filename[38] == "." and filename[39:] in syntaxhighlight.LANGUAGES:
if age > max_age_uncompressed:
self.debug("compressing: %s/%s" % (section, filename))
worker = process(["/bin/bzip2", fullname])
worker.wait()
compressed_count += 1
else:
uncompressed_count += 1
elif len(filename) > 42 and filename[38] == "." and filename[-4] == "." and filename[39:-4] in syntaxhighlight.LANGUAGES:
if filename.endswith(".bz2"):
if age > max_age_compressed:
self.debug("purging: %s/%s" % (section, filename))
cursor.execute("INSERT INTO purged (sha1) VALUES (%s)", (section + filename[:-4],))
purged_paths.append(fullname)
else:
compressed_count += 1
elif filename.endswith(".ctx"):
self.debug("deleting context file: %s/%s" % (section, filename))
os.unlink(fullname)
self.debug("uncompressed=%d / compressed=%d / purged=%d" % (uncompressed_count, compressed_count, len(purged_paths)))
if purged_paths:
for path in purged_paths: os.unlink(path)
cursor.execute("DELETE FROM codecontexts USING purged WHERE codecontexts.sha1=purged.sha1")
db.commit()
db.close()
def revparse(self, name):
git = process([
configuration.executables.GIT, 'rev-parse', '--verify', '--quiet',
name
],
stdout=PIPE,
stderr=PIPE,
cwd=self.path)
stdout, stderr = git.communicate()
if git.returncode == 0: return stdout.strip()
else:
raise GitError("'git rev-parse' failed: %s" % stderr.strip(),
ref=name,
repository=self)
def lstree(remote, regexp=None):
if regexp: name_check = lambda item: bool(regexp.match(item[1]))
else: name_check = lambda item: True
git = process([configuration.executables.GIT, 'ls-remote', remote],
stdout=PIPE,
stderr=PIPE,
cwd="/tmp")
stdout, stderr = git.communicate()
if git.returncode == 0:
return filter(name_check,
(line.split() for line in stdout.splitlines()))
else:
raise Exception, "'git ls-remote' failed: %s" % stderr.strip()
def lint(code, comment):
# write input code to tmp file
fp = temp_file.format(tmp=temp_dir)
with open(fp, "w") as fh:
fh.write(code)
fh.close()
# lint it
out, err = process(["pep8", "--count", abspath(fp)],
stdout=PIPE,
stderr=STDOUT).communicate()
if out == b'':
comment("Good adherence to PEP8")
return 1
else:
comment(out)
return 1 - len(str(out).split("\n")) / len(code.split("\n"))
def runCustom(self, cwd, command, *arguments, **kwargs):
argv = [configuration.executables.GIT, command]
argv.extend(arguments)
stdin_data = kwargs.get("input")
if stdin_data is None: stdin = None
else: stdin = PIPE
env = {}
env.update(environ)
env.update(kwargs.get("env", {}))
if "GIT_DIR" in env: del env["GIT_DIR"]
git = process(argv, stdin=stdin, stdout=PIPE, stderr=PIPE, cwd=cwd, env=env)
stdout, stderr = git.communicate(stdin_data)
if kwargs.get("check_errors", True):
if git.returncode == 0:
if kwargs.get("include_stderr", False):
return stdout + stderr
else:
return stdout
else: raise Exception, "'%s' failed: %s (in %s)" % (" ".join(argv), stderr.strip(), cwd)
else:
return git.returncode, stdout, stderr
def mergebase(self, commit_or_commits, db=None):
if db and isinstance(commit_or_commits, Commit):
cursor = db.cursor()
cursor.execute("SELECT mergebase FROM mergebases WHERE commit=%s", (commit_or_commits.getId(db),))
try:
return cursor.fetchone()[0]
except:
result = self.mergebase(commit_or_commits)
cursor.execute("INSERT INTO mergebases (commit, mergebase) VALUES (%s, %s)", (commit_or_commits.getId(db), result))
return result
try: sha1s = commit_or_commits.parents
except: sha1s = map(str, commit_or_commits)
assert len(sha1s) >= 2
git = process([configuration.executables.GIT, 'merge-base'] + sha1s,
stdout=PIPE, stderr=PIPE, cwd=self.path)
stdout, stderr = git.communicate()
if git.returncode == 0: return stdout.strip()
else: raise Exception, "'git merge-base' failed: %s" % stderr.strip()
def run(self):
try:
batch = process(
[configuration.executables.GIT, 'cat-file', '--batch'],
stdin=PIPE,
stdout=PIPE,
stderr=STDOUT,
cwd=self.repository.path)
stdout, stderr = batch.communicate("\n".join(self.sha1s.keys()) +
"\n")
gitobjects = []
for sha1, commit_id in self.sha1s.items():
line, stdout = stdout.split("\n", 1)
try:
object_sha1, object_type, object_size = line.split(" ")
except:
raise Exception, "unexpected header line: %r" % line
assert object_sha1 == sha1, "%s != %s" % (object_sha1, sha1)
assert object_type == "commit"
object_size = int(object_size)
object_data = stdout[:object_size]
stdout = stdout[object_size + 1:]
gitobjects.append(
(GitObject(object_sha1, object_type, object_size,
object_data), commit_id))
self.gitobjects = gitobjects
except:
self.error = format_exc()
def __compact(self):
import syntaxhighlight
now = time.time()
max_age_uncompressed = 7 * 24 * 60 * 60
max_age_compressed = 90 * 24 * 60 * 60
uncompressed_count = 0
compressed_count = 0
purged_paths = []
db = dbutils.Database()
cursor = db.cursor()
cursor.execute("CREATE TEMPORARY TABLE purged (sha1 CHAR(40) PRIMARY KEY)")
cache_path = configuration.services.HIGHLIGHT["cache_dir"]
for section in sorted(os.listdir(cache_path)):
if len(section) == 2:
for filename in os.listdir("%s/%s" % (cache_path, section)):
fullname = "%s/%s/%s" % (cache_path, section, filename)
age = now - os.stat(fullname).st_mtime
if len(filename) > 38 and filename[38] == "." and filename[39:] in syntaxhighlight.LANGUAGES:
if age > max_age_uncompressed:
self.debug("compressing: %s/%s" % (section, filename))
worker = process(["/bin/bzip2", fullname])
worker.wait()
compressed_count += 1
else:
uncompressed_count += 1
elif (
len(filename) > 42
and filename[38] == "."
and filename[-4] == "."
and filename[39:-4] in syntaxhighlight.LANGUAGES
):
if filename.endswith(".bz2"):
if age > max_age_compressed:
self.debug("purging: %s/%s" % (section, filename))
cursor.execute("INSERT INTO purged (sha1) VALUES (%s)", (section + filename[:-4],))
purged_paths.append(fullname)
else:
compressed_count += 1
elif filename.endswith(".ctx"):
self.debug("deleting context file: %s/%s" % (section, filename))
os.unlink(fullname)
self.debug(
"uncompressed=%d / compressed=%d / purged=%d"
% (uncompressed_count, compressed_count, len(purged_paths))
)
if purged_paths:
for path in purged_paths:
os.unlink(path)
cursor.execute("DELETE FROM codecontexts USING purged WHERE codecontexts.sha1=purged.sha1")
db.commit()
db.close()
def __compact(self):
import syntaxhighlight
cache_dir = configuration.services.HIGHLIGHT["cache_dir"]
if not os.path.isdir(cache_dir):
# Newly installed system that hasn't highlighted anything.
return 0, 0, 0, 0
self.info("cache compacting started")
now = time.time()
max_age_uncompressed = 7 * 24 * 60 * 60
max_age_compressed = 90 * 24 * 60 * 60
uncompressed_count = 0
compressed_count = 0
purged_paths = []
db = dbutils.Database.forSystem()
cursor = db.cursor()
cursor.execute(
"CREATE TEMPORARY TABLE purged (sha1 CHAR(40) PRIMARY KEY)")
cursor.execute(
"INSERT INTO purged (sha1) SELECT DISTINCT sha1 FROM codecontexts"
)
for section in sorted(os.listdir(cache_dir)):
if len(section) == 2:
for filename in os.listdir("%s/%s" % (cache_dir, section)):
fullname = "%s/%s/%s" % (cache_dir, section, filename)
age = now - os.stat(fullname).st_mtime
parts = filename.split(".")
if len(parts) < 2 \
or len(parts[0]) != 38 \
or parts[1] not in syntaxhighlight.LANGUAGES:
os.unlink(fullname)
continue
sha1 = section + parts[0]
if parts[-1] == "bz2":
if age > max_age_compressed:
self.debug("purging: %s/%s" %
(section, filename))
purged_paths.append(fullname)
else:
cursor.execute(
"DELETE FROM purged WHERE sha1=%s",
(sha1, ))
compressed_count += 1
elif parts[-1] == "ctx":
self.debug("deleting context file: %s/%s" %
(section, filename))
os.unlink(fullname)
else:
cursor.execute("DELETE FROM purged WHERE sha1=%s",
(sha1, ))
if age > max_age_uncompressed:
self.debug("compressing: %s/%s" %
(section, filename))
worker = process(["/bin/bzip2", fullname])
worker.wait()
compressed_count += 1
else:
uncompressed_count += 1
self.info(
"cache compacting finished: uncompressed=%d / compressed=%d / purged=%d"
% (uncompressed_count, compressed_count, len(purged_paths)))
if purged_paths:
for path in purged_paths:
os.unlink(path)
cursor.execute("SELECT COUNT(*) FROM purged")
purged_contexts = cursor.fetchone()[0]
cursor.execute("""DELETE
FROM codecontexts
WHERE sha1 IN (SELECT sha1
FROM purged)""")
db.commit()
db.close()
return uncompressed_count, compressed_count, len(
purged_paths), purged_contexts
def readObject(repository_path, object_type, object_sha1):
git = process([configuration.executables.GIT, 'cat-file', object_type, object_sha1], stdout=PIPE, stderr=PIPE, cwd=repository_path)
stdout, stderr = git.communicate()
if git.returncode == 0: return stdout
else: raise Exception, "'git cat-file' failed: %s" % stderr.strip()
def iscommit(self, name):
git = process([configuration.executables.GIT, 'cat-file', '-t', name],
stdout=PIPE, stderr=PIPE, cwd=self.path)
stdout, stderr = git.communicate()
if git.returncode == 0: return stdout.strip() == "commit"
else: return False
def revparse(self, name):
git = process([configuration.executables.GIT, 'rev-parse', '--verify', '--quiet', name],
stdout=PIPE, stderr=PIPE, cwd=self.path)
stdout, stderr = git.communicate()
if git.returncode == 0: return stdout.strip()
else: raise Exception, "'git rev-parse' failed: %s" % stderr.strip()
def branch(self, name, startpoint):
git = process([configuration.executables.GIT, 'branch', name, startpoint],
stdout=PIPE, stderr=PIPE, cwd=self.path)
stdout, stderr = git.communicate()
if git.returncode != 0: raise Exception, stderr
def __startBatchCheck(self):
if self.__batchCheck is None:
self.__batchCheck = process([configuration.executables.GIT, 'cat-file', '--batch-check'],
stdin=PIPE, stdout=PIPE, stderr=STDOUT, cwd=self.path)
def __compact(self):
import syntaxhighlight
cache_dir = configuration.services.HIGHLIGHT["cache_dir"]
if not os.path.isdir(cache_dir):
# Newly installed system that hasn't highlighted anything.
return 0, 0, 0, 0
self.info("cache compacting started")
now = time.time()
max_age_uncompressed = 7 * 24 * 60 * 60
max_age_compressed = 90 * 24 * 60 * 60
uncompressed_count = 0
compressed_count = 0
purged_paths = []
db = dbutils.Database.forSystem()
cursor = db.cursor()
cursor.execute("CREATE TEMPORARY TABLE purged (sha1 CHAR(40) PRIMARY KEY)")
cursor.execute("INSERT INTO purged (sha1) SELECT DISTINCT sha1 FROM codecontexts")
for section in sorted(os.listdir(cache_dir)):
if len(section) == 2:
for filename in os.listdir("%s/%s" % (cache_dir, section)):
fullname = "%s/%s/%s" % (cache_dir, section, filename)
age = now - os.stat(fullname).st_mtime
parts = filename.split(".")
if len(parts) < 2 \
or len(parts[0]) != 38 \
or parts[1] not in syntaxhighlight.LANGUAGES:
os.unlink(fullname)
continue
sha1 = section + parts[0]
if parts[-1] == "bz2":
if age > max_age_compressed:
self.debug("purging: %s/%s" % (section, filename))
purged_paths.append(fullname)
else:
cursor.execute("DELETE FROM purged WHERE sha1=%s", (sha1,))
compressed_count += 1
elif parts[-1] == "ctx":
self.debug("deleting context file: %s/%s" % (section, filename))
os.unlink(fullname)
else:
cursor.execute("DELETE FROM purged WHERE sha1=%s", (sha1,))
if age > max_age_uncompressed:
self.debug("compressing: %s/%s" % (section, filename))
worker = process(["/bin/bzip2", fullname])
worker.wait()
compressed_count += 1
else:
uncompressed_count += 1
self.info("cache compacting finished: uncompressed=%d / compressed=%d / purged=%d"
% (uncompressed_count, compressed_count, len(purged_paths)))
if purged_paths:
for path in purged_paths: os.unlink(path)
cursor.execute("SELECT COUNT(*) FROM purged")
purged_contexts = cursor.fetchone()[0]
cursor.execute("""DELETE
FROM codecontexts
WHERE sha1 IN (SELECT sha1
FROM purged)""")
db.commit()
db.close()
return uncompressed_count, compressed_count, len(purged_paths), purged_contexts
def __compact(self):
import syntaxhighlight
cache_dir = configuration.services.HIGHLIGHT["cache_dir"]
if not os.path.isdir(cache_dir):
# Newly installed system that hasn't highlighted anything.
return 0, 0, 0, 0
self.info("cache compacting started")
now = time.time()
max_age_uncompressed = 7 * 24 * 60 * 60
max_age_compressed = 90 * 24 * 60 * 60
uncompressed_count = 0
compressed_count = 0
purged_paths = []
db = dbutils.Database()
cursor = db.cursor()
cursor.execute(
"CREATE TEMPORARY TABLE purged (sha1 CHAR(40) PRIMARY KEY)")
cursor.execute(
"INSERT INTO purged (sha1) SELECT DISTINCT sha1 FROM codecontexts"
)
for section in sorted(os.listdir(cache_dir)):
if len(section) == 2:
for filename in os.listdir("%s/%s" % (cache_dir, section)):
fullname = "%s/%s/%s" % (cache_dir, section, filename)
age = now - os.stat(fullname).st_mtime
if len(filename
) > 38 and filename[38] == "." and filename[
39:] in syntaxhighlight.LANGUAGES:
cursor.execute("DELETE FROM purged WHERE sha1=%s",
(section + filename[:38], ))
if age > max_age_uncompressed:
self.debug("compressing: %s/%s" %
(section, filename))
worker = process([find_bzip2(), fullname])
worker.wait()
compressed_count += 1
else:
uncompressed_count += 1
elif len(filename
) > 41 and filename[38] == "." and filename[
-4] == "." and filename[
39:-4] in syntaxhighlight.LANGUAGES:
if filename.endswith(".bz2"):
if age > max_age_compressed:
self.debug("purging: %s/%s" %
(section, filename))
purged_paths.append(fullname)
else:
cursor.execute(
"DELETE FROM purged WHERE sha1=%s",
(section + filename[:38], ))
compressed_count += 1
elif filename.endswith(".ctx"):
self.debug("deleting context file: %s/%s" %
(section, filename))
os.unlink(fullname)
else:
os.unlink(fullname)
self.info(
"cache compacting finished: uncompressed=%d / compressed=%d / purged=%d"
% (uncompressed_count, compressed_count, len(purged_paths)))
if purged_paths:
for path in purged_paths:
os.unlink(path)
cursor.execute("SELECT COUNT(*) FROM purged")
purged_contexts = cursor.fetchone()[0]
cursor.execute(
"DELETE FROM codecontexts USING purged WHERE codecontexts.sha1=purged.sha1"
)
db.commit()
db.close()
return uncompressed_count, compressed_count, len(
purged_paths), purged_contexts