def run(self, parse_now=True):
"""Run git-fast-export."""
import_marks = self.write_marks()
export_marks = p4gf_tempfile.new_temp_file(prefix='fe-marks-')
# Note that we do not ask Git to attempt to detect file renames or
# copies, as this seems to lead to several bugs, including one that
# loses data. For now, the safest option is to translate the file
# operations exactly as they appear in the commit. This also makes the
# round-trip conversion safer.
cmd = ['git', 'fast-export', '--no-data']
if self.ctx.find_copy_rename_enabled:
cmd.extend(self.ctx.find_copy_rename_args)
cmd.append("--import-marks={}".format(import_marks.name))
cmd.append("--export-marks={}".format(export_marks.name))
if self.last_old_commit:
cmd.append("{}..{}".format(self.last_old_commit, self.last_new_commit))
elif isinstance(self.last_new_commit, list):
cmd.extend(list(set(self.last_new_commit)))
else:
cmd.append(self.last_new_commit)
LOG.debug('cmd={}'.format(cmd))
try:
# work around pylint bug where it doesn't know check_output() returns encoded bytes
result = p4gf_proc.popen_binary(cmd)
self.script = result['out']
self.read_marks(export_marks)
if parse_now:
self.parse_commands()
finally:
import_marks.close()
export_marks.close()
def write_marks(self):
"""Write a text file with list of every known commit sha1.
"Known" here means our Git Fusion knows about it and it has been
copied to Perforce.".
"""
log = LOG.getChild('marks')
marksfile = p4gf_tempfile.new_temp_file(prefix='fastexport-')
sha1_list = p4gf_object_type.known_commit_sha1_list(self.ctx)
# If configured to run unpacked, do so. Even to the point of unpacking
# incoming packfiles. This allows for some time optimizations at the
# (great!) expense of disk space.
if not self.ctx.git_autopack:
p4gf_git.unpack_objects()
# Ensure hashes are unique and refer to existing objects.
sha1_list = _prune_missing_objects(sha1_list, self.ctx.repo)
mark_num = 0
for sha1 in sha1_list:
# Don't tell git-fast-export about last_new_commit if we want to
# force git-fast-export to export it.
if self.force_export_last_new_commit and sha1 == self.last_new_commit:
continue
mark_num += 1
content = ":{} {}\n".format(mark_num, sha1)
marksfile.write(content.encode())
log.debug(content)
marksfile.flush()
return marksfile
def _create_database(self):
"""Use sqlite3 maintain separate lists of unique MD5 and tree paths"""
db_file = p4gf_tempfile.new_temp_file(prefix='repo_size_',
suffix='.db',
delete=False)
self.sql_db = sqlite3.connect(database=db_file.name,
isolation_level="EXCLUSIVE")
self.db_file_name = db_file.name
self.sql_db.execute("PRAGMA synchronous = OFF")
self.sql_db.execute("CREATE TABLE md5 (key TEXT PRIMARY KEY)")
self.sql_db.execute("CREATE TABLE tree (key TEXT PRIMARY KEY)")
def __init__(self, ctx):
self.ctx = ctx
self.script = p4gf_tempfile.new_temp_file(prefix='fastimport-')
self.timezone = ctx.timezone
self.__tzname = None
self.project_root_path_length = len(ctx.contentlocalroot)
self._line_count = 0
self._byte_count = 0
self.username_map = dict()
self.usermap = p4gf_usermap.UserMap(ctx.p4gf,
ctx.email_case_sensitivity)
self.lfs_files = {}
self.text_pointers = []
def __enter__(self):
"""Enter the context."""
if not self.__temp:
self.__temp = p4gf_tempfile.new_temp_file(mode='w+',
encoding='UTF-8',
prefix='http-output-',
delete=False)
self.__stdout = sys.stdout
self.__stderr = sys.stderr
sys.stdout.flush()
sys.stderr.flush()
sys.stdout = self.__temp
sys.stderr = self.__temp
LOG.debug("stdout/stderr redirecting to %s...", self.__temp.name)
return self
def spec_file_path(self):
"""Lazy-create, then reuse over and over, a single temp file to hold
our fake changelist spec.
"""
if self._spec_file_path:
return self._spec_file_path
# Don't bother if we're just PASS/FAIL/None
if not self.needs_spec_file():
self._spec_file_path = ''
return self._spec_file_path
self._spec_file = p4gf_tempfile.new_temp_file(
prefix='preflight-commit-', delete=False)
self._spec_file_path = self._spec_file.name
return self._spec_file_path
def read_request_data(environ):
"""Read the incoming request data to a temporary file.
Handles both WSGI and CGI environments.
:param dict environ: WSGI request environment.
:return: name of temporary file containing request data.
"""
# Read the input from the client.
incoming = environ['wsgi.input']
stdin_fobj = p4gf_tempfile.new_temp_file(prefix='http-client-input-', delete=False)
LOG.debug('read_request_data() writing stdin to %s', stdin_fobj.name)
if is_cgi():
# Running in CGI mode as a WSGI application. In a hosted CGI
# environment, the matter of content-length and transfer-encoding
# is handled for us by the server. We simply read the input until
# the EOF is encountered.
shutil.copyfileobj(incoming, stdin_fobj)
else:
# Running within the WSGI simple server.
# For more information on the idiosyncrasies within WSGI 1.0, see
# http://blog.dscpl.com.au/2009/10/details-on-wsgi-10-amendmentsclarificat.html
try:
content_length = int(environ.get('CONTENT_LENGTH', 0))
except ValueError:
content_length = 0
method = environ['REQUEST_METHOD']
if TE_HEADER in environ and environ[TE_HEADER] == 'chunked':
reader = ChunkedTransferReader(stdin_fobj)
reader.read(incoming)
elif content_length and (method == "POST" or method == "PUT"):
# To avoid blocking forever reading input from the client, must
# read _only_ the number of bytes specified in the request
# (which happens to permit HTTP/1.1 keep-alive connections).
while content_length > 0:
length = min(65536, content_length)
buf = incoming.read(length)
if not buf:
break
stdin_fobj.write(buf)
content_length -= len(buf)
stdin_fobj.close()
return stdin_fobj.name
def read_p4changes_from_depot_path(ctx):
"""Read 'p4 changes -l' text and scan for changlists with "push state: complete"."""
tmp_file = p4gf_tempfile.new_temp_file(mode='w', prefix='annotate-changes', delete=False)
regex = re.compile(r'^', re.MULTILINE)
for branch_chunk in ctx.iter_branch_chunks():
with ctx.switched_to_union(branch_chunk):
p4_result = ctx.p4run('changes', '-l', '//{}/...'.format(ctx.p4.client),
log_warnings=logging.WARN)
for rr in p4_result:
tmp_file.write("Change {} on \n".format(rr['change']))
desc = re.sub(regex, ' ', rr['desc'])
tmp_file.write("{}\n".format(desc))
tmp_file.close()
complete_change_nums = read_p4changes_from_text_file(tmp_file.name)
if DEBUG:
print(_("p4 changes data for this repo written to: '{}'.").format(tmp_file.name))
else:
os.remove(tmp_file.name)
return complete_change_nums
def read_p4files_from_depot_path(p4, repo):
"""Write the depot path for the repo's commits to a temporary file.
Then call read_p4files_from_file_path."""
tmp_file = p4gf_tempfile.new_temp_file(mode='w', prefix='annotate-files', delete=False)
commits_path = '//.git-fusion/objects/repos/{}/commits/...'.format(repo)
p4_result = p4.run('files', '-e', commits_path)
for rr in p4_result:
depot_path = rr.get('depotFile')
if not depot_path:
continue
tmp_file.write("{}\n".format(depot_path))
tmp_file.close()
d = read_p4files_from_file_path(tmp_file.name)
if DEBUG:
print(_("p4 files data for this repo written to: '{}'.").format(tmp_file.name))
else:
os.remove(tmp_file.name)
return d
def get_gitlogs_from_git(git_dir):
"""Create file with git log output.
git log --graph --decorate --oneline --all
"""
tmp_file = p4gf_tempfile.new_temp_file(mode='w', prefix='annotate-gitlog', delete=False)
if DEBUG:
print(_("get_gitlogs_from_git: gitdir='{}'.").format(git_dir))
if not os.path.exists(git_dir):
LOG.warning("Git repository {} missing stopping.".format(git_dir))
print(_("Git repository {} missing stopping.").format(git_dir))
sys.exit(1)
# it's not the end of the world if the git repo disappears, just recreate it
else:
# Ensure the pack config settings are set to desired values.
git_dir = '--git-dir={}'.format(git_dir)
cmd = ['git', git_dir, 'log', '--graph', '--decorate', '--oneline', '--all']
subprocess.call(cmd, stdout=tmp_file)
tmp_file.close()
if DEBUG:
print(_("git logs data for this repo written to: '{}'.").format(tmp_file.name))
return tmp_file.name
def run_fast_import(self):
"""Run git-fast-import to create the git commits.
Returns: a list of commits. Each line is formatted as
a change number followed by the SHA1 of the commit.
The returned list is also written to a file called marks.
"""
with Timer(OVERALL):
with Timer(RUN):
LOG.debug("running git fast-import")
# tell git-fast-import to export marks to a temp file
self.script.flush()
marks_file = p4gf_tempfile.new_temp_file(prefix='marks-')
try:
cmd = [
'git', 'fast-import', '--quiet',
'--export-marks=' + marks_file.name
]
ec = p4gf_proc.wait(cmd, stdin=self.script.name)
if ec:
_log_crash_report(
'git-fast-import failed for {}'.format(
self.ctx.config.repo_name))
raise CalledProcessError(ec, NTR('git fast-import'))
# read the exported marks from file and return result
with open(marks_file.name, "r") as marksfile:
marks = [l.strip() for l in marksfile.readlines()]
if LOG.getChild('marks').isEnabledFor(logging.DEBUG3):
LOG.getChild('marks').debug3(
'git-fast-import returned marks ct={}\n'.format(
len(marks)) + '\n'.join(marks))
return marks
finally:
self.script.close()
marks_file.close()
