def _create_file(p4, client_name, local_path, file_content): """Create and submit a file. Write a file to the local Git Fusion workspace and then add and submit to Perforce. NOP if file already exists in Perforce after a 'p4 sync'. """ filename = os.path.basename(local_path) with p4gf_util.restore_client(p4, client_name): try: with p4.at_exception_level(p4.RAISE_NONE): # Sync the file and ensure we really have it. p4.run('sync', '-q', local_path) results = p4.run('have', local_path) if not results: LOG.debug("_write_file(): {} does not exist, will create...".format(local_path)) # Perms are probably read-only, need to remove before writing. if os.path.exists(local_path): os.remove(local_path) else: p4gf_util.ensure_parent_dir(local_path) with open(local_path, 'w') as mf: mf.write(file_content) desc = _("Creating initial '{filename}' file via p4gf_init.py")\ .format(filename=filename) with p4gf_util.NumberedChangelist(p4=p4, description=desc) as nc: nc.p4run('add', local_path) nc.submit() LOG.debug("_write_file(): successfully created {}".format(local_path)) _info(_("File '{path}' created.").format(path=local_path)) else: _info(_("File '{path}' already exists.").format(path=local_path)) except P4.P4Exception as e: LOG.warning('error setting up {file} file: {e}' .format(file=filename, e=str(e)))
def _install_tag(repo, fname): """Given the path of a tag copied from Perforce object cache, copy the tag to the repository, with the appropriate name and SHA1. There may be multiple lightweight tags associated with the same SHA1, in which case multiple tags will be created. Arguments: repo -- pygit2 repository fname -- clientFile attr for sync'd tag """ sha1 = fname[-42:].replace('/', '') LOG.debug("_install_tag() examining {}...".format(sha1)) with open(fname, 'rb') as f: contents = f.read() try: zlib.decompress(contents) # Must be an annotated tag... blob_path = os.path.join('.git', 'objects', sha1[:2], sha1[2:]) p4gf_util.ensure_parent_dir(blob_path) os.link(fname, blob_path) tag_obj = repo.get(sha1) tag_name = tag_obj.name LOG.debug("_install_tag() annotated tag {}".format(tag_name)) _create_tag_ref(repo, tag_name, sha1) except zlib.error: # Lightweight tags are stored simply as the tag name, but # there may be more than one name for a single SHA1. tag_names = contents.decode('UTF-8') for name in tag_names.splitlines(): LOG.debug("_install_tag() lightweight tag {}".format(name)) _create_tag_ref(repo, name, sha1)
def _configure_logger(config, section, name=None, ident=None): """Configure the named logger (or the root logger if name is None). Use provided settings, which likely came from _effective_config(). :param dict config: logger settings (will be modified). :param str section: name of logging section (e.g. general). :param str name: name of the logger to configure (defaults to root logger). :param str ident: syslog identity, if handler is 'syslog'. """ # pylint: disable=too-many-branches _deconfigure_logger(name) formatter = None if 'handler' in config: val = config.pop('handler') if val.startswith('syslog'): words = val.split(maxsplit=1) if len(words) > 1: handler = P4GFSysLogHandler(address=words[1], ident=ident) else: handler = P4GFSysLogHandler(ident=ident) formatter = P4GFSysLogFormatter() elif val == 'console': handler = logging.StreamHandler() else: sys.stderr.write( _('Git Fusion: unrecognized log handler: {}\n').format(val)) handler = logging.StreamHandler() elif 'filename' in config: fpath = config.pop('filename') p4gf_util.ensure_parent_dir(fpath) handler = logging.FileHandler(fpath, 'a', 'utf-8') if fpath.endswith('.xml'): formatter = XmlFormatter() _rotate_log_file(fpath, section, config) else: handler = logging.StreamHandler() # Always remove these fake logging levels fs = config.pop('format', None) dfs = config.pop('datefmt', None) config.pop(_max_size_mb_name, None) config.pop(_retain_count_name, None) fs = _include_process_id(config, fs) if formatter is None: # Build the formatter if one has not already been. formatter = BaseFormatter(fs, dfs) handler.setFormatter(formatter) logger = logging.getLogger(name) logger.propagate = False logger.addHandler(handler) logger.setLevel(config.pop('root').upper()) # Set the logging levels based on the remaining settings for key, val in config.items(): logging.getLogger(key).setLevel(val.upper())
def _create_tag_ref(repo, name, sha1): """Create a single tag reference in the repository.""" if not name or not sha1: LOG.warning("_create_tag_ref() invalid params: ({}, {})".format( name, sha1)) return if repo.get(sha1) is None: LOG.warning("_create_tag_ref() unknown object: {}".format(sha1)) return tag_file = os.path.join('.git', 'refs', 'tags', name) p4gf_util.ensure_parent_dir(tag_file) with open(tag_file, 'w') as f: f.write(sha1)
def _log_file_lock(name): """Acquire exclusive access to the lock for rotating the logs. :param str name: name of the logging section (e.g. general) """ path = "{home}/locks/log_{name}.lock".format(home=p4gf_const.P4GF_HOME, name=name) p4gf_util.ensure_parent_dir(path) fd = os.open(path, os.O_CREAT | os.O_WRONLY) fcntl.flock(fd, fcntl.LOCK_EX) os.write(fd, bytes(str(os.getpid()), "utf-8")) try: yield finally: os.close(fd)
def write_git_object_from_sha1(view_repo, object_sha1, target_path): """Utility to extract commit,tree,blob,tag objects from a git (packed/unpacked) object-store, and write git object to file path. The target_path will be written with a re-constructed git object extracted via pygit2 :param: view_repo pygit2 repository :param: object_sha1 sha1 in git object store :param: target_path path of new git object """ data = cat_file(object_sha1, view_repo) p4gf_util.ensure_parent_dir(target_path) try: with open(target_path, 'wb') as compressed: compress = zlib.compressobj() compressed.write(compress.compress(data)) compressed.write(compress.flush()) except Exception as e: LOG.exception("Failed to write sha1='%s' from repo to path '%s'", object_sha1, target_path) raise RuntimeError( _("Failed to write sha1='{sha1}' from repo to path {path}.\n{exception}" ).format(sha1=object_sha1, path=target_path, exception=e))
def _log_crash_report(errmsg): """Capture the fast-import crash report to a separate log file. :type errmsg: str :param errmsg: error message to write to Git Fusion log. """ log = logging.getLogger('failures') if not log.isEnabledFor(logging.ERROR): return log.error(errmsg) # For each crash report we find, dump its contents. # In theory we clean up after a crash so there should be only one. cwd = os.getcwd() for entry in os.listdir('.git'): if entry.startswith('fast_import_crash_'): report_path = os.path.join(cwd, '.git', entry) date = datetime.datetime.now() date_str = date.isoformat().replace(':', '').split('.')[0] log_path = p4gf_const.P4GF_FAILURE_LOG.format( P4GF_DIR=p4gf_const.P4GF_HOME, prefix='git-fast-import-', date=date_str) p4gf_util.ensure_parent_dir(log_path) # Compress the file to help in preserving its integrity. gz_path = log_path + '.gz' log.error( 'Compressing fast-import crash report to {}'.format(gz_path)) with open(report_path, 'rb') as fin, gzip.open(gz_path, 'wb') as fout: while True: b = fin.read(100 * 1024) if not len(b): break fout.write(b) fout.flush()
def _add_tag(ctx, name, sha1, edit_list, add_list): """Add a tag to the object cache. If adding another lightweight tag that refers to the same object, edit the file rather than add. """ LOG.debug("_add_tag() adding tag {}".format(name)) fpath = os.path.join(ctx.gitlocalroot, _client_path(ctx, sha1)) if os.path.exists(fpath): # Overwriting an existing tag? Git prohibits that. # But, another lightweight tag of the same object is okay. # Sanity check if this is a lightweight tag of an annotated # tag and reject with a warning. with open(fpath, 'rb') as f: contents = f.read() try: zlib.decompress(contents) return except zlib.error: pass # it's a lightweight tag, just append the name with open(fpath, 'ab') as f: f.write(b'\n') f.write(name.encode('UTF-8')) edit_list.append(fpath) else: obj = ctx.repo.get(sha1) if obj.type == pygit2.GIT_OBJ_TAG: LOG.debug("_add_tag() annotated tag {}".format(name)) p4gf_git.write_git_object_from_sha1(ctx.repo, sha1, fpath) else: # Lightweight tags can be anything: commit, tree, blob LOG.debug("_add_tag() lightweight tag {}".format(name)) p4gf_util.ensure_parent_dir(fpath) with open(fpath, 'wb') as f: f.write(name.encode('UTF-8')) add_list.append(fpath)
def flush(self): """compress the last file, hash it and stick it in the repo Now that we've got the complete file contents, the header can be created and used along with the spooled content to create the sha1 and zlib compressed blob content. Finally that is written into the .git/objects dir. """ if not self.rev: return size = self.tempfile.tell() if size > 0 and self.rev.is_symlink(): # p4 print adds a trailing newline, which is no good for symlinks. self.tempfile.seek(-1, 2) b = self.tempfile.read(1) if b[0] == 10: size = self.tempfile.truncate(size - 1) self.tempfile.seek(0) self.total_byte_count += size self.printed_rev_count += 1 compressed = tempfile.NamedTemporaryFile(delete=False, dir=self.tempdir, prefix='p2g-blob-') compress = zlib.compressobj() # pylint doesn't understand dynamic definition of sha1 in hashlib # pylint: disable=E1101 sha1 = hashlib.sha1() # pylint:disable=W1401 # disable complaints about the null. We need that. # add header first header = ("blob " + str(size) + "\0").encode() compressed.write(compress.compress(header)) sha1.update(header) # then actual contents chunksize = 4096 while True: chunk = self.tempfile.read(chunksize) if chunk: compressed.write(compress.compress(chunk)) sha1.update(chunk) else: break # pylint: enable=E1101 compressed.write(compress.flush()) compressed.close() digest = sha1.hexdigest() self.rev.sha1 = digest blob_path_tuple = _sha1_to_blob_path_tuple(self.rev.sha1) if not os.path.exists(blob_path_tuple.path): if not os.path.exists(blob_path_tuple.dir): os.makedirs(blob_path_tuple.dir) shutil.move(compressed.name, blob_path_tuple.path) else: os.remove(compressed.name) #self.revs.append(self.rev) symlink_path = _depot_rev_to_symlink( depot_path = self.rev.depot_path , rev = self.rev.revision , symlink_dir = self.symlink_dir ) p4gf_util.ensure_parent_dir(symlink_path) e = os.path.islink(symlink_path) if not e: os.symlink(blob_path_tuple.path, symlink_path) _debug3('Printed {e} {blob} @{ch:<5} {rev:<50} {symlink}' , blob = blob_path_tuple.path , symlink = symlink_path , rev = self.rev.rev_path() , ch = self.rev.change , e = 'e' if e else ' ') self.rev = None