def refresh_thread():
if self.g:
self.list[0]=list(self.g.untracked_files)
#self.view['tableview1'].reload()
self.list[1]=porcelain.status(self.g.path).unstaged
#self.view['tableview1'].reload()
self.list[2]=porcelain.status(self.g.path).staged['add']
#self.view['tableview1'].reload()
self.list[3]=porcelain.status(self.g.path).staged['delete']
#self.view['tableview1'].reload()
self.list[4]=porcelain.status(self.g.path).staged['modify']
#self.view['tableview1'].reload()
self.list[5]=list(self.g.tracked_files-set(itertools.chain(*self.list[1:4])))
def test_status(self):
"""Integration test for `status` functionality."""
# Commit a dummy file then modify it
fullpath = os.path.join(self.repo.path, 'foo')
with open(fullpath, 'w') as f:
f.write('origstuff')
porcelain.add(repo=self.repo.path, paths=['foo'])
porcelain.commit(repo=self.repo.path, message=b'test status',
author=b'', committer=b'')
# modify access and modify time of path
os.utime(fullpath, (0, 0))
with open(fullpath, 'wb') as f:
f.write(b'stuff')
# Make a dummy file and stage it
filename_add = 'bar'
fullpath = os.path.join(self.repo.path, filename_add)
with open(fullpath, 'w') as f:
f.write('stuff')
porcelain.add(repo=self.repo.path, paths=filename_add)
results = porcelain.status(self.repo)
self.assertEqual(results.staged['add'][0], filename_add.encode('ascii'))
self.assertEqual(results.unstaged, [b'foo'])
def test_get_untracked_paths(self):
with open(os.path.join(self.repo.path, '.gitignore'), 'w') as f:
f.write('ignored\n')
with open(os.path.join(self.repo.path, 'ignored'), 'w') as f:
f.write('blah\n')
with open(os.path.join(self.repo.path, 'notignored'), 'w') as f:
f.write('blah\n')
self.assertEqual(
set(['ignored', 'notignored', '.gitignore']),
set(porcelain.get_untracked_paths(self.repo.path, self.repo.path,
self.repo.open_index())))
self.assertEqual(set(['.gitignore', 'notignored']),
set(porcelain.status(self.repo).untracked))
self.assertEqual(set(['.gitignore', 'notignored', 'ignored']),
set(porcelain.status(self.repo, ignored=True)
.untracked))
def _confirm_dangerous():
repo = _get_repo()
status=porcelain.status(repo.path)
if any(status.staged.values()+status.unstaged):
force=raw_input('WARNING: there are uncommitted modified files and/or staged changes. These could be overwritten by this command. Continue anyway? [y/n] ')
if not force=='y':
raise Exception('User cancelled dangerous operation')
def test_status(self):
"""Integration test for `status` functionality."""
# Commit a dummy file then modify it
fullpath = os.path.join(self.repo.path, "foo")
with open(fullpath, "w") as f:
f.write("origstuff")
porcelain.add(repo=self.repo.path, paths=["foo"])
porcelain.commit(repo=self.repo.path, message=b"test status", author=b"", committer=b"")
# modify access and modify time of path
os.utime(fullpath, (0, 0))
with open(fullpath, "wb") as f:
f.write(b"stuff")
# Make a dummy file and stage it
filename_add = "bar"
fullpath = os.path.join(self.repo.path, filename_add)
with open(fullpath, "w") as f:
f.write("stuff")
porcelain.add(repo=self.repo.path, paths=filename_add)
results = porcelain.status(self.repo)
self.assertEqual(results.staged["add"][0], filename_add.encode("ascii"))
self.assertEqual(results.unstaged, [b"foo"])
def git_status(args):
if len(args) == 0:
repo = _get_repo()
status = porcelain.status(repo.repo)
print status
else:
print command_help['git_staged']
def commit(self,sender):
if list(itertools.chain(*porcelain.status(self.g.path).staged.itervalues())):
self.g=self._get_repo()
user=self.view['user'].text
email=self.view['email'].text
message=self.view['message'].text
author = "{0} <{1}>".format(user, email)
porcelain.commit(self.g.path,message,author,author)
console.hud_alert('committed')
self.view['message'].text=''
self.refresh()
else:
console.hud_alert('nothing to commit!',icon='error')
def git_status(args):
if len(args) == 0:
repo = _get_repo()
status = porcelain.status(repo.repo.path)
print('STAGED')
for k,v in status.staged.iteritems():
if v:
print(k,v)
print('UNSTAGED LOCAL MODS')
print(status.unstaged)
else:
print(command_help['status'])
def git_status(args):
if len(args) == 0:
repo = _get_repo()
status = porcelain.status(repo.repo.path)
print('STAGED')
for k, v in iteritems(status.staged):
if v:
print(k, v)
print('UNSTAGED LOCAL MODS')
print(status.unstaged)
else:
print(command_help['status'])
def git_status(args):
if len(args) == 0:
repo = _get_repo()
status = porcelain.status(repo.repo.path)
print 'STAGED'
for k,v in status.staged.iteritems():
if v:
print k,v
print 'UNSTAGED LOCAL MODS'
print status.unstaged
else:
print command_help['status']
def git_status(args):
if len(args) == 0:
repo = _get_repo()
status = porcelain.status(repo.repo.path)
print 'STAGED'
for k, v in status.staged.iteritems():
if v:
print k, v
print 'UNSTAGED LOCAL MODS'
print status.unstaged
else:
print command_help['status']
def gitDoMyTest(self):
skillName = 'FritzBox'
rep: Repo = Repo(f'skills/{skillName}')
self.logInfo(f'got {rep} in {rep.path}')
stat = status(rep)
self.logInfo(f'statstaged {stat.staged}')
self.logInfo(f'statuntrack {stat.untracked}')
self.logInfo(f'statunstag {stat.unstaged}')
rep.stage(stat.unstaged + stat.untracked)
self.logInfo(f'commit {commit(repo=rep, message="pushed by AliceSK")}')
self.logInfo(
f'push {push(repo=rep, remote_location=self.getRemote(), refspecs=b"master")}'
)
def commit(self, sender):
if list(
itertools.chain(
*porcelain.status(self.g.path).staged.itervalues())):
self.g = self._get_repo()
user = self.view['user'].text
email = self.view['email'].text
message = self.view['message'].text
author = "{0} <{1}>".format(user, email)
porcelain.commit(self.g.path, message, author, author)
console.hud_alert('committed')
self.view['message'].text = ''
self.refresh()
else:
console.hud_alert('nothing to commit!', icon='error')
def git_status(self):
from dulwich import porcelain as git
try:
repo = git.Repo.discover()
status = git.status(repo=repo)
except dulwich.errors.NotGitRepository:
raise ContextError("{} is not a valid git repository".format(
self.cwd))
for state in ('add', 'modify', 'delete'):
if status.staged[state]:
self.is_dirty = True
break
if status.unstaged:
self.is_dirty = True
self._git_status = status
def git_reset_repo_to_origin():
try:
repo = porcelain.Repo(database)
porcelain.fetch(repo)
tree_head_id = repo[repo[b'refs/heads/master'].tree].id
tree_origin_master_id = repo[
repo[b'refs/remotes/origin/master'].tree].id
store = repo.object_store
list_all_files_head = list_all_files(store, tree_head_id)
list_all_files_origin_master = list_all_files(store,
tree_origin_master_id)
deleted_files = list(
set(list_all_files_head) - set(list_all_files_origin_master))
# print(deleted_files)
if deleted_files != []:
for all in deleted_files:
file_path = os.path.join(database, all.decode('utf-8'))
os.remove(file_path)
status = porcelain.status(repo)
repo.stage(status.unstaged)
porcelain.commit(repo, message="delete files")
###working###
porcelain.reset(repo, "hard", treeish=b"refs/remotes/origin/master")
porcelain.clean(repo=repo, target_dir=database)
resolve_divergence()
########
return True
except MaxRetryError:
print('MaxRetryError')
return False
except ProtocolError:
print('ProtocolError')
return False
def _get_commit():
"""Return current head commit hash if you run inside git repository."""
if Reporter._get_package_version() != "local":
return "0" * 40
clean_status = porcelain.GitStatus(staged={
"delete": [],
"add": [],
"modify": []
},
unstaged=[],
untracked=[])
repo_path = str(Path(__file__).parents[2])
head = dulwich.repo.Repo(repo_path).head().decode()
if porcelain.status(repo_path) == clean_status:
return head
else:
return "%s (dirty)" % head
def update(repo_url, logger=None):
if logger:
logger.info("Updating docs for: %s", repo_url)
else:
print("Updating docs for: %s" % repo_url)
try:
shutil.rmtree(package_folder_name)
except:
pass
repo = porcelain.clone(package_git_url, package_folder_name)
refspecs = b"HEAD:refs/heads/%s" %package_git_branch
porcelain.pull(repo, package_git_url, refspecs = refspecs)
repo_url = repo_url.split("\n")[0]
repo_name = repo_url.split('/')[-1]
readme_md, readme_name = loadReadme(repo_url)
readme_md, readme_title = addTitleToReadme(readme_md, repo_url)
readme_md = addEditLink(readme_md, readme_name, repo_url)
updateMenu(readme_md, repo_name, repo)
updateReadmePage(readme_md, repo_name, repo)
updateLandingPage(readme_title, repo)
staged = porcelain.status(repo).staged
if (len(staged['add']) == 0) and (len(staged['modify']) == 0):
if logger:
logger.info("No changes to commit")
else:
print("No changes to commit")
else:
if logger:
logger.info("Commiting changes for %s", repo_name)
else:
print("Commiting changes for %s" % repo_name)
porcelain.commit(repo, b"Updated docs for %s" % repo_name)
porcelain.push(repo, package_git_url, refspecs = refspecs)
try:
shutil.rmtree(package_folder_name)
except Exception, err:
if logger:
logger.exception(err)
else:
print(err)
def test():
default_remote_location = 'https://*****:*****@[email protected]/OpenGitspace/meta'
repo_path = './data/OpenGitspace'
if is_git_repo(repo_path):
repo = Repo(repo_path)
else:
repo = Repo.init(repo_path)
branches = list_remote_branch(default_remote_location)
branches = list(filter(lambda x: x[0] == 'n', branches))
print(branches)
print('local_branches:', list_branch(repo))
create_head(repo)
test_branch = '000_test'
create_branch(repo, test_branch)
switch_branch(repo, test_branch)
pull_remote_branch(repo, default_remote_location, 'test')
push_local_branch(repo, test_branch, default_remote_location, test_branch)
status = porcelain.status(repo)
paths = status.untracked + status.unstaged
print(status)
def git_push_to_origin(ui, admin, file_list, message, worker_text):
# local_appdata = os.getenv('LOCALAPPDATA')
# credentials_file = os.path.join(os.getenv('LOCALAPPDATA'),"LaMA", "credentials","developer_credentials.txt")
if admin == True:
access_token = get_access_token('developer')
else:
access_token = get_access_token('user')
repo = porcelain.open_repo(database)
if admin == True:
status = porcelain.status(repo)
repo.stage(status.unstaged + status.untracked)
# print(status.unstaged)
# print(status.untracked)
if status.unstaged == [] and status.untracked == []:
# information_window("Es wurden keine Änderungen gefunden.")
return False
for file in file_list:
file_path = os.path.join(database, file)
porcelain.add(repo, paths=file_path)
ui.label.setText("{} (27%)".format(worker_text))
if admin == True:
mode = 'Administrator'
else:
mode = 'User'
porcelain.commit(repo,
message="New Update ({0}) - {1}".format(mode, message))
ui.label.setText("{} (84%)".format(worker_text))
porcelain.push(
repo,
"https://*****:*****@github.com/chrisiweb/lama_latest_update.git".
format(access_token), "master")
ui.label.setText("{} (100%)".format(worker_text))
return True
async def main(reactor):
git = Repo(".")
st = status(git)
if any(st.staged.values()) or st.unstaged:
print("unclean checkout; aborting")
raise SystemExit(1)
v = create_new_version(git)
if "--no-tag" in sys.argv:
print(v)
return
print("Existing tags: {}".format(" ".join(existing_tags(git))))
print("New tag will be {}".format(v))
# the "tag time" is seconds from the epoch .. we quantize these to
# the start of the day in question, in UTC.
now = datetime.now()
s = now.utctimetuple()
ts = int(
time.mktime(
time.struct_time(
(s.tm_year, s.tm_mon, s.tm_mday, 0, 0, 0, 0, s.tm_yday, 0))))
tag_create(
repo=git,
tag=v.encode("utf8"),
author=author.encode("utf8"),
message="Release {}".format(v).encode("utf8"),
annotated=True,
objectish=b"HEAD",
sign=author.encode("utf8"),
tag_time=ts,
tag_timezone=0,
)
print("Tag created locally, it is not pushed")
print("To push it run something like:")
print(" git push origin {}".format(v))
def get_sha_and_dirtiness(
prompt_on_dirty: bool = True) -> Optional[Tuple[str, bool]]:
try:
git_status = status()
except NotGitRepository:
return None
dirty = False
def to_str(string: Union[str, bytes]) -> str:
if isinstance(string, str):
return string
return string.decode("utf8")
def print_files(filenames: Iterable[Union[str, bytes]]) -> None:
print("\n".join(f" - {to_str(filename)}" for filename in filenames))
if git_status.untracked:
print("Those files are untracked:", file=stderr)
print_files(git_status.untracked)
dirty = True
if git_status.unstaged:
print("Those files are unstaged:", file=stderr)
print_files(git_status.unstaged)
dirty = True
if any(git_status.staged.values()):
print("Those files are uncommited:", file=stderr)
print_files(chain(*git_status.staged.values()))
dirty = True
if dirty:
print("Are you sure you want to continue [y/n]? ", end="")
answer = input()
if answer != "y":
exit(1)
repo = Repo(".")
sha = to_str(repo.head())
return sha, dirty
def status(ctx):
# Get status
st = gitp.status(ctx.obj['repo'])
added_new = (name.decode('utf-8') for name in st.staged['add'])
added_modified = (name.decode('utf-8') for name in st.staged['modify'])
added_deleted = (name.decode('utf-8') for name in st.staged['delete'])
unstaged = (name.decode('utf-8') for name in st.unstaged)
untracked = st.untracked
#click.echo(repr(st))
if True: # <-- will be for --long
click.echo('On branch ' + active_branch(ctx.obj['repo_obj']))
st_section(
'Changes to be committed',
('use "git reset HEAD <file>..." to unstage', ),
itertools.chain(
('deleted: ' + i for i in added_deleted),
('modified: ' + i for i in added_modified),
('new file: ' + i for i in added_new),
),
ANSI['green'],
)
st_section(
'Changes not staged for commit',
(
'use "git add <file>..." to update what will be committed',
'use "git checkout -- <file>..." to discard changes in working directory',
),
unstaged,
ANSI['red'],
)
st_section(
'Untracked files',
('use "git add <file>..." to include in what will be committed', ),
untracked,
ANSI['red'],
)
def staged_files(repository):
staged = porcelain.status(repository.dulwich_repo).staged
return staged['modify'] + staged['add']
def staged_files(repository):
staged = porcelain.status(repository.dulwich_repo).staged
return staged['modify'] + staged['add']
def main(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
base_dir=default_base_dir,
comment=default_comment,
persisted=False,
recursive=False,
no_strip_timestamp=False,
page=False):
"""
track_getconfig.py
Description:
Store running or persisted domain configuration in a local git
repository for auditing purposes.
Usage:
:::bash
$ mast contrib/track_getconfig.py --appliances <HOSTNAMES> --credentials <USER:PASS> --base-dir tmp/config
Parameters:
* `-a, --appliances` - The hostname(s), ip addresse(s), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases please see the comments in `hosts.conf` located
in `$MAST_HOME/etc/default`.
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password` and should be provided in a space-seperated list
if multiple are provided. If you would prefer to not use plain-text
passwords, you can use the output of
`$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-b, --base-dir`: The base directory where to store the downloaded
files. Files will actually be stored in a subdirectory of `base_dir`
named after the hostname in the form of `base_dir/<hostname>`
* `-p, --persisted`: If specified, the persisted configuration will
be retrieved as opposed to the running configuration (which is the
default)
* `-r, --recursive`: If specified, the configuration will be retrieved
recursively to the extent of the facilities provided by DataPower
* `-N, --no-strip-timestamp`: If specified, the timestamp will not be
stripped from the XML document, This is done because we are tracking
this information with git and stripping this out allows us to alert
on any changes in the repository as opposed to special-casing the
difference in timestamp.
* `-P, --page`: If specified, page the output when too long to display
at once
"""
base_dir = os.path.abspath(base_dir)
if not os.path.exists(base_dir):
os.makedirs(base_dir)
try:
repo = git.Repo(base_dir)
repo.head()
except NotGitRepository:
print "Initializing git repository"
git.init(base_dir)
git.add(base_dir)
git.commit(base_dir, message="Initial Commit")
except KeyError:
git.add(base_dir)
git.commit(base_dir, message="Initial Commit")
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout=timeout,
check_hostname=check_hostname)
for appliance in env.appliances:
appliance_directory = os.path.join(base_dir, appliance.hostname)
if not os.path.exists(appliance_directory):
os.mkdir(appliance_directory)
for domain in appliance.domains:
config = appliance.get_config(domain=domain,
recursive=recursive,
persisted=persisted)
config = config.pretty
if no_strip_timestamp:
pass
else:
config = re.sub(r"^.*?<dp:timestamp>.*?</dp:timestamp>.*?$",
r"",
config,
flags=re.MULTILINE)
filename = os.path.join(appliance_directory,
"{}.xml".format(domain))
with open(filename, "wb") as fout:
fout.write(config)
git.add(base_dir)
print git.status(base_dir)
git.commit(base_dir, message=comment)
tmp = StringIO()
git.show(base_dir, outstream=tmp)
tmp.seek(0)
out = highlight(tmp.read(), DiffLexer(), TerminalFormatter())
if page:
pprint.page(out)
else:
print out
def status(self, silent=False):
repo = self
msg = git.status(repo)
if not silent:
print(msg)
return msg
def main(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
base_dir=default_base_dir,
comment=default_comment,
page=False,
no_highlight_diff=False):
"""
track_autoconfig.py
Description:
Store persisted domain configuration in a local git repository
for auditing purposes.
Usage:
:::bash
$ mast contrib/track_autoconfig.py --appliances <HOSTNAMES> --credentials <USER:PASS> --base-dir tmp/config
Parameters:
* `-a, --appliances` - The hostname(s), ip addresse(s), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases please see the comments in `hosts.conf` located
in `$MAST_HOME/etc/default`.
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password` and should be provided in a space-seperated list
if multiple are provided. If you would prefer to not use plain-text
passwords, you can use the output of
`$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-b, --base-dir`: The base directory where to store the downloaded
files. Files will actually be stored in a subdirectory of `base_dir`
named after the hostname in the form of `base_dir/<hostname>`
* `-p, --page`: If specified, page the output when too long to display
at once
* `-N, --no-highlight-diff`: If specified, the output of the diff will
be syntax-highlighted
"""
base_dir = os.path.abspath(base_dir)
if not os.path.exists(base_dir):
os.makedirs(base_dir)
try:
repo = git.Repo(base_dir)
first_sha1 = repo.head()
print first_sha1
except NotGitRepository:
print "Initializing git repository"
git.init(base_dir)
git.add(base_dir)
first_sha1 = git.commit(base_dir, message="Initial Commit")
print first_sha1
except KeyError:
git.add(base_dir)
git.commit(base_dir, message="Initial Commit")
print
pull_autoconfig(appliances=appliances,
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
base_dir=base_dir)
git.add(base_dir)
print git.status(base_dir)
second_sha1 = git.commit(base_dir, message=comment)
print second_sha1
print "\n\nDIFF\n\n"
tmp = StringIO()
git.show(base_dir, outstream=tmp)
tmp.seek(0)
if no_highlight_diff:
out = tmp.read()
else:
out = highlight(tmp.read(), DiffLexer(), TerminalFormatter())
if page:
pprint.page(out)
else:
print out
def get_status(self): if os.path.exists(self.repo_path): print self.repo_path print p.status(self.repo_path) else: print "Repo does not exist"
def main():
print(porcelain.status())
def on_modified(event):
try:
log(f"hey buddy, {event.src_path} has been modified")
#log(aaa)
#log(os.getcwd())
os.chdir(path)
log(os.getcwd())
#log("curr dir changed")
#os.system("git status")
basename=os.path.basename(path)
if os.name == 'nt':
log("G:/My Drive/sync/company_notebook/"+basename)
if os.path.isdir("G:/My Drive/sync/company_notebook/"+basename):
log("its a dir")
os.system("rd /s /q \"G:/My Drive/sync/company_notebook/"+basename+"\"")
log("deleted")
os.system("mkdir \"G:/My Drive/sync/company_notebook/"+basename+"\"")
log("gdrive dir created")
os.system("git status > \"G:/My Drive/sync/company_notebook/"+basename+"/status\"")
log("status loged")
os.system("git diff > \"G:/My Drive/sync/company_notebook/"+basename+"/diff\"")
log("diff loged")
repo=porcelain.open_repo(path)
log("repo has been set")
status=porcelain.status(repo)
log("status got")
log('staged - added:', status.staged['add'], 'modified:', status.staged['modify'], 'deleted: ', status.staged['delete'])
log('unstaged: ', status.unstaged)
#log 'untracked: ', status.untracked
#untracked not working on version 0.9.7
#log 'untracked: '+str(repo.untracked_files)
log('untracked: ', status.untracked)
for file in status.untracked:
#>xcopy /s python\ftp_upload_file.pyc "G:\My Drive\sync\company_notebook\scripts\python\"
if os.path.dirname(file) == '':
log( "xcopy /s \""+file+"\" \"G:\\My Drive\\sync\\company_notebook\\"+basename+"\\untracked\\\"")
os.system("cmd.exe /c xcopy /s \""+file+"\" \"G:\\My Drive\\sync\\company_notebook\\"+basename+"\\untracked\\\"")
else:
log( "xcopy /s \""+file+"\" \"G:\\My Drive\\sync\\company_notebook\\"+basename+"\\untracked\\"+os.path.dirname(file)+"\\\"")
os.system("cmd.exe /c xcopy /s \""+file+"\" \"G:\\My Drive\\sync\\company_notebook\\"+basename+"\\untracked\\"+os.path.dirname(file)+"\\\"")
else :
os.system("rclone purge \"mobile_rclone:/sync/private_mobile/"+basename+"\"")
os.system("rclone mkdir \"mobile_rclone:/sync/private_mobile/"+basename+"\"")
os.system("rclone mkdir \"mobile_rclone:/sync/private_mobile/"+basename+"/untracked\"")
os.system("mkdir \"../"+basename+"_tmp\"")
os.system("git status > \"../"+basename+"_tmp/status\"")
os.system("git diff > \"../"+basename+"_tmp/diff\"")
os.system("rclone copy ../"+basename+"_tmp/diff \"mobile_rclone:/sync/private_mobile/"+basename+"/\"")
os.system("rclone copy ../"+basename+"_tmp/status \"mobile_rclone:/sync/private_mobile/"+basename+"/\"")
os.system("rm -rf \"../"+basename+"_tmp\"")
repo=porcelain.open_repo(path)
status=porcelain.status(repo)
log('staged - added:', status.staged['add'], 'modified:', status.staged['modify'], 'deleted: ', status.staged['delete'])
log('unstaged: ', status.unstaged)
#log 'untracked: ', status.untracked
#untracked not working on version 0.9.7
#log 'untracked: '+str(repo.untracked_files)
log('untracked: ', status.untracked)
for file in status.untracked:
#>xcopy /s python\ftp_upload_file.pyc "G:\My Drive\sync\company_notebook\scripts\python\"
log("rclone copy \""+file+"\" \"mobile_rclone:/sync/private_mobile/"+basename+"/untracked/"+os.path.dirname(file)+"\"")
os.system("rclone copy \""+file+"\" \"mobile_rclone:/sync/private_mobile/"+basename+"/untracked/"+os.path.dirname(file)+"\"")
#log( "xcopy /s \""+file+"\" \"G:\\My Drive\\sync\\company_notebook\\"+basename+"\\"+os.path.dirname(file)+"\\\"")
#os.system("xcopy /s \""+file+"\" \"G:\\My Drive\\sync\\company_notebook\\"+basename+"\\"+os.path.dirname(file)+"\\\"")
#shutil.copy(event.src_path, "G:/My Drive/sync")
log("ready: ", str(datetime.now()))
except BaseException:
os._exit(1)
except:
print >> sys.stderr, "trying to commit resulted in a failure"
traceback.print_exc(file=sys.stderr)
print "something went wrong"
"""
commit = porcelain.commit(repo=repoPath,
author=author,
committer=committer,
message=message_bytes)
print >> sys.stderr, commit
print "next steps:\nrun '|git log' to show the history"
# the commit id is in the commit variable - try to print it nicely to give feedback
#porcelain.print_commit(commit)
elif command == "status":
print >> sys.stderr, "status"
status = porcelain.status(repoPath)
# status(tracked_changes, unstaged_changes, untracked_changes)
results = []
result = collections.OrderedDict() # {
result["add"] = status[0]["add"]
result["modify"] = status[0]["modify"]
result["delete"] = status[0]["delete"]
result["unstaged_changes"] = status[1] #unstaged
result["untracked_changes"] = status[2] #unstaged
results.append(result)
splunk.Intersplunk.outputResults(results)
elif command == "treechanges":
porcelain.get_tree_changes(repoPath)
elif command == "branch":
"""
def dirty(self):
"""Check if there are likely local user modifications to the app repository."""
status_ = status(self)
return bool(
any(bool(_) for _ in status_.staged.values()) or status_.unstaged)
def check_for_changes(database):
repo = porcelain.open_repo(database)
status = porcelain.status(repo)
return status.unstaged, status.untracked
command = f"{args.script} {args.scriptParameters} --src-dir={source_path} --dst-dir={destination_path}"
# FIXME fix race condition with transformer for print statements
termcolor.colored(f"Running transformer with command '{command}'", "yellow")
process = subprocess.Popen(
[command],
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
)
output, err = process.communicate()
sys.stdout.write(output)
sys.stderr.write(err)
status = porcelain.status(dulwich_repo)
dulwich_repo.stage(status.untracked)
dulwich_repo.stage(status.unstaged)
porcelain.commit(
dulwich_repo,
args.pullRequestName,
"grace-production <*****@*****.**>",
)
logging_verb1 = "Would copy"
logging_verb2 = "would be"
logging_color = "yellow"
if not args.dryRun:
logging_verb1 = "Copied"
def testDulwich(self):
# init remote and repo dir
remote = os.path.join(self.wdir, 'git_remote')
if os.path.exists(remote):
del_tree(remote)
os.mkdir(remote)
path = os.path.join(self.wdir, 'git_test')
if os.path.exists(path):
del_tree(path)
os.mkdir(path)
# start at remote
self.assertReturnsNonZero(status_git, path=remote)
self.assertReturnsZero(init_git, path=remote)
first_file = 'first_file'
first_contents = 'Hello to repo!'
with open(os.path.join(remote, first_file), 'w') as file:
file.write(first_contents)
self.assertIn(first_file, status(remote).untracked)
self.assertReturnsZero(add_git, path=remote)
self.assertIn(first_file.encode(), status(remote).staged['add'])
self.assertReturnsZero(add_and_commit_git,
'remote_commit',
path=remote)
self.assertReturnsZero(branch_git, 'other', path=remote)
if os.name == 'posix':
self.assertReturnsZero(checkout_git, 'other', path=remote)
# switch to repo
self.assertReturnsZero(clone_git, remote, path=path)
if os.name == 'posix':
self.assertReturnsZero(checkout_git, 'master', path=path)
self.assertReturnsZero(pull_git, remote, path=path)
self.assertReturnsZero(add_and_commit_git, 'empty_commit', path=path)
with open(os.path.join(path, 'first_file'), 'r') as file:
read_first_contents = file.read()
self.assertEqual(first_contents, read_first_contents)
append_contents = ' And hello to remote!'
with open(os.path.join(path, first_file), 'a') as file:
file.write(append_contents)
second_contents = 'Hello to remote!'
second_file = 'second_file'
with open(os.path.join(path, second_file), 'w') as file:
file.write(second_contents)
self.assertIn(first_file.encode(), status(path).unstaged)
self.assertIn(second_file, status(path).untracked)
self.assertReturnsZero(status_git, path=path)
self.assertReturnsZero(add_git, path=path)
self.assertReturnsZero(status_git, path=path)
self.assertIn(first_file.encode(), status(path).staged['modify'])
self.assertIn(second_file.encode(), status(path).staged['add'])
self.assertReturnsZero(add_and_commit_git, 'repo_commit', path=path)
tag = 'test_tag'
self.assertReturnsZero(tag_git, tag, path=path)
self.assertIn(tag.encode(), tag_list(path))
self.assertReturnsZero(push_git, remote, path=path)
# switch back to remote
if os.name == 'posix':
self.assertReturnsZero(checkout_git, 'master', path=remote)
with open(os.path.join(remote, first_file), 'r') as file:
self.assertEqual(file.read(), first_contents + append_contents)
with open(os.path.join(remote, second_file), 'r') as file:
self.assertEqual(file.read(), second_contents)
def has_uncommitted_changes(self):
if(porcelain.status(self.g.path).unstaged or
porcelain.status(self.g.path).staged['add'] or
porcelain.status(self.g.path).staged['modify'] or
porcelain.status(self.g.path).staged['delete']):
return True
def has_uncommitted_changes(self):
if (porcelain.status(self.g.path).unstaged
or porcelain.status(self.g.path).staged['add']
or porcelain.status(self.g.path).staged['modify']
or porcelain.status(self.g.path).staged['delete']):
return True
def get_status(self):
if os.path.exists(self.repo_path):
print self.repo_path
print p.status(self.repo_path)
else:
print "Repo does not exist"
def main(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
base_dir=default_base_dir,
comment=default_comment,
page=False,
no_highlight_diff=False):
"""
track_autoconfig.py
Description:
Store persisted domain configuration in a local git repository
for auditing purposes.
Usage:
:::bash
$ mast contrib/track_autoconfig.py --appliances <HOSTNAMES> --credentials <USER:PASS> --base-dir tmp/config
Parameters:
* `-a, --appliances` - The hostname(s), ip addresse(s), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases please see the comments in `hosts.conf` located
in `$MAST_HOME/etc/default`.
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password` and should be provided in a space-seperated list
if multiple are provided. If you would prefer to not use plain-text
passwords, you can use the output of
`$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-b, --base-dir`: The base directory where to store the downloaded
files. Files will actually be stored in a subdirectory of `base_dir`
named after the hostname in the form of `base_dir/<hostname>`
* `-p, --page`: If specified, page the output when too long to display
at once
* `-N, --no-highlight-diff`: If specified, the output of the diff will
be syntax-highlighted
"""
base_dir = os.path.abspath(base_dir)
if not os.path.exists(base_dir):
os.makedirs(base_dir)
try:
repo = git.Repo(base_dir)
first_sha1 = repo.head()
print first_sha1
except NotGitRepository:
print "Initializing git repository"
git.init(base_dir)
git.add(base_dir)
first_sha1 = git.commit(base_dir, message="Initial Commit")
print first_sha1
except KeyError:
git.add(base_dir)
git.commit(base_dir, message="Initial Commit")
print
pull_autoconfig(appliances=appliances,
credentials=credentials,
timeout=timeout,
no_check_hostname=no_check_hostname,
base_dir=base_dir)
git.add(base_dir)
print git.status(base_dir)
second_sha1 = git.commit(base_dir, message=comment)
print second_sha1
print "\n\nDIFF\n\n"
tmp = StringIO()
git.show(base_dir, outstream=tmp)
tmp.seek(0)
if no_highlight_diff:
out = tmp.read()
else:
out = highlight(tmp.read(), DiffLexer(), TerminalFormatter())
if page:
pprint.page(out)
else:
print out
def test_empty(self):
results = porcelain.status(self.repo)
self.assertEqual({"add": [], "delete": [], "modify": []}, results.staged)
self.assertEqual([], results.unstaged)
def is_dirty():
"""Check for uncommitted changes. True if dirty."""
repo = git.Repo(os.getcwd())
s = git.status(repo)
return any(s.staged.values() + [s.unstaged])
def performCommit(localRepo, bookid, bookinfo, bookroot, bookfiles):
has_error = False
staged = []
added = []
ignored = []
# convert url paths to os specific paths
repo_home = pathof(localRepo)
repo_home = repo_home.replace("/", os.sep)
repo_path = os.path.join(repo_home, "epub_" + bookid)
book_home = pathof(bookroot)
book_home = book_home.replace("/", os.sep)
# convert from bookpaths to os relative file paths
filepaths = []
for bkpath in bookfiles:
afile = pathof(bkpath)
afile = afile.replace("/", os.sep)
filepaths.append(afile)
cdir = os.getcwd()
if os.path.exists(repo_path):
# handle updating the staged files and commiting and tagging
# first collect info to determine files to delete form repo
# current tag, etc
os.chdir(repo_path)
# determine the new tag
tags = porcelain.list_tags(repo='.')
tagname = "V%04d" % (len(tags) + 1)
tagmessage = "Tag: " + tagname
message = "updating to " + tagname
# extra parameters must be passed as bytes if annotated is true
tagname = utf8_str(tagname)
message = utf8_str(message)
tagmessage = utf8_str(tagmessage)
# delete files that are no longer needed from staging area
tracked = []
tracked = porcelain.ls_files(repo='.')
files_to_delete = []
for afile in tracked:
afile = pathof(afile)
if afile not in filepaths:
if afile not in ["mimetype", ".gitignore", ".bookinfo"]:
files_to_delete.append(afile)
if len(files_to_delete) > 0:
porcelain.rm(repo='.', paths=files_to_delete)
# copy over current files
copy_book_contents_to_destination(book_home, filepaths, repo_path)
(staged, unstaged, untracked) = porcelain.status(repo='.')
files_to_update = []
for afile in unstaged:
afile = pathof(afile)
files_to_update.append(afile)
for afile in untracked:
afile = pathof(afile)
files_to_update.append(afile)
(added, ignored) = porcelain.add(repo='.', paths=files_to_update)
commit_sha1 = porcelain.commit(repo='.',
message=message,
author=_SIGIL,
committer=_SIGIL)
# create annotated tags so we can get a date history
tag = porcelain.tag_create(repo='.',
tag=tagname,
message=tagmessage,
annotated=True,
author=_SIGIL)
os.chdir(cdir)
add_bookinfo(repo_path, bookinfo, bookid, unicode_str(tagname))
else:
# this will be an initial commit to this repo
tagname = b"V0001"
tagmessage = b'First Tag'
message = b"Initial Commit"
os.makedirs(repo_path)
add_gitignore(repo_path)
add_gitattributes(repo_path)
cdir = os.getcwd()
os.chdir(repo_path)
r = porcelain.init(path='.', bare=False)
staged = copy_book_contents_to_destination(book_home, filepaths,
repo_path)
(added, ignored) = porcelain.add(repo='.', paths=staged)
# it seems author, committer, messages, and tagname only work with bytes if annotated=True
commit_sha1 = porcelain.commit(repo='.',
message=message,
author=_SIGIL,
committer=_SIGIL)
tag = porcelain.tag_create(repo='.',
tag=tagname,
message=tagmessage,
annotated=True,
author=_SIGIL)
os.chdir(cdir)
add_bookinfo(repo_path, bookinfo, bookid, unicode_str(tagname))
result = "\n".join(added)
result = result + "***********" + "\n".join(ignored)
if not has_error:
return result
return ''
def is_dirty():
"""Check for uncommitted changes. True if dirty."""
repo = git.Repo(os.getcwd())
s = git.status(repo)
return any(s.staged.values() + [s.unstaged])
def test_empty(self):
results = porcelain.status(self.repo)
self.assertEqual(
{'add': [], 'delete': [], 'modify': []},
results.staged)
self.assertEqual([], results.unstaged)
current_path = current_path + os.sep
print(current_path)
# print os.walk(current_path).next()[1]
def get_repositories(current_path) :
repositories = {}
for directory in os.walk(current_path).next()[1] :
repository_path = current_path+directory+'/'
if os.path.isdir(repository_path+'.git') :
repositories[directory] = repository_path
return repositories
# repositories = get_repositories(current_path)
def get_immediate_subdirectories(a_dir):
return [name for name in os.listdir(a_dir)
if os.path.isdir(os.path.join(a_dir, name))]
directories = get_immediate_subdirectories(current_path)
for item in directories :
print(item)
path = os.path.join(current_path, item)
print(path)
if os.path.isdir(os.path.join(path, '.git')):
r = Repo(path)
print(git.status(r))
def main(appliances=[],
credentials=[],
timeout=120,
no_check_hostname=False,
base_dir=default_base_dir,
comment=default_comment,
persisted=False,
recursive=False,
no_strip_timestamp=False,
page=False):
"""
track_getconfig.py
Description:
Store running or persisted domain configuration in a local git
repository for auditing purposes.
Usage:
:::bash
$ mast contrib/track_getconfig.py --appliances <HOSTNAMES> --credentials <USER:PASS> --base-dir tmp/config
Parameters:
* `-a, --appliances` - The hostname(s), ip addresse(s), environment name(s)
or alias(es) of the appliances you would like to affect. For details
on configuring environments please see the comments in
`environments.conf` located in `$MAST_HOME/etc/default`. For details
on configuring aliases please see the comments in `hosts.conf` located
in `$MAST_HOME/etc/default`.
* `-c, --credentials`: The credentials to use for authenticating to the
appliances. Should be either one set to use for all appliances
or one set for each appliance. Credentials should be in the form
`username:password` and should be provided in a space-seperated list
if multiple are provided. If you would prefer to not use plain-text
passwords, you can use the output of
`$ mast-system xor <username:password>`.
* `-t, --timeout`: The timeout in seconds to wait for a response from
an appliance for any single request. __NOTE__ Program execution may
halt if a timeout is reached.
* `-n, --no-check-hostname`: If specified SSL verification will be turned
off when sending commands to the appliances.
* `-b, --base-dir`: The base directory where to store the downloaded
files. Files will actually be stored in a subdirectory of `base_dir`
named after the hostname in the form of `base_dir/<hostname>`
* `-p, --persisted`: If specified, the persisted configuration will
be retrieved as opposed to the running configuration (which is the
default)
* `-r, --recursive`: If specified, the configuration will be retrieved
recursively to the extent of the facilities provided by DataPower
* `-N, --no-strip-timestamp`: If specified, the timestamp will not be
stripped from the XML document, This is done because we are tracking
this information with git and stripping this out allows us to alert
on any changes in the repository as opposed to special-casing the
difference in timestamp.
* `-P, --page`: If specified, page the output when too long to display
at once
"""
base_dir = os.path.abspath(base_dir)
if not os.path.exists(base_dir):
os.makedirs(base_dir)
try:
repo = git.Repo(base_dir)
repo.head()
except NotGitRepository:
print "Initializing git repository"
git.init(base_dir)
git.add(base_dir)
git.commit(base_dir, message="Initial Commit")
except KeyError:
git.add(base_dir)
git.commit(base_dir, message="Initial Commit")
check_hostname = not no_check_hostname
env = datapower.Environment(appliances,
credentials,
timeout=timeout,
check_hostname=check_hostname)
for appliance in env.appliances:
appliance_directory = os.path.join(base_dir, appliance.hostname)
if not os.path.exists(appliance_directory):
os.mkdir(appliance_directory)
for domain in appliance.domains:
config = appliance.get_config(domain=domain,
recursive=recursive,
persisted=persisted)
config = config.pretty
if no_strip_timestamp:
pass
else:
config = re.sub(r"^.*?<dp:timestamp>.*?</dp:timestamp>.*?$",
r"",
config,
flags=re.MULTILINE)
filename = os.path.join(appliance_directory,
"{}.xml".format(domain))
with open(filename, "wb") as fout:
fout.write(config)
git.add(base_dir)
print git.status(base_dir)
git.commit(base_dir, message=comment)
tmp = StringIO()
git.show(base_dir, outstream=tmp)
tmp.seek(0)
out = highlight(tmp.read(), DiffLexer(), TerminalFormatter())
if page:
pprint.page(out)
else:
print out
def status(self):
return porcelain.status(self.repo)