def list_local_for_remote_sync(base_destination, list_of_files):
# check if files all come from the same project folder
configdic = config.read_bitconfig()
local_path = configdic["local_path"]
size_local = len(local_path.split("/"))
parent_folder = []
check_project = []
for i in list_of_files:
f = os.path.abspath(i)
parent_folder.append(f.split("/")[size_local])
check_project.append(f.split("/")[size_local + 1])
check_project = list(set(check_project))
if len(check_project) > 1:
print("Found more than one project:\n")
for p in check_project:
print(p)
sys.stdout.flush()
sys.exit(0)
else:
project_name = check_project[0]
parent_folder = parent_folder[0]
target_project = parent_folder + "/" + project_name
base_destination = base_destination + parent_folder + "/" + project_name
upload_dic = {}
subfolders = [base_destination]
check = base_destination.split("/")
for i in range(len(check)):
c = "/".join(check[:i - len(check)])
subfolders.append(c)
for f in list_of_files:
full = os.path.abspath(f)
if CheckFoldersCon(local_path, full):
if os.path.isdir(full):
subfol = base_destination + "/" + os.path.basename(full)
upload_dic[full] = base_destination + "/" + os.path.basename(
full)
subfolders.append(subfol)
elif os.path.isfile(full):
tfile = base_destination + full.split(local_path + "/" +
target_project)[1]
upload_dic[full] = tfile
subfolders.append(tfile.split(os.path.basename(tfile))[0])
else:
upload_dic[full] = base_destination + "/" + os.path.basename(
full)
else:
print(
"%s is either a link to %s or is not on your projects path. This file\
will not be syncronized." % (f, full))
sys.stdout.flush()
subfolders = list(set(subfolders))
subfolders = [xx for xx in subfolders if len(xx) > 0]
subfolders.sort()
return upload_dic, subfolders, base_destination, parent_folder
def ownCloud_download(gitssh=None, pick_a_date=None):
configdic = config.read_bitconfig()
for r in downloadreqs:
while configdic[r] == None:
configdic=config.check_reqs([r],configdic,config_file=None, \
gitssh=gitssh)
local_path = os.path.abspath(configdic["local_path"])
size_local = len(local_path.split("/"))
f = os.path.abspath(str(pick_a_date))
parent_folder = f.split("/")[size_local]
project_name = f.split("/")[size_local + 1]
target_project = parent_folder + "/" + project_name
base_destination = get_owncloud_base_folder(configdic,
target_project,
getfolder=True,
pick_a_date=pick_a_date)
# login to owncloud
try:
oc = owncloud.Client(configdic["owncloud_address"])
oc.login(configdic["owncloud_user"], configdic["owncloud_pass"])
except:
print("Could not login to ownCloud.\nPlease make sure you are giving \
the right address to your owncloud and using the right login credentials."
)
sys.exit(0)
oc.get_directory_as_zip(base_destination, pick_a_date + ".zip")
oc.logout()
print("Downloaded %s.zip" % pick_a_date)
sys.stdout.flush()
def ownCloud_create_folder(gitssh=None, pick_a_date=None, days_to_share=None):
configdic = config.read_bitconfig()
for r in downloadreqs:
while configdic[r] == None:
configdic=config.check_reqs([r],configdic,config_file=None, \
gitssh=gitssh)
local_path = os.path.abspath(configdic["local_path"])
size_local = len(local_path.split("/"))
f = os.path.abspath(str(pick_a_date))
parent_folder = f.split("/")[size_local]
project_name = f.split("/")[size_local + 1]
target_project = parent_folder + "/" + project_name
base_destination = get_owncloud_base_folder(configdic,
target_project,
create_folder=True,
pick_a_date=pick_a_date)
# login to owncloud
try:
oc = owncloud.Client(configdic["owncloud_address"])
oc.login(configdic["owncloud_user"], configdic["owncloud_pass"])
except:
print("Could not login to ownCloud.\nPlease make sure you are giving \
the right address to your owncloud and using the right login credentials."
)
sys.exit(0)
check = base_destination.split("/")
print(check)
for i in range(len(check) + 1):
c = "/".join(check[:i])
print(c)
try:
oc.file_info(c)
except:
oc.mkdir(c)
# Time stamp for expiration date
tshare = datetime.date.today()
tshare = tshare + datetime.timedelta(days=int(days_to_share))
tshare = time.mktime(tshare.timetuple())
link_info = oc.share_file_with_link(base_destination,
expiration=tshare,
public_upload=True)
private_link = get_ownCloud_links(link_info, configdic["owncloud_address"])
oc.logout()
def read_remote_config(sshLogin, remotePass, forceImport=None):
remote_address = sshLogin.split("@")[1]
if os.path.isfile(".bit_config.%s" % remote_address):
if not forceImport:
print("Using previously collected remote config.")
else:
get_remote_config(sshLogin, remotePass)
else:
get_remote_config(sshLogin, remotePass)
remote_config=config.read_bitconfig(showit=None,bit_config=".bit_config.%s" \
%remote_address)
return remote_config
def main():
import argparse
parser = argparse.ArgumentParser(description="bit, [b]ermuda [i]nformation [t]riangle.\
bit is a git-based tool for the management of code and data. It uses git for code versioning\
and ownCloud for storing and exchanging data. It saves storage by avoiding versioning\
of data while logging changes in associated git wikis.",\
formatter_class = argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-i", "--input", nargs='*', help="Input files")
parser.add_argument("-s", "--subfolder", help="Subfolder to be created.", default=None)
parser.add_argument("-m", "--message",nargs='*', help="Message to write on log file.", default=None)
parser.add_argument("-d", "--pick_a_date", help="Pick an existing date folder to transfer data to/from. Format=YYYY-MM-DD", default=None)
parser.add_argument("-c", "--create_folder", help="Create dropbox folder for user to upload data.", action="store_true")
parser.add_argument("-g", "--getfolder", help="Downloads a folder as zip file. Requires --pick_a_date. Defaults base_folder=upload:download to download", action="store_true")
parser.add_argument("-t", "--days_to_share", help="Number of days you wish to share this folder further.", default=21)
parser.add_argument("--issue", help="Issue to comment on with --message and owncloud data links", default=None)
parser.add_argument("--scripts",help="Needs -i and -m. Simultaneously sync the scripts.user folder when uploading data.", action="store_true")
parser.add_argument("--start", help="Project name of the format. PI_PROJECT_NAME. Initiates a project. This will create the required local folders and respective git repositories.", default=None)
parser.add_argument("--stdfolders",nargs='*', help="Folders to be created in addition to scripts.user and and wiki.user when a project is started.", default=["tmp","slurm_logs"])
parser.add_argument("--adduser",help="Add a user to a project creating his scripts.user and wiki.user folder",action="store_true")
parser.add_argument("--sync", nargs='*', help="Files or folders to syncronize with remote server using rsync over ssh.",default=None)
parser.add_argument("--sync_to", help="Destination server to sync to in the form: <user_name>@<server.address>", default=None)
parser.add_argument("--sync_from", help="Destination server to sync from in the form: <user_name>@<server.address>", default=None)
parser.add_argument("--cpus",help="Number of CPUs/channels to open for rsync.", default=1)
parser.add_argument("--forceRemote", help="If syncing from or to a remoter server force the import of a remote 'bit_config'.", action="store_true")
parser.add_argument("--gitssh", help="Use your git SSH keys.", action="store_true")
parser.add_argument("--config", help="Generate a config file.", action="store_true")
args = parser.parse_args()
if args.sync:
if args.sync_to:
calls=rsync.rsync_to(args.sync_to, args.sync, forceImport=args.forceRemote, \
sync_to=True, sync_from=False)
elif args.sync_from:
calls=rsync.rsync_from(args.sync_from, args.sync, forceImport=args.forceRemote, \
sync_to=False, sync_from=True)
pool=mp.Pool(int(args.cpus))
funclist=[]
for call in calls:
out=pool.apply_async(worker,[call])
funclist.append(out)
results=[]
for ff in funclist:
res=ff.get()
print(res)
results.append(res)
if args.config:
print("Setting up your config file.")
sys.stdout.flush()
config.make_bitconfig()
sys.exit(0)
# initate a project
if args.start:
configdic=config.read_bitconfig()
for r in config.start_reqs:
if r != "user_group":
while configdic[r] == None:
configdic=config.check_reqs([r],configdic,config_file=None, gitssh=None)
local_path=os.path.abspath(configdic["local_path"])
full_path=os.path.abspath(args.start)
project_name=os.path.basename(full_path)
# check format projects_folder/group_head/project_name
checkf=len(local_path.split("/"))
if len(full_path.split("/")) != len(local_path.split("/"))+2:
print("The path (%s) to this project does not obey the structure and/or defined local path (%s). Check the reference structure:\n%s" \
%(full_path,local_path,config.structure) )
sys.stdout.flush()
sys.exit(0)
# have the user rechecking that the the string for the project name is really correct
checks=None
while checks not in ["Y","N"]:
checks=str(raw_input("Is the label %s in agreement with the structure PF_project_name where PF stands for the initials of the Parent_Folder? (Y/N) " \
%project_name )) or None
if checks=="N":
sys.exit(0)
# create the repo
github_api=config.get_github_api(configdic["github_address"])
github_api=github_api+configdic["github_organization"]+"/repos"
create_call=["curl","-u",configdic["github_user"]+":"+configdic["github_pass"]\
,github_api,"-d",'{"name":"'+project_name+'","private": "true",\
"auto_init": true }']
p = Popen(create_call, stdout=PIPE, stdin=PIPE, stderr=STDOUT)
print(p.communicate()[0].rstrip())
sys.stdout.flush()
# clone the repo and the wiki by initiating this user
raw_input("\n\n*************\n\nPlease go to %s/%s/%s/wiki and click on 'Create the first page' and then 'Save Page'.\n\nPress Enter once you have saved the first wiki page.\n\n*************\n\n" \
%(configdic["github_address"],configdic["github_organization"],project_name) )
config.init_user(full_path,configdic["github_address"],configdic["github_organization"],\
project_name,github_user=configdic["github_user"],\
github_pass=configdic["github_pass"],gitssh=args.gitssh)
# create additional folders
for f in args.stdfolders:
if not os.path.exists(full_path+"/"+f):
os.makedirs(full_path+"/"+f)
if configdic["user_group"]:
os.chmod(full_path, stat.S_IRWXU)
user_group=configdic["user_group"].split(",")
try:
for use in user_group:
call=["setfacl","-m","user:%s:rwx" %use,full_path]
out=Popen(call, stdout=PIPE, stdin=PIPE, stderr=STDOUT)
print(out.communicate()[0].rstrip())
except:
print("Failed to setfacls.")
sys.stdout.flush()
local_path_owner=os.stat(local_path)
local_path_owner=local_path_owner.st_uid
#os.chown(full_path,local_path_owner,-1)
sys.exit(0)
if args.adduser:
configdic=config.read_bitconfig()
for r in config.start_reqs:
while configdic[r] == None:
configdic=config.check_reqs([r],configdic,config_file=None, gitssh=args.gitssh)
local_path=os.path.abspath(configdic["local_path"])
if args.start:
full_path=os.path.abspath(args.start)
else:
full_path=os.path.abspath(os.getcwd())
project_name=os.path.basename(full_path)
# check format projects_folder/group_head/project_name
checkf=len(local_path.split("/"))
if len(full_path.split("/")) != len(local_path.split("/"))+2:
print("The path (%s) to this project does not obey the structure and/or defined local path (%s). Check the reference structure:\n%s" %(full_path,local_path,structure))
sys.stdout.flush()
sys.exit(0)
config.init_user(full_path,configdic["github_address"],configdic["github_organization"],project_name,github_user=configdic["github_user"],github_pass=configdic["github_pass"],gitssh=args.gitssh)
sys.exit(0)
if args.input:
if not args.message:
print("ERROR\nYou need to use -m to leave a message in the logs.")
sys.exit()
oc.ownCloud_upload(input_files=args.input,message=args.message,gitssh=args.gitssh,days_to_share=args.days_to_share,scripts=args.scripts,issue=args.issue, subfolder=args.subfolder,pick_a_date=args.pick_a_date)
sys.exit(0)
if args.create_folder:
oc.ownCloud_create_folder(gitssh=args.gitssh,pick_a_date=args.pick_a_date,days_to_share=args.days_to_share)
sys.exit(0)
if args.getfolder:
if not args.pick_a_date:
print("--getfolder implies --pick_a_date.\nPlease use -d in combination with -g.\nThank you!")
sys.exit(0)
oc.ownCloud_download(gitssh=args.gitssh,pick_a_date=args.pick_a_date)
sys.exit(0)
sys.exit(0)
def rsync_from(sshLogin,
rsync_files,
forceImport=None,
sync_to=False,
sync_from=True):
remotePass=str(getpass.getpass(prompt="Please give in your password for %s: "\
%sshLogin.split("@")[1] ))
configdic = config.read_bitconfig()
local_path = configdic["local_path"]
remote_config = read_remote_config(sshLogin, remotePass, forceImport)
remote_path = remote_config["local_path"]
# get remote dirs and subdirs
sync_dic, subfolders, path_to_project, parent_folder=list_local_for_remote_sync(\
remote_path, rsync_files)
check_remote = []
for f in sync_dic:
check_remote.append(sync_dic[f])
check_remote = " ".join(check_remote)
# check if files exist on remote
temp = tempfile.NamedTemporaryFile()
check_remote_files = "\'for f in " + check_remote + " ; do if [ -f $f ]; then echo $f; fi; done\' > " + temp.name
check_remote_files = "sshpass -p " + str(remotePass) + " ssh " + str(
sshLogin) + " " + check_remote_files
os.system(check_remote_files)
res = temp.readlines()
temp.close()
resFiles = [s.strip("\n") for s in res]
# check if folders exist on remote
temp = tempfile.NamedTemporaryFile()
check_remote_folder = "\'for f in " + check_remote + " ; do if [ -d $f ]; then echo $f; find $f -type d -print; fi; done\' > " + temp.name
check_remote_folder = "sshpass -p " + str(remotePass) + " ssh " + str(
sshLogin) + " " + check_remote_folder
os.system(check_remote_folder)
res = temp.readlines()
temp.close()
resFolders = [s.strip("\n") + "/" for s in res]
list_folders_contents = " ".join(resFolders)
temp = tempfile.NamedTemporaryFile()
check_remote_folder = "\'for f in " + list_folders_contents + " ; do cd $f;\
for c in $(ls); do cc=$(readlink -f $c); if [ -f $cc ]; then echo $cc; fi; done; done\' > " + temp.name
check_remote_folder = "sshpass -p " + str(remotePass) + " ssh " + str(
sshLogin) + " " + check_remote_folder
os.system(check_remote_folder)
res = temp.readlines()
temp.close()
resAllFiles = [s.strip("\n") for s in res]
res = [resFiles, resAllFiles] #resFolders
res = [item for sublist in res for item in sublist]
res = list(set(res))
showres = "\n".join(res)
print("The following targets could be found on the remote server:\n%s" %
showres)
sys.stdout.flush()
inv_sync_dic = {}
for r in res:
inv_sync_dic[r] = local_path + "/" + r.split(remote_path)[1]
lenLocalPath = len(os.path.abspath(local_path).split("/"))
for remfold in resFolders:
ltf = local_path + "/" + remfold.split(remote_path)[1]
if not os.path.exists(ltf):
os.makedirs(ltf)
if len(os.path.abspath(ltf).split("/")) in [
lenLocalPath + 1, lenLocalPath + 2
]:
if configdic["user_group"]:
os.chmod(ltf, stat.S_IRWXU)
user_group = configdic["user_group"]
try:
for use in user_group:
call = ["setfacl", "-m", "user:%s:rwx" % use, ltf]
out = Popen(call,
stdout=PIPE,
stdin=PIPE,
stderr=STDOUT)
print(out.communicate()[0].rstrip())
except:
print("Failed to setfacls.")
sys.stdout.flush()
local_path_owner = os.stat(local_path)
local_path_owner = local_path_owner.st_uid
os.chown(ltf, local_path_owner, -1)
sync_from_calls = []
for f in inv_sync_dic:
call='rsync -tlzhP --rsh="sshpass -p %s ssh -o \
StrictHostKeyChecking=no -l %s" %s:%s %s' %(str(remotePass), \
str(sshLogin.split("@")[0]), str(sshLogin.split("@")[1]), \
f, inv_sync_dic[f] )
sync_from_calls.append(call)
#os.system(call)
return sync_from_calls
def ownCloud_upload(input_files=None,
message=None,
gitssh=None,
days_to_share=None,
scripts=None,
issue=None,
subfolder=None,
pick_a_date=None):
if type(message) == list:
message = [str(xx) for xx in message]
message = " ".join(message)
else:
message = str(message)
configdic = config.read_bitconfig()
for r in config.requirements:
if not gitssh:
if r not in ["user_group"]:
while configdic[r] == None:
configdic=config.check_reqs([r],configdic,config_file=None, \
gitssh=None)
else:
if r not in ["github_user", "github_pass", "user_group"]:
while configdic[r] == None:
configdic=config.check_reqs([r],configdic,config_file=None, \
gitssh=gitssh)
local_path = os.path.abspath(configdic["local_path"])
# check if files all come from the same project folder
size_local = len(local_path.split("/"))
parent_folder = []
check_project = []
for i in input_files:
f = os.path.abspath(i)
parent_folder.append(f.split("/")[size_local])
check_project.append(f.split("/")[size_local + 1])
check_project = list(set(check_project))
if len(check_project) > 1:
print("Found more than one project:\n")
for p in check_project:
print(p)
sys.stdout.flush()
sys.exit(0)
else:
project_name = check_project[0]
parent_folder = parent_folder[0]
target_project = parent_folder + "/" + project_name
base_destination = get_owncloud_base_folder(configdic,
target_project,
subfolder=subfolder,
pick_a_date=pick_a_date)
upload_dic, subfolders = list_upload(base_destination, input_files)
# login to owncloud
try:
oc = owncloud.Client(configdic["owncloud_address"])
oc.login(configdic["owncloud_user"], configdic["owncloud_pass"])
except:
print("Could not login to ownCloud.\nPlease make sure you are giving \
the right address to your owncloud and using the right login credentials."
)
sys.exit(0)
# create required subfolders in ownCloud
for fold in subfolders:
try:
oc.file_info(fold)
except:
oc.mkdir(fold)
# Upload files
if len(upload_dic) > 1:
print("Uploading %s files.." % str(len(upload_dic)))
sys.stdout.flush()
else:
print("Uploading %s file.." % str(len(upload_dic)))
sys.stdout.flush()
skipped_files = []
for f in upload_dic:
file_handle = open(f, 'r', 8192)
file_handle.seek(0, os.SEEK_END)
size = file_handle.tell()
file_handle.seek(0)
if size == 0:
skipped_files.append(os.path.basename(f))
print("\t%s is empty. Skipping .. " % str(f))
sys.stdout.flush()
continue
if size > 1879048192:
print("\t%s\t(chunked)" % str(upload_dic[f]))
sys.stdout.flush()
oc.put_file(upload_dic[f], f)
else:
print("\t%s" % str(upload_dic[f]))
sys.stdout.flush()
oc.put_file(upload_dic[f], f, chunked=False)
print("Finished uploading.")
# Time stamp for expiration date
tshare = datetime.date.today()
tshare = tshare + datetime.timedelta(days=int(days_to_share))
tshare = time.mktime(tshare.timetuple())
link_info = oc.share_file_with_link(base_destination, expiration=tshare)
private_link = get_ownCloud_links(link_info, configdic["owncloud_address"])
oc.logout()
# Go to wiki folder and make a git sync
print("Logging changes..")
sys.stdout.flush()
user_name = getpass.getuser()
os.chdir(local_path + "/" + target_project + "/wiki." + user_name)
files_to_add = os.listdir(local_path + "/" + target_project + "/wiki." +
user_name)
git.git_sync(files_to_add,"bit sync",configdic["github_address"],\
configdic["github_organization"],project_name+".wiki",\
github_user=configdic["github_user"],github_pass=configdic["github_pass"],\
gitssh=gitssh)
# Write log file
if len(skipped_files) > 0:
skipped_files = ", ".join(skipped_files)
skipped_files = "\n\n(skipped: %s)" % skipped_files
else:
skipped_files = ""
logfile = "uploads.md"
logtext="\n\n##### ["+base_destination.split("/")[3]+"\t::\t"+user_name+"]("+private_link+") : "\
+str(" ".join(message))+"\n"+\
str(datetime.datetime.now()).split(".")[0]+", "+str(", ".join(input_files))\
+skipped_files
log = open(logfile, "a")
log.write(logtext)
log.close()
# push the log
git.git_add(["uploads.md"])
git.git_commit(message)
git.git_push(configdic["github_address"],configdic["github_organization"],\
project_name+".wiki",github_user=configdic["github_user"],\
github_pass=configdic["github_pass"],gitssh=gitssh)
if scripts:
print("Syncronizing your code..")
sys.stdout.flush()
os.chdir(local_path + "/" + target_project + "/scripts." + user_name)
#files_to_add=os.listdir(local_path+"/"+target_project+"/scripts."+user_name)
#git.git_sync(files_to_add,message,configdic["github_address"],\
git.git_sync(["-A"],message,configdic["github_address"],\
configdic["github_organization"],project_name,\
github_user=configdic["github_user"],\
github_pass=configdic["github_pass"],gitssh=gitssh)
if issue:
for r in ["github_user", "github_pass"]:
while configdic[r] == None:
configdic=config.check_reqs([r],configdic,config_file=None, \
gitssh=None)
publink = str(link_info).split("url=")[1].split(",")[0]
issueMSG="Public link: %s; Private link: %s; Commit message: %s" \
%(publink, private_link,message)
git.git_write_comment(issueMSG,config.get_github_api(configdic["github_address"]),\
configdic["github_organization"],project_name,str(issue),\
github_user=configdic["github_user"],github_pass=configdic["github_pass"])