def createTreeFolder( graphList ):
redate = re.compile( "[0-9]{4}-[0-9]{2}-[0-9]{2}" )
rename = re.compile( "[a-z_]+" )
for graph in graphList:
# check
if isdir( graph ) or '.dot' not in graph:
continue
print "processing", graph
path,graphname = split( graph )
try:
date = redate.findall( graphname )[0]
name = rename.findall( graphname )[0]
except:
continue
classname = name[0].upper()+name[1:]+'Network'
dest = join( path , classname , date )
mkpath( dest )
move( graph, join( dest, "graph.dot" ) )
def downloadDot( savepath, nettype, wget, createTree ):
mkpath( savepath )
try:
os.chdir( savepath )
except:
print "bad path"
sys.exit(1)
print "Downloading...."
if not wget:
command = "wget -r -nd -A "+nettype+"* "+os.path.join( "http://www.trustlet.org/datasets/", nettype )+os.path.sep
else:
command = wget+" -r -nd -A "+nettype+"* "+os.path.join( "http://www.trustlet.org/datasets/", nettype )+os.path.sep
print command
try:
os.system( command )
except:
print "To use this script you must have wget installed"
sys.exit(1)
if createTree:
createTreeFolder( os.listdir( '.' ) )
return
def createTreeFolder(graphList):
redate = re.compile("[0-9]{4}-[0-9]{2}-[0-9]{2}")
rename = re.compile("[a-z_]+")
for graph in graphList:
# check
if isdir(graph) or '.dot' not in graph:
continue
print "processing", graph
path, graphname = split(graph)
try:
date = redate.findall(graphname)[0]
name = rename.findall(graphname)[0]
except:
continue
classname = name[0].upper() + name[1:] + 'Network'
dest = join(path, classname, date)
mkpath(dest)
move(graph, join(dest, "graph.dot"))
n = Network.Network()
for user in users:
n.add_node(user)
#print user
for user in users:
if not exists(userpath%user):
print "Doesn't exist profile of "+user
print userpath%user
continue
profile = parse(userpath%user)
try:
certs = profile.getElementsByTagName('certs')[0]
except IndexError:
# no votes
continue
#print "len(certs)",len(certs.getElementsByTagName('cert'))
for cert in certs.getElementsByTagName('cert'):
subj = str(cert.getAttribute('subj'))
level = str(cert.getAttribute('level'))
#print ">",user, subj, level
n.add_edge(user,subj,{'value':level})
helpers.mkpath(addsep(dataset))
write_dot(n,join(addsep(dataset),'graph.dot'))
def main():
if 'help' in sys.argv or '--help' in sys.argv:
print __doc__
return
# set comment
if '-c' in sys.argv[1:-1]:
i = sys.argv.index('-c')
usercomment = sys.argv[i + 1]
del sys.argv[i + 1]
del sys.argv[i]
else:
usercomment = ''
if sys.argv[1:] and sys.argv[1][0] != '-':
basepath = path.realpath(sys.argv[1])
else:
basepath = HOME
hiddenpath = path.join(basepath, HIDDENDIR)
datasetspath = path.join(basepath, DIR)
#remove old links - why?
#if path.islink(path.join(HOME,HIDDENDIR)):
# os.remove(path.join(HOME,HIDDENDIR))
#if path.islink(path.join(HOME,DIR)):
# os.remove(path.join(HOME,DIR))
if basepath != HOME:
path1 = path.join(HOME, HIDDENDIR)
path2 = path.join(HOME, DIR)
assert not path.islink(path1) and not path.exists(
path1), 'remove ' + path1
assert not path.islink(path2) and not path.exists(
path2), 'remove ' + path2
assert not os.system('ln -s "%s" "%s"' % (hiddenpath, path1))
assert not os.system('ln -s "%s" "%s"' % (datasetspath, path2))
sys.argv = set(sys.argv)
if '--verbose' in sys.argv:
sys.argv.add('-v')
upload = not '--no-upload' in sys.argv
#files updated from svn
file_updated = set()
if not path.isdir(hiddenpath) or not path.isdir(
path.join(hiddenpath, '.svn')):
# dir checking
os.chdir(HOME)
if path.exists(hiddenpath):
shutil.rmtree(hiddenpath)
assert not os.system(SVNCO % hiddenpath)
else:
os.chdir(hiddenpath)
#timestamp update ..> I can use ts of the last execution of sync?
lastupload = load_upload_timestamp()
assert not os.system(SVNUP), 'Update failied. ' + CONFLICT
for dirpath, dirs, files in os.walk(hiddenpath):
if not '.svn' in dirpath:
file_updated |= set([
path.join(dirpath, x) for x in files
if mtime(path.join(dirpath, x)) > lastupload
])
added = updated = uploaded = merged = deleted_server = deleted_local = 0
#print file_updated
# update files modifiedy both from server and from client
for p1 in file_updated:
p2 = p1.replace(hiddenpath, datasetspath)
mkpath(os.path.split(p2)[0])
if p1.endswith('.c2') and os.path.isfile(p2):
# sure that p2 is updated
safe_merge(p2)
if p1.endswith('.c2') and mtime(p2) > lastupload:
# modified from last time (both server and client) -> merge
if merge_cache([p1, p2], p1):
shutil.copy(p1, p2)
print 'merged', relative_path(p2, DIR)[1]
merged += 1
else:
print "I'm updating (coping) from server", relative_path(
p2, DIR)[
1] # ¡¡¡ it might a lost update, non c2 files only !!!
shutil.copy(p1, p2) # hiddenpath -> datasetpath
updated += 1
# from svn to datasets
for dirpath, dirnames, filenames in os.walk(hiddenpath):
if '.svn' in dirpath:
continue
destbasepath = dirpath.replace(hiddenpath, datasetspath)
for filename in filenames:
#print filename
srcpath = path.join(dirpath, filename)
dstpath = path.join(destbasepath, filename)
# relative path, without HOME/(.)datasetspath
rpath = relative_path(srcpath, HIDDENDIR)[1]
#print '<<< File:',rpath
if path.isfile(dstpath):
# Manage removed files
if removed(srcpath):
print 'I\'m removing', rpath
os.remove(dstpath)
rm_empty_path(destbasepath) # ¿¿¿!!!
deleted_server += 1
continue
if not srcpath in file_updated and mtime(
dstpath) > lastupload:
#updated from client and not from server
print 'Upload', rpath
uploaded += 1
shutil.copy(dstpath, srcpath)
elif filename[0] != PREFIX and not re_svnconflict.match(
filename) and not filename.endswith(
'.mine') and not removed(
srcpath) and not filename.endswith('.lock'):
#adding
print 'adding file', rpath
added += 1
#print srcpath,dstpath
# create missing directoriesx
mkpath(destbasepath)
shutil.copy(srcpath, dstpath)
if added:
print '# of added files from server:', added
if updated:
print '# of updated files:', updated
if uploaded:
print '# of uploaded files:', uploaded
if merged:
print '# of merged files:', merged
# from datasetspath to svn
# to adding files
os.chdir(hiddenpath)
#copy *new* files to upload dir (hiddenpath) and remove removed files
for dirpath, dirnames, filenames in os.walk(datasetspath):
#print '>>',dirpath
destbasepath = dirpath.replace(datasetspath, hiddenpath)
if path.sep + PREFIX in dirpath:
# not upload dirs +*
print 'Directory %s will not uploaded' % path.split(dirpath)[1]
continue
# merge locally c2 files ( name.\d+.c2 into name.c2 )
for filename in filenames:
srcpath = path.join(dirpath, filename)
dstpath = path.join(destbasepath, filename)
# Manage removed files
if removed(srcpath):
print 'I\'m removing', dstpath
os.remove(srcpath)
rm_empty_path(dirpath)
deleted_local += 1
continue
# if dstpath in file_updated -» yet merged
if srcpath.endswith('.c2') and path.exists(
srcpath) and dstpath not in file_updated:
#print 'Merge:',srcpath
safe_merge(srcpath)
for filename in filenames:
srcpath = path.join(dirpath, filename)
if not path.exists(srcpath):
continue #merging c2 might erase files
dstpath = path.join(destbasepath, filename)
# relative path, without HOME/(.)datasetspath
rpath = relative_path(srcpath, DIR)[1]
#print '>>> File:',rpath
if filename.startswith(PREFIX) or re_svnconflict.match(filename) \
or filename.endswith('.mine') or filename.endswith('~') \
or filename.endswith('.lock'):
# not upload files _*, svn file and backup files
print 'File %s will not uploaded' % rpath
continue
if not path.isfile(dstpath):
print 'adding file', rpath
added += 1
mkpath(destbasepath)
svnaddpath(destbasepath)
shutil.copy(srcpath, destbasepath)
svnadd(dstpath)
#print '<',srcpath,srcpath in updatedc2
#print updatedc2
comment = ''
if added:
comment += ' Added %d files.' % added
if uploaded:
comment += ' Uploaded %d files.' % uploaded
if deleted_local:
comment += ' Deleted %d files.' % deleted_local
if usercomment:
comment += ' ' + usercomment
if comment and '-v' in sys.argv:
print 'Commit comment:', comment
assert not os.system(SVNCI % comment), CONFLICT
if added:
print '# of added files to server repository:', added
if deleted_server or deleted_local:
print '# of deleted files:', deleted_local + deleted_server
# os.chdir(CURDIR)
# this is the last thing to do because a crash after save_upload_ts()
# without upload new data will cause a lost of them
save_upload_timestamp()
def main():
if 'help' in sys.argv or '--help' in sys.argv:
print __doc__
return
# set comment
if '-c' in sys.argv[1:-1]:
i = sys.argv.index('-c')
usercomment = sys.argv[i+1]
del sys.argv[i+1]
del sys.argv[i]
else:
usercomment = ''
if sys.argv[1:] and sys.argv[1][0]!='-':
basepath = path.realpath(sys.argv[1])
else:
basepath = HOME
hiddenpath = path.join(basepath,HIDDENDIR)
datasetspath = path.join(basepath,DIR)
#remove old links - why?
#if path.islink(path.join(HOME,HIDDENDIR)):
# os.remove(path.join(HOME,HIDDENDIR))
#if path.islink(path.join(HOME,DIR)):
# os.remove(path.join(HOME,DIR))
if basepath != HOME:
path1 = path.join(HOME,HIDDENDIR)
path2 = path.join(HOME,DIR)
assert not path.islink(path1) and not path.exists(path1),'remove '+path1
assert not path.islink(path2) and not path.exists(path2),'remove '+path2
assert not os.system('ln -s "%s" "%s"' % (hiddenpath,path1))
assert not os.system('ln -s "%s" "%s"' % (datasetspath,path2))
sys.argv = set(sys.argv)
if '--verbose' in sys.argv:
sys.argv.add('-v')
upload = not '--no-upload' in sys.argv
#files updated from svn
file_updated = set()
if not path.isdir(hiddenpath) or not path.isdir(path.join(hiddenpath,'.svn')):
# dir checking
os.chdir(HOME)
if path.exists(hiddenpath):
shutil.rmtree(hiddenpath)
assert not os.system(SVNCO % hiddenpath)
else:
os.chdir(hiddenpath)
#timestamp update ..> I can use ts of the last execution of sync?
lastupload = load_upload_timestamp()
assert not os.system(SVNUP),'Update failied. '+CONFLICT
for dirpath,dirs,files in os.walk(hiddenpath):
if not '.svn' in dirpath:
file_updated |= set([path.join(dirpath,x) for x in files
if mtime(path.join(dirpath,x))>lastupload])
added = updated = uploaded = merged = deleted_server = deleted_local = 0
#print file_updated
# update files modifiedy both from server and from client
for p1 in file_updated:
p2 = p1.replace(hiddenpath,datasetspath)
mkpath(os.path.split(p2)[0])
if p1.endswith('.c2') and os.path.isfile(p2):
# sure that p2 is updated
safe_merge(p2)
if p1.endswith('.c2') and mtime(p2)>lastupload:
# modified from last time (both server and client) -> merge
if merge_cache([p1,p2],p1):
shutil.copy(p1,p2)
print 'merged',relative_path(p2,DIR)[1]
merged += 1
else:
print "I'm updating (coping) from server",relative_path(p2,DIR)[1] # ¡¡¡ it might a lost update, non c2 files only !!!
shutil.copy(p1,p2) # hiddenpath -> datasetpath
updated += 1
# from svn to datasets
for dirpath,dirnames,filenames in os.walk(hiddenpath):
if '.svn' in dirpath:
continue
destbasepath = dirpath.replace(hiddenpath,datasetspath)
for filename in filenames:
#print filename
srcpath = path.join(dirpath,filename)
dstpath = path.join(destbasepath,filename)
# relative path, without HOME/(.)datasetspath
rpath = relative_path(srcpath,HIDDENDIR)[1]
#print '<<< File:',rpath
if path.isfile(dstpath):
# Manage removed files
if removed(srcpath):
print 'I\'m removing',rpath
os.remove(dstpath)
rm_empty_path(destbasepath) # ¿¿¿!!!
deleted_server += 1
continue
if not srcpath in file_updated and mtime(dstpath)>lastupload:
#updated from client and not from server
print 'Upload',rpath
uploaded += 1
shutil.copy(dstpath,srcpath)
elif filename[0]!=PREFIX and not re_svnconflict.match(filename) and not filename.endswith('.mine') and not removed(srcpath) and not filename.endswith('.lock'):
#adding
print 'adding file',rpath
added += 1
#print srcpath,dstpath
# create missing directoriesx
mkpath(destbasepath)
shutil.copy(srcpath,dstpath)
if added:
print '# of added files from server:',added
if updated:
print '# of updated files:',updated
if uploaded:
print '# of uploaded files:',uploaded
if merged:
print '# of merged files:',merged
# from datasetspath to svn
# to adding files
os.chdir(hiddenpath)
#copy *new* files to upload dir (hiddenpath) and remove removed files
for dirpath,dirnames,filenames in os.walk(datasetspath):
#print '>>',dirpath
destbasepath = dirpath.replace(datasetspath,hiddenpath)
if path.sep+PREFIX in dirpath:
# not upload dirs +*
print 'Directory %s will not uploaded' % path.split(dirpath)[1]
continue
# merge locally c2 files ( name.\d+.c2 into name.c2 )
for filename in filenames:
srcpath = path.join(dirpath,filename)
dstpath = path.join(destbasepath,filename)
# Manage removed files
if removed(srcpath):
print 'I\'m removing',dstpath
os.remove(srcpath)
rm_empty_path(dirpath)
deleted_local += 1
continue
# if dstpath in file_updated -» yet merged
if srcpath.endswith('.c2') and path.exists(srcpath) and dstpath not in file_updated:
#print 'Merge:',srcpath
safe_merge(srcpath)
for filename in filenames:
srcpath = path.join(dirpath,filename)
if not path.exists(srcpath):
continue #merging c2 might erase files
dstpath = path.join(destbasepath,filename)
# relative path, without HOME/(.)datasetspath
rpath = relative_path(srcpath,DIR)[1]
#print '>>> File:',rpath
if filename.startswith(PREFIX) or re_svnconflict.match(filename) \
or filename.endswith('.mine') or filename.endswith('~') \
or filename.endswith('.lock'):
# not upload files _*, svn file and backup files
print 'File %s will not uploaded' % rpath
continue
if not path.isfile(dstpath):
print 'adding file',rpath
added += 1
mkpath(destbasepath)
svnaddpath(destbasepath)
shutil.copy(srcpath,destbasepath)
svnadd(dstpath)
#print '<',srcpath,srcpath in updatedc2
#print updatedc2
comment = ''
if added:
comment += ' Added %d files.' % added
if uploaded:
comment += ' Uploaded %d files.' % uploaded
if deleted_local:
comment += ' Deleted %d files.' % deleted_local
if usercomment:
comment += ' ' + usercomment
if comment and '-v' in sys.argv:
print 'Commit comment:', comment
assert not os.system(SVNCI % comment),CONFLICT
if added:
print '# of added files to server repository:',added
if deleted_server or deleted_local:
print '# of deleted files:',deleted_local+deleted_server
# os.chdir(CURDIR)
# this is the last thing to do because a crash after save_upload_ts()
# without upload new data will cause a lost of them
save_upload_timestamp()
