def createTreeFolder( graphList ): redate = re.compile( "[0-9]{4}-[0-9]{2}-[0-9]{2}" ) rename = re.compile( "[a-z_]+" ) for graph in graphList: # check if isdir( graph ) or '.dot' not in graph: continue print "processing", graph path,graphname = split( graph ) try: date = redate.findall( graphname )[0] name = rename.findall( graphname )[0] except: continue classname = name[0].upper()+name[1:]+'Network' dest = join( path , classname , date ) mkpath( dest ) move( graph, join( dest, "graph.dot" ) )
def downloadDot( savepath, nettype, wget, createTree ): mkpath( savepath ) try: os.chdir( savepath ) except: print "bad path" sys.exit(1) print "Downloading...." if not wget: command = "wget -r -nd -A "+nettype+"* "+os.path.join( "http://www.trustlet.org/datasets/", nettype )+os.path.sep else: command = wget+" -r -nd -A "+nettype+"* "+os.path.join( "http://www.trustlet.org/datasets/", nettype )+os.path.sep print command try: os.system( command ) except: print "To use this script you must have wget installed" sys.exit(1) if createTree: createTreeFolder( os.listdir( '.' ) ) return
def createTreeFolder(graphList): redate = re.compile("[0-9]{4}-[0-9]{2}-[0-9]{2}") rename = re.compile("[a-z_]+") for graph in graphList: # check if isdir(graph) or '.dot' not in graph: continue print "processing", graph path, graphname = split(graph) try: date = redate.findall(graphname)[0] name = rename.findall(graphname)[0] except: continue classname = name[0].upper() + name[1:] + 'Network' dest = join(path, classname, date) mkpath(dest) move(graph, join(dest, "graph.dot"))
n = Network.Network() for user in users: n.add_node(user) #print user for user in users: if not exists(userpath%user): print "Doesn't exist profile of "+user print userpath%user continue profile = parse(userpath%user) try: certs = profile.getElementsByTagName('certs')[0] except IndexError: # no votes continue #print "len(certs)",len(certs.getElementsByTagName('cert')) for cert in certs.getElementsByTagName('cert'): subj = str(cert.getAttribute('subj')) level = str(cert.getAttribute('level')) #print ">",user, subj, level n.add_edge(user,subj,{'value':level}) helpers.mkpath(addsep(dataset)) write_dot(n,join(addsep(dataset),'graph.dot'))
def main(): if 'help' in sys.argv or '--help' in sys.argv: print __doc__ return # set comment if '-c' in sys.argv[1:-1]: i = sys.argv.index('-c') usercomment = sys.argv[i + 1] del sys.argv[i + 1] del sys.argv[i] else: usercomment = '' if sys.argv[1:] and sys.argv[1][0] != '-': basepath = path.realpath(sys.argv[1]) else: basepath = HOME hiddenpath = path.join(basepath, HIDDENDIR) datasetspath = path.join(basepath, DIR) #remove old links - why? #if path.islink(path.join(HOME,HIDDENDIR)): # os.remove(path.join(HOME,HIDDENDIR)) #if path.islink(path.join(HOME,DIR)): # os.remove(path.join(HOME,DIR)) if basepath != HOME: path1 = path.join(HOME, HIDDENDIR) path2 = path.join(HOME, DIR) assert not path.islink(path1) and not path.exists( path1), 'remove ' + path1 assert not path.islink(path2) and not path.exists( path2), 'remove ' + path2 assert not os.system('ln -s "%s" "%s"' % (hiddenpath, path1)) assert not os.system('ln -s "%s" "%s"' % (datasetspath, path2)) sys.argv = set(sys.argv) if '--verbose' in sys.argv: sys.argv.add('-v') upload = not '--no-upload' in sys.argv #files updated from svn file_updated = set() if not path.isdir(hiddenpath) or not path.isdir( path.join(hiddenpath, '.svn')): # dir checking os.chdir(HOME) if path.exists(hiddenpath): shutil.rmtree(hiddenpath) assert not os.system(SVNCO % hiddenpath) else: os.chdir(hiddenpath) #timestamp update ..> I can use ts of the last execution of sync? lastupload = load_upload_timestamp() assert not os.system(SVNUP), 'Update failied. ' + CONFLICT for dirpath, dirs, files in os.walk(hiddenpath): if not '.svn' in dirpath: file_updated |= set([ path.join(dirpath, x) for x in files if mtime(path.join(dirpath, x)) > lastupload ]) added = updated = uploaded = merged = deleted_server = deleted_local = 0 #print file_updated # update files modifiedy both from server and from client for p1 in file_updated: p2 = p1.replace(hiddenpath, datasetspath) mkpath(os.path.split(p2)[0]) if p1.endswith('.c2') and os.path.isfile(p2): # sure that p2 is updated safe_merge(p2) if p1.endswith('.c2') and mtime(p2) > lastupload: # modified from last time (both server and client) -> merge if merge_cache([p1, p2], p1): shutil.copy(p1, p2) print 'merged', relative_path(p2, DIR)[1] merged += 1 else: print "I'm updating (coping) from server", relative_path( p2, DIR)[ 1] # ¡¡¡ it might a lost update, non c2 files only !!! shutil.copy(p1, p2) # hiddenpath -> datasetpath updated += 1 # from svn to datasets for dirpath, dirnames, filenames in os.walk(hiddenpath): if '.svn' in dirpath: continue destbasepath = dirpath.replace(hiddenpath, datasetspath) for filename in filenames: #print filename srcpath = path.join(dirpath, filename) dstpath = path.join(destbasepath, filename) # relative path, without HOME/(.)datasetspath rpath = relative_path(srcpath, HIDDENDIR)[1] #print '<<< File:',rpath if path.isfile(dstpath): # Manage removed files if removed(srcpath): print 'I\'m removing', rpath os.remove(dstpath) rm_empty_path(destbasepath) # ¿¿¿!!! deleted_server += 1 continue if not srcpath in file_updated and mtime( dstpath) > lastupload: #updated from client and not from server print 'Upload', rpath uploaded += 1 shutil.copy(dstpath, srcpath) elif filename[0] != PREFIX and not re_svnconflict.match( filename) and not filename.endswith( '.mine') and not removed( srcpath) and not filename.endswith('.lock'): #adding print 'adding file', rpath added += 1 #print srcpath,dstpath # create missing directoriesx mkpath(destbasepath) shutil.copy(srcpath, dstpath) if added: print '# of added files from server:', added if updated: print '# of updated files:', updated if uploaded: print '# of uploaded files:', uploaded if merged: print '# of merged files:', merged # from datasetspath to svn # to adding files os.chdir(hiddenpath) #copy *new* files to upload dir (hiddenpath) and remove removed files for dirpath, dirnames, filenames in os.walk(datasetspath): #print '>>',dirpath destbasepath = dirpath.replace(datasetspath, hiddenpath) if path.sep + PREFIX in dirpath: # not upload dirs +* print 'Directory %s will not uploaded' % path.split(dirpath)[1] continue # merge locally c2 files ( name.\d+.c2 into name.c2 ) for filename in filenames: srcpath = path.join(dirpath, filename) dstpath = path.join(destbasepath, filename) # Manage removed files if removed(srcpath): print 'I\'m removing', dstpath os.remove(srcpath) rm_empty_path(dirpath) deleted_local += 1 continue # if dstpath in file_updated -» yet merged if srcpath.endswith('.c2') and path.exists( srcpath) and dstpath not in file_updated: #print 'Merge:',srcpath safe_merge(srcpath) for filename in filenames: srcpath = path.join(dirpath, filename) if not path.exists(srcpath): continue #merging c2 might erase files dstpath = path.join(destbasepath, filename) # relative path, without HOME/(.)datasetspath rpath = relative_path(srcpath, DIR)[1] #print '>>> File:',rpath if filename.startswith(PREFIX) or re_svnconflict.match(filename) \ or filename.endswith('.mine') or filename.endswith('~') \ or filename.endswith('.lock'): # not upload files _*, svn file and backup files print 'File %s will not uploaded' % rpath continue if not path.isfile(dstpath): print 'adding file', rpath added += 1 mkpath(destbasepath) svnaddpath(destbasepath) shutil.copy(srcpath, destbasepath) svnadd(dstpath) #print '<',srcpath,srcpath in updatedc2 #print updatedc2 comment = '' if added: comment += ' Added %d files.' % added if uploaded: comment += ' Uploaded %d files.' % uploaded if deleted_local: comment += ' Deleted %d files.' % deleted_local if usercomment: comment += ' ' + usercomment if comment and '-v' in sys.argv: print 'Commit comment:', comment assert not os.system(SVNCI % comment), CONFLICT if added: print '# of added files to server repository:', added if deleted_server or deleted_local: print '# of deleted files:', deleted_local + deleted_server # os.chdir(CURDIR) # this is the last thing to do because a crash after save_upload_ts() # without upload new data will cause a lost of them save_upload_timestamp()
def main(): if 'help' in sys.argv or '--help' in sys.argv: print __doc__ return # set comment if '-c' in sys.argv[1:-1]: i = sys.argv.index('-c') usercomment = sys.argv[i+1] del sys.argv[i+1] del sys.argv[i] else: usercomment = '' if sys.argv[1:] and sys.argv[1][0]!='-': basepath = path.realpath(sys.argv[1]) else: basepath = HOME hiddenpath = path.join(basepath,HIDDENDIR) datasetspath = path.join(basepath,DIR) #remove old links - why? #if path.islink(path.join(HOME,HIDDENDIR)): # os.remove(path.join(HOME,HIDDENDIR)) #if path.islink(path.join(HOME,DIR)): # os.remove(path.join(HOME,DIR)) if basepath != HOME: path1 = path.join(HOME,HIDDENDIR) path2 = path.join(HOME,DIR) assert not path.islink(path1) and not path.exists(path1),'remove '+path1 assert not path.islink(path2) and not path.exists(path2),'remove '+path2 assert not os.system('ln -s "%s" "%s"' % (hiddenpath,path1)) assert not os.system('ln -s "%s" "%s"' % (datasetspath,path2)) sys.argv = set(sys.argv) if '--verbose' in sys.argv: sys.argv.add('-v') upload = not '--no-upload' in sys.argv #files updated from svn file_updated = set() if not path.isdir(hiddenpath) or not path.isdir(path.join(hiddenpath,'.svn')): # dir checking os.chdir(HOME) if path.exists(hiddenpath): shutil.rmtree(hiddenpath) assert not os.system(SVNCO % hiddenpath) else: os.chdir(hiddenpath) #timestamp update ..> I can use ts of the last execution of sync? lastupload = load_upload_timestamp() assert not os.system(SVNUP),'Update failied. '+CONFLICT for dirpath,dirs,files in os.walk(hiddenpath): if not '.svn' in dirpath: file_updated |= set([path.join(dirpath,x) for x in files if mtime(path.join(dirpath,x))>lastupload]) added = updated = uploaded = merged = deleted_server = deleted_local = 0 #print file_updated # update files modifiedy both from server and from client for p1 in file_updated: p2 = p1.replace(hiddenpath,datasetspath) mkpath(os.path.split(p2)[0]) if p1.endswith('.c2') and os.path.isfile(p2): # sure that p2 is updated safe_merge(p2) if p1.endswith('.c2') and mtime(p2)>lastupload: # modified from last time (both server and client) -> merge if merge_cache([p1,p2],p1): shutil.copy(p1,p2) print 'merged',relative_path(p2,DIR)[1] merged += 1 else: print "I'm updating (coping) from server",relative_path(p2,DIR)[1] # ¡¡¡ it might a lost update, non c2 files only !!! shutil.copy(p1,p2) # hiddenpath -> datasetpath updated += 1 # from svn to datasets for dirpath,dirnames,filenames in os.walk(hiddenpath): if '.svn' in dirpath: continue destbasepath = dirpath.replace(hiddenpath,datasetspath) for filename in filenames: #print filename srcpath = path.join(dirpath,filename) dstpath = path.join(destbasepath,filename) # relative path, without HOME/(.)datasetspath rpath = relative_path(srcpath,HIDDENDIR)[1] #print '<<< File:',rpath if path.isfile(dstpath): # Manage removed files if removed(srcpath): print 'I\'m removing',rpath os.remove(dstpath) rm_empty_path(destbasepath) # ¿¿¿!!! deleted_server += 1 continue if not srcpath in file_updated and mtime(dstpath)>lastupload: #updated from client and not from server print 'Upload',rpath uploaded += 1 shutil.copy(dstpath,srcpath) elif filename[0]!=PREFIX and not re_svnconflict.match(filename) and not filename.endswith('.mine') and not removed(srcpath) and not filename.endswith('.lock'): #adding print 'adding file',rpath added += 1 #print srcpath,dstpath # create missing directoriesx mkpath(destbasepath) shutil.copy(srcpath,dstpath) if added: print '# of added files from server:',added if updated: print '# of updated files:',updated if uploaded: print '# of uploaded files:',uploaded if merged: print '# of merged files:',merged # from datasetspath to svn # to adding files os.chdir(hiddenpath) #copy *new* files to upload dir (hiddenpath) and remove removed files for dirpath,dirnames,filenames in os.walk(datasetspath): #print '>>',dirpath destbasepath = dirpath.replace(datasetspath,hiddenpath) if path.sep+PREFIX in dirpath: # not upload dirs +* print 'Directory %s will not uploaded' % path.split(dirpath)[1] continue # merge locally c2 files ( name.\d+.c2 into name.c2 ) for filename in filenames: srcpath = path.join(dirpath,filename) dstpath = path.join(destbasepath,filename) # Manage removed files if removed(srcpath): print 'I\'m removing',dstpath os.remove(srcpath) rm_empty_path(dirpath) deleted_local += 1 continue # if dstpath in file_updated -» yet merged if srcpath.endswith('.c2') and path.exists(srcpath) and dstpath not in file_updated: #print 'Merge:',srcpath safe_merge(srcpath) for filename in filenames: srcpath = path.join(dirpath,filename) if not path.exists(srcpath): continue #merging c2 might erase files dstpath = path.join(destbasepath,filename) # relative path, without HOME/(.)datasetspath rpath = relative_path(srcpath,DIR)[1] #print '>>> File:',rpath if filename.startswith(PREFIX) or re_svnconflict.match(filename) \ or filename.endswith('.mine') or filename.endswith('~') \ or filename.endswith('.lock'): # not upload files _*, svn file and backup files print 'File %s will not uploaded' % rpath continue if not path.isfile(dstpath): print 'adding file',rpath added += 1 mkpath(destbasepath) svnaddpath(destbasepath) shutil.copy(srcpath,destbasepath) svnadd(dstpath) #print '<',srcpath,srcpath in updatedc2 #print updatedc2 comment = '' if added: comment += ' Added %d files.' % added if uploaded: comment += ' Uploaded %d files.' % uploaded if deleted_local: comment += ' Deleted %d files.' % deleted_local if usercomment: comment += ' ' + usercomment if comment and '-v' in sys.argv: print 'Commit comment:', comment assert not os.system(SVNCI % comment),CONFLICT if added: print '# of added files to server repository:',added if deleted_server or deleted_local: print '# of deleted files:',deleted_local+deleted_server # os.chdir(CURDIR) # this is the last thing to do because a crash after save_upload_ts() # without upload new data will cause a lost of them save_upload_timestamp()