def get_command():
cmd = None
try:
from subprocess import Popen, PIPE
p = Popen('uniconvertor', shell=True, stdout=PIPE, stderr=PIPE).wait()
if p==0 :
cmd = 'uniconvertor'
else:
p = Popen('uniconv', shell=True, stdout=PIPE, stderr=PIPE).wait()
if p==0 :
cmd = 'uniconv'
except ImportError:
from popen2 import Popen3
p = Popen3('uniconv', True).wait()
if p!=32512 : cmd = 'uniconv'
p = Popen3('uniconvertor', True).wait()
if p!=32512 : cmd = 'uniconvertor'
if cmd == None:
# there's no succeffully-returning uniconv command; try to get the module directly (on Windows)
try:
# cannot simply import uniconvertor, it aborts if you import it without parameters
import imp
imp.find_module("uniconvertor")
except ImportError:
sys.stderr.write(_('You need to install the UniConvertor software.\n'+\
'For GNU/Linux: install the package python-uniconvertor.\n'+\
'For Windows: download it from\n'+\
'http://sk1project.org/modules.php?name=Products&product=uniconvertor\n'+\
'and install into your Inkscape\'s Python location\n'))
sys.exit(1)
cmd = 'python -c "import uniconvertor; uniconvertor.uniconv_run();"'
return cmd
def pipe(cmd, str=None):
from popen2 import Popen3 as Popen
bufsize = 4096
MAX = 65536*16
if str and len(str) > 4096: # XXX: Hardcoded Linux 2.4, 2.6 pipe buffer size
bufsize = len(str)
pipe = Popen(cmd, True, bufsize)
if not str is None:
pipe.tochild.write(str)
pipe.tochild.close()
out = ""
err = ""
while True:
str = pipe.fromchild.read()
if str:
out += str
code = pipe.poll()
if code > -1:
err = pipe.childerr.read()
break
if len(out) >= MAX:
err = "Output exceeds %s bytes and has been truncated"
break
return (code, out, err)
def pipe(cmd, str=None):
from popen2 import Popen3 as Popen
bufsize = 4096
MAX = 65536 * 16
if str and len(
str) > 4096: # XXX: Hardcoded Linux 2.4, 2.6 pipe buffer size
bufsize = len(str)
pipe = Popen(cmd, True, bufsize)
if not str is None:
pipe.tochild.write(str)
pipe.tochild.close()
out = ""
err = ""
while True:
str = pipe.fromchild.read()
if str:
out += str
code = pipe.poll()
if code > -1:
err = pipe.childerr.read()
break
if len(out) >= MAX:
err = "Output exceeds %s bytes and has been truncated" % MAX
break
return (code, out, err)
def assert_search_syslog_call(self, pattern, callable_object, *args,
**kw_args):
"""
Passes if re.search(pattern, SYSLOG_CONTENT) doesn't return None
after callable_object(*args, **kw_args).
self.assert_search_syslog_call("X", syslog.syslog, "XYZ") # => pass
self.assert_search_syslog_call("X", syslog.syslog, "ABC") # => fail
"""
if not hasattr(sys.modules[__name__], "syslog"):
self.omit("syslog isn't supported on this environment")
self.assert_callable(callable_object)
mark = 'Pikzie: %.20f' % random.random()
syslog.syslog(mark)
log_file = "/var/log/messages"
command = ["tail", "-F", log_file]
try:
from subprocess import Popen, PIPE
messages = Popen(command, stdout=PIPE, close_fds=True)
except ImportError:
from popen2 import Popen3
messages = Popen3(command)
messages.stdout = messages.fromchild
fcntl.fcntl(
messages.stdout, fcntl.F_SETFL,
os.O_NONBLOCK | fcntl.fcntl(messages.stdout, fcntl.F_GETFL))
def search(pattern):
if isinstance(pattern, str):
pattern = re.compile(pattern)
content = b''
timeout = 1.5
while len(select.select([messages.stdout], [], [],
timeout)[0]) > 0:
timeout = 0.1
added_content = messages.stdout.read()
if not added_content:
break
content += added_content
if re.search(pattern, str(content)):
return
message = \
"expected: <%s> is found in <%s>\n" \
" content: <%s>" % \
(pp.format_re(pattern),
pp.format(log_file),
pp.format(content))
self.fail(message)
try:
search(re.escape(mark))
result = callable_object(*args, **kw_args)
search(pattern)
finally:
os.kill(messages.pid, signal.SIGINT)
messages.wait()
def cmdexec(cmd):
""" Executes a command in a subshell and returns (return_value, (stdout, stderr)). """
if (sys.platform == "win32"):
cmd = "(%s) 2>&1" % (cmd)
result = 0
stderr_output = []
stdout_output = cmdoutput(cmd)
else:
my_popen3 = Popen3(cmd, True)
# wait until the sub process has finished
while (my_popen3.poll() == -1):
sleep(0.01)
stderr_output = my_popen3.fromchild.readlines()
stdout_output = my_popen3.childerr.readlines()
# read the result value
result = my_popen3.poll()
if (my_popen3.fromchild != None):
my_popen3.fromchild.close()
if (my_popen3.childerr != None):
my_popen3.childerr.close()
if (my_popen3.tochild != None):
my_popen3.tochild.close()
return (result, (stdout_output, stderr_output))
def run(command_format, prog_name, uniconv_format):
outfile = tempfile.mktemp(uniconv_format)
command = command_format + outfile
msg = None
# In order to get a return code from the process, we use subprocess.Popen
# if it's available (Python 2.4 onwards) and otherwise use popen2.Popen3
# (Unix only). As the Inkscape package for Windows includes Python 2.5,
# this should cover all supported platforms.
try:
try:
from subprocess import Popen, PIPE
p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE)
rc = p.wait()
out = p.stdout.read()
err = p.stderr.read()
except ImportError:
try:
from popen2 import Popen3
p = Popen3(command, True)
p.wait()
rc = p.poll()
out = p.fromchild.read()
err = p.childerr.read()
except ImportError:
# shouldn't happen...
msg = "Neither subprocess.Popen nor popen2.Popen3 is available"
if rc and msg is None:
msg = "%s failed:\n%s\n%s\n" % (prog_name, out, err)
except Exception, inst:
msg = "Error attempting to run %s: %s" % (prog_name, str(inst))
def run(self):
if self.delay:
self.log.debug("delaying for %.3f secs: '%s'" %
(self.delay, self.cmd))
time.sleep(self.delay)
self.log.debug("program starting '%s'" % self.cmd)
p = Popen3(self.cmd, True)
if self.stdin:
if p.poll() == -1:
p.tochild.write(self.stdin)
p.tochild.flush()
p.tochild.close()
#log.debug("wrote '%s' to child" % self.stdin)
else:
self.log.error("child exited before stdin was written to")
done = False
while not done and self.killtime > 0:
time.sleep(0.2)
if p.poll() != -1:
done = True
self.killtime -= 0.2
if not done and self.killsig != -1:
try:
os.kill(p.pid, self.killsig)
self.killed = True
except OSError, e:
self.log.exception("problem killing")
self.exception = e
return
def bettersystem(command, stdout=None, stderr=None):
"""Not quite finished, sadly."""
import select
from popen2 import Popen3
stdout = stdout or sys.stdout
stderr = stderr or sys.stderr
p = Popen3(command, capturestderr=True)
p_out, p_err = p.fromchild, p.childerr
fd_out = p_out.fileno()
fd_err = p_err.fileno()
while 1:
if p.poll() != -1:
break
rlist, _, _ = select.select([p_out, p_err], [], [])
if not rlist:
break
if p_out in rlist:
output = os.read(fd_out, 1024)
if output == '':
p.wait()
else:
stdout.write(output)
stdout.flush()
if p_err in rlist:
output = os.read(fd_err, 1024)
if output == '':
p.wait()
else:
stderr.write(output)
stderr.flush()
return p.wait()
def runMarkdown(input):
markdown = Popen3(MARKDOWN)
markdown.tochild.write(input)
markdown.tochild.close()
output = markdown.fromchild.read()
status = markdown.wait()
assert normalExitStatus(status), "Markdown exited unusually"
return output
def mail(subject, msg_content, to="*****@*****.**"):
# Opening the senmail process
from popen2 import Popen3
sendmail = Popen3('mail %s -s "%s"' % (to, subject), True)
# "Body" of the mail
sendmail.tochild.write("%s\n" % msg_content)
# "Closing" the mail process
sendmail.tochild.close()
def open_pipe(command, mode):
"""Opens a popen3 pipe to COMMAND in MODE.
Returns (infile, outfile, errfile, waiter); waiter
should be passed to wait_on_pipe."""
if platform_with_popen3_class:
kid = Popen3(command, True)
return kid.tochild, kid.fromchild, kid.childerr, (kid, command)
else:
inf, outf, errf = os.popen3(command, mode)
return inf, outf, errf, None
def RunExternal(command, str_stdin=""):
"""Run an external command
@param command: String of the command to execute
@param stdin: String to put put in stdin
@return: (str(stdout), str(stderr)) of command
Returns the stdout and stderr
"""
logging.info("Running external command: %s" % command)
popen_inst = Popen3(command, True)
logging.debug("stdin = %s" % str_stdin)
str_stdout = str_stderr = ""
while 1:
read_from_child = -1
if not popen_inst.tochild.closed:
(rlist, wlist, xlist) = select([popen_inst.fromchild, popen_inst.childerr], \
[popen_inst.tochild], [])
else:
(rlist, wlist,
xlist) = select([popen_inst.fromchild, popen_inst.childerr], [],
[])
if popen_inst.fromchild in rlist:
tmpread = popen_inst.fromchild.read(4096)
read_from_child = len(tmpread)
str_stdout += tmpread
if popen_inst.childerr in rlist:
tmpread = popen_inst.childerr.read(4096)
read_from_child += len(tmpread)
str_stderr += tmpread
if popen_inst.tochild in wlist and len(str_stdin) > 0:
popen_inst.tochild.write(str_stdin[:min([len(str_stdin), 4096])])
str_stdin = str_stdin[min([len(str_stdin), 4096]):]
read_from_child += 1
elif popen_inst.tochild in wlist:
popen_inst.tochild.close()
#logging.debug("len(str_stdin) = %i, read_from_child = %i, rlist = %s, wlist = %s", len(str_stdin), read_from_child, rlist, wlist)
if popen_inst.poll() != -1 and len(str_stdin) == 0 and (
read_from_child == -1 or read_from_child == 0):
break
logging.debug("Exit code: %i", popen_inst.wait())
logging.debug("stdout: %s", str_stdout)
logging.debug("strerr: %s", str_stderr)
return str_stdout, str_stderr
def send_nsca(self,
code,
message,
ncsahost,
hostname=node(),
service=None):
"""
Send data via send_nsca for passive service checks
"""
# Execute send_nsca
from popen2 import Popen3
command = "send_nsca -H %s" % ncsahost
p = Popen3(command, capturestderr=True)
# Service check
if service:
print >> p.tochild, "%s %s %s %s %s" % (
hostname, service, code, message, self.perfdata_string())
# Host check, omit service_description
else:
print >> p.tochild, "%s %s %s %s" % (hostname, code, message,
self.perfdata_string())
# Send eof
# TODO, support multiple statuses ?
p.tochild.close()
# Save output incase we have an error
nsca_output = ''
for line in p.fromchild.readlines():
nsca_output += line
# Wait for send_nsca to exit
returncode = p.wait()
returncode = os.WEXITSTATUS(returncode)
# Problem with running nsca
if returncode != 0:
if returncode == 127:
raise Exception("Could not find send_nsca in path")
else:
raise Exception("returncode: %i\n%s" %
(returncode, nsca_output))
return 0
def __init__ (self, binary = 'peez2', common_arguments = '2> /dev/null',
debug = DEBUGGING_STATUS,
partition_scheme = MINIMAL_PARTITION_SCHEME):
""" Detect locale and scan for drives. """
self.__binary = binary
self.__common_arguments = common_arguments
self.__debug = debug
self.__partition_scheme = partition_scheme
self.__locale = getdefaultlocale () [0]
self.__drives = self.__scan_drives ()
# Disable this attribute when auto-partitioning is mature enough:
self.__ONLY_MANUALLY = False
# Every partitioning command executed will be also written here:
p = Popen3 ('if [ -e /tmp/guadalinex-express.commands ]; ' + \
'then rm /tmp/guadalinex-express.commands; fi')
p.wait ()
def __call_peez2 (self, args = '', input = ''):
""" Execute I{peez2} with arguments provided, if any. It is also
possible to specify an input. """
command = self.__binary + ' ' + args + ' ' + self.__common_arguments
command = 'LANGUAGE=C ' + command
if '' != input:
command = 'echo -e "' + input + '" | ' + command
if self.__debug:
stderr.write ('__call_peez2: command "' + command + '" executed.\n')
child = Popen3 (command, False, 1048576)
# child.wait ()
return {'out': child.fromchild,
'in': child.tochild,
'err': child.childerr}
def run(command_format, prog_name):
svgfile = tempfile.mktemp(".svg")
command = command_format % svgfile
msg = None
# ps2pdf may attempt to write to the current directory, which may not
# be writeable, so we switch to the temp directory first.
try:
os.chdir(tempfile.gettempdir())
except Exception:
pass
# In order to get a return code from the process, we use subprocess.Popen
# if it's available (Python 2.4 onwards) and otherwise use popen2.Popen3
# (Unix only). As the Inkscape package for Windows includes Python 2.5,
# this should cover all supported platforms.
try:
try:
from subprocess import Popen, PIPE
p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE)
rc = p.wait()
out = p.stdout.read()
err = p.stderr.read()
except ImportError:
try:
from popen2 import Popen3
p = Popen3(command, True)
p.wait()
rc = p.poll()
out = p.fromchild.read()
err = p.childerr.read()
except ImportError:
# shouldn't happen...
msg = "Neither subprocess.Popen nor popen2.Popen3 is available"
if msg is None:
if rc:
msg = "%s failed:\n%s\n%s\n" % (prog_name, out, err)
elif err:
sys.stderr.write(
"%s executed but logged the following error:\n%s\n%s\n" %
(prog_name, out, err))
except Exception, inst:
msg = "Error attempting to run %s: %s" % (prog_name, str(inst))
def run():
child = Popen3([js, script])
while True:
bytes = child.fromchild.read(4096)
if bytes:
protocol.dataReceived(bytes)
else:
break
exitCode = child.wait()
if os.WIFSIGNALED(exitCode):
result.addError(
self, (Exception,
"JavaScript interpreter exited due to signal %d" %
(os.WTERMSIG(exitCode), ), None))
else:
exitStatus = os.WEXITSTATUS(exitCode)
if exitStatus:
result.addError(
self, (Exception,
"JavaScript interpreter had error exit: %d" %
(exitStatus, ), None))
def __init__(self, command, fc=True):
'''
Creates object _and_ runs a command
command - a command to run
fc - `fast call` - whether to run via fork'n'exec (True)
or in the subshell (False)
'''
if fc:
command = command.split()
inst = Popen3(command, True, -1)
(o, i, e) = (inst.fromchild, inst.tochild, inst.childerr)
self.pid = inst.pid
self.elines = e.readlines()
self.lines = o.readlines()
ret = inst.wait()
if WIFEXITED(ret):
self.ret = WEXITSTATUS(ret)
else:
self.ret = 255
i.close()
o.close()
e.close()
self.edata = ""
self.data = ""
for i in self.lines:
self.data += i
for i in self.elines:
self.edata += i
def chkupdn(hostList):
upList = []
dnList = []
for hostname in hostList:
print 'checking', hostname
if rshCmd == 'ssh':
rshArgs = '-x -n'
else:
rshArgs = '-n'
cmd = "%s %s %s /bin/echo hello" % (rshCmd, rshArgs, hostname)
runner = Popen3(cmd, 1, 0)
runout = runner.fromchild
runerr = runner.childerr
runin = runner.tochild
runpid = runner.pid
up = 0
try:
# (readyFDs,unused1,unused2) = select([runout,runerr],[],[],9)
(readyFDs, unused1, unused2) = select([runout], [], [], 9)
except:
print 'select failed'
readyFDs = []
for fd in readyFDs: # may have runout and runerr sometimes
line = fd.readline()
if line and line.startswith('hello'):
up = 1
else:
pass
if up:
upList.append(hostname)
else:
dnList.append(hostname)
try:
kill(runpid, SIGKILL)
except:
pass
return (upList, dnList)
def bettersystem(command, stdout=None, stderr=None):
"""Select-based version of commands.getstatusoutput. stdout and stderr
are stream or stream-like objects. Returns the exit status."""
stdout = stdout or sys.stdout
stderr = stderr or sys.stderr
p = Popen3(command, capturestderr=True)
p_out, p_err = p.fromchild, p.childerr
fd_out = p_out.fileno()
fd_err = p_err.fileno()
out_finished = False
err_finished = False
while 1:
rlist, _, _ = select.select([p_out, p_err], [], [])
if not rlist:
break
if p_out in rlist:
output = os.read(fd_out, 1024)
if output == '':
out_finished = True
else:
stdout.write(output)
stdout.flush()
if p_err in rlist:
output = os.read(fd_err, 1024)
if output == '':
err_finished = True
else:
stderr.write(output)
stderr.flush()
if out_finished and err_finished and p.poll() != -1:
break
return p.wait()
def run(out):
langs = ["c", "ruby", "shell", "mysql"]
if len(sys.argv) > 1:
newlangs = []
for lang in sys.argv[1:]:
if lang in langs:
newlangs.append(lang)
else:
print >> sys.stderr, 'Ignoring language %s' % lang
langs = newlangs
total_passed = 0
total_tests = 0
cwd = os.getcwd()
for lang in langs:
print >> out, "Testing %s...\n%s" % (lang, "-" * 70)
os.chdir('%s/%s' % (os.path.dirname(__file__), lang))
print >> out, 'Building...'
p = Popen3('./build')
out.write(p.fromchild.read())
status = p.wait()
testcount = 4
successcount = 0
linecount = 0
if status:
print >> out, 'Failed to build.'
else:
if lang == 'c': testcount = 3
if lang == 'mysql': testcount = 5
if lang == 'shell': testcount = 8
command = os.popen('./testrig', 'r')
while True:
line = command.readline()
if not line: break
linecount += 1
if linecount > testcount:
print >> out, 'Error: saw too many lines. Make sure you aren\'t printing anything.'
successcount = 0
break
if lang == 'python':
if (line.strip() == 'True'): successcount += 1
else:
if (line.strip() == 'correct'): successcount += 1
command.close()
os.chdir(cwd)
print >> out, "Passed %i out of %i tests.\n" % (successcount,
testcount)
total_passed += successcount
total_tests += testcount
os.chdir(cwd)
return total_passed, total_tests
def __init__(self, command):
self.pipe = Popen3(command, capturestderr=True)
self.command = command
def main():
op = create_option_parser()
(options, args) = op.parse_args()
if len(args) != 1:
print "Insert one (and only one) file to process\n"
op.print_help()
sys.exit(2)
fn = args[0]
lang, date, type_ = mwlib.explode_dump_filename(fn)
g = sg.load(fn)
g.time_slice_subgraph(start=options.start, end=options.end)
g.invert_edge_attr('weight', 'length')
vn = len(g.g.vs) # number of vertexes
en = len(g.g.es) # number of edges
timr = Timr()
if options.as_table:
tablr = Tablr()
tablr.start(1024 * 32, lang)
if options.group or options.users_role or options.histogram:
for group_name, group_attr in groups.iteritems():
g.defineClass(group_name, group_attr)
print ' * %s : nodes number : %d' % (group_name,
len(g.classes[group_name]))
else:
g.defineClass('all', {})
print " * filename: %s" % (fn, )
print " * lang: %s" % (lang, )
print " * date: %s" % (date, )
if options.details:
with Timr("details"):
print " * nodes number: %d" % (vn, )
print " * edges number: %d" % (en, )
nodes_with_outdegree = len(g.g.vs.select(_outdegree_ge=1))
nodes_with_indegree = len(g.g.vs.select(_indegree_ge=1))
self_loop_edges = len([edge for edge in g.g.es \
if edge.target == edge.source])
print " * nodes with out edges number: %d (%6f%%)" % (
nodes_with_outdegree, 100. * nodes_with_outdegree / vn)
print " * nodes with in edges number: %d (%6f%%)" % (
nodes_with_indegree, 100. * nodes_with_indegree / vn)
print " * max weights on edges : %s" % top(g.g.es['weight'])
print " * self-loop edges: %d" % self_loop_edges
#print " * diameter : %6f" % g.g.diameter(weights='length')
#print " * average weight : %6f" % numpy.average(g.g.es['weight'])
if options.density or options.reciprocity:
with Timr('density&reciprocity'):
for cls, vs in g.classes.iteritems():
if not len(vs) > 1:
continue
subgraph = vs.subgraph()
print " * %s : density : %.10f" % (cls, subgraph.density())
print " * %s : reciprocity : %.10f" % (cls,
subgraph.reciprocity())
if options.degree:
with Timr('degree'):
g.g.vs['indegree'] = g.g.degree(type=ig.IN)
g.g.vs['outdegree'] = g.g.degree(type=ig.OUT)
for cls, vs in g.classes.iteritems():
if not vs:
continue
ind = numpy.array(vs['indegree'])
outd = numpy.array(vs['outdegree'])
print " * %s : mean IN degree (no weights): %f" % (
cls, numpy.average(ind))
print " * %s : mean OUT degree (no weights): %f" % (
cls, numpy.average(outd))
print " * %s : max IN degrees (no weights): %s" % (cls,
top(ind))
print " * %s : max OUT degrees (no weights): %s" % (cls,
top(outd))
print " * %s : stddev IN degree (no weights): %f" % (
cls, numpy.sqrt(numpy.var(ind)))
print " * %s : stddev OUT degree (no weights): %f" % (
cls, numpy.sqrt(numpy.var(outd)))
if options.transitivity:
##print " * transitivity: %f" % (nx.transitivity(g), )
pass
if options.summary:
# don't use with --as-table
print " * summary: %s" % (g.g.summary(), )
if options.distance:
with Timr('split clusters'):
vc = g.g.clusters()
size_clusters = vc.sizes()
giant = vc.giant()
print " * length of 5 max clusters: %s" % top(size_clusters)
#print " * #node in 5 max clusters/#all nodes: %s" % top(
# [1.*cluster_len/vn for cluster_len in size_clusters])
if options.distance:
with Timr('distance'):
gg = sg.Graph(giant)
print " * average distance in the giant component: %f" % \
gg.averageDistance(weight='length')
print " * average hops in the giant component: %f" % \
gg.averageDistance()
#print "Average distance 2: %f" % giant.average_path_length(True,
# False)
if options.efficiency:
with Timr('efficiency'):
print " * efficiency: %f" % g.efficiency(weight='length')
##TODO: compute for centrality only if "all" or "degree"
if (options.plot or options.histogram or options.power_law
or options.centrality):
with Timr('set weighted indegree'):
g.set_weighted_degree()
if options.centrality:
timr.start('centrality')
centralities = options.centrality.split(',')
if 'all' in centralities:
centralities = 'betweenness,pagerank,degree'.split(',')
if set(centralities).difference(
'betweenness,pagerank,degree'.split(',')):
logging.error('Unknown centrality')
sys.exit(0)
if "betweenness" in centralities:
print >> sys.stderr, "betweenness"
g.g.vs['bw'] = g.g.betweenness(weights='length', directed=True)
#g.g.vs['ev'] = g.g.evcent(weights='weight') # eigenvector centrality
if 'pagerank' in centralities:
print >> sys.stderr, "pagerank"
g.g.vs['pr'] = g.g.pagerank(weights='weight') # pagerank
if 'degree' in centralities:
print >> sys.stderr, "outdegree"
g.set_weighted_degree(type=ig.OUT)
#total_weights = sum(g.g.es['weight'])
max_edges = vn * (vn - 1)
for cls, vs in g.classes.iteritems():
if not vs:
continue
if "betweenness" in centralities:
norm_betweenness = numpy.array(g.classes[cls]['bw']) \
/ max_edges
print " * %s : average betweenness : %.10f" % (
cls, numpy.average(norm_betweenness))
print " * %s : stddev betweenness : %.10f" % (
cls, numpy.sqrt(numpy.var(norm_betweenness)))
print " * %s : max betweenness: %s" % (
cls, top(numpy.array(g.classes[cls]['bw']) / max_edges))
#print " * Average eigenvector centrality : %6f" % numpy.average(
# g.vs['ev'])
if 'pagerank' in centralities:
print " * %s : average pagerank : %.10f" % (
cls, numpy.average(g.classes[cls]['pr']))
print " * %s : stddev pagerank : %.10f" % (
cls, numpy.sqrt(numpy.var(g.classes[cls]['pr'])))
print " * %s : max pagerank: %s" % (cls,
top(g.classes[cls]['pr']))
if 'degree' in centralities:
wi = g.classes[cls]['weighted_indegree']
print " * %s : average IN degree centrality (weighted): %.10f" % (
cls, numpy.average(wi))
print " * %s : stddev IN degree centrality (weighted): %.10f" % (
cls, numpy.sqrt(numpy.var(wi)))
print " * %s : max IN degrees centrality (weighted): %s" % (
cls, top(wi))
del wi
wo = g.classes[cls]['weighted_outdegree']
print " * %s : average OUT degree centrality (weighted) : %.10f" %\
(cls, numpy.average(wo))
print " * %s : stddev OUT degree centrality (weighted) : %.10f" % \
(cls, numpy.sqrt(numpy.var(wo)))
print " * %s : max OUT degrees centrality (weighted): %s" % (
cls, top(wo))
del wo
timr.stop('centrality')
if options.power_law:
with Timr('power law'):
for cls, vs in g.classes.iteritems():
if not vs:
continue
indegrees = vs['weighted_indegree']
try:
alpha_exp = ig.statistics.power_law_fit(indegrees, xmin=6)
print " * %s : alpha exp IN degree distribution : %10f " %\
(cls, alpha_exp)
except ValueError:
print >> sys.stderr,\
" * %s : alpha exp IN degree distribution : ERROR" %\
(cls,)
if options.histogram:
list_with_index = lambda degrees, idx: [(degree, idx)
for degree in degrees
if degree]
all_list = []
nogrp_indegrees = g.g.vs.select(sysop_ne=True,
bureaucrat_ne=True,
steward_ne=True,
founder_ne=True,
bot_ne=True)['weighted_indegree']
all_list += list_with_index(nogrp_indegrees, 1)
sysops_indegrees = g.classes['sysop']['weighted_indegree']
all_list += list_with_index(sysops_indegrees, 2)
burs_indegrees = g.classes['bureaucrat']['weighted_indegree']
all_list += list_with_index(burs_indegrees, 3)
stewards_indegrees = g.classes['steward']['weighted_indegree']
all_list += list_with_index(stewards_indegrees, 4)
founders_indegrees = g.classes['founder']['weighted_indegree']
all_list += list_with_index(founders_indegrees, 5)
bots_indegrees = g.classes['bot']['weighted_indegree']
all_list += list_with_index(bots_indegrees, 6)
if options.gnuplot:
f = open('hist.dat', 'w')
else:
f = open('%swiki-%s-hist.dat' % (lang, date), 'w')
all_list.sort(reverse=True)
for indegree, grp in all_list:
for _ in range(grp - 1):
print >> f, 0,
print >> f, indegree,
for _ in range(grp, 6):
print >> f, 0,
print >> f, ""
f.close()
if options.gnuplot:
from popen2 import Popen3
process = Popen3('gnuplot hist.gnuplot')
process.wait()
os.rename('hist.png', '%swiki-%s-hist.png' % (lang, date))
os.rename('hist.dat', '%swiki-%s-hist.dat' % (lang, date))
if options.plot:
## TODO: evaluate if this can be done with
## http://bazaar.launchpad.net/~igraph/igraph/0.6-main/revision/2018
with Timr('plot'):
import math
## filter:
#print len(g.g.vs), len(g.g.es)
#g.set_weighted_degree(type=ig.OUT)
#g.g = g.g.subgraph(g.g.vs.select(weighted_indegree_ge=10,
# weighted_outdegree_ge=1))
#g.g.write_graphml('itwiki-20100729-stub-meta-history_in10_out1.graphml')
#print len(g.g.vs), len(g.g.es)
bots = g.g.vs.select(bot=True)
bots['color'] = ('purple', ) * len(bots)
logging.debug('bots: ok')
anonyms = g.g.vs.select(anonymous=True)
anonyms['color'] = ('blue', ) * len(anonyms)
sysops = g.g.vs.select(sysop=True)
sysops['color'] = ('yellow', ) * len(sysops)
bur_sysops = g.g.vs.select(bureaucrat=True, sysop=True)
bur_sysops['color'] = ('orange', ) * len(bur_sysops)
g.g.vs['size'] = [
math.sqrt(v['weighted_indegree'] + 1) * 10 for v in g.g.vs
]
logging.debug('plot: begin')
ig.plot(g.g,
target=lang + "_general.png",
bbox=(0, 0, 8000, 8000),
edge_color='grey',
layout='drl')
logging.debug('plot: end')
weights = g.g.es['weight']
max_weight = max(weights)
g.g.es['color'] = [(255. * e['weight'] / max_weight, 0., 0.)
for e in g.g.es]
g.g.es['width'] = weights
ig.plot(g.g,
target=lang + "_weighted_edges.png",
bbox=(0, 0, 4000, 2400),
layout='fr',
vertex_label=' ')
if options.as_table:
tablr.stop()
#tablr.printHeader()
#tablr.printData()
tablr.saveInDjangoModel()
if options.adjacency:
giant = g.g.clusters().giant()
#destAdj = "%s/%swiki-%s-adj.csv" % (os.path.split(fn)[0], lang, date)
destAdj = "%swiki-%s-adj.csv" % (lang, date)
#destRec = "%s/%swiki-%s-rec.csv" % (os.path.split(fn)[0], lang, date)
destRec = "%swiki-%s-rec.csv" % (lang, date)
sg.Graph(giant).writeAdjacencyMatrix(destAdj, 'username')
sg.Graph(giant).writeReciprocityMatrix('username', destRec)
if options.users_role:
l = g.get_user_class('username',
('anonymous', 'bot', 'bureaucrat', 'sysop'))
#destUR = "%s/%swiki-%s-ur.csv" % (os.path.split(fn)[0], lang, date)
destUR = "%swiki-%s-ur.csv" % (lang, date)
with open(destUR, 'w') as f:
for username, role in sorted(l):
print >> f, "%s,%s" % (username, role)
from random import shuffle
#destCls = "%s/%swiki-%s-%%s.csv" % (os.path.split(fn)[0], lang, date)
destCls = "%swiki-%s-%%s.csv" % (lang, date)
for cls in ('anonymous', 'bot', 'bureaucrat', 'sysop', 'normal_user'):
users = g.classes[cls]['username']
shuffle(users)
with open(destCls % cls, 'w') as f:
for username in users:
print >> f, \
("%s,http://vec.wikipedia.org/w/index.php?title=" + \
"Discussion_utente:%s&action=history&offset=" + \
"20100000000001") % (username, username)
def __init__(self, cmd, capturestderr=False, bufsize=-1):
Popen3.__init__(self, cmd, capturestderr, bufsize)
def _run_child(self, cmd):
"Set process group and run child."
os.setpgrp()
Popen3._run_child(self, cmd)
def __init__(self, command):
from popen2 import Popen3
self.pipe = Popen3(command, capturestderr=True)
self.command = command
def call_peez2(args=''):
""" Execute "peez2" with arguments provided, if any. """
child = Popen3(binary + ' ' + common_arguments + ' ' + args)
return {'out': child.fromchild, 'in': child.tochild, 'err': child.childerr}
def call_inkscape(self, verbs, ids=None):
u"""Calls Inkscape to perform inkscape operations on the document.
Note that this destroys and recreates self.document, so any node
you previously collected is no longer valid. Any manipulation you
performed on self.document previous to calling ``call_inkscape``
will be stored, but there is no use in searching for a particular
node, calling ``call_inkscape`` and afterwards manipulating said node,
since this node belongs to a document no longer valid. It *is* possible
to store collections of id's, since those are more persistent between
subsequent calls of ``call_inkscape``.
Note also that ``call_inkscape`` is a rather costly operation: the
current document is stored as a temporary file on disk, another
Inkscape process is loaded into memory, this process loads, manipulates
and saves the temporary file, and then this temporary file is read and
parsed again to retrieve the DOM tree. It is therefore advisable to
perform as much Inkscape commands in one go as possible.
To perform an Inkscape command, you have to specify a *verb*, and an id
of an object on which to perform this action. Both verbs and ids are
strings. If you have just a single object with a single operation,
you can specify both ``verbs`` and ``ids`` as a single string,
respectively.
If you have a bunch of objects on which you want to perform the same
operation, you can specify ``verbs`` as a single string, and ``ids``
as a list of strings.
If you have a bunch of objects on which you have to perform a bunch of
different operations, you can ommit ``ids`` and specify ``verbs`` as
a list of (verb, id)-tuples.
And finally, to give you the most ammount of flexibility imaginable,
you can specify your own list of optional arguments to submit to
Inkscape if you specify ``verbs`` as a single string and ommit ``ids``.
Examples::
def effect(self):
# ...
# Transform a single text element to a group of paths:
self.call_inkscape("ObjectToPath", "text1234")
# Transform several text elements into groups of paths:
self.call_inkscape("ObjectToPath", ["text1", "text2", "text3"])
# Perform several different operations:
self.call_inkscape([("EditClone", "path1234"),
("ObjectToPath", "text5678")])
# Do arbitrary stuff (remove unused definitions, duplicate
# a text and convert the initial version to a path):
self.call_inkscape(
"--vacuum-defs"
" --select=text1234 --verb=EditDuplicate"
" --verb=ObjectToPath")
# ...
"""
err_file = open("/tmp/inkscape_rps_cmd.txt" , "w")
fd, tmp = tempfile.mkstemp(".svg", text=True)
try:
self.document.write(tmp)
cmd = self.inkscape_path + " --file=\"%s\"" % tmp
if ids:
if isinstance(ids, basestring):
cmd += " --select="+ids
cmd += " --verb="+verbs
else:
for id in ids:
cmd += " --select="+id
cmd += " --verb="+verbs
else:
if isinstance(verbs, basestring):
cmd += " --verb=" + verbs
else:
for tverb, tid in verbs:
cmd += " --select="+tid
cmd += " --verb="+tverb
cmd += " --verb=FileSave --verb=FileClose"
err_file.write('CMD: ' + cmd)
try:
from subprocess import Popen, PIPE
p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
rc = p.wait()
out = p.stdout.read()
err = p.stderr.read()
except ImportError:
from popen2 import Popen3
p = Popen3(cmd, True)
p.wait()
rc = p.poll()
out = p.fromchild.read()
err = p.childerr.read()
self.parse(tmp)
self.getposinlayer()
self.getselected()
self.getdocids()
finally:
err_file.close()
os.close(fd)
os.remove(tmp)
def __init__(self, cmd, capturestderr=False, bufsize=-1, env=os.environ):
self._env = env
Popen3.__init__(self, cmd, capturestderr, bufsize)
def mpiexec():
global totalProcs, nextRange, argvCopy, configLines, configIdx, appnum
global validGlobalArgs, globalArgs, validLocalArgs, localArgSets
validGlobalArgs = { '-l' : 0, '-usize' : 1, '-gdb' : 0, '-gdba' : 1, '-bnr' : 0, '-tv' : 0,
'-if' : 1, '-machinefile' : 1, '-kx' : 0, '-s' : 1,
'-gn' : 1, '-gnp' : 1, '-ghost' : 1, '-gpath' : 1, '-gwdir' : 1,
'-gsoft' : 1, '-garch' : 1, '-gexec' : 1,
'-genvall' : 0, '-genv' : 2, '-genvnone' : 0,
'-genvlist' : 1 }
validLocalArgs = { '-n' : 1, '-np' : 1, '-host' : 1, '-path' : 1, '-wdir' : 1,
'-soft' : 1, '-arch' : 1,
'-envall' : 0, '-env' : 2, '-envnone' : 0, '-envlist' : 1 }
globalArgs = {}
localArgSets = {}
localArgSets[0] = []
totalProcs = 0
nextRange = 0
configLines = []
configIdx = 0
xmlForArgsets = []
appnum = 0
if len(argv) < 2 or argv[1] == '-h' or argv[1] == '-help' or argv[1] == '--help':
usage()
fullDirName = path.abspath(path.split(argv[0])[0]) # normalize for platform also
mpdrun = path.join(fullDirName,'mpdrun.py')
if not access(mpdrun,X_OK):
print 'mpiexec: cannot execute mpdrun %s' % mpdrun
exit(0);
if argv[1] == '-file':
if len(argv) != 3:
usage()
xmlFilename = argv[2]
globalArgs['-kx'] = 1
else:
if argv[1] == '-gdba':
if len(argv) != 3:
print '-gdba must be used only with a jobid'
usage()
execvpe(mpdrun,[mpdrun,'-ga',argv[2]],environ)
elif argv[1] == '-configfile':
if len(argv) != 3:
usage()
configFileFD = osopen(argv[2],O_RDONLY)
configFile = fdopen(configFileFD,'r',0)
configLines = configFile.readlines()
configLines = [ x.strip() + ' : ' for x in configLines if x[0] != '#' ]
tempargv = []
for line in configLines:
shOut = Popen3("/bin/sh -c 'for a in $*; do echo _$a; done' -- %s" % (line))
for shline in shOut.fromchild:
tempargv.append(shline[1:].strip()) # 1: strips off the leading _
tempargv = [argv[0]] + tempargv[0:-1] # strip off the last : I added
collect_args(tempargv)
else:
collect_args(argv)
machineFileInfo = read_machinefile(globalArgs['-machinefile'])
xmlDOC = xml.dom.minidom.Document()
xmlCPG = xmlDOC.createElement('create-process-group')
xmlDOC.appendChild(xmlCPG)
for k in localArgSets.keys():
handle_argset(localArgSets[k],xmlDOC,xmlCPG,machineFileInfo)
xmlCPG.setAttribute('totalprocs', str(totalProcs) ) # after handling argsets
if globalArgs['-l']:
xmlCPG.setAttribute('output', 'label')
if globalArgs['-if']:
xmlCPG.setAttribute('net_interface', globalArgs['-if'])
if globalArgs['-s']:
xmlCPG.setAttribute('stdin_goes_to_who', globalArgs['-s'])
if globalArgs['-bnr']:
xmlCPG.setAttribute('doing_bnr', '1')
if globalArgs['-gdb']:
xmlCPG.setAttribute('gdb', '1')
if globalArgs['-tv']:
xmlCPG.setAttribute('tv', '1')
submitter = getpwuid(getuid())[0]
xmlCPG.setAttribute('submitter', submitter)
xmlFilename = '/tmp/%s_tempxml_%d' % (submitter,getpid())
try: unlink(xmlFilename)
except: pass
xmlFileFD = osopen(xmlFilename,O_CREAT|O_WRONLY|O_EXCL,0600)
xmlFile = fdopen(xmlFileFD,'w',0)
print >>xmlFile, xmlDOC.toprettyxml(indent=' ')
# print xmlDOC.toprettyxml(indent=' ') #### RMB: TEMP DEBUG
xmlFile.close()
if globalArgs['-kx']:
execvpe(mpdrun,[mpdrun,'-f',xmlFilename],environ)
else:
execvpe(mpdrun,[mpdrun,'-delxmlfile',xmlFilename],environ)
print 'mpiexec: exec failed for %s' % mpdrun
exit(0);
def __init__(self, cmd, capturestderr=False, bufsize=-1, env=os.environ):
self._env = env
Popen3.__init__(self, cmd, capturestderr, bufsize)
import inkex
cmd = None
try:
from subprocess import Popen, PIPE
p = Popen('uniconvertor', shell=True, stdout=PIPE, stderr=PIPE).wait()
if p == 0:
cmd = 'uniconvertor'
else:
p = Popen('uniconv', shell=True, stdout=PIPE, stderr=PIPE).wait()
if p == 0:
cmd = 'uniconv'
except ImportError:
from popen2 import Popen3
p = Popen3('uniconv', True).wait()
if p != 32512: cmd = 'uniconv'
p = Popen3('uniconvertor', True).wait()
if p != 32512: cmd = 'uniconvertor'
if cmd == None:
# there's no succeffully-returning uniconv command; try to get the module directly (on Windows)
try:
# cannot simply import uniconvertor, it aborts if you import it without parameters
import imp
imp.find_module("uniconvertor")
except ImportError:
inkex.localize()
inkex.errormsg(_('You need to install the UniConvertor software.\n'+\
'For GNU/Linux: install the package python-uniconvertor.\n'+\
'For Windows: download it from\n'+\
ghbaf = gethostbyaddr(fipaddr2)
except:
print "*** gethostbyaddr failed for remote hosts's IP %s" % (
fipaddr2)
except:
print "*** gethostbyname_ex failed for host %s" % (fqhn2)
# see if we can run /bin/date on remote hosts
if not do_ssh:
exit(0)
for host in hostsFromFile:
cmd = "ssh %s -x -n /bin/echo hello" % (host)
if verbose:
print 'trying: %s' % (cmd)
runner = Popen3(cmd, 1, 0)
runout = runner.fromchild
runerr = runner.childerr
runin = runner.tochild
runpid = runner.pid
try:
(readyFDs, unused1, unused2) = select([runout], [], [], 9)
except Exception, data:
print 'select 1 error: %s ; %s' % (data.__class__, data)
exit(-1)
if len(readyFDs) == 0:
print '** ssh timed out to %s' % (host)
line = ''
failed = 0
if runout in readyFDs:
line = runout.readline()
def auto_partition (self, drive, steps = None, \
do_it = ACTUAL_PARTITIONING):
""" Make 3 partitions automatically on the specified C{device}. When
C{progress_bar} is not C{None}, it is updated dinamically as the
partitioning process goes on. """
result = None
if steps is not None:
status = 0.1
steps.put ('%f|Iniciando el proceso de particionado automático...' %
status)
status = status + 0.1
if drive.has_key ('info'):
if drive ['info'].has_key ('primary'):
if drive ['info'] ['primary'] < 2:
# Make 3 new primary partitions?
pass
components = self.__partition_scheme.keys ()
stop = False
# We suppose that there is no extended partition:
ext_part = False
# Initially, every new partition will be logical:
try_primary = False
if drive.has_key ('info'):
if drive ['info'].has_key ('ext'):
if drive ['info'] ['ext'] > 0:
# Actually, there is already an extended partition:
ext_part = True
for part in components:
if self.__debug:
stderr.write ('auto_partition: part = "' + str (part) + '".\n')
if try_primary:
# Create a primary partition:
required = int (round (self.__partition_scheme [part]) * 1.02)
info = self.__get_info (drive ['id'], required)
type = 'primaria'
else:
if ext_part:
# A new logical partition is created. It has 2% more space
# so "recycle" partitioning method is enabled if user
# decides to reinstall in the same drive:
required = int (round (self.__partition_scheme [part]) * 1.02)
info = self.__get_info (drive ['id'], required, '-j')
type = 'lógica'
else:
# It is necessary to create an extended partition
# (with 8% more space -- 6% of 3 partitions and 2% more to be sure):
required = int (round (sum (self.__partition_scheme.values ()) * 1.08))
info = self.__get_info (drive ['id'], required, '-x')
components.append (part)
type = 'extendida'
if steps is not None:
steps.put ('%f|Creando una partición %s de %s...' % \
(status, type, beautify_size (int (required) * 1024 * 1024)))
status = status + 0.1
# Now we have to decide which option is better:
if info.has_key ('opts'):
if self.__debug:
stderr.write ('auto_partition: has_key ("opts").\n')
options = info ['opts']
else:
if self.__debug:
stderr.write ('auto_partition: NO has_key ("opts").\n')
if try_primary:
# Definitively, no more partitions can be created.
# Stop partitioning:
if self.__debug:
stderr.write ('auto_partition: stopped!\n')
stop = True
break
else:
# Next partitions should be primary, or not be at all:
if self.__debug:
stderr.write ('auto_partition: switching to primary.\n')
components.append (part)
try_primary = True
continue
what = -1
i = 1
while -1 == what and i <= len (options):
if 'CR' == options [i - 1] [1] [:2]:
what = i
i = i + 1
i = 1
while -1 == what and i <= len (options):
if 'RE' == options [i - 1] [1] [:2]:
what = i
i = i + 1
if what is -1:
if self.__debug:
stderr.write ('auto_partition: there are no valid options.\n')
if try_primary:
# Definitively, no more partitions can be created.
# Stop partitioning:
if self.__debug:
stderr.write ('auto_partition: stopped!\n')
stop = True
break
else:
# Next partitions should be primary, or not be at all:
if self.__debug:
stderr.write ('auto_partition: switching to primary.\n')
components.append (part)
try_primary = True
continue
else:
if try_primary:
info = self.__get_info (drive ['id'], required,
'-i', str (what) + '\n')
else:
if not ext_part:
info = self.__get_info (drive ['id'], required,
'-x -i', str (what) + '\n')
else:
info = self.__get_info (drive ['id'], required,
'-j -i', str (what) + '\n')
if info.has_key ('commands'):
c = info ['commands']
p = Popen3 ('echo "Creando ' + str (part) +
'..." >> /tmp/guadalinex-express.commands')
p.wait ()
subprogress = 0.2 / (len (c) + 1)
for i in c:
if steps is not None:
steps.put ('%f|Creando una partición %s de %s...' % \
(status, type, beautify_size (int (required) * 1024 * 1024)))
status = status + subprogress
# Print the commands:
if self.__debug:
stderr.write ('auto_partition: command: "' +
i.strip () + '" executed.\n')
p = Popen3 ('echo "' + i.strip () +
'" >> /tmp/guadalinex-express.commands')
p.wait ()
if do_it:
# Do it! Execute commands to make partitions!
if 'parted ' in i and exists ('/proc/partitions'):
partitions_file = file ('/proc/partitions')
previous_partitions = partitions_file.read ()
partitions_file.close ()
previous_checksum = crc32 (previous_partitions)
# Execute the command:
p = Popen3 (i)
p.wait ()
# Let the system be aware of the changes:
if 'parted ' in i and exists ('/proc/partitions'):
current_checksum = previous_checksum
while current_checksum is previous_checksum:
sleep (1)
partitions_file = file ('/proc/partitions')
current_partitions = partitions_file.read ()
partitions_file.close ()
current_checksum = crc32 (current_partitions)
sleep (5)
if info.has_key ('metacoms'):
mc = info ['metacoms']
for i in mc:
if self.__debug:
stderr.write ('# ' + i)
if self.__debug:
stderr.write ("info.has_key ('dest') = " +
str (info.has_key ('dest')) +
'; ext_part = ' + str (ext_part) + '.\n')
if ext_part:
if info.has_key ('dest'):
if result is None:
result = {}
result [(info ['dest']).strip ()] = part.strip ()
if self.__debug:
stderr.write (str (part.strip ()) + \
' added as ' + \
str ((info ['dest']).strip ()) + '\n')
else:
ext_part = True
# During formatting and copying, "root" is known as "/",
# and "home" is known as "/home", so it is necessary to
# change them before passing mount point associations to
# the backend:
if steps is not None:
steps.put ('%f|Terminando el proceso de particionado...' %
status)
if stop:
result = 'STOPPED'
else:
for i in result.keys ():
if 'root' == result [i].lower ():
result [i] = '/'
elif 'home' == result [i].lower ():
result [i] = '/home'
if self.__debug:
stderr.write ('auto_partition: result = "' + str (result) + '".\n')
return result
