def _get_api_key(self, username, private_key_string):
# Normally we try not to import modules anywhere but globally,
# but I'd like to keep this xmlrpc client usable without repy files
# available when the user provides their api key and doesn't require
# it to be retrieved.
try:
import repyhelper
import repyportability
repyhelper.translate_and_import("rsa.repy")
except ImportError, e:
raise SeattleClearinghouseError("Unable to get API key from SeattleClearinghouse " +
"because a required python or repy module " +
"cannot be found:" + str(e))
def _get_api_key(self, username, private_key_string):
# Normally we try not to import modules anywhere but globally,
# but I'd like to keep this xmlrpc client usable without repy files
# available when the user provides their api key and doesn't require
# it to be retrieved.
try:
import repyhelper
import repyportability
repyhelper.translate_and_import("rsa.repy")
except ImportError, e:
raise SeattleClearinghouseError("Unable to get API key from SeattleGENI " +
"because a required python or repy module " +
"cannot be found:" + str(e))
def add_dy_support(_context):
# Add dylink support
repyhelper.translate_and_import("dylink.repy", callfunc = 'initialize')
# The dy_* functions are only added to the namespace after init_dylink is called.
init_dylink(_context,{})
# Make our own `dy_import_module_symbols` and add it to the context.
# It is not currently possible to use the real one (details at ticket #1046)
def _dy_import_module_symbols(module,new_callfunc="import"):
new_context = _context['dy_import_module'](module, new_callfunc)._context
# Copy in the new symbols into our namespace.
for symbol in new_context:
if symbol not in _context: # Prevent the imported object from destroying our namespace.
_context[symbol] = new_context[symbol]
_context['dy_import_module_symbols'] = _dy_import_module_symbols
def add_dy_support(_context):
# Add dylink support
repyhelper.translate_and_import("dylink.repy", callfunc='initialize')
# The dy_* functions are only added to the namespace after init_dylink is called.
init_dylink(_context, {})
# Make our own `dy_import_module_symbols` and add it to the context.
# It is not currently possible to use the real one (details at ticket #1046)
def _dy_import_module_symbols(module, new_callfunc="import"):
new_context = _context['dy_import_module'](module,
new_callfunc)._context
# Copy in the new symbols into our namespace.
for symbol in new_context:
if symbol not in _context: # Prevent the imported object from destroying our namespace.
_context[symbol] = new_context[symbol]
_context['dy_import_module_symbols'] = _dy_import_module_symbols
7 * * * * /usr/bin/python /home/seattle/opendhtputget.py > /home/seattle/cron_log.opendhtputget
"""
import time
import os
import socket
import sys
import traceback
import threading
import random
import send_gmail
import integrationtestlib
import repyhelper
repyhelper.translate_and_import("/home/monzum/opendhtputget_new/openDHTadvertise.repy")
# Armon: This is to replace using the time command with getruntime
import nonportable
# event for communicating when the lookup is done or timedout
lookup_done_event = threading.Event()
lookup_timeout_fired = 0
def lookup_timedout():
"""
<Purpose>
Waits for lookup_done_event and notifies the folks on the
notify_list (global var) of the lookup timeout.
Amended 5 July, 2009 to be an extension to time.repy
Description:
This module is an implementation of time_interface.repy
Contacts a server running time_server.repy to get the current time from an
NTP because only the server can communicate with an NTP.
To use this module, make one call to time_updatetime() to get the
time from the server. This function also implicitly sets the time. Then
call time_gettime() every time the current time is needed.
"""
repyhelper.translate_and_import('time_interface.repy')
repyhelper.translate_and_import('advertise.repy')
repyhelper.translate_and_import('random.repy')
repyhelper.translate_and_import('sockettimeout.repy')
# This function contacts the server to get the time from a NTP
def tcp_time_updatetime(localport):
"""
<Purpose>
Opens a connection with a server hosting time_server.repy, which obtains the
current time via a NTP, then calls time_settime(float(currenttime)) to set
import shutil
# clean up any left over data...
try:
shutil.rmtree('sourcepathtest')
except (OSError, IOError):
# it's okay if it doesn't exist...
pass
os.mkdir('sourcepathtest')
# prepend this to the Python path...
sys.path = ['sourcepathtest'] + sys.path
# remove the file if it's there...
if os.path.exists('rhtest_filetests_repy.py'):
os.remove('rhtest_filetests_repy.py')
# write files there
repyhelper.translate_and_import('rhtest_filetests.repy')
# This should work...
try:
rhtest_filetests_exists()
except NameError:
print "Failed to import rhtest_filetests when using sourcepath test"
# and the file should be in the current directory.
if not os.path.exists('rhtest_filetests_repy.py'):
print "The rhtest_filetests.repy file was not preprocessed to the current directory because 'rhtest_filetests_repy.py' does not exist"
contact some random subset of NTP servers to get and store the local time.
Then, to get the actual time, call time_gettime() which will return
the current time (in seconds). time_gettime() can be called at any point
after having called time_updatetime(localport) since time_gettime() simply
calculates how much time has elapsed since the local time was originally
acquired from one of the NTP servers.
Note that time_gettime() will raise TimeError if no NTP server responded or
if time_updatetime(localport) was never previously called. If time_gettime()
fails, then time_updatetime(localport) can be called again to sample time
from another random set of NTP servers.
"""
repyhelper.translate_and_import('time_interface.repy')
# Use for random sampling...
repyhelper.translate_and_import('random.repy')
timeservers = ["time-a.nist.gov", "time-b.nist.gov",
"time-a.timefreq.bldrdoc.gov", "time-b.timefreq.bldrdoc.gov",
"time-c.timefreq.bldrdoc.gov", "utcnist.colorado.edu", "time.nist.gov",
"nist.netservicesgroup.com"]
#BUG: Do I need to compensate for the time taken to contact the time server? (#353)
def ntp_time_updatetime(localport):
of random bytes, this is not a permanent fix (the extraction
of random bytes from the float is not portable). The change will
likely be made to random_randombytes (since calls os.urandom will
likely be restricted to a limited number of bytes).
TODO -
random_randombytes will remained but serve as a helper function
to collect the required number of bytes. Calls to randombytes
will be restricted to a set number of bytes at a time, since
allowing an arbitrary request to os.urandom would circumvent
performance restrictions.
TODO -
_random_long_to_bytes will no longer be needed.
"""
repyhelper.translate_and_import('math.repy')
def random_randombytes(num_bytes, random_float=None):
"""
<Purpose>
Return a string of length num_bytes, made of random bytes
suitable for cryptographic use (because randomfloat draws
from a os provided random source).
*WARNING* If python implements float as a C single precision
floating point number instead of a double precision then
there will not be 53 bits of data in the coefficient.
<Arguments>
num_bytes:
""" Test the callfunc arg of the translations, to make sure it actually gets used """ import repyhelper import test_utils TESTFILE = 'rhtest_callfunc.r2py' #Make sure we have fresh translations per test run test_utils.cleanup_translations([TESTFILE]) modname = repyhelper.translate(TESTFILE, callfunc='plankton') a = __import__(modname) ### Try again with translate_and_import #Allow retranslation test_utils.cleanup_translations([TESTFILE]) repyhelper.translate_and_import(TESTFILE, callfunc='plankton') test_utils.cleanup_translations([TESTFILE])
of random bytes, this is not a permanent fix (the extraction
of random bytes from the float is not portable). The change will
likely be made to random_randombytes (since calls os.urandom will
likely be restricted to a limited number of bytes).
TODO -
random_randombytes will remained but serve as a helper function
to collect the required number of bytes. Calls to randombytes
will be restricted to a set number of bytes at a time, since
allowing an arbitrary request to os.urandom would circumvent
performance restrictions.
TODO -
_random_long_to_bytes will no longer be needed.
"""
repyhelper.translate_and_import('math.repy')
def random_randombytes(num_bytes, random_float=None):
"""
<Purpose>
Return a string of length num_bytes, made of random bytes
suitable for cryptographic use (because randomfloat draws
from a os provided random source).
*WARNING* If python implements float as a C single precision
floating point number instead of a double precision then
there will not be 53 bits of data in the coefficient.
<Arguments>
num_bytes:
The number of bytes to request from os.urandom.
def add_dy_support(_context):
"""
<Purpose>
Enable usage of repy's dynamic library linking. This should only
be called on the module-level.
<Arguments>
_context:
The context that dylink's functions should be inserted into.
<Side Effects>
Public functions from dylink.repy will be inserted into _context.
_context should be globals() for a module.
<Exceptions>
Exception is raised when a module import fails.
<Returns>
None
"""
# Add dylink support
repyhelper.translate_and_import("dylink.r2py", callfunc='initialize')
# The dy_* functions are only added to the namespace after init_dylink is called.
init_dylink(_context, {})
original_import_module = _context['dy_import_module']
def _new_dy_import_module_symbols(module, callfunc="import"):
# Remember the path we are currently in. We need to change to
# this script's dir (assuming it also contains dylink.r2py and
# rest of the Repy runtime and libraries) so that dylink is
# able to link in code from the runtime.
# This is required due to Repy safety measures that inhibit
# dylink to access files outside of its directory.
# Once dylink is done, we return to the previously-current
# working dir.
previous_cwd = os.getcwd()
repyportability_dir = os.path.dirname(os.path.realpath(__file__))
os.chdir(repyportability_dir)
# If we are using repyportability, we want to check all pythonpath for
# the file we are looking to import.
COMMON_EXTENSIONS = ["", ".py", ".repy", ".py.repy", ".pp", ".r2py"]
# Check all combination of filepath with file extension and try to import the
# file if we have found it.
for pathdir in sys.path:
possiblefilenamewithpath = os.path.join(pathdir, module)
# If we have found a path, then we can import the module and
# return so we do not continue to look in other paths.
if os.path.isfile(possiblefilenamewithpath):
filenamewithpath = possiblefilenamewithpath
importedmodule = original_import_module(
filenamewithpath, callfunc)
os.chdir(previous_cwd)
return importedmodule
# If we don't find the file, we just call down to dylink, and
# let it raise the appropriate error.
try:
importedmodule = original_import_module(module, callfunc)
return importedmodule
except:
raise
finally:
os.chdir(previous_cwd)
_context['dy_import_module'] = _new_dy_import_module_symbols
# Make our own `dy_import_module_symbols` and add it to the context.
# It is not currently possible to use the real one (details at ticket #1046)
def _dy_import_module_symbols(module, new_callfunc="import"):
new_context = _context['dy_import_module'](module,
new_callfunc)._context
# Copy in the new symbols into our namespace.
for symbol in new_context:
if symbol not in _context: # Prevent the imported object from destroying our namespace.
_context[symbol] = new_context[symbol]
_context['dy_import_module_symbols'] = _dy_import_module_symbols
callargs = []
"""
<Program Name>
advertise.repy
<Started>
October 14, 2008
<Author>
Justin Cappos
<Purpose>
Module which allows clients to send advertise queries to various servers.
"""
repyhelper.translate_and_import('listops.repy')
repyhelper.translate_and_import('centralizedadvertise.repy')
repyhelper.translate_and_import('centralizedadvertise_v2.repy')
repyhelper.translate_and_import('DORadvertise.repy')
repyhelper.translate_and_import('parallelize.repy')
repyhelper.translate_and_import('udpcentralizedadvertise.repy')
# All the names of services we can support.
# As of January 2012, openDHT is no longer a default service.
_advertise_all_services = ("central", "central_v2", "DOR")
nodemanager_announce_context = {}
for service in _advertise_all_services:
nodemanager_announce_context["skip" + service] = 0
nodemanager_announce_context["previous" + service + "skip"] = 1
nodemanager_announce_context_lock = getlock()
<Author>
Eric Kimbrel
<Purpose>
Send out emails if fewer than 8 timeservers are running
"""
import sys
import send_gmail
import integrationtestlib
import traceback
# use repy helper to bring in advertise.repy
import repyhelper
repyhelper.translate_and_import('advertise.repy')
def main():
# initialize the gmail module
success, explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
#add Eric Kimbrel to the email notify list
integrationtestlib.notify_list.append("*****@*****.**")
try:
integrationtestlib.log("Looking up time_servers")
### Deleting this file forces regeneration of a repy translation
from repyportability import *
import repyhelper
mycontext = repyhelper.get_shared_context()
callfunc = 'import'
callargs = []
"""
<Program Name>
time.repy
<Author>
Eric Kimbrel
<Started>
Jul 2, 2009
<Purpose>
replaces the previous time.repy by use of the active interface
time_interface.repy and the implementors ntp_time.repy and tcp_time.repy
see time_interface.repy for details
"""
repyhelper.translate_and_import('ntp_time.repy')
repyhelper.translate_and_import('tcp_time.repy')
### Automatically generated by repyhelper.py ### /chalmers/users/vikbergl/distributedGit/Byzantine-/demokit/time.repy
"""
from repyportability import *
import hashlib
import getopt
import tarfile
import time
import sys
import os
import repyhelper
import signal
import select
# Get the advertisement methods
repyhelper.translate_and_import("centralizedadvertise.repy")
# This dictionary can be used to hardcode hostname to IPs
# Otherwise, we fallback onto advertisement
HOSTS = {}
PORT = 50000
# Finds all available seattle testbeds
def find_testbeds():
servers = centralizedadvertise_lookup("SEATTLE_TESTBEDS")
return servers
# Finds the IP's of a specific testbed
def find_testbed(name):
ip_addrs = centralizedadvertise_lookup("SEATTLE_TESTBED_"+name)
return ip_addrs
- notify_list, a list of strings with emails denoting who will be
emailed when something goes wrong
This script takes no arguments. A typical use of this script is to
have it run periodically using something like the following crontab line:
7 * * * * /usr/bin/python /home/seattle/centralizedputget.py > /home/seattle/cron_log.centralizedputget
"""
import urllib
import send_gmail
import integrationtestlib
import urllib2
import subprocess
import repyhelper
repyhelper.translate_and_import('serialize.repy')
username = '******'
apikey = '1X3YFBLPTKVSI8DQHWJZ0NR92645ECUA'
userdata = {username: {'apikey': apikey}}
ERROR_EMAIL_SUBJECT = "Selexor monitoring test failure!"
SELEXOR_PAGE = "https://selexor.poly.edu:8888/"
def retrieve_url(url, data=None):
print "retrieve_url"
args = ["curl", url, "--insecure"]
if data:
args += ['--data', '"' + data + '"']
from repyportability import *
import repyhelper
mycontext = repyhelper.get_shared_context()
callfunc = 'import'
callargs = []
"""
Author: Conrad Meyer
Start Date: Wed Dec 9 2009
Description:
Advertisements to the Digital Object Registry run by CNRI.
"""
repyhelper.translate_and_import('sockettimeout.repy')
repyhelper.translate_and_import('httpretrieve.repy')
repyhelper.translate_and_import('xmlparse.repy')
DORadvertise_FORM_LOCATION = "http://geni.doregistry.org/SeattleGENI/HashTable"
class DORadvertise_XMLError(Exception):
"""
Exception raised when the XML recieved from the Digital Object Registry
server does not match the structure we expect.
"""
pass
class DORadvertise_BadRequest(Exception):
"""
Test to see if restrictions can be initialized and restored from within Python.
"""
from repyportability import *
import repyhelper
import sys
# in the call to this, it will do 'exitall' if the restrictions are in place
repyhelper.translate_and_import("rhtest_printifreadisfast.r2py")
# We need to flush stdout (this won't be done for us unless it goes to the
# terminal).
sys.stdout.flush()
Amended 5 July, 2009 to be an extension to time.repy
Description:
This module is an implementation of time_interface.repy
Contacts a server running time_server.repy to get the current time from an
NTP because only the server can communicate with an NTP.
To use this module, make one call to time_updatetime() to get the
time from the server. This function also implicitly sets the time. Then
call time_gettime() every time the current time is needed.
"""
repyhelper.translate_and_import('time_interface.repy')
repyhelper.translate_and_import('advertise.repy')
repyhelper.translate_and_import('random.repy')
repyhelper.translate_and_import('sockettimeout.repy')
# This function contacts the server to get the time from a NTP
def tcp_time_updatetime(localport):
"""
<Purpose>
Opens a connection with a server hosting time_server.repy, which obtains the
current time via a NTP, then calls time_settime(float(currenttime)) to set
the current time to the received value form the server.
"""
<Purpose>
Ensures that private IP addresses trigger proper exceptions.
See #1280.
"""
import repyhelper
repyhelper.translate_and_import("geoip_client.repy")
geoip_init_client()
private_ips = ["10.0.0.0", "172.16.0.0", "192.168.0.0"]
for ip in private_ips:
try:
geoip_record_by_addr(ip)
except Exception, e:
if "Not a public IP address" not in str(e):
raise
else:
raise Exception("Private IP address silently failed!")
os.rename(EXPOSED_READLINE_SO_FN, HIDDEN_READLINE_SO_FN)
# Needed for parsing user commands and executing command functions
import seash_dictionary
# Used for re-enabling modules on the last run
import seash_modules
# To be able to catch certain exceptions thrown throughout the program
import seash_exceptions
import seash_helper
import repyhelper
repyhelper.translate_and_import("time.repy")
import traceback
import os.path # fix path names when doing upload, loadkeys, etc.
def command_loop(test_command_list):
# If a test command list is passed, filter the tab completion warning
if test_command_list:
warnings.filterwarnings("ignore", "Auto tab completion is off, because it is not available on your operating system.",
ImportWarning)
"""
<Purpose>
Ensures that private IP addresses trigger proper exceptions.
See #1280.
"""
import repyhelper
repyhelper.translate_and_import("geoip_client.repy")
geoip_init_client()
private_ips = ["10.0.0.0", "172.16.0.0", "192.168.0.0"]
for ip in private_ips:
try:
geoip_record_by_addr(ip)
except Exception, e:
if "Not a public IP address" not in str(e):
raise
else:
raise Exception("Private IP address silently failed!")
Evan Meagher
<Start Date>
Nov 26, 2009
<Description>
XMl-RPC client for remote GeoIP server. Given an IP:port of a GeoIP
XML-RPC server, allows location lookup of hostnames and IP addresses.
<Usage>
client = geoip_client(server_address)
Where server_address is the ip address of a remote GeoIP XMl-RPC server.
"""
repyhelper.translate_and_import('parallelize.repy')
repyhelper.translate_and_import('xmlrpc_client.repy')
"""
Initialize global GeoIP XML-RPC client object to None
Note: client is stored in wrapper list to avoid using mycontext dict
"""
geoip_clientlist = []
def geoip_init_client(url=[
"http://geoipserver.poly.edu:12679", "http://geoipserver2.poly.edu:12679"
]):
"""
<Purpose>
Create a new GeoIP XML-RPC client object.
<Started>
August 19, 2009
<Authors>
Yafete Yemuru
Conrad Meyer
<Purpose>
Provides a method for retrieving content from web servers using the HTTP
protocol. The content can be accessed as a file like object, or saved to
a file or returned as a string.
"""
repyhelper.translate_and_import('urlparse.repy')
repyhelper.translate_and_import('sockettimeout.repy')
repyhelper.translate_and_import('urllib.repy')
class HttpConnectionError(Exception):
"""
Error indicating that the web server has unexpectedly dropped the
connection.
"""
class HttpBrokenServerError(Exception):
"""
Author: Justin Cappos
Description
Should import all names from an included python program
No output indicates success
"""
myname = __name__
import repyhelper
repyhelper.translate_and_import('rhtestname_oddnames.repy')
# these should all be defined now...
name1
name2
_name3
# don't change the __ mappings because bad things happen!
assert (myname == __name__)
###Helper Methods###
import seash_global_variables
import seash_exceptions
import repyhelper
#repyhelper.translate_and_import("nmclient.repy")
import fastnmclient
# Use local clock for time if there is no network connectivity
import time
repyhelper.translate_and_import("time.repy")
repyhelper.translate_and_import("rsa.repy")
repyhelper.translate_and_import("listops.repy")
repyhelper.translate_and_import("parallelize.repy")
repyhelper.translate_and_import("domainnameinfo.repy")
repyhelper.translate_and_import(
"advertise.repy") # used to do OpenDHT lookups
repyhelper.translate_and_import(
"geoip_client.repy") # used for `show location`
import repyhelper
repyhelper.translate_and_import('domainnameinfo.repy')
if __name__ == '__main__':
assert (domainnameinfo_gethostlocation('amazon.uk') == 'United Kingdom')
assert (domainnameinfo_gethostlocation('microsoft.us') == 'United States')
# we don't care about what the url contains (except the end)
assert (domainnameinfo_gethostlocation(
'asd.fasdf.as.df.asdf.asd.fas.microsoft.us') == 'United States')
try:
# invalid, we should get a TypeError
domainnameinfo_gethostlocation(False)
except TypeError:
pass
else:
print "Error, didn't get exception when passing in False..."
try:
# invalid, we should get a TypeError
domainnameinfo_gethostlocation('google.com')
except UnknownHostLocationError:
pass
else:
print "Hmm, can I really locate google.com?"
import sys
import os
import os.path
# Used to import repy scripts without polluting the current directory.
import shutil
import tempfile
import repyhelper
repycachedir = tempfile.mkdtemp()
sys.path = [repycachedir] + sys.path
repyhelpercachedir = repyhelper.set_importcachedir(repycachedir)
repyhelper.translate_and_import("rsa.repy")
repyhelper.translate_and_import("signeddata.repy")
repyhelper.translate_and_import("time.repy")
shutil.rmtree(repycachedir)
# Armon: The port that should be used to update our time using NTP
TIME_PORT = 51345
def get_file_hash(filename):
fileobj = file(filename, 'rb')
filedata = fileobj.read()
fileobj.close()
return sha_hexhash(filedata)
"""
Test to see if restrictions can be initialized and restored from within Python.
"""
from repyportability import *
initialize_restrictions("restrictions.veryslowread")
import repyhelper
import sys
override_restrictions()
# in the call to this, it will do 'exitall' if the restrictions are in place
repyhelper.translate_and_import("rhtest_printifreadisfast.r2py")
# We need to flush stdout (this won't be done for us unless it goes to the
# terminal).
sys.stdout.flush()
# clean up any left over data...
try:
shutil.rmtree('importcachetest')
except (OSError, IOError):
# it's okay if it doesn't exist...
pass
os.mkdir('importcachetest')
# append this to the Python path...
sys.path = sys.path + ['importcachetest']
# write files there
repyhelper.set_importcachedir('importcachetest')
repyhelper.translate_and_import('rhtestrecursion_1.repy')
# This should work...
try:
# note: this is a function from rhtest_recursion_1. I'm not calling it...
one
except NameError:
print "Failed to import rhtest_recursion_1 when using importcachetest"
# This should work...
try:
# note: this is a function from rhtest_recursion_2. I'm not calling it...
two
except NameError:
print "Failed to import rhtest_recursion_2 when using importcachetest"
from repyportability import *
import repyhelper
mycontext = repyhelper.get_shared_context()
callfunc = 'import'
callargs = []
"""
Author: Armon Dadgar, Eric Kimbrel
Start date: March 27, 2009
Description: Abstracts the task of looking up and advertising servers and forwarders.
"""
repyhelper.translate_and_import('advertise.repy')
# What should we register as?
NAT_FORWARDER_ADVERTISE_KEY = "__NAT__FORWARDER__"
NAT_SRV_PREFIX = "__NAT_SRV__"
# Limit of forwarder lookups
NAT_MAX_LOOKUP = 50
NAT_ADVERTISE_INTERVAL = 60
NAT_ADVERTISE_TTL = 3*NAT_ADVERTISE_INTERVAL
# Pools the advertisements, so that they can be done in one thread
# Maps key-> value
NAT_ADVERTISE_POOL = {}
import runonce
# for getruntime...
import nonportable
# for harshexit
import harshexit
import traceback
import servicelogger
repyhelper.translate_and_import('sha.repy')
repyhelper.translate_and_import('rsa.repy')
repyhelper.translate_and_import('ShimStackInterface.py')
# Armon: To handle user preferrences with respect to IP's and Interfaces
# I will re-use the code repy uses in emulcomm
import emulcomm
# One problem we need to tackle is should we wait to restart a failed service
# or should we constantly restart it. For advertisement and status threads,
# I've chosen to wait before restarting... For worker and accepter, I think
# it's essential to keep restarting them as often as we can...
#
# these variables help us to track when we've started and whether or not we
<Changes>
2009-04-26 Michael Phan-Ba <*****@*****.**>
* Initial release
2009-05-24 Michael Phan-Ba <*****@*****.**>
* Added change log
* Fixed base64 name error
* Set property svn:keyword to "Id"
"""
repyhelper.translate_and_import('base64.repy')
repyhelper.translate_and_import('xmlparse.repy')
class xmlrpc_common_Binary(object):
"""
<Purpose>
Wrapper class for base64-encoded binary data in XML-RPC requests and
responses. This class is used when sending and receiving binary
data through XML-RPC.
<Side Effects>
None.
<Example Use>
blob = xmlrpc_common_Binary("\x00\x01\x00")
"""
Test the removing of multiple files in a bundle through the command line.
"""
import bundle_test_helper
import repyhelper
repyhelper.translate_and_import("bundle.repy")
ORIGINAL_BUNDLE_FN = "test_readonly.bundle.repy"
# We make a local copy of the test bundle so that we
# don't affect the execution of other tests
TEST_BUNDLE_FN = "test.bundle.repy"
FILENAMES = ["src1copy", "src2copy", "src3copy"]
# This must be done in binary mode. Otherwise, differences of filesize due to
# \r\n and \n on different OSes will cause the bundler to look for data at the
# wrong places.
_bundle_copy_file(ORIGINAL_BUNDLE_FN, TEST_BUNDLE_FN)
bundle_test_helper.run_program("bundler.py", ["remove", TEST_BUNDLE_FN] + FILENAMES)
# We shouldn't see ANY files in this list...
# pragma out {}
bundle = bundle_Bundle(TEST_BUNDLE_FN, "r")
print bundle.list()
bundle.close()
# Now run the test script!
bundle_test_helper.run_repy_program(TEST_BUNDLE_FN)
experiment manager.
Start date: September 7th 2008
The design goals of this version are to be secure, simple, and reliable (in
that order).
"""
# JAC: This is the only change to the file over the nmclient.repy version...
# for signing the data we send to the node manager
import fastsigneddata
import repyhelper
# session wrapper (breaks the stream into messages)
# an abstracted "itemized data communication" in a separate API
repyhelper.translate_and_import("session.repy")
#allow nat layer
repyhelper.translate_and_import("NATLayer_rpc.repy")
# makes connections time out
repyhelper.translate_and_import("sockettimeout.repy")
# For rsa key conversion.
repyhelper.translate_and_import("rsa.repy")
repyhelper.translate_and_import("time.repy")
# The idea is that this module returns "node manager handles". A handle
# may be used to communicate with a node manager and issue commands. If the
# caller wants to have a set of node managers with the same state, this can
<Program Name>
xmlrpc_client.py
<Started>
May 3, 2009
<Author>
Michael Phan-Ba
<Purpose>
Implements the client-side XML-RPC protocol.
"""
repyhelper.translate_and_import('urlparse.repy')
repyhelper.translate_and_import('httpretrieve.repy')
repyhelper.translate_and_import('xmlrpc_common.repy')
class xmlrpc_client_Client(object):
"""
<Purpose>
XML-RPC client implementation.
<Side Effects>
None.
<Example Use>
client = xmlrpc_client_Client("http://phpxmlrpc.sourceforge.net/server.php")
print client.send_request("examples.getStateName", (1,))
import shutil
import socket # we'll make it so we don't hang...
import tempfile
import traceback # For exception logging if the servicelogger fails.
import runonce
import harshexit # Used for portablekill
import portable_popen
# Import servicelogger to do logging
import servicelogger
# This gives us do_sleep
import misc
repyhelper.translate_and_import("signeddata.repy")
repyhelper.translate_and_import("sha.repy")
# Armon: The port that should be used to update our time using NTP
TIME_PORT = 51234
TIME_PORT_2 = 42345
softwareurl = "http://seattlesoftwareupdater.poly.edu/updatesite/"
# embedded this because it seems easier to update it along with this file
# Every computer running Seattle will have this same public key, and will trust
# files signed by this key.
softwareupdatepublickey = {'e':82832270266597330072676409661763231354244983360850404742185516224735762244569727906889368190381098316859532462559839005559035695542121011189767114678746829532642015227757061325811995458461556243183965254348908097559976740460038862499279411461302045605434778587281242796895759723616079286531587479712074947611, 'n':319621204384190529645831372818389656614287850643207619926347176392517761801427609535545760457027184668587674034177692977122041230031985031724339016854308623931563908276376263003735701277100364224187045110833742749159504168429702766032353498688487937836208653017735915837622736764430341063733201947629404712911592942893299407289815035924224344585640141382996031910529762480483482480840200108190644743566141062967857181966489101744929170144756204101501136046697030104623523067263295405505628760205318871212056879946829241448986763757070565574197490565540710448548232847380638562809965308287901471553677400477022039092783245720343246522144179191881098268618863594564939975401607436281396130900640289859459360314214324155479461961863933551434423773320970748327521097336640702078449006530782991443968680573263568609595969967079764427272827202433035192418494908184888678872217792993640959292902948045622147093326912328933981365394795535990933982037636876825043938697362285277475661382202880481400699819441979130858152032120174957606455858082332914545153781708896942610940094268714863253465554125515897189179557899347310399568254877069082016414203023408461051519104976942275899720740657969311479534442473551582563833145735116565451064388421}
# Whether the nodemanager should be told not to daemonize when it is restarted.
# This is only to assist our automated tests.
of random bytes, this is not a permanent fix (the extraction
of random bytes from the float is not portable). The change will
likely be made to random_randombytes (since calls os.urandom will
likely be restricted to a limited number of bytes).
TODO -
random_randombytes will remained but serve as a helper function
to collect the required number of bytes. Calls to randombytes
will be restricted to a set number of bytes at a time, since
allowing an arbitrary request to os.urandom would circumvent
performance restrictions.
TODO -
_random_long_to_bytes will no longer be needed.
"""
repyhelper.translate_and_import("math.repy")
def random_randombytes(num_bytes, random_float=None):
"""
<Purpose>
Return a string of length num_bytes, made of random bytes
suitable for cryptographic use (because randomfloat draws
from a os provided random source).
*WARNING* If python implements float as a C single precision
floating point number instead of a double precision then
there will not be 53 bits of data in the coefficient.
<Arguments>
num_bytes:
"""
Test the filename parameter of the translation calls, to make sure errors are
handled well.
"""
import repyhelper
try:
repyhelper.translate_and_import('asdkfjaNONEXISTANT_FILEkasfj')
except ValueError:
pass
else:
print "Was able to import a nonexistant file (no dir)..."
try:
repyhelper.translate_and_import('./asdkfjaVALID_DIR_NONEXISTANT_FILEkasfj')
except ValueError:
pass
else:
print "Was able to import a nonexistant file in a valid dir..."
try:
repyhelper.translate_and_import('asaINVALID_DIRkasd/adjNONEXISTANT_FILEkaj')
except ValueError:
pass
"""
Loads a series of keys and make sures seash recognizes them correctly.
"""
import seash
import sys
import repyhelper
repyhelper.translate_and_import('rsa.repy')
orig_stdout = sys.stdout
# Prepare the expected results.
# This list of tuples dictates whether the specified guest has its
# public/private keys loaded.
# Each tuple represents: (guestname, publickey, privatekey)
guest_has_keys = [('guest2', True, True), ('guest0', True, False),
('guest1', False, True)]
sys.stdout = open("loadkeys_test_results.txt", "w")
# show keys
# Guestname Publickey/None Privatekey/None
# None is printed only if that key isn't loaded.
# e.g.:
# guest0 {'e': 65537L, 'n': 899L} {'q': 31L, 'p': 29L, 'd': 593L}
# guest1 None {'q': 31L, 'p': 29L, 'd': 593L}
# guest2 {'e': 65537L, 'n': 899L} None
for (guest, haspublic, hasprivate) in guest_has_keys:
print guest,
# clean up any left over data...
try:
shutil.rmtree('importcachetest')
except (OSError, IOError):
# it's okay if it doesn't exist...
pass
os.mkdir('importcachetest')
# append this to the Python path...
sys.path = sys.path + ['importcachetest']
# write files there
repyhelper.set_importcachedir('importcachetest')
repyhelper.translate_and_import('rhtestrecursion_1.r2py')
# This should work...
try:
# note: this is a function from rhtest_recursion_1. I'm not calling it...
one
except NameError:
print "Failed to import rhtest_recursion_1 when using importcachetest"
# This should work...
try:
# note: this is a function from rhtest_recursion_2. I'm not calling it...
two
except NameError:
print "Failed to import rhtest_recursion_2 when using importcachetest"
"""
Performs initialization for the bundle unit tests.
It should create bundles that contains 3 files: src1, src2, and src3.
"""
import bundle_test_helper
import repyhelper
repyhelper.translate_and_import('bundle.repy')
# Tests shouldn't modify this file directly
# They should make a copy if they need to modify this file.
TEST_BUNDLE_FN = 'test_readonly.bundle.repy'
TEST_EMBED_BUNDLE_FN = 'test_embedded_readonly.bundle.repy'
EMBED_SCRIPT_FN = 'testscript.repy'
TEST_FILENAMES = ['src1', 'src2', 'src3']
TEST_COPY_FILENAMES = ['src1copy', 'src2copy', 'src3copy']
# Make sure the bundle doesn't exist to test bundle creation
bundle_test_helper.remove_files_from_directory([TEST_BUNDLE_FN])
bundle_test_helper.prepare_test_sourcefiles()
test_bundle = bundle_Bundle(TEST_BUNDLE_FN, 'w')
test_bundle.add_files(TEST_COPY_FILENAMES)
test_bundle.close()
test_bundle = bundle_Bundle(TEST_EMBED_BUNDLE_FN, 'w', srcfn=EMBED_SCRIPT_FN)
test_bundle.add_files(TEST_COPY_FILENAMES)
""" Test the callargs parameter of the translation calls, to make sure it actually gets used """ import repyhelper import test_utils TESTFILE = "rhtest_callargs2.r2py" #Make sure we have fresh translations per test run test_utils.cleanup_translations([TESTFILE]) repyhelper.translate_and_import(TESTFILE, callargs=['a', 'samoas', 'c']) if num_callargs() is not 3: print "translate_and_import had wrong number of callargs:", num_callargs() test_utils.cleanup_translations([TESTFILE])
emailed when something goes wrong
This script takes no arguments. A typical use of this script is to
have it run periodically using something like the following crontab line:
7 * * * * /usr/bin/python /home/seattle/centralizedputget.py > /home/seattle/cron_log.centralizedputget
"""
import urllib
import send_gmail
import integrationtestlib
import urllib2
import subprocess
import repyhelper
repyhelper.translate_and_import('serialize.repy')
username = '******'
apikey = '1X3YFBLPTKVSI8DQHWJZ0NR92645ECUA'
userdata = {username: {'apikey': apikey}}
ERROR_EMAIL_SUBJECT = "Selexor monitoring test failure!"
SELEXOR_PAGE = "https://selexor.poly.edu:8888/"
def retrieve_url(url, data=None):
print "retrieve_url"
args = ["curl", url, "--insecure"]
if data:
Evan Meagher
<Start Date>
Nov 26, 2009
<Description>
XMl-RPC client for remote GeoIP server. Given an IP:port of a GeoIP
XML-RPC server, allows location lookup of hostnames and IP addresses.
<Usage>
client = geoip_client(server_address)
Where server_address is the ip address of a remote GeoIP XMl-RPC server.
"""
repyhelper.translate_and_import('parallelize.repy')
repyhelper.translate_and_import('xmlrpc_client.repy')
"""
Initialize global GeoIP XML-RPC client object to None
Note: client is stored in wrapper list to avoid using mycontext dict
"""
geoip_clientlist = []
def geoip_init_client(url=["http://geoipserver.poly.edu:12679", "http://geoipserver2.poly.edu:12679"]):
"""
<Purpose>
Create a new GeoIP XML-RPC client object.
<Changes>
2009-04-26 Michael Phan-Ba <*****@*****.**>
* Initial release
2009-05-24 Michael Phan-Ba <*****@*****.**>
* Added change log
* Fixed base64 name error
* Set property svn:keyword to "Id"
"""
repyhelper.translate_and_import('base64.repy')
repyhelper.translate_and_import('xmlparse.repy')
class xmlrpc_common_Binary(object):
"""
<Purpose>
Wrapper class for base64-encoded binary data in XML-RPC requests and
responses. This class is used when sending and receiving binary
data through XML-RPC.
<Side Effects>
None.
- acquires log output from vessels and stores them in state for user
This is a general interface with connecting with remote Seattle instances
directly, but the functionality is geared toward the needs of the
autograder.
"""
import sys
import nmAPI
from repyportability import *
import repyhelper
repyhelper.translate_and_import("nmclient.repy")
repyhelper.translate_and_import("rsa.repy")
repyhelper.translate_and_import("parallelize.repy")
# global state
key = {}
vesselinfo = {}
# the default Seattle instance to connect to
DEFAULT_NODEMANAGER_PORT = 1224
# alpers - adds a vessel and its information to the overall vessel dict
# (from seash.mix); this is strictly internal.
def add_vessel(longname, vesselhandle):
vesselinfo[longname] = {}
vesselinfo[longname]['handle'] = vesselhandle
mycontext = repyhelper.get_shared_context()
callfunc = 'import'
callargs = []
"""
Author: Justin Cappos
Start Date: July 8, 2008
Description:
Advertisements to a central server (similar to openDHT)
"""
repyhelper.translate_and_import('session.repy')
# I'll use socket timeout to prevent hanging when it takes a long time...
repyhelper.translate_and_import('sockettimeout.repy')
repyhelper.translate_and_import('serialize.repy')
# Hmm, perhaps I should make an initialization call instead of hardcoding this?
# I suppose it doesn't matter since one can always override these values
servername = "advertiseserver.poly.edu"
# This port is updated to use the new port (legacy port is 10101)
serverport = 10102
class CentralAdvertiseError(Exception):
import shutil
# clean up any left over data...
try:
shutil.rmtree('importcachetest')
except (OSError, IOError):
# it's okay if it doesn't exist...
pass
os.mkdir('importcachetest')
# append this to the Python path...
sys.path = sys.path + ['importcachetest']
# write files there
repyhelper.set_importcachedir('importcachetest')
# NOTICE THE RELATIVE PATH NAME!!!
repyhelper.translate_and_import('./rhtest_filetests.r2py')
# This should work...
try:
rhtest_filetests_exists()
except NameError:
print "Failed to import rhtest_filetests when using importcachetest"
# and the file should be in importcachetest...
if not os.path.exists('importcachetest/rhtest_filetests_repy.py'):
print "The rhtest_filetests.r2py file was not preprocessed to importcache test because 'importcachetest/rhtest_filetests_repy.py' does not exist"
PyCrypto which is authored by Dwayne C. Litzenberger
Seattle's origional rsa.repy which is Authored by:
Adapted by Justin Cappos from the version by:
author = "Sybren Stuvel, Marloes de Boer and Ivo Tamboer"
<Purpose>
This is the main interface for using the ported RSA implementation.
<Notes on port>
The random function can not be defined with the initial construction of
the RSAImplementation object, it is hard coded into number_* functions.
"""
repyhelper.translate_and_import('random.repy')
repyhelper.translate_and_import('pycryptorsa.repy')
def rsa_gen_pubpriv_keys(bitsize):
"""
<Purpose>
Will generate a new key object with a key size of the argument
bitsize and return it. A recommended value would be 1024.
<Arguments>
bitsize:
The number of bits that the key should have. This
means the modulus (publickey - n) will be in the range
[2**(bitsize - 1), 2**(bitsize) - 1]
callfunc = 'import'
callargs = []
"""
Author: Armon Dadgar, Eric Kimbrel
Start Date: January 22nd, 2009
Description:
Provides a method of transferring data to machines behind firewalls or Network Address Translation (NAT).
"""
repyhelper.translate_and_import('NAT_advertisement.repy')
repyhelper.translate_and_import('Multiplexer.repy')
"""
This block of code declares and sets the constant values used for the forwarder RPC interface
NAT RPC requests are dictionaries that have special fields
A typical request would be like the following:
# Server requests to de-register
rpc_req = {"id":0,"request":"de_reg_serv"}
# Forwarder responds with success message
rpc_resp = {"id":0,"status":True,"value":None}
print "Was able to set the cache dir to be something besides a string..."
try:
repyhelper.set_importcachedir('asdkfjaNONEXISTANT_DIRasfj')
except TypeError:
pass
else:
print "Was able to set my cache to a nonexistant dir..."
try:
repyhelper.set_importcachedir('..')
except ValueError:
pass
else:
print "Was able to set my cache to be a dir that's not in the python path..."
try:
repyhelper.translate_and_import('./asdkfjaVALID_DIR_NONEXISTANT_FILEkasfj')
except ValueError:
pass
else:
print "Was able to import a nonexistant file in a valid dir..."
try:
repyhelper.translate_and_import(
'asaINVALID_DIRkasd/adjNONEXISTANT_FILEkaj')
except ValueError:
pass
else:
print "Was able to import a nonexistant file in an invalid dir..."
This module is adapted from code in seash which had similar functionality.
NOTE (for the programmer using this module). It's really important to
write concurrency safe code for the functions they provide us. It will not
work to write:
def foo(...):
mycontext['count'] = mycontext['count'] + 1
YOU MUST PUT A LOCK AROUND SUCH ACCESSES.
"""
# I use this to get unique identifiers.
repyhelper.translate_and_import('uniqueid.repy')
class ParallelizeError(Exception):
"""An error occurred when operating on a parallelized task"""
# This has information about all of the different parallel functions.
# The keys are unique integers and the entries look like this:
# {'abort':False, 'callfunc':callfunc, 'callargs':callargs,
# 'targetlist':targetlist, 'availabletargetpositions':positionlist,
# 'runninglist':runninglist, 'result':result}
#
# abort is used to determine if future events should be aborted.
# callfunc is the function to call
"""
Test adding a string through the API.
"""
import bundle_test_helper
import repyhelper
repyhelper.translate_and_import('bundle.repy')
TEST_BUNDLE_FN = 'test.bundle.repy'
# Make sure the bundle doesn't exist to test bundle creation
bundle_test_helper.remove_files_from_directory([TEST_BUNDLE_FN])
TEST_STRING = 'this is a test string'
TEST_STRING_FN = 'teststring'
bundle = bundle_Bundle(TEST_BUNDLE_FN, 'w')
bundle.add_string(TEST_STRING_FN, TEST_STRING)
bundle.close()
# Make sure the added file is inside the bundle
bundle = bundle_Bundle(TEST_BUNDLE_FN, 'r')
if not TEST_STRING_FN in bundle.list():
print "Added string not found when listing bundle contents"
# Make sure added file matches original file contents
added_file_contents = bundle.extract_to_string(TEST_STRING_FN)
if added_file_contents != TEST_STRING:
print "Added string contents does not match string contents"
# Now run the test script!
<Purpose>
This file provides the encryption functionality for rsa.repy.
This code has been left as close to origional as possible with
the notes about changes made to enable for easier modification
if pycrypto is updated.
It contains:
pubkey.py
RSA.py
_RSA.py
_slowmath.py
number.py
"""
repyhelper.translate_and_import('random.repy')
#
# pubkey.py : Internal functions for public key operations
#
# Part of the Python Cryptography Toolkit
#
# Distribute and use freely; there are no restrictions on further
# dissemination and usage except those imposed by the laws of your
# country of residence. This software is provided "as is" without
# warranty of fitness for use or suitability for any purpose, express
# or implied. Use at your own risk or not at all.
#
"""
<Modified>
###Helper Methods###
import seash_global_variables
import seash_exceptions
import repyhelper
#repyhelper.translate_and_import("nmclient.repy")
import fastnmclient
# Use local clock for time if there is no network connectivity
import time
repyhelper.translate_and_import("time.repy")
repyhelper.translate_and_import("rsa.repy")
repyhelper.translate_and_import("listops.repy")
repyhelper.translate_and_import("parallelize.repy")
repyhelper.translate_and_import("domainnameinfo.repy")
repyhelper.translate_and_import("advertise.repy") # used to do OpenDHT lookups
repyhelper.translate_and_import("geoip_client.repy") # used for `show location`
repyhelper.translate_and_import("serialize.repy") # used for loadstate and savestate
Amended 5 July, 2009 to be an extension to time.repy
Description:
This module is an implementation of time_interface.repy
Contacts a server running time_server.repy to get the current time from an
NTP because only the server can communicate with an NTP.
To use this module, make one call to time_updatetime() to get the
time from the server. This function also implicitly sets the time. Then
call time_gettime() every time the current time is needed.
"""
repyhelper.translate_and_import("time_interface.repy")
repyhelper.translate_and_import("advertise.repy")
repyhelper.translate_and_import("random.repy")
repyhelper.translate_and_import("sockettimeout.repy")
# This function contacts the server to get the time from a NTP
def tcp_time_updatetime(localport):
"""
<Purpose>
Opens a connection with a server hosting time_server.repy, which obtains the
current time via a NTP, then calls time_settime(float(currenttime)) to set
the current time to the received value form the server.
