def expand_macro(self, formatter, name, args):
if args is None:
return "No diagram text defined!"
diagramText = str(args).strip()
# First, encode the data.
data = urllib.urlencode({"message" : diagramText, "style" : self.style, "paginate" : 0, "paper" : "letter", "landscape" : 0, "page" : 1, "format" : "png"})
# Now get that file-like object again, remembering to mention the data.
f = urllib.urlopen("http://www.websequencediagrams.com/index.php", data)
# Read the results back.
s = f.read()
f.close()
s = s[1:-1]
seqargs = s.split(",")
locargs = seqargs[0].split(":")
loc = locargs[1].strip()
loc = loc[1:-1]
s = StringIO()
s.write("{{{\n")
s.write("#!html\n")
s.write("<img src='http://www.websequencediagrams.com/%s'>" % loc)
s.write("\n}}}\n")
v = s.getvalue()
s.close()
return wiki_to_html(v, self.env, formatter.req)
def getSavedFormInputForEdit(self, header=False, delimiter=','):
""" returns saved as CSV text """
sbuf = StringIO()
writer = csvwriter(sbuf, delimiter=delimiter)
names = self.getColumnNames()
titles = self.getColumnTitles()
if header:
encoded_titles = []
for t in titles:
if isinstance(t, unicode):
t = t.encode('utf-8')
encoded_titles.append(t)
writer.writerow(encoded_titles)
for row in self.getSavedFormInput():
def get_data(row, i):
data = row.get(i, '')
if self._is_file_data(data):
return data.filename
if isinstance(data, unicode):
return data.encode('utf-8')
return data
writer.writerow([get_data(row, i) for i in names])
res = sbuf.getvalue()
sbuf.close()
return res
def _export_dashboard(request, dashboard_id):
"""
Exports dashboard as multi-sheet Excel workbook.
This is a helper method for export_dashboard and export_shared_dashboard
below. Renders an export without requiring login, should not be exposed
directly via a URL pattern.
"""
dashboard = Dashboard.objects.get(id=dashboard_id)
stream = StringIO()
workbook = xlwt.Workbook()
for metric in dashboard.metric_set.all():
metric.export(workbook)
workbook.save(stream)
response = HttpResponse(
stream.getvalue(),
mimetype='application/vnd.ms-excel'
)
response['Content-Disposition'] = 'attachment; filename="%s-%s.xls"' \
% (slugify(dashboard.name), date.today())
stream.close()
return response
def close(self):
if self.mode == 'w':
disposition = 'attachment; filename="%s"' % self.filename
self.hosted_file._root._browser.put(
self.url, self.getvalue(),
self.content_type, {'Content-Disposition' : disposition})
StringIO.close(self)
def dump_fixtures(request):
output = StringIO()
fixture = request.GET.get('fixture', None)
try:
if fixture:
call_command('dumpdata', fixture, '--indent=2', stdout=output)
else:
call_command('dumpdata', '--indent=2', stdout=output)
data = output.getvalue()
output.close()
if fixture:
file_label = 'fixtures_%s_%s' % (fixture, datetime.datetime.now().strftime('%d-%b-%Y_%H-%M'))
else:
file_label = 'fixtures_all_%s' % datetime.datetime.now().strftime('%d-%b-%Y_%H-%M')
response = HttpResponse(data, content_type="application/json")
response['Content-Disposition'] = 'attachment; filename=%s' % file_label
return response
except:
dest = request.META.get('HTTP_REFERER', '/')
messages.info(request, 'Fixture name not recognized: %s' % fixture)
return HttpResponseRedirect(dest)
def __init__(self, config):
stream = StringIO()
stream.write("Current platform settings:\n")
configfile.pretty_print(config.get_all_platforms(), stream)
self.description = stream.getvalue()
stream.close()
self.config = config
def write_out(self, fde):
"""
Write out full packet chain
"""
# TODO: impl is just outright terrible.
# Fix it in any way shape or form i don't care
sio = StringIO()
seq_id = self.start_seq_id
net_total_written = 0
total_written = 0
last_total_written = 0xffffff
for label, field in self.fields:
written = field.write_out(sio, label='\t%s' % label)
total_written += written
net_total_written += written
if total_written >= 0xffffff:
self._write_packet_header(0xffffff, seq_id, fde)
fde.write(sio.read(0xffffff))
remaining_bytes = sio.read()
sio.close()
sio = StringIO(remaining_bytes)
last_total_written = total_written
total_written -= 0xffffff
seq_id += 1
if last_total_written == 0xffffff:
self._write_packet_header(total_written, seq_id, fde)
sio.seek(0)
fde.write(sio.read(total_written))
net_total_written += total_written
return (net_total_written, seq_id)
class WebDAVFile(File):
def __init__(self, name, storage, mode):
self._name = name
self._storage = storage
self._mode = mode
self._is_dirty = False
self.file = StringIO()
self._is_read = False
@property
def name(self):
return self._name
@property
def size(self):
if not hasattr(self, '_size'):
self._size = self._storage.size(self._name)
return self._size
def read(self, num_bytes=None):
if not self._is_read:
self.file = self._storage._read(self._name)
self._is_read = True
return self.file.read(num_bytes)
def write(self, content):
if 'w' not in self._mode:
raise AttributeError("File was opened for read-only access.")
self.file = StringIO(content)
self._is_dirty = True
self._is_read = True
def close(self):
self.file.close()
def get(self,filename):
name,sep,ext = filename.rpartition(".")
if not sep:
img_name = ext
else:
img_name = name
try:
img_file = self.fs.get_version(filename=img_name)
img = img_file.read()
except gridfs.errors.NoFile:
raise tornado.web.HTTPError(500, 'image is not found ')
resize = self.get_argument('_re', None)
if resize :
width, resep, height = resize.rpartition("x")
output = StringIO()
output.write(img)
output.seek(0)
im = Image.open(output)
format = im.format
# size = im.size
# logging.info("format is %s ,size is %s" %(format,size))
im = im.resize((int(width),int(height)), Image.ANTIALIAS)
tmp = StringIO()
im.save(tmp, format)
img = tmp.getvalue()
tmp.close()
output.close()
self.add_header('Content-Type',img_file.content_type)
self.write(img)
self.finish()
def test_packAndUnpackVerySmallIBMFloats(self):
"""
The same test as test_packAndUnpackIBMFloat just for small numbers
because they might suffer more from the inaccuracies.
"""
# Some random seeds.
seeds = [123, 1592, 4482, 601, 1, 783, 6849]
endians = ['<', '>']
# Loop over all combinations.
for seed in seeds:
# Generate 50000 random floats from -10000 to +10000.
np.random.seed(seed)
data = 1E-5 * np.random.ranf(50000)
# Convert to float64 in case native floats are different to be
# able to utilize double precision.
data = np.require(data, 'float64')
# Loop over little and big endian.
for endian in endians:
# Pack.
f = StringIO()
DATA_SAMPLE_FORMAT_PACK_FUNCTIONS[1](f, data, endian)
# Jump to beginning and read again.
f.seek(0, 0)
new_data = DATA_SAMPLE_FORMAT_UNPACK_FUNCTIONS[1](f,
len(data), endian)
f.close()
# A relative tolerance of 1E-6 is considered good enough.
rms1 = rms(data, new_data)
self.assertEqual(True, rms1 < 1E-6)
def test_packAndUnpackIBMSpecialCases(self):
"""
Tests the packing and unpacking of several powers of 16 which are
problematic because they need separate handling in the algorithm.
"""
endians = ['>', '<']
# Create the first 10 powers of 16.
data = []
for i in xrange(10):
data.append(16 ** i)
data.append(-16 ** i)
data = np.array(data)
# Convert to float64 in case native floats are different to be
# able to utilize double precision.
data = np.require(data, 'float64')
# Loop over little and big endian.
for endian in endians:
# Pack.
f = StringIO()
DATA_SAMPLE_FORMAT_PACK_FUNCTIONS[1](f, data, endian)
# Jump to beginning and read again.
f.seek(0, 0)
new_data = DATA_SAMPLE_FORMAT_UNPACK_FUNCTIONS[1](f,
len(data), endian)
f.close()
# Test both.
np.testing.assert_array_equal(new_data, data)
def CharacterDataHandler(self, data):
if self.write_to == 'Name':
data = data.strip()
self.nvpair.name = data
elif self.write_to == 'Value':
data = data.strip()
self.nvpair.value = data
elif self.write_to == 'DataSpace':
data = data.strip()
self.coordsys.dataspace = xform_codes.code[data]
elif self.write_to == 'TransformedSpace':
data = data.strip()
self.coordsys.xformspace = xform_codes.code[data]
elif self.write_to == 'MatrixData':
# conversion to numpy array
from StringIO import StringIO
c = StringIO(data)
self.coordsys.xform = np.loadtxt(c)
c.close()
elif self.write_to == 'Data':
da_tmp = self.img.darrays[-1]
da_tmp.data = read_data_block(da_tmp.encoding, da_tmp.endian, \
da_tmp.ind_ord, da_tmp.datatype, \
da_tmp.dims, data)
elif self.write_to == 'Label':
self.label.label = data.strip()
def test_packAndUnpackIBMFloat(self):
"""
Packing and unpacking IBM floating points might yield some inaccuracies
due to floating point rounding errors.
This test tests a large number of random floating point numbers.
"""
# Some random seeds.
seeds = [1234, 592, 459482, 6901, 0, 7083, 68349]
endians = ['<', '>']
# Loop over all combinations.
for seed in seeds:
# Generate 50000 random floats from -10000 to +10000.
np.random.seed(seed)
data = 200000.0 * np.random.ranf(50000) - 100000.0
# Convert to float64 in case native floats are different to be
# able to utilize double precision.
data = np.require(data, 'float64')
# Loop over little and big endian.
for endian in endians:
# Pack.
f = StringIO()
DATA_SAMPLE_FORMAT_PACK_FUNCTIONS[1](f, data, endian)
# Jump to beginning and read again.
f.seek(0, 0)
new_data = DATA_SAMPLE_FORMAT_UNPACK_FUNCTIONS[1](f,
len(data), endian)
f.close()
# A relative tolerance of 1E-6 is considered good enough.
rms1 = rms(data, new_data)
self.assertEqual(True, rms1 < 1E-6)
def readFile():
global curId
script = StringIO(QuestScripts.SCRIPT)
def readLine():
return script.readline().replace('\r', '')
gen = tokenize.generate_tokens(readLine)
line = getLineOfTokens(gen)
while line is not None:
if line == []:
line = getLineOfTokens(gen)
continue
if line[0] == 'ID':
parseId(line)
elif curId is None:
notify.error('Every script must begin with an ID')
else:
lineDict[curId].append(line)
line = getLineOfTokens(gen)
script.close()
def get_results(self):
"""Get analysis results.
@return: data.
"""
root = self._get_root(container="cuckoo", create=False)
if not os.path.exists(root):
return False
zip_data = StringIO()
zip_file = ZipFile(zip_data, "w", ZIP_DEFLATED)
root_len = len(os.path.abspath(root))
for root, dirs, files in os.walk(root):
archive_root = os.path.abspath(root)[root_len:]
for name in files:
path = os.path.join(root, name)
archive_name = os.path.join(archive_root, name)
zip_file.write(path, archive_name, ZIP_DEFLATED)
zip_file.close()
data = xmlrpclib.Binary(zip_data.getvalue())
zip_data.close()
return data
def put(self, path, v, wait='short'):
c = self.con
filesize = len(v)
f = StringIO(v)
c.setopt(pycurl.INFILESIZE, filesize)
c.setopt(pycurl.PUT, 1)
c.setopt(pycurl.READFUNCTION, FileReader(f).read_callback)
# For the TIMEOUT, it will depends if we are waiting for a long query or not
# long:data_timeout, like for huge broks receptions
# short:timeout, like for just "ok" connection
if wait == 'short':
c.setopt(c.TIMEOUT, self.timeout)
else:
c.setopt(c.TIMEOUT, self.data_timeout)
# if proxy:
# c.setopt(c.PROXY, proxy)
# Pycurl want a list of tuple as args
c.setopt(c.URL, str(self.uri + path))
# Ok now manage the response
response = StringIO()
c.setopt(pycurl.WRITEFUNCTION, response.write)
# c.setopt(c.VERBOSE, 1)
try:
c.perform()
except pycurl.error, error:
errno, errstr = error
f.close()
raise HTTPException('Connection error to %s : %s' % (self.uri, errstr))
def upload_analyzer(self):
"""Upload analyzer to guest.
@return: operation status.
"""
zip_data = StringIO()
zip_file = ZipFile(zip_data, "w", ZIP_DEFLATED)
root = os.path.join("analyzer", self.platform)
root_len = len(os.path.abspath(root))
if not os.path.exists(root):
log.error("No valid analyzer found at path: %s" % root)
return False
for root, dirs, files in os.walk(root):
archive_root = os.path.abspath(root)[root_len:]
for name in files:
path = os.path.join(root, name)
archive_name = os.path.join(archive_root, name)
zip_file.write(path, archive_name, ZIP_DEFLATED)
zip_file.close()
data = xmlrpclib.Binary(zip_data.getvalue())
zip_data.close()
log.debug("Uploading analyzer to guest (ip=%s)" % self.ip)
self.server.add_analyzer(data)
class MockRequest(object):
def __init__(self, utoken, uri=None):
self._utoken = utoken
self.uri = uri
self._finishedDeferreds = []
self.stream = StringIO()
self.args = {}
def getSession(self):
return MockSession(self._utoken)
def finish(self):
pass
def write(self, data):
self.stream.write(data)
def clear(self):
self.stream.close()
self.stream = StringIO()
def notifyFinish(self):
finished = Deferred()
self._finishedDeferreds.append(finished)
return finished
def setResponseCode(self, code, error):
self.code = code
self.error = error
def __str__(self):
return "\nrequest:args = %s\ndata = %s\n" % (self.args, self.stream.getvalue())
def read_bed(bed_file):
"""
Parses the bed file specified into `pd.DataFrame`
:param bed_file:
:return:
:rtype: `pd.DataFrame`
"""
f = open(bed_file, 'r')
try:
s = StringIO()
# Filter out all lines that do not start with "chr" as BED files are allowed to contain some junk
for line in f:
if line.startswith('chr'):
s.write(line)
s.seek(0)
regions = pd.read_csv(s, sep="\t", header=None)
finally:
f.close()
s.close()
regions.columns = BED_COLUMNS[:len(regions.columns)]
if len(regions.name) != len(regions.name.drop_duplicates()):
raise Exception('Input BED file {0!r} contains duplicate values in name column. '
'Please ensure the names of the regions are unique'.format(bed_file))
if 'name' in regions.columns:
regions = regions.set_index('name')
return Regions(regions)
def send(self, content = '', code = 200, message = 'OK', headers = {}):
if not 'Content-Type' in headers:
headers['Content-Type'] = 'text/html; charset=utf-8'
#headers['Expires'] = 'Sun, 25 Jan 1988 13:15:00 GMT'
#headers['Last-Modified'] = 'Sun, 25 Jan 1988 13:15:00 GMT'
#headers['Cache-Control'] = 'must-revalidate'
#headers['X-Powered-By'] = 'jambe'
headers['Content-Length'] = len(content)
if 'gzip' in self.headers.get('Accept-Encoding', 'none').split(','):
from StringIO import StringIO
import gzip
headers['Content-Encoding'] = 'gzip'
f = StringIO()
gzf = gzip.GzipFile(mode = "wb", fileobj = f, compresslevel = 1)
gzf.write(content)
gzf.close()
content = f.getvalue()
f.close()
new_content_length = len(content)
self.log_message('Gzip: old-length=%s, new-length=%s' % (headers['Content-Length'], new_content_length))
headers['Content-Length'] = new_content_length
self.send_response(code, message)
for key in headers:
self.send_header(key, headers[key])
self.end_headers()
if self.command != 'HEAD' and code >= 200 and code not in (204, 304):
self.wfile.write(content)
def execute_from_socket(self,params):
"""
Executes an arbitrary python command through the socket, and returns the console
output
"""
dc=self.__determineContext__(params).dict
# setup a buffer to capture response, temporarily grab stdio
params['request']['protocol'].transport.write('<Python< '+params['command']+'\n\r')
rbuffer = StringIO()
sys.stdout = rbuffer
try: exec(params['command'],dc)
except:
params['request']['protocol'].transport.write('>Python> ERROR\n\r')
params['request']['protocol'].transport.loseConnection()
return
# exec command has side-effect of adding builtins; remove them
if dc.has_key('__builtins__'):
del dc['__builtins__']
# update data context
# remember to restore the original stdout!
sys.stdout = sys.__stdout__
# requests variables from the directory and writes to user.
params['request']['protocol'].transport.write('>Code>')
for var in dc:
params['request']['protocol'].transport.write('>Var>' + var + ' is ' + str(type(var)) + ' and is equal to ' + str(dc[var]))
params['request']['protocol'].transport.write('>Code>')
# output the response buffer to the HTTP request
params['request']['protocol'].transport.write('>Python> '+rbuffer.getvalue()+'\n\r')
params['request']['protocol'].transport.loseConnection()
rbuffer.close()
def fill_file(result):
url = result
#print url
f = get_page(url)
#print f
if f == "":
return "a"
f = f.read()
##removes tags
f = html_to_text(f)
##decoding to ASCII
encoding = 'utf-8'
try:
ustr = f.decode(encoding)
except:
pass
return f
b = StringIO()
old = sys.stdout
try:
sys.stdout = b
html2text.wrapwrite(html2text.html2text(ustr, url))
finally: sys.stdout = old
text = b.getvalue()
b.close()
return text
def parse_pdf_pdfminer(self, f, fpath):
try:
laparams = LAParams()
laparams.all_texts = True
rsrcmgr = PDFResourceManager()
pagenos = set()
if self.dedup:
self.dedup_store = set()
self.handler.print_header(fpath)
page_num = 0
for page in PDFPage.get_pages(f, pagenos, check_extractable=True):
page_num += 1
retstr = StringIO()
device = TextConverter(rsrcmgr, retstr, laparams=laparams)
interpreter = PDFPageInterpreter(rsrcmgr, device)
interpreter.process_page(page)
data = retstr.getvalue()
retstr.close()
self.parse_page(fpath, data, page_num)
self.handler.print_footer(fpath)
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
self.handler.print_error(fpath, e)
def convert_df_to_njson_and_upload(self, df, destination_blobname, metadata={}, tmp_bucket='isb-cgc-dev'):
log.info("Converting dataframe into a new-line delimited JSON file")
file_to_upload = StringIO()
for i, rec in df.iterrows():
file_to_upload.write(rec.convert_objects(convert_numeric=False).to_json() + "\n")
file_to_upload.seek(0)
bucket = self.client.get_bucket(tmp_bucket)
upload_blob = storage.blob.Blob(destination_blobname, bucket=bucket)
upload_blob.upload_from_string(file_to_upload.getvalue())
# set blob metadata
if metadata:
log.info("Setting object metadata")
upload_blob.metadata = metadata
upload_blob.patch()
file_to_upload.close()
# check if the uploaded blob exists. Just a sanity check
if upload_blob.exists():
log.info("The uploaded file {0} has size {1} bytes.".format(destination_blobname, upload_blob.size))
return True
else:
raise Exception('File upload failed - {0}.'.format(destination_blobname))
def _createCrumbs(self, locations, img=""):
"""
Assemble the breadcrumbs.
"""
buff = StringIO()
count = 0
size = len(locations) - 1
for title, url in locations:
log.debug("title: %s, url: %s", title, url)
# Make the last page non-linkable.
if size == count: url = ""
if count > 0:
if img != "":
buff.write('<img class="arrow" src="%s" alt="»" />'\
% img)
else:
buff.write('<span class="arrow">»</span>')
if url:
buff.write('<a class="title" href="%s">%s</a>' % (url, title))
else:
buff.write('<span class="title">%s</span>' % title)
count += 1
crumbs = buff.getvalue()
buff.close()
return crumbs
class _PythonCapturer(object):
def __init__(self, stdout=True):
if stdout:
self._original = sys.stdout
self._set_stream = self._set_stdout
else:
self._original = sys.stderr
self._set_stream = self._set_stderr
self._stream = StringIO()
self._set_stream(self._stream)
def _set_stdout(self, stream):
sys.stdout = stream
def _set_stderr(self, stream):
sys.stderr = stream
def release(self):
# Original stream must be restored before closing the current
self._set_stream(self._original)
self._stream.flush()
output = self._stream.getvalue()
self._stream.close()
return output
def update_addon(uid,wow_dir):
"""Downloads latest version of addon and extract it into wow directory.
uid: Wow interface addon addon id
wow_dir: Wow addon directory"""
url = 'http://www.wowinterface.com/patcher/info-%d.xml' % uid
dom = minidom.parse(urllib2.urlopen(url))
if dom.getElementsByTagName('error'):
if int(dom.getElementsByTagName('id')[0].firstChild.nodeValue) == 403:
print 'The file is still being checked by mods, update will be downloaded next time you run this script.' #This function shouldn't print.
return False
else:
print 'Please give this info to the addon author: <%d> - %s' % (int(dom.getElementsByTagName('id')[0].firstChild.nodeValue),
str(dom.getElementsByTagName('message')[0].firstChild.nodeValue))
return False
file_location = str(dom.getElementsByTagName('UIFileURL')[0].firstChild.nodeValue)
size = int(dom.getElementsByTagName('UISize')[0].firstChild.nodeValue)
if size > 1048576: #If size is lager then 1mb
print 'Downloading big file, this may take more then few seconds' #This function shouldn't print. This is just a workaround. Again.
f = urllib2.urlopen(file_location)
data = StringIO(f.read())
f.close()
data = zipfile.ZipFile(data)
addon_dirs = []
for f in data.namelist():
dir = str(f.split('/',1)[0])
if not (dir in addon_dirs):
addon_dirs.append(dir)
wuiup_removedir(os.path.join(wow_dir, dir))
wuiup_unzip(data,wow_dir)
data.close()
return True
def __call__(self, parser, namespace, values, option_string=None):
# create the help string and store it into a string
from StringIO import StringIO
fstr = StringIO()
try:
parser.print_help(file=fstr)
help_str = fstr.getvalue()
finally:
fstr.close()
# create the regular expression to match the desciption
descmatch = r'{0}(.+?){0}(.+?){0}'
# escape possible dangerous characters
esplit_string = re.escape(split_string)
re_desc = re.compile(descmatch.format(esplit_string),
flags=re.DOTALL)
# select the case according to which option_string is selected
if option_string == '-h':
to_sub = r'\1'
elif option_string == '--help':
to_sub = r'\1\2'
print(re_desc.sub(to_sub, help_str))
parser.exit()
class BaseTest(TestCase):
def setUp(self):
pass
""" ignore output buffer """
self.stdout = StringIO()
self.stderr = StringIO()
# self.new_command = new.Command()
# app_dir = __import__(self.new_command.usercommand)
# self.app_dir = app_dir.__path__[0]
def tearDown(self):
pass
""" ignore output buffer """
self.stdout.close()
self.stderr.close()
def new_setUp(self):
pass
""" """
# os.mkdir(os.path.join(self.app_dir, "static"))
# os.mkdir(os.path.join(self.app_dir, "static", "js"))
#
# self.assertRaises(
# (SystemExit, ValueError),
# self.new_command.execute,
# "gencmd",
## stdout=self.stdout,
## stderr=self.stderr
# )
def new_tearDown(self):
pass
def get_PDB_universe(pdbcode):
# Get file from PDB
filename = 'pdb'+str(pdbcode)+'.ent.gz'
ftp = FTP('ftp.wwpdb.org')
ftp.login()
ftp.cwd('pub/pdb/data/structures/all/pdb')
gzipfile = StringIO() # buffer for retrieved file
ftp.retrbinary('RETR {}'.format(filename), gzipfile.write)
ftp.quit()
# unzip PDB file
gzipfile.seek(0) # reset StringIO object for reading
with gzip.GzipFile(pdbcode, 'rb', 0, gzipfile) as unzipper:
pdbcontent = unzipper.read()
gzipfile.close()
# Is there no way to create a Universe directly from a text object?
fh = tempfile.NamedTemporaryFile(suffix='.pdb', mode='w')
fh.write(pdbcontent)
fh.flush()
# create universe
u = mda.Universe(fh.name)
# clean up and return
fh.close()
return u
geometry.setAttribute('GeometryType', 'ORIGIN_DxDyDz')
grid.appendChild(geometry)
# create DataItem for origin
origin_data_item = doc.createElement('DataItem')
origin_data_item.setAttribute('Dimensions', '3')
origin_data_item.setAttribute('Format', 'XML')
origin_values = doc.createTextNode('0.0 0.0 0.0')
origin_data_item.appendChild(origin_values)
geometry.appendChild(origin_data_item)
# create DataItem for dxdydz
dxdydz_data_item = doc.createElement('DataItem')
dxdydz_data_item.setAttribute('Dimensions', '3')
dxdydz_data_item.setAttribute('Format', 'XML')
dxdydz_values = doc.createTextNode('1.0 1.0 1.0')
dxdydz_data_item.appendChild(dxdydz_values)
geometry.appendChild(dxdydz_data_item)
# create Attribute for values
attribute = doc.createElement('Attribute')
attribute.setAttribute('Name', 'temperature')
attribute.setAttribute('Center', 'Node')
grid.appendChild(attribute)
# create DataItem for Attribute
data_item = doc.createElement('DataItem')
data_item.setAttribute('Format', 'XML')
data_item.setAttribute('Dimensions', dim_str)
data_text = doc.createTextNode(data_str.getvalue())
data_item.appendChild(data_text)
attribute.appendChild(data_item)
data_str.close()
print(doc.toprettyxml(indent=' '))
def export_reports(self,
format='json',
report_id=None,
raw_or_label='raw',
raw_or_label_headers='raw',
export_checkbox_labels='false',
decimal_character=None,
df_kwargs=None):
"""
Export a report of the Project
Notes
-----
Parameters
----------
report_id : the report ID number provided next to the report name
on the report list page
format : (``'json'``), ``'csv'``, ``'xml'``, ``'df'``
Format of returned data. ``'json'`` returns json-decoded
objects while ``'csv'`` and ``'xml'`` return other formats.
``'df'`` will attempt to return a ``pandas.DataFrame``.
raw_or_label : raw [default], label - export the raw coded values or
labels for the options of multiple choice fields
raw_or_label_headers : raw [default], label - (for 'csv' format 'flat'
type only) for the CSV headers, export the variable/field names
(raw) or the field labels (label)
export_checkbox_labels : true, false [default] - specifies the format of
checkbox field values specifically when exporting the data as labels
(i.e., when rawOrLabel=label). When exporting labels, by default
(without providing the exportCheckboxLabel flag or if
exportCheckboxLabel=false), all checkboxes will either have a value
'Checked' if they are checked or 'Unchecked' if not checked.
But if exportCheckboxLabel is set to true, it will instead export
the checkbox value as the checkbox option's label (e.g., 'Choice 1')
if checked or it will be blank/empty (no value) if not checked.
If rawOrLabel=false, then the exportCheckboxLabel flag is ignored.
decimal_character : If specified, force all numbers into same decimal
format. You may choose to force all data values containing a
decimal to have the same decimal character, which will be applied
to all calc fields and number-validated text fields. Options
include comma ',' or dot/full stop '.', but if left blank/null,
then it will export numbers using the fields' native decimal format.
Simply provide the value of either ',' or '.' for this parameter.
Returns
-------
Per Redcap API:
Data from the project in the format and type specified
Ordered by the record (primary key of project) and then by event id
"""
ret_format = format
if format == 'df':
from pandas import read_csv
ret_format = 'csv'
pl = self.__basepl(content='report', format=ret_format)
keys_to_add = (report_id, raw_or_label, raw_or_label_headers,
export_checkbox_labels, decimal_character)
str_keys = ('report_id', 'rawOrLabel', 'rawOrLabelHeaders',
'exportCheckboxLabel', 'decimalCharacter')
for key, data in zip(str_keys, keys_to_add):
if data:
pl[key] = data
response, _ = self._call_api(pl, 'exp_report')
if format in ('json', 'csv', 'xml'):
return response
elif format == 'df':
if not df_kwargs:
if self.is_longitudinal():
df_kwargs = {
'index_col': [self.def_field, 'redcap_event_name']
}
else:
df_kwargs = {'index_col': self.def_field}
buf = StringIO(response)
df = self.read_csv(buf, **df_kwargs)
buf.close()
return df
def import_records(self,
to_import,
overwrite='normal',
format='json',
return_format='json',
return_content='count',
date_format='YMD',
force_auto_number=False):
"""
Import data into the RedCap Project
Parameters
----------
to_import : array of dicts, csv/xml string, ``pandas.DataFrame``
:note:
If you pass a csv or xml string, you should use the
``format`` parameter appropriately.
:note:
Keys of the dictionaries should be subset of project's,
fields, but this isn't a requirement. If you provide keys
that aren't defined fields, the returned response will
contain an ``'error'`` key.
overwrite : ('normal'), 'overwrite'
``'overwrite'`` will erase values previously stored in the
database if not specified in the to_import dictionaries.
format : ('json'), 'xml', 'csv'
Format of incoming data. By default, to_import will be json-encoded
return_format : ('json'), 'csv', 'xml'
Response format. By default, response will be json-decoded.
return_content : ('count'), 'ids', 'nothing'
By default, the response contains a 'count' key with the number of
records just imported. By specifying 'ids', a list of ids
imported will be returned. 'nothing' will only return
the HTTP status code and no message.
date_format : ('YMD'), 'DMY', 'MDY'
Describes the formatting of dates. By default, date strings
are formatted as 'YYYY-MM-DD' corresponding to 'YMD'. If date
strings are formatted as 'MM/DD/YYYY' set this parameter as
'MDY' and if formatted as 'DD/MM/YYYY' set as 'DMY'. No
other formattings are allowed.
force_auto_number : ('False') Enables automatic assignment of record IDs
of imported records by REDCap. If this is set to true, and auto-numbering
for records is enabled for the project, auto-numbering of imported records
will be enabled.
Returns
-------
response : dict, str
response from REDCap API, json-decoded if ``return_format`` == ``'json'``
"""
pl = self.__basepl('record')
if hasattr(to_import, 'to_csv'):
# We'll assume it's a df
buf = StringIO()
if self.is_longitudinal():
csv_kwargs = {
'index_label': [self.def_field, 'redcap_event_name']
}
else:
csv_kwargs = {'index_label': self.def_field}
to_import.to_csv(buf, **csv_kwargs)
pl['data'] = buf.getvalue()
buf.close()
format = 'csv'
elif format == 'json':
pl['data'] = json.dumps(to_import, separators=(',', ':'))
else:
# don't do anything to csv/xml
pl['data'] = to_import
pl['overwriteBehavior'] = overwrite
pl['format'] = format
pl['returnFormat'] = return_format
pl['returnContent'] = return_content
pl['dateFormat'] = date_format
pl['forceAutoNumber'] = force_auto_number
response = self._call_api(pl, 'imp_record')[0]
if 'error' in response:
raise RedcapError(str(response))
return response
def export_records(self,
records=None,
fields=None,
forms=None,
events=None,
raw_or_label='raw',
event_name='label',
format='json',
export_survey_fields=False,
export_data_access_groups=False,
df_kwargs=None,
export_checkbox_labels=False,
filter_logic=None,
date_begin=None,
date_end=None):
"""
Export data from the REDCap project.
Parameters
----------
records : list
array of record names specifying specific records to export.
by default, all records are exported
fields : list
array of field names specifying specific fields to pull
by default, all fields are exported
forms : list
array of form names to export. If in the web UI, the form
name has a space in it, replace the space with an underscore
by default, all forms are exported
events : list
an array of unique event names from which to export records
:note: this only applies to longitudinal projects
raw_or_label : (``'raw'``), ``'label'``, ``'both'``
export the raw coded values or labels for the options of
multiple choice fields, or both
event_name : (``'label'``), ``'unique'``
export the unique event name or the event label
format : (``'json'``), ``'csv'``, ``'xml'``, ``'df'``
Format of returned data. ``'json'`` returns json-decoded
objects while ``'csv'`` and ``'xml'`` return other formats.
``'df'`` will attempt to return a ``pandas.DataFrame``.
export_survey_fields : (``False``), True
specifies whether or not to export the survey identifier
field (e.g., "redcap_survey_identifier") or survey timestamp
fields (e.g., form_name+"_timestamp") when surveys are
utilized in the project.
export_data_access_groups : (``False``), ``True``
specifies whether or not to export the
``"redcap_data_access_group"`` field when data access groups
are utilized in the project.
:note: This flag is only viable if the user whose token is
being used to make the API request is *not* in a data
access group. If the user is in a group, then this flag
will revert to its default value.
df_kwargs : dict
Passed to ``pandas.read_csv`` to control construction of
returned DataFrame.
by default, ``{'index_col': self.def_field}``
export_checkbox_labels : (``False``), ``True``
specify whether to export checkbox values as their label on
export.
filter_logic : string
specify the filterLogic to be sent to the API.
date_begin : datetime
for the dateRangeStart filtering of the API
date_end : datetime
for the dateRangeEnd filtering snet to the API
Returns
-------
data : list, str, ``pandas.DataFrame``
exported data
"""
ret_format = format
if format == 'df':
ret_format = 'csv'
pl = self.__basepl('record', format=ret_format)
fields = self.backfill_fields(fields, forms)
keys_to_add = (records, fields, forms, events, raw_or_label,
event_name, export_survey_fields,
export_data_access_groups, export_checkbox_labels)
str_keys = ('records', 'fields', 'forms', 'events', 'rawOrLabel',
'eventName', 'exportSurveyFields',
'exportDataAccessGroups', 'exportCheckboxLabel')
for key, data in zip(str_keys, keys_to_add):
if data:
if key in ('fields', 'records', 'forms', 'events'):
for i, value in enumerate(data):
pl["{}[{}]".format(key, i)] = value
else:
pl[key] = data
if date_begin:
pl["dateRangeBegin"] = date_begin.strftime('%Y-%m-%d %H:%M:%S')
if date_end:
pl["dateRangeEnd"] = date_end.strftime('%Y-%m-%d %H:%M:%S')
if filter_logic:
pl["filterLogic"] = filter_logic
response, _ = self._call_api(pl, 'exp_record')
if format in ('json', 'csv', 'xml'):
return response
elif format == 'df':
if not df_kwargs:
if self.is_longitudinal():
df_kwargs = {
'index_col': [self.def_field, 'redcap_event_name']
}
else:
df_kwargs = {'index_col': self.def_field}
buf = StringIO(response)
df = self.read_csv(buf, **df_kwargs)
buf.close()
return df
def test2(self):
"""Test: configuration file and some other files."""
os.mkdir("t2")
handle = open("t2/rose-t.conf", "wb")
handle.write(r"""title=all day breakfast
[sausage]
number=3
type=pork and apple
[toast]
butter=yes
number=2
type=brown
[tomato]
number=1
type=grilled
""")
handle.close()
os.mkdir("t2/bin")
handle = open("t2/bin/make-breakfast", "wb")
handle.write(r"""#!/bin/sh
echo "Making breakfast $@"
""")
handle.close()
os.chmod("t2/bin/make-breakfast", 0755)
os.mkdir("t2/etc")
for key, val in (("sausage", "10 fat sausages"),
("bread", "slice bread"), ("tomato", "a red tomato")):
handle = open(os.path.join("t2/etc", key), "wb")
handle.write(val + "\n")
handle.close()
conf_tree = self.config_tree_loader("t2", "rose-t.conf")
string_io = StringIO()
self.config_dumper(conf_tree.node, string_io)
self.test(
"t2.node", string_io.getvalue(), r"""title=all day breakfast
[sausage]
number=3
type=pork and apple
[toast]
butter=yes
number=2
type=brown
[tomato]
number=1
type=grilled
""")
string_io.close()
conf_dir = os.path.join(os.getcwd(), "t2")
self.test(
"t2.files", conf_tree.files, {
"bin/make-breakfast": conf_dir,
"etc/sausage": conf_dir,
"etc/bread": conf_dir,
"etc/tomato": conf_dir
})
self.test("t2.conf_dirs", conf_tree.conf_dirs, [conf_dir])
class BrighticsPythonRunner(object):
NO_EXCEPTION = (False, "")
def __init__(self, use_spark=False):
self._stdout = StringIO()
self._is_exception = self.NO_EXCEPTION
self._init_executer()
import brightics.brightics_data_api as data_api
import brightics.common.data.utils as data_util
from brightics.common.utils import check_required_parameters
self._globals = {
'make_data_path_from_key': data_util.make_data_path_from_key,
'get_data_info': data_api.get_data_info,
'get_data_status': data_api.get_data_status,
'get_data': data_api.get_data,
'list_status': data_api.list_status,
'view_data': data_api.view_data,
'view_schema': data_api.view_schema,
'write_data': data_api.write_data,
'delete_data': data_api.delete_data,
'put_data': data_api.put_data,
'read_parquet': data_api.read_parquet,
'read_redis': data_api.read_redis,
'check_required_parameters': check_required_parameters
}
if use_spark:
sc, sparkSession, sqlContext = brtc_java_gateway.init_spark_context(
)
self._globals['sc'] = sc
self._globals['spark'] = sparkSession
self._globals['sqlContext'] = sqlContext
signal.signal(signal.SIGINT, self._interrupt_handler)
def __del__(self):
self._reset_stdout()
self._stdout.close()
self._exec.close()
def run(self, code):
try:
self._switch_stdout()
return self._exec.send(code), self._is_exception
except StopIteration:
self._init_executer()
return self._exec.send(code), self._is_exception
finally:
self._reset_stdout()
def _init_executer(self):
self._exec = self._executer()
next(self._exec)
def _executer(self):
while True:
code = (yield self._stdout.getvalue())
self._is_exception = self.NO_EXCEPTION
try:
code_tree = ast.parse(code)
exec_code = code_tree.body[:-1]
single_code = code_tree.body[-1:] # last node of code
exec_code_object = compile(ast.Module(exec_code), '<string>',
'exec')
interactive_code_object = compile(ast.Interactive(single_code),
'<string>', 'single')
with redirect_stderr():
exec(exec_code_object, self._globals)
exec(interactive_code_object, self._globals)
except BrighticsCoreException as bce:
raise bce
except Exception as e:
self._stdout.write(traceback.format_exc())
self._is_exception = (True,
traceback.format_exception_only(
type(e), e)[-1])
def _switch_stdout(self):
self._stdout.truncate(0)
self._stdout.seek(0)
self.old_stdout = sys.stdout
sys.stdout = self._stdout
def _reset_stdout(self):
try:
sys.stdout = self.old_stdout
except:
sys.stdout = sys.__stdout__
def _interrupt_handler(self, signum, frame):
"""
signum: the signal number
frame: current stack frame (None or a frame object)
"""
raise Exception("User Interrupt")
def analyze(self):
pb = self.pb
page = pb.get_page_source()
# preconditions
assert page is not None and len(page) > 0, ""
self.page_compressed = zlib.compress(page.encode('utf-8'))
if not self.quietly:
sys.stdout.write("Page data compressed: self.page_compressed..\n")
sys.stdout.flush()
soup = BS(page,'html.parser')
# precondition
assert soup != None, "No Page!"
#main page
if not self.quietly:
sys.stdout.write("..Main..")
sys.stdout.flush()
#analyze main page ===========
# reward
frame = soup.select(".NS-projects-reward")
proj_reward_append = self.projects_reward_result.append
if len(frame) > 0:
#
for card in frame:
#money
money_f = card.select(".money")
if len(money_f) > 0:
money = filter_number(money_f[0].text.strip())
else:
money = 0.0
#backers
backers_f = card.select(".num-backers")
if len(backers_f) > 0:
num_backers = filter_number(backers_f[0].text.strip())
else:
num_backers = 0.0
#description
desc_f = card.select(".desc")
if len(desc_f) > 0:
description = desc_f[0].text.strip()
else:
description = ""
#delivery
delivery_f = card.select("time")
if len(delivery_f) > 0:
delivery_estimated = delivery_f[0]['datetime']
else:
delivery_estimated = ""
#limited
limited_f = card.select(".limited-number")
if len(limited_f) > 0:
limited_num = int(re.findall(r"of ([0-9]+)",limited_f[0].text)[0])
else:
limited_num = 0
proj_reward_append([
money,
num_backers,
description,
delivery_estimated,
limited_num,
])
#for
# collect images
frame = soup.select("img.fit")
image_fnames_append = self.image_fnames.append
images_append = self.images.append
if len(frame) > 0:
for imgf in frame:
src = imgf['src']
src = re.sub(r"\?.*$","",src)
image_fnames_append(src)
images_append("") #basically nothing is apppended
if self.has_image:
inque = list_to_queue(self.image_fnames)
outque = Queue.Queue()
tasks = []
# parallel processing
for i in range(inque.qsize()):
imageD = ImageDownloader(inque,outque,self.quietly,True)
imageD.setDaemon(True)
tasks.append(imageD)
imageD.start()
inque.join() #wait till being finished
for task in tasks:
task.stop()
outlist = queue_to_list(outque)
self.images = copy.deepcopy(outlist)
# replace string list to binary data list
# video (source file name, length)s
frame = soup.select("video.has_webm")
self.video_fname = "na"
v_url = ""
if len(frame) > 0:
sources = frame[0].select("source")
if len(sources) > 0:
self.video_has = True
for source in sources:
v_url = source['src']
if v_url.endswith(".mp4"):
if v_url.find('high') > 0:
self.video_has_high = 1
self.video_fname = v_url
else:
self.video_has_base = 1
self.video_fname = v_url # if base exists, replace by it.
if self.video_has_high > 0 or self.video_has_base > 0:
url = self.video_fname
# video duration
r = requests.get(url,stream = True)
a = r.raw.read(2000) #2kb buffer
b = StringIO()
b.write(a)
c = mp4.File(b)
self.video_length = c.duration
b.close()
r.close()
# collect full description
frame = soup.select(".full-description")
rv = ""
if len(frame) > 0:
desc = frame[0].text
rv = re.sub(r"\n\n\n+","\n",desc)
try:
rv = rv.strip()
except:
pass
self.full_description = rv
# collect risk
frame = soup.select("#risks")
rv = ""
if len(frame) > 0:
desc = frame[0].text
rv = re.sub(r"\n\n\n+","\n",desc)
try:
rv = rv.strip()
except:
pass
self.risks = rv
# Facebook
frame = soup.select("li.facebook.mr2 .count")
waiting = 1
while 1:
if len(frame) > 0:
try:
facebook_count = int(frame[0].text) #error prone
break
except:
if not self.quietly:
sys.stdout.write("[facebook waiting...%d]\n"%waiting)
sys.stdout.flush()
time.sleep(waiting)
waiting += 1
temp_soup_facebook = BS(pb.get_page_source(),'html.parser')
frame = temp_soup_facebook.select("li.facebook.mr2 .count")
else:
self.facebook_like = 0
break
if waiting >= 10:
if not self.quietly:
sys.stdout.write(" [facebook error] ")
sys.stdout.flush()
self.facebook_like = -1 #means, error
break
# ============================
if not self.quietly:
sys.stdout.write("OK\n")
sys.stdout.flush()
#backers =====================
#btn = pb.css_selector_element("#backers_nav")
soup = None
if not self.quietly:
sys.stdout.write("..Visiting backers data..")
sys.stdout.flush()
try:
headers = { 'User-Agent' : 'Mozilla/5.0 (X11; U; Linux i686) Gecko/20071127 Firefox/2.0.0.11',
'connection' : 'close',
'charset' : 'utf-8'}
params = {'format':'json'}
con = requests.get(self.url,headers=headers,params=params)
j = con.json()
board = j['running_board']
s = BS(board,'html.parser')
eles = s.select('a#backers_nav data')
if len(eles) > 0:
t = eles[0].text.replace(",","")
n = int(t)
pn = n/40 + 2
else:
n = 0
pn = 0
self.pagination = pn
backer_url = self.url+"/backers"
pb.goto(backer_url)
page = 1
# measure current backers
p = pb.get_page_source()
s = BS(p,'html.parser')
frame = s.select("div.NS_backers__backing_row .meta a")
prevc = len(frame)
# init
nowc = 0
# pagination
while 1:
pb.scroll_down()
# measure current backers
p = pb.get_page_source()
s = BS(p,'html.parser')
frame = s.select("div.NS_backers__backing_row .meta a")
nowc = len(frame)
if not self.quietly:
sys.stdout.write("b")
sys.stdout.flush()
if nowc > prevc:
prevc = nowc
else:
break
self.get_backers(pb)
#get backers
p = pb.get_page_source()
s = BS(p,'html.parser')
frame = s.select("div.NS_backers__backing_row .meta a")
backers_append = self.backers.append
if len(frame) > 0:
for backer in frame:
profile_url = "%s"%(backer['href'])
backer_name = backer.text
backers_append((profile_url,backer_name,))
def write_summary_report(coverage_module, project, modules):
summary = StringIO()
coverage_module.report(modules, file=summary)
project.write_report("coverage", summary.getvalue())
summary.close()
print('There was an error when trying the input: input' + str(_i) + '.txt')
print('Your code outputted: ', codeOut.getvalue())
print('Your code gave an error: ', codeErr.getvalue())
print('This is the system error: ', sys.exc_info())
exit()
_code_out = codeOut.getvalue()
if _code_out != _out:
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
sys.stdin = sys.__stdin__
print('Wrong Answer!')
print('For the input: input' + str(_i) + ".txt" )
print('Your code gave output:"' + _code_out + '"')
print('But the desired output was:"' + _out + '"')
exit()
else:
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
sys.stdin = sys.__stdin__
print(_i, 'passed.')
codeOut.close()
codeErr.close()
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
sys.stdin = sys.__stdin__
print('Accepted')
print('Your code passed all 299 tests')
def show_error():
f = StringIO()
traceback.print_exc(file=f)
log.error(f.getvalue())
f.close()
class Cmd(object):
def __init__(self, cmd, chdir='', indir='', stdin=None, stdout=None, stderr=None, exitcode=0, reference=None):
self.cmd = cmd
self.chdir = chdir
self.indir = indir
self.reference = reference
# set and rewind stdin
self.stdin = stdin
if stdin:
self.stdin = StringIO()
for f in stdin:
self.stdin.write(f + '\n')
self.stdin.seek(0)
self.stdout = stdout
if stdout:
if chdir:
self.stdout = []
for o in stdout:
self.stdout.append(os.path.join(chdir, o))
elif indir:
self.stdout = []
for o in stdout:
self.stdout.append(os.path.join(indir, o))
self.stderr = stderr
self.exitcode = exitcode
def __str__(self):
return self.cmd
def argv(self):
args = []
if self.chdir:
args.extend(['-d', self.chdir])
args.extend(self.cmd.split())
return args
def cmdline(self):
args = ['x2y']
args.extend(self.argv())
return args
def run(self):
rval = Cmd(self.cmd)
argv = self.argv()
stdout = StringIO()
stderr = StringIO()
rval.exitcode = x2y.x2y.main(argv, stdin=self.stdin, stdout=stdout, stderr=stderr)
stdout_value = stdout.getvalue()
stderr_value = stderr.getvalue()
if stdout_value:
rval.stdout = stdout_value.strip().split('\n')
if stderr_value:
rval.stderr = stderr_value.strip().split('\n')
stdout.close()
stderr.close()
if self.stdin:
self.stdin.close()
return rval
def run_as_process(self):
rval = Cmd(self.cmd)
try:
cmd = self.cmdline()
os.environ['COVERAGE_PROCESS_START'] = '1'
env = os.environ.copy()
env['COVERAGE_FILE'] = '.coverage.%s' % (self.cmd.replace('/', '-').replace(' ', '-'))
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
except Exception as e:
pytest.fail(msg='Cmd: run: exception running %s: %s' % (cmd, e))
else:
stdout, stderr = p.communicate()
if stdout:
rval.stdout = stdout.decode().strip().split('\n')
if stderr:
rval.stderr = stderr.decode().strip().split('\n')
rval.exitcode = p.wait()
return rval
def outfile(self):
argv = self.argv()
i = 0
while i < len(argv):
if argv[i] == '-o':
return argv[i + 1]
return None
def files_match(self):
outfile = self.outfile()
with open(outfile, 'rb') as fp:
out_bytes = bytearray(fp.read())
with open(self.reference, 'rb') as fp:
ref_bytes = bytearray(fp.read())
if out_bytes == ref_bytes:
return True
pyf.fail(msg='Files differ: %s %s' % (self.reference, outfile))
return False
def _choices_as_array(self):
from StringIO import StringIO
valuebuffer = StringIO(self.list_values)
choices = [[item.strip(), item.strip()] for item in valuebuffer.readlines()]
valuebuffer.close()
return choices
def open_json_file():
sio = StringIO()
sio.write(json.dumps(TEST_DATA))
sio.seek(0)
yield sio
sio.close()
def handle(self, *args, **options):
month = options['month']
year = options['year']
###################
### NEW SERVERS ###
###################
if month == 1:
inidate = date(year - 1, 12, 1)
else:
inidate = date(year, month - 1, 1)
# Sites that haven't been canceled (end_date is null) and do not have a billing (PO) associated
if Site.objects.filter(start_date__month=inidate.month,
start_date__year=inidate.year,
end_date__isnull=True,
billing__isnull=True).exists():
LOGGER.error(
"Sites not cancelled were found without billing after a month")
# Billings of sites that haven't been canceled (end_date is null) and haven't been sent to finance yet
new_sites_billing = Billing.objects.filter(
date_sent_to_finance__isnull=True, site__end_date__isnull=True)
################
### RENEWALS ###
################
# Billings of sites that haven't been canceled (end_date is null), that hasn't expressed to want to cancel
# their subscription, and that started in the actual month of a previous year
renewal_sites_billing = Billing.objects.filter(
site__start_date__month=month,
site__start_date__lt=date(year, 1, 1),
site__end_date__isnull=True)
if not (new_sites_billing.exists() or renewal_sites_billing.exists()):
return # Nothing to send
###################
### SEND REPORT ###
###################
po_files = map(
lambda x:
("%s%s" %
(slugify(x.purchase_order_number), splitext(x.purchase_order.name)
[1]), x.purchase_order.read(), 'application/other'),
new_sites_billing | renewal_sites_billing)
new_billing = map(
lambda x: [
x.site.id, x.site.name, x.group, x.purchase_order_number, x.
site.start_date, x.site.type.price, x.site.start_date,
calcendperiod(x.site.start_date)
], new_sites_billing)
renewals_billing = map(
lambda x: [
x.site.id, x.site.name, x.group, x.purchase_order_number, x.
site.start_date, x.site.type.price,
x.site.start_date.replace(year=year),
calcendperiod(x.site.start_date.replace(year=year)), x.
date_modified > (date.today() - timedelta(days=100))
], renewal_sites_billing)
header = [
'id', 'Name', 'PO raised by', 'PO number', 'Created at', 'Cost',
'Period start', 'Period end'
]
new_billing = [header] + new_billing
renewals_billing = [header + ['Have they uploaded a new PO?']
] + renewals_billing
stream_new = StringIO()
stream_renewal = StringIO()
writer_new = csv.writer(stream_new)
writer_renewal = csv.writer(stream_renewal)
for billing in new_billing:
writer_new.writerow(billing)
for billing in renewals_billing:
writer_renewal.writerow(billing)
EmailMessage(
subject="Monthly Financial Report MWS3 - %s %i" %
(month_name[month], year),
body=
"Hello,\n\nAttached you can find the monthly report spreadsheet for new servers and for renewals. "
"You will also find all the corresponding purchase orders. The cost codes for MWS3 are:\n\n"
"Cost centre for Managed Web Service = VCBQ\nTransaction code for Managed Web Service = LRED\n"
"Internal code = VCBQ GAAB LRED\nExternal code = VCBQ GAAA LRED\n\nBest regards,\n\nMWS3 Team.\n",
from_email="Managed Web Service Support <%s>" % getattr(
settings, 'EMAIL_MWS3_SUPPORT', '*****@*****.**'),
to=[settings.FINANCE_EMAIL],
bcc=[settings.EMAIL_MWS3_SUPPORT],
headers={
'Return-Path':
getattr(settings, 'EMAIL_MWS3_SUPPORT',
'*****@*****.**')
},
attachments=[('mws3sites_new.csv', stream_new.getvalue(),
'application/vnd.ms-excel'),
('mws3sites_renewals.csv', stream_renewal.getvalue(),
'application/vnd.ms-excel')] + po_files).send()
new_sites_billing.update(date_sent_to_finance=timezone.now().date())
renewal_sites_billing.update(
date_sent_to_finance=timezone.now().date())
stream_new.close()
stream_renewal.close()
class Form:
""" Form class to use reportlab to generate pdf """
def __init__(self, pdfname="ocrform.pdf", margintop=50, marginsides=50, **kw):
""" Form initialization """
self.pdfpath = kw.get("pdfpath", pdfname)
self.verbose = kw.get("verbose", 0)
self.font = kw.get("typeface", Courier)
self.fontsize = kw.get("fontsize", 13)
self.IObuffer = StringIO()
self.canvas = Canvas(self.IObuffer, pagesize = A4)
self.width, self.height = A4
self.x = marginsides
self.lastx = marginsides
self.marginsides = marginsides
self.margintop = margintop
self.y = self.height - margintop
self.lasty = self.height - margintop
self.num = 1
def barcode(self, uuid):
""" Generate barcode of uuid """
barcode = code128.Code128(str(uuid), barWidth=1, barHeight=20)
barcode.drawOn(self.canvas, self.lastx, self.lasty)
self.lasty = self.lasty - 20
self.y = self.lasty
def decorate(self):
""" Decorates the the form with the markers needed to align the form later """
c = self.canvas
c.rect(20, 20, 20, 20, fill=1)
c.rect(self.width - 40, 20, 20, 20, fill=1)
c.rect(20, self.height - 40, 20, 20, fill=1)
c.rect(self.width/2 - 10, 20, 20, 20, fill=1)
c.rect(20, self.height/2 - 10, 20, 20, fill=1)
c.rect(self.width - 40, self.height - 40, 20, 20, fill=1)
c.rect(self.width - 40, self.height/2 - 10, 20, 20, fill=1)
def print_text(self, lines, fontsize=12, gray=0, seek=0, continuetext=0, style="default"):
""" Give the lines to be printed as a list, set the font and grey level """
c = self.canvas
self.fontsize = fontsize
if style == "center":
self.x = self.width / 2
if seek > (self.width-(self.marginsides + self.fontsize)):
seek = 0
if seek != 0:
self.x = self.x + seek
if continuetext == 1:
self.x = self.lastx + seek
if seek == 0:
self.y = self.y + fontsize
for line in lines:
if style == "center":
self.x = self.x - (len(line)) * self.fontsize / 2
if style == "right":
self.x = self.width - (self.marginsides + len(line) * self.fontsize)
if (self.width - self.marginsides - self.lastx) < 200:
self.x = self.marginsides
if continuetext == 1:
self.y = self.y - 2 * fontsize
if (self.y - self.fontsize) < 50:
self.set_new_page()
t = c.beginText(self.x, self.y)
t.setFont(Helvetica, fontsize)
t.setFillGray(gray)
t.textOut(line)
c.drawText(t)
self.y = self.y - fontsize
self.lastx = t.getX()
self.lasty = self.y
self.x = self.marginsides
def draw_check_boxes(self, boxes=1, completeline=0, lines=0, seek=0, continuetext=0, fontsize=0, gray=0, style="", isdate=0):
""" Function to draw check boxes default no of boxes = 1 """
c = self.canvas
c.setLineWidth(0.90)
c.setStrokeGray(gray)
if style == "center":
self.x = self.width / 2
elif style == "right":
self.x = self.width - self.marginsides - self.fontsize
if seek > (self.width - (self.marginsides + self.fontsize)):
seek = 0
if (self.y - self.fontsize) < 40:
self.set_new_page()
if continuetext == 1:
self.y = self.y + self.fontsize
self.x = self.lastx
else:
self.x = self.marginsides
if seek != 0:
self.x = self.x + seek
if fontsize == 0:
fontsize = self.fontsize
else:
self.fontsize = fontsize
if completeline == 1:
boxes = int(self.width / self.fontsize)
for i in range(boxes):
c.rect(self.x, self.y, self.fontsize, self.fontsize)
self.x = self.x + self.fontsize
if self.x > (self.width - (self.marginsides + self.fontsize)):
break
self.lastx = self.x
self.x = self.marginsides
self.y = self.y - self.fontsize
if isdate:
t = c.beginText(self.x, self.y)
t.setFont(Helvetica, 13)
t.setFillGray(0)
t.textOut(" D D M M Y Y Y Y")
c.drawText(t)
self.y = self.y - fontsize
self.lastx = t.getX()
self.lasty = self.y
self.lastx = self.x
self.x = self.marginsides
self.y = self.y - 13
def draw_circle(self, boxes=1, completeline=0, lines=0, seek=0, continuetext=0, fontsize=0, gray=0, style=""):
""" Draw circles on the form """
c = self.canvas
c.setLineWidth(0.90)
c.setStrokeGray(gray)
if style == "center":
self.x = self.width / 2
elif style == "right":
self.x = self.width - self.marginsides - self.fontsize
if seek > (self.width - (self.marginsides + self.fontsize)):
seek = 0
if (self.y - self.fontsize) < 40:
self.set_new_page()
if continuetext == 1:
self.y = self.y + self.fontsize
self.x = self.lastx
else:
self.x = self.marginsides
if seek != 0:
self.x = self.x + seek
if fontsize == 0:
fontsize = self.fontsize
else:
self.fontsize = fontsize
if completeline == 1:
boxes = int(self.width / self.fontsize)
for i in range(boxes):
c.circle(self.x + self.fontsize/2, self.y+self.fontsize/2, self.fontsize/2, fill = 0)
self.x = self.x + self.fontsize
if self.x > (self.width - (self.marginsides + self.fontsize)):
break
self.lastx = self.x
self.x = self.marginsides
self.y = self.y - self.fontsize
def draw_line(self, gray=0):
""" Function to draw a straight line """
c = self.canvas
c.setStrokeGray(gray)
c.setLineWidth(0.40)
self.y = self.y - (self.fontsize)
c.line(self.x, self.y, self.width - self.x, self.y)
self.y = self.y - (self.fontsize)
def set_new_page(self):
"""
All changes are forgotten when a showPage() has been executed.
They have to be set again.
"""
self.num += 1
c = self.canvas
c.showPage()
self.decorate()
self.x = self.marginsides
self.lastx = self.marginsides
self.y = self.height - self.margintop
self.print_text([str("Page "+ str(self.num))], fontsize=8, style="right")
self.x = self.marginsides
self.lastx = self.x
self.y = self.y - 32
def set_title(self, title = "FORM"):
""" Sets the title of the pdf. """
c = self.canvas.setTitle(title)
def save(self):
""" Saves the form """
self.canvas.save()
pdf = self.IObuffer.getvalue()
self.IObuffer.close()
return pdf
def generate(self, zetacount=2, polarizationcount=1,
tailnorm=(0.16, 0.3, 0.6), energysplit=0.1, tolerance=1.0e-3,
referencefile=None, referenceindex=None, rcutpol_rel=1.0,
rcutmax=20.0,
rcharpol_rel=None,
vconf_args=(12.0, 0.6), txt='-',
include_energy_derivatives=False,
#lvalues=None, # XXX clean up some of these!
jvalues=None,
l_pol=None):
"""Generate an entire basis set.
This is a high-level method which will return a basis set
consisting of several different basis vector types.
Parameters:
===================== =================================================
``zetacount`` Number of basis functions per occupied orbital
``polarizationcount`` Number of polarization functions
``tailnorm`` List of tail norms for split-valence scheme
``energysplit`` Energy increase defining confinement radius (eV)
``tolerance`` Tolerance of energy split (eV)
``referencefile`` gpw-file used to generate polarization function
``referenceindex`` Index in reference system of relevant atom
``rcutpol_rel`` Polarization rcut relative to largest other rcut
``rcutmax`` No cutoff will be greater than this value
``vconf_args`` Parameters (alpha, ri/rc) for conf. potential
``txt`` Log filename or '-' for stdout
===================== =================================================
Returns a fully initialized Basis object.
"""
if txt == '-':
txt = sys.stdout
elif txt is None:
txt = devnull
if isinstance(tailnorm, float):
tailnorm = (tailnorm,)
if 1 + len(tailnorm) < max(polarizationcount, zetacount):
raise ValueError(
'Needs %d tail norm values, but only %d are specified' %
(max(polarizationcount, zetacount) - 1, len(tailnorm)))
textbuffer = StringIO()
class TeeStream: # quick hack to both write and save output
def __init__(self, out1, out2):
self.out1 = out1
self.out2 = out2
def write(self, string):
self.out1.write(string)
self.out2.write(string)
txt = TeeStream(txt, textbuffer)
if vconf_args is not None:
amplitude, ri_rel = vconf_args
g = self.generator
rgd = self.rgd
njcore = g.njcore
n_j = g.n_j[njcore:]
l_j = g.l_j[njcore:]
f_j = g.f_j[njcore:]
if jvalues is None:
jvalues = []
sortkeys = []
for j in range(len(n_j)):
if f_j[j] == 0 and l_j[j] != 0:
continue
jvalues.append(j)
sortkeys.append(l_j[j])
# Now order jvalues by l
#
# Use a stable sort so the energy ordering within each
# angular momentum is guaranteed to be preserved
args = np.argsort(sortkeys, kind='mergesort')
jvalues = np.array(jvalues)[args]
fulljvalues = [njcore + j for j in jvalues]
if isinstance(energysplit, float):
energysplit = [energysplit] * len(jvalues)
title = '%s Basis functions for %s' % (g.xcname, g.symbol)
print(title, file=txt)
print('=' * len(title), file=txt)
singlezetas = []
energy_derivative_functions = []
multizetas = [[] for i in range(zetacount - 1)]
polarization_functions = []
splitvalencedescr = 'split-valence wave, fixed tail norm'
derivativedescr = 'derivative of sz wrt. (ri/rc) of potential'
for vj, fullj, esplit in zip(jvalues, fulljvalues, energysplit):
l = l_j[vj]
n = n_j[vj]
assert n > 0
orbitaltype = str(n) + 'spdf'[l]
msg = 'Basis functions for l=%d, n=%d' % (l, n)
print(file=txt)
print(msg + '\n', '-' * len(msg), file=txt)
print(file=txt)
if vconf_args is None:
adverb = 'sharply'
else:
adverb = 'softly'
print('Zeta 1: %s confined pseudo wave,' % adverb, end=' ', file=txt)
u, e, de, vconf, rc = self.rcut_by_energy(fullj, esplit,
tolerance,
vconf_args=vconf_args)
if rc > rcutmax:
rc = rcutmax # scale things down
if vconf is not None:
vconf = g.get_confinement_potential(amplitude, ri_rel * rc,
rc)
u, e = g.solve_confined(fullj, rc, vconf)
print('using maximum cutoff', file=txt)
print('rc=%.02f Bohr' % rc, file=txt)
else:
print('fixed energy shift', file=txt)
print('DE=%.03f eV :: rc=%.02f Bohr' % (de * Hartree,
rc), file=txt)
if vconf is not None:
print(('Potential amp=%.02f :: ri/rc=%.02f' %
(amplitude, ri_rel)), file=txt)
phit_g = self.smoothify(u, l)
bf = BasisFunction(l, rc, phit_g,
'%s-sz confined orbital' % orbitaltype)
norm = np.dot(g.dr, phit_g * phit_g)**.5
print('Norm=%.03f' % norm, file=txt)
singlezetas.append(bf)
zetacounter = iter(xrange(2, zetacount + 1))
if include_energy_derivatives:
assert zetacount > 1
zeta = zetacounter.next()
print('\nZeta %d: %s' % (zeta, derivativedescr), file=txt)
vconf2 = g.get_confinement_potential(amplitude,
ri_rel * rc * .99, rc)
u2, e2 = g.solve_confined(fullj, rc, vconf2)
phit2_g = self.smoothify(u2, l)
dphit_g = phit2_g - phit_g
dphit_norm = np.dot(rgd.dr_g, dphit_g * dphit_g) ** .5
dphit_g /= dphit_norm
descr = '%s-dz E-derivative of sz' % orbitaltype
bf = BasisFunction(l, rc, dphit_g, descr)
energy_derivative_functions.append(bf)
for i, zeta in enumerate(zetacounter):
print('\nZeta %d: %s' % (zeta, splitvalencedescr), file=txt)
# Unresolved issue: how does the lack of normalization
# of the first function impact the tail norm scheme?
# Presumably not much, since most interesting stuff happens
# close to the core.
rsplit, norm, splitwave = rsplit_by_norm(rgd, l, phit_g,
tailnorm[i]**2.0,
txt)
descr = '%s-%sz split-valence wave' % (orbitaltype,
'0sdtq56789'[zeta])
bf = BasisFunction(l, rsplit, phit_g - splitwave, descr)
multizetas[i].append(bf)
if polarizationcount > 0 or l_pol is not None:
if l_pol is None:
# Now make up some properties for the polarization orbital
# We just use the cutoffs from the previous one times a factor
# Find 'missing' values in lvalues
lvalues = [l_j[vj] for vj in jvalues]
for i in range(max(lvalues) + 1):
if list(lvalues).count(i) == 0:
l_pol = i
break
else:
l_pol = max(lvalues) + 1
# Find the last state with l=l_pol - 1, which will be the state we
# base the polarization function on
for vj, fullj, bf in zip(jvalues[::-1], fulljvalues[::-1],
singlezetas[::-1]):
if bf.l == l_pol - 1:
fullj_pol = fullj
rcut = bf.rc * rcutpol_rel
break
else:
raise ValueError('The requested value l_pol=%d requires l=%d '
'among valence states' % (l_pol, l_pol - 1))
rcut = min(rcut, rcutmax)
msg = 'Polarization function: l=%d, rc=%.02f' % (l_pol, rcut)
print('\n' + msg, file=txt)
print('-' * len(msg), file=txt)
# Make a single Gaussian for polarization function.
#
# It is known that for given l, the sz cutoff defined
# by some fixed energy is strongly correlated to the
# value of the characteristic radius which best reproduces
# the wave function found by interpolation.
#
# We know that for e.g. d orbitals:
# rchar ~= .37 rcut[sz](.3eV)
# Since we don't want to spend a lot of time finding
# these value for other energies, we just find the energy
# shift at .3 eV now
u, e, de, vconf, rc_fixed = self.rcut_by_energy(fullj_pol,
.3, 1e-2,
6., (12., .6))
default_rchar_rel = .25
# Defaults for each l. Actually we don't care right now
rchar_rels = {}
if rcharpol_rel is None:
rcharpol_rel = rchar_rels.get(l_pol, default_rchar_rel)
rchar = rcharpol_rel * rc_fixed
gaussian = QuasiGaussian(1.0 / rchar**2, rcut)
psi_pol = gaussian(rgd.r_g) * rgd.r_g**(l_pol + 1)
norm = np.dot(rgd.dr_g, psi_pol * psi_pol) ** .5
psi_pol /= norm
print('Single quasi Gaussian', file=txt)
msg = 'Rchar = %.03f*rcut = %.03f Bohr' % (rcharpol_rel, rchar)
adjective = 'Gaussian'
print(msg, file=txt)
type = '%s-type %s polarization' % ('spdfg'[l_pol], adjective)
bf_pol = BasisFunction(l_pol, rcut, psi_pol, type)
polarization_functions.append(bf_pol)
for i in range(polarizationcount - 1):
npol = i + 2
msg = '\n%s: %s' % (['Secondary', 'Tertiary', 'Quaternary',
'Quintary', 'Sextary', 'Septenary'][i],
splitvalencedescr)
print(msg, file=txt)
rsplit, norm, splitwave = rsplit_by_norm(rgd, l_pol, psi_pol,
tailnorm[i],
txt)
descr = ('%s-type split-valence polarization %d'
% ('spdfg'[l_pol], npol))
bf_pol = BasisFunction(l_pol, rsplit, psi_pol - splitwave,
descr)
polarization_functions.append(bf_pol)
bf_j = []
bf_j.extend(singlezetas)
bf_j.extend(energy_derivative_functions)
for multizeta_list in multizetas:
bf_j.extend(multizeta_list)
bf_j.extend(polarization_functions)
rcmax = max([bf.rc for bf in bf_j])
# The non-equidistant grids are really only suited for AE WFs
d = 1.0 / 64
equidistant_grid = np.arange(0.0, rcmax + d, d)
ng = len(equidistant_grid)
for bf in bf_j:
# We have been storing phit_g * r, but we just want phit_g
bf.phit_g = divrl(bf.phit_g, 1, rgd.r_g)
gcut = min(int(1 + bf.rc / d), ng - 1)
assert equidistant_grid[gcut] >= bf.rc
assert equidistant_grid[gcut - 1] <= bf.rc
bf.rc = equidistant_grid[gcut]
# Note: bf.rc *must* correspond to a grid point (spline issues)
bf.ng = gcut + 1
# XXX all this should be done while building the basis vectors,
# not here
# Quick hack to change to equidistant coordinates
spline = rgd.spline(bf.phit_g, rgd.r_g[rgd.floor(bf.rc)], bf.l,
points=100)
bf.phit_g = np.array([spline(r) * r**bf.l
for r in equidistant_grid[:bf.ng]])
bf.phit_g[-1] = 0.
basis = Basis(g.symbol, self.name, False)
basis.ng = ng
basis.d = d
basis.bf_j = bf_j
basis.generatordata = textbuffer.getvalue().strip()
basis.generatorattrs = {'version': version}
textbuffer.close()
return basis
def close(self):
buf = self.getvalue()
store, key = self._target
store[key] = buf
StringIO.close(self)
def test_mlp(n_nodes=[74484,100,100,100,4], # input-hidden-nodees
datasets='lhrhadvs_sample_data.mat', # load data
# activation: # sigmoid function: T.nnet.sigmoid, hyperbolic tangent function: T.tanh, Rectified Linear Unit: relu1
batch_size = 40, n_epochs = 300, learning_rate=0.001,activation = T.tanh,
beginAnneal= 50, min_annel_lrate = 1e-4, decay_rate = 0.0005, momentum_val=0.01,
# Select optimizer 'Grad' for GradientDescentOptimizer, 'Adam' for AdamOptimizer, 'Rmsp' for RMSPropOptimizer
optimizer_algorithm='Grad',
# Parameters for the node-wise control of weight sparsity
# if you have three hidden layer, the number of target Hoyer's sparseness should be same
tg_hspset=[0.7, 0.7, 0.5], # Target sparsity
max_beta=[0.05, 0.95, 0.7], # Maximum beta changes
# Parameters for the layer-wise control of weight sparsity
# tg_hspset=[0.7, 0.7, 0.5], # Target sparsity
# max_beta=[0.05, 0.95, 0.8], # Maximum beta changes
beta_lrates = 1e-2, L2_reg = 1e-4,
# flag_nodewise =1 is the node-wise control of weight sparsity
# flag_nodewise =0 is the layer-wise control of weight sparsity
flag_nodewise = 0,
# Save path
sav_path = '/Users/bspl/Downloads/dnnwsp-master/Theano_code', # a directory to save dnnwsp result
):
########################################## Input data #################################################
datasets=sio.loadmat(datasets) # load datasets
############# lhrhadvs_sample_data.mat #############
# train_x = 240 volumes x 74484 voxels
# train_x = 240 volumes x 1 [0:left-hand clenching task, 1:right-hand clenching task, 2:auditory task, 3:visual task]
# test_x = 120 volumes x 74484 voxels
# test_y = 120 volumes x 1 [0:left-hand clenching task, 1:right-hand clenching task, 2:auditory task, 3:visual task]
############################################################
train_x = datasets['train_x']; train_y = datasets['train_y'];
test_x = datasets['test_x']; test_y = datasets['test_y'];
train_set_x = theano.shared(numpy.asarray(train_x, dtype=theano.config.floatX))
train_set_y = T.cast(theano.shared(train_y.flatten(),borrow=True),'int32')
test_set_x = theano.shared(numpy.asarray(test_x, dtype=theano.config.floatX))
test_set_y = T.cast(theano.shared(test_y.flatten(),borrow=True),'int32')
# compute number of minibatches for training, validation and testing
n_train_batches = int(train_set_x.get_value(borrow=True).shape[0] / batch_size)
n_test_batches = int(test_set_x.get_value(borrow=True).shape[0] / batch_size)
########################################## Build model #################################################
print('... building the model')
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x')
y = T.ivector('y') # the labels are presented as 1D vector of [int] labels
l1_penalty_layer = T.fvector() # L1-norm regularization parameter
ln_rate = T.scalar(name='learning_rate') # learning rate
momentum = T.scalar(name='momentum')
rng = numpy.random.RandomState(1234)
# construct the MLP class
classifier = MLP(
rng=rng,
input=x,
n_nodes = n_nodes,
activation = activation,
)
# cost function
cost = (classifier.negative_log_likelihood(y))
# L1 regularization term for either node-wise or layer-wise control
if flag_nodewise==1:
for i in range(len(n_nodes)-2):
node_size = n_nodes[i+1]; tg_index = np.arange((i * node_size),((i + 1) * node_size));
cost += (T.dot(abs(classifier.hiddenLayer[i].W),l1_penalty_layer[tg_index])).sum();
else:
for i in range(len(n_nodes)-2):
cost += l1_penalty_layer[i] * classifier.L1[i]
# L2 regularization
cost += L2_reg * classifier.L2_sqr
# compiling a Theano function `train_model` that returns the cost, but
# in the same time updates the parameter of the model based on the rules
# defined in `updates`
updates =[];
# Select optimizer 'Grad' for GradientDescentOptimizer, 'Adam' for AdamOptimizer, 'Rmsp' for RMSPropOptimizer
if optimizer_algorithm=='Grad':
gparams = [T.grad(cost, param) for param in classifier.params]
for param, gparam, oldparam in zip(classifier.params, gparams, classifier.oldparams):
delta = ln_rate * gparam + momentum * oldparam
updates.append((param, param - delta))
updates.append((oldparam, delta))
elif optimizer_algorithm=='Adam':
updates = adam(cost, classifier.params, learning_rate)
elif optimizer_algorithm=='Rmsp' :
updates = RMSprop(cost, classifier.params, learning_rate)
train_model = theano.function(
inputs=[index, l1_penalty_layer,ln_rate,momentum],
outputs=[cost,classifier.errors(y),classifier.mse(batch_size,n_nodes[-1],y)],
updates=updates,
givens={
x: train_set_x[index * batch_size: (index + 1) * batch_size],
y: train_set_y[index * batch_size: (index + 1) * batch_size]
},
allow_input_downcast = True,
on_unused_input = 'ignore'
)
updates_test = []
for hiddenlayer in classifier.hiddenLayer:
for i in range(1):
updates_test.append( hiddenlayer.updates[i] )
test_model = theano.function(
inputs=[index],
outputs=[classifier.errors(y),classifier.mse(batch_size,n_nodes[-1],y)],
updates=updates_test,
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]
}
)
########################################## Learning model #################################################
print('... training')
test_score = 0.
start_time = timeit.default_timer()
epoch = 0; done_looping = False
# Define variables to save/check training model
train_errors = np.zeros(n_epochs); test_errors = np.zeros(n_epochs);
train_mse = np.zeros(n_epochs); test_mse = np.zeros(n_epochs);
lrs = np.zeros(n_epochs); lrate_list = np.zeros(n_epochs);
if flag_nodewise==1:
hsp_avg_vals =[]; L1_beta_avg_vals=[]; all_hsp_vals =[]; all_L1_beta_vals=[];
L1_beta_vals = np.zeros(np.sum(n_nodes[1:(len(n_nodes)-1)]));
cnt_hsp_val = np.zeros(len(n_nodes)-2);
cnt_beta_val = np.zeros(len(n_nodes)-2);
for i in range(len(n_nodes)-2):
hsp_avg_vals.append(np.zeros((n_epochs,n_nodes[i+1])));
L1_beta_avg_vals.append(np.zeros((n_epochs,n_nodes[i+1])));
all_hsp_vals.append(np.zeros((n_epochs,n_nodes[i+1])));
all_L1_beta_vals.append(np.zeros((n_epochs,n_nodes[i+1])));
else:
all_hsp_vals = np.zeros((n_epochs,len(n_nodes)-2));
all_L1_beta_vals = np.zeros((n_epochs,len(n_nodes)-2));
L1_beta_vals= np.zeros(len(n_nodes)-2)
cnt_hsp_val = np.zeros(len(n_nodes)-2);
###################
# start training
###################
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
minibatch_all_avg_error = []; minibatch_all_avg_mse = []
# minibatch based training
for minibatch_index in range(n_train_batches):
disply_text = StringIO();
minibatch_avg_cost, minibatch_avg_error, minibatch_avg_mse = train_model(minibatch_index, L1_beta_vals,learning_rate,momentum_val)
minibatch_all_avg_error.append(minibatch_avg_error)
minibatch_all_avg_mse.append(minibatch_avg_mse)
# Node-wise or layer-wise control of weight sparsity
if flag_nodewise==1:
for i in range(len(n_nodes)-2):
node_size = n_nodes[i+1]; tg_index = np.arange((i * node_size),((i + 1) * node_size));
[all_hsp_vals[i][epoch-1], L1_beta_vals[tg_index]] = hsp_fnc(L1_beta_vals[tg_index],classifier.hiddenLayer[i].W,max_beta[i],tg_hspset[i],beta_lrates,flag_nodewise);
all_L1_beta_vals[i][epoch-1]= L1_beta_vals[tg_index];
else:
for i in range(len(n_nodes)-2):
[cnt_hsp_val[i], L1_beta_vals[i]] = hsp_fnc(L1_beta_vals[i],classifier.hiddenLayer[i].W,max_beta[i],tg_hspset[i],beta_lrates,flag_nodewise);
# iteration number
iter = (epoch - 1) * n_train_batches + minibatch_index
# test it on the test set
test_losses = []; test_mses = []
for i in range(n_test_batches):
test_losses.append(test_model(i)[0])
test_mses.append(test_model(i)[1])
test_score = numpy.mean(test_losses);
# Begin Annealing
if beginAnneal == 0:
learning_rate = learning_rate * 1.0
elif epoch > beginAnneal:
learning_rate = max(min_annel_lrate, (-decay_rate*epoch + (1+decay_rate*beginAnneal)) * learning_rate )
# Save variables to check training
train_errors[epoch-1] = np.mean(minibatch_all_avg_error)*100
test_errors[epoch-1] = test_score*100
train_mse[epoch-1] = np.mean(minibatch_all_avg_mse)
test_mse[epoch-1] = np.mean(test_mses)
# Node-wise or layer-wise control of weight sparsity to display the current state of training
if flag_nodewise ==1:
disply_text.write("Node-wise control, epoch %i/%d, Tr.err= %.2f, Ts.err= %.2f, lr = %.6f, " % (epoch,n_epochs,train_errors[epoch-1],test_errors[epoch-1],learning_rate))
for layer_idx in range(len(n_nodes)-2):
cnt_hsp_val[layer_idx] = np.mean(all_hsp_vals[layer_idx][epoch-1])
cnt_beta_val[layer_idx] = np.mean(all_L1_beta_vals[layer_idx][epoch-1])
else:
disply_text.write("Layer-wise control, epoch %i/%d, Tr.err= %.2f, Ts.err= %.2f, lr = %.6f, " % (epoch,n_epochs,train_errors[epoch-1],test_errors[epoch-1],learning_rate))
all_hsp_vals[epoch-1,:] = cnt_hsp_val;
all_L1_beta_vals[epoch-1,:] = L1_beta_vals;
cnt_beta_val = L1_beta_vals;
for layer_idx in range(len(n_nodes)-2):
if (layer_idx==len(n_nodes)-3):
disply_text.write("hsp_l%d = %.2f/%.2f, beta_l%d = %.2f" % (layer_idx+1,cnt_hsp_val[layer_idx],tg_hspset[layer_idx],layer_idx+1,cnt_beta_val[layer_idx]))
else:
disply_text.write("hsp_l%d = %.2f/%.2f, beta_l%d = %.2f, " % (layer_idx+1,cnt_hsp_val[layer_idx],tg_hspset[layer_idx],layer_idx+1,cnt_beta_val[layer_idx]))
# Display variables
print(disply_text.getvalue())
disply_text.close()
lrs[epoch-1] = learning_rate
########################################## Save variables #################################################
# make a new directory to save data
if not os.path.exists(sav_path):
os.makedirs(sav_path)
end_time = timeit.default_timer()
cst_time = (end_time - start_time) / 60.
print(sys.stderr, ('\n The code for file ' + os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.)))
sav_text = StringIO();
for layer_idx in range(len(n_nodes)-2):
if layer_idx==len(n_nodes)-3:
sav_text.write("%d" % (n_nodes[layer_idx+1]))
else:
sav_text.write("%d-" % (n_nodes[layer_idx+1]))
if flag_nodewise==1:
sav_name = '%s/mlp_rst_node_%s.mat' % (sav_path,sav_text.getvalue())
else:
sav_name = '%s/mlp_rst_layer_%s.mat' % (sav_path,sav_text.getvalue())
sav_text.close()
data_variable = {};
for i in range(len(n_nodes)-1):
if (i==len(n_nodes)-2):
W_name = "w%d" %(i+1); b_name = "b%d" % (i+1);
data_variable[W_name] = classifier.logRegressionLayer.W.get_value(borrow=True)
data_variable[b_name] = classifier.logRegressionLayer.b.get_value(borrow=True)
else:
W_name = "w%d" %(i+1); b_name = "b%d" % (i+1)
data_variable[W_name] = classifier.hiddenLayer[i].W.get_value(borrow=True)
data_variable[b_name] = classifier.hiddenLayer[i].b.get_value(borrow=True)
data_variable['hsp_vals'] = all_hsp_vals;
data_variable['L1_vals'] = all_L1_beta_vals;
data_variable['train_errors'] = train_errors; data_variable['test_errors'] = test_errors;
data_variable['l_rate'] = lrs;
data_variable['momtentum'] = momentum_val; data_variable['beginAnneal'] = beginAnneal; data_variable['decay_lr'] = decay_rate;
data_variable['beta_lrates'] = beta_lrates; data_variable['max_beta'] = max_beta; data_variable['tg_hspset'] = tg_hspset;
data_variable['batch_size'] = batch_size; data_variable['n_epochs'] = n_epochs; data_variable['min_annel_lrate'] = min_annel_lrate;
data_variable['n_nodes'] = n_nodes; data_variable['lrate_list'] = lrate_list;
sio.savemat(sav_name,data_variable)
print('...done!')
session = SessionManager.Session()
username = args.user_promote[0]
level = int(args.user_promote[1])
user = session.query(User).filter(User.name == username).one()
user.level = level
session.commit()
print('Update successfully')
SessionManager.Session.remove()
elif args.db_init:
Base.metadata.create_all(SessionManager.engine)
fp = StringIO()
alembic_config = Config('./alembic.ini', stdout=fp)
command.heads(alembic_config)
content = fp.getvalue()
fp.close()
revision_hash = re.search('^([0-9a-f]+)\s\(head\)', content, re.U).group(1)
print('set current revision {0}'.format(revision_hash))
new_alembic_config = Config('./alembic.ini')
command.stamp(new_alembic_config, revision=revision_hash)
print('Database initialized')
elif args.cover:
fr = open('./config/config.yml', 'r')
config = yaml.load(fr)
download_location = config['download']['location']
session = SessionManager.Session()
cur = session.query(Bangumi)
resp_cookies = None
file_downloader = FileDownloader()
for bangumi in cur:
def check_gradient(self,
inputs=None,
outputs=None,
stream=sys.stdout,
mode='auto'):
"""Compare the OpenMDAO-calculated gradient with one calculated
by straight finite-difference. This provides the user with a way
to validate his derivative functions (apply_deriv and provideJ.)
inputs: (optional) iter of str or None
Names of input variables. The calculated gradient will be
the matrix of values of the output variables with respect
to these input variables. If no value is provided for inputs,
they will be determined based on the parameters of
the Driver corresponding to this workflow.
outputs: (optional) iter of str or None
Names of output variables. The calculated gradient will be
the matrix of values of these output variables with respect
to the input variables. If no value is provided for outputs,
they will be determined based on the objectives and constraints
of the Driver corresponding to this workflow.
stream: (optional) file-like object or str
Where to write to, default stdout. If a string is supplied,
that is used as a filename. If None, no output is written.
mode: (optional) str
Set to 'forward' for forward mode, 'adjoint' for adjoint mode,
or 'auto' to let OpenMDAO determine the correct mode.
Defaults to 'auto'.
Returns the finite difference gradient, the OpenMDAO-calculated
gradient, and a list of suspect inputs/outputs.
"""
parent = self.parent
# tuples cause problems
if inputs:
inputs = list(inputs)
if outputs:
outputs = list(outputs)
if isinstance(stream, basestring):
stream = open(stream, 'w')
close_stream = True
else:
close_stream = False
if stream is None:
stream = StringIO()
J = self.calc_gradient(inputs, outputs, mode=mode, force_regen=True)
Jbase = self.calc_gradient(inputs,
outputs,
mode='fd',
force_regen=True)
print >> stream, 24 * '-'
print >> stream, 'Calculated Gradient'
print >> stream, 24 * '-'
print >> stream, J
print >> stream, 24 * '-'
print >> stream, 'Finite Difference Comparison'
print >> stream, 24 * '-'
print >> stream, Jbase
# This code duplication is needed so that we print readable names for
# the constraints and objectives.
if inputs is None:
if hasattr(parent, 'list_param_group_targets'):
inputs = parent.list_param_group_targets()
input_refs = []
for item in inputs:
if len(item) < 2:
input_refs.append(item[0])
else:
input_refs.append(item)
# Should be caught in calc_gradient()
else: # pragma no cover
msg = "No inputs given for derivatives."
self.scope.raise_exception(msg, RuntimeError)
else:
input_refs = inputs
if outputs is None:
outputs = []
output_refs = []
if hasattr(parent, 'get_objectives'):
obj = [
"%s.out0" % item.pcomp_name
for item in parent.get_objectives().values()
]
outputs.extend(obj)
output_refs.extend(parent.get_objectives().keys())
if hasattr(parent, 'get_constraints'):
con = [
"%s.out0" % item.pcomp_name
for item in parent.get_constraints().values()
]
outputs.extend(con)
output_refs.extend(parent.get_constraints().keys())
if len(outputs) == 0: # pragma no cover
msg = "No outputs given for derivatives."
self.scope.raise_exception(msg, RuntimeError)
else:
output_refs = outputs
out_width = 0
for output, oref in zip(outputs, output_refs):
out_val = self.scope.get(output)
out_names = _flattened_names(oref, out_val)
out_width = max(out_width, max([len(out) for out in out_names]))
inp_width = 0
for input_tup, iref in zip(inputs, input_refs):
if isinstance(input_tup, str):
input_tup = [input_tup]
inp_val = self.scope.get(input_tup[0])
inp_names = _flattened_names(str(iref), inp_val)
inp_width = max(inp_width, max([len(inp) for inp in inp_names]))
label_width = out_width + inp_width + 4
print >> stream
print >> stream, label_width*' ', \
'%-18s %-18s %-18s' % ('Calculated', 'FiniteDiff', 'RelError')
print >> stream, (label_width + (3 * 18) + 3) * '-'
suspect_limit = 1e-5
error_n = error_sum = 0
error_max = error_loc = None
suspects = []
i = -1
io_pairs = []
for output, oref in zip(outputs, output_refs):
out_val = self.scope.get(output)
for out_name in _flattened_names(oref, out_val):
i += 1
j = -1
for input_tup, iref in zip(inputs, input_refs):
if isinstance(input_tup, basestring):
input_tup = (input_tup, )
inp_val = self.scope.get(input_tup[0])
for inp_name in _flattened_names(iref, inp_val):
j += 1
calc = J[i, j]
finite = Jbase[i, j]
if finite and calc:
error = (calc - finite) / finite
else:
error = calc - finite
error_n += 1
error_sum += abs(error)
if error_max is None or abs(error) > abs(error_max):
error_max = error
error_loc = (out_name, inp_name)
if abs(error) > suspect_limit or isnan(error):
suspects.append((out_name, inp_name))
print >> stream, '%*s / %*s: %-18s %-18s %-18s' \
% (out_width, out_name, inp_width, inp_name,
calc, finite, error)
io_pairs.append(
"%*s / %*s" %
(out_width, out_name, inp_width, inp_name))
print >> stream
if error_n:
print >> stream, 'Average RelError:', error_sum / error_n
print >> stream, 'Max RelError:', error_max, 'for %s / %s' % error_loc
if suspects:
print >> stream, 'Suspect gradients (RelError > %s):' % suspect_limit
for out_name, inp_name in suspects:
print >> stream, '%*s / %*s' \
% (out_width, out_name, inp_width, inp_name)
print >> stream
if close_stream:
stream.close()
# return arrays and suspects to make it easier to check from a test
return Jbase.flatten(), J.flatten(), io_pairs, suspects
print >> deadXMLBuf, ' </action>'
print >> deadXMLBuf, " </actions>"
def termChar(s):
if len(compose.char(s)) > 1:
return xmlChar(compose.terminators.get(compose.char(s), "?"))
return xmlChar(compose.terminators.get(compose.char(s), compose.char(s)))
print >> deadXMLBuf
print >> deadXMLBuf
print >> deadXMLBuf, ' <terminators>'
for m in sorted(dmm, mod_order):
if m != tuple():
print >> deadXMLBuf, ' <when state="%s" output="%s"/>' % (
'_'.join(m), ''.join([xmlChar(terminators.get(n, "?"))
for n in m]))
for ss in sorted(compose.states):
C = ''.join([termChar(s) for s in ss])
s = '_'.join(list(ss))
print >> deadXMLBuf, ' <when state="%s" output="%s"/>' % (s, C)
print >> deadXMLBuf, ' </terminators>'
deadXMLCode = deadXMLBuf.getvalue()
deadXMLBuf.close()
if __name__ == "__main__":
print deadXMLCode
class TestOSTConnector(unittest.TestCase):
"""
Class to test the IM connectors
"""
def setUp(self):
self.log = StringIO()
self.handler = logging.StreamHandler(self.log)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
self.handler.setFormatter(formatter)
logging.RootLogger.propagate = 0
logging.root.setLevel(logging.ERROR)
logger = logging.getLogger('CloudConnector')
logger.setLevel(logging.DEBUG)
logger.propagate = 0
for handler in logger.handlers:
logger.removeHandler(handler)
logger.addHandler(self.handler)
def tearDown(self):
self.handler.flush()
self.log.close()
self.log = StringIO()
self.handler.close()
@staticmethod
def get_ost_cloud():
cloud_info = CloudInfo()
cloud_info.type = "OpenStack"
cloud_info.protocol = "https"
cloud_info.server = "server.com"
cloud_info.port = 5000
inf = MagicMock()
inf.id = "1"
one_cloud = OpenStackCloudConnector(cloud_info, inf)
return one_cloud
@patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver')
def test_10_concrete(self, get_driver):
radl_data = """
network net ()
system test (
cpu.arch='x86_64' and
cpu.count>=1 and
memory.size>=512m and
net_interface.0.connection = 'net' and
net_interface.0.dns_name = 'test' and
disk.0.os.name = 'linux' and
disk.0.image.url = 'ost://server.com/ami-id' and
disk.0.os.credentials.username = '******'
)"""
radl = radl_parse.parse_radl(radl_data)
radl_system = radl.systems[0]
auth = Authentication([{
'id': 'ost',
'type': 'OpenStack',
'username': '******',
'password': '******',
'tenant': 'tenant',
'host': 'https://server.com:5000'
}])
ost_cloud = self.get_ost_cloud()
driver = MagicMock()
get_driver.return_value = driver
node_size = MagicMock()
node_size.ram = 512
node_size.price = 1
node_size.disk = 1
node_size.vcpus = 1
node_size.name = "small"
driver.list_sizes.return_value = [node_size]
concrete = ost_cloud.concreteSystem(radl_system, auth)
self.assertEqual(len(concrete), 1)
self.assertNotIn("ERROR",
self.log.getvalue(),
msg="ERROR found in log: %s" % self.log.getvalue())
@patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver')
def test_20_launch(self, get_driver):
radl_data = """
network net1 (outbound = 'yes' and provider_id = 'public' and outports = '8080')
network net2 ()
system test (
cpu.arch='x86_64' and
cpu.count=1 and
memory.size=512m and
net_interface.0.connection = 'net1' and
net_interface.0.dns_name = 'test' and
net_interface.1.connection = 'net2' and
disk.0.os.name = 'linux' and
disk.0.image.url = 'ost://server.com/ami-id' and
disk.0.os.credentials.username = '******' and
disk.1.size=1GB and
disk.1.device='hdb' and
disk.1.mount_path='/mnt/path'
)"""
radl = radl_parse.parse_radl(radl_data)
radl.check()
auth = Authentication([{
'id': 'ost',
'type': 'OpenStack',
'username': '******',
'password': '******',
'tenant': 'tenant',
'host': 'https://server.com:5000'
}])
ost_cloud = self.get_ost_cloud()
driver = MagicMock()
get_driver.return_value = driver
node_size = MagicMock()
node_size.ram = 512
node_size.price = 1
node_size.disk = 1
node_size.vcpus = 1
node_size.name = "small"
driver.list_sizes.return_value = [node_size]
net = MagicMock()
net.name = "public"
driver.ex_list_networks.return_value = [net]
sg = MagicMock()
sg.name = "sg"
driver.ex_create_security_group.return_value = sg
driver.ex_list_security_groups.return_value = []
driver.ex_create_security_group_rule.return_value = True
keypair = MagicMock()
keypair.public_key = "public"
driver.create_key_pair.return_value = keypair
driver.features = {'create_node': ['ssh_key']}
node = MagicMock()
node.id = "ost1"
node.name = "ost1name"
driver.create_node.return_value = node
res = ost_cloud.launch(InfrastructureInfo(), radl, radl, 1, auth)
success, _ = res[0]
self.assertTrue(success, msg="ERROR: launching a VM.")
self.assertNotIn("ERROR",
self.log.getvalue(),
msg="ERROR found in log: %s" % self.log.getvalue())
@patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver')
def test_30_updateVMInfo(self, get_driver):
radl_data = """
network net (outbound = 'yes')
system test (
cpu.arch='x86_64' and
cpu.count=1 and
memory.size=512m and
net_interface.0.connection = 'net' and
net_interface.0.dns_name = 'test' and
disk.0.os.name = 'linux' and
disk.0.image.url = 'one://server.com/1' and
disk.0.os.credentials.username = '******' and
disk.0.os.credentials.password = '******'
)"""
radl = radl_parse.parse_radl(radl_data)
radl.check()
auth = Authentication([{
'id': 'ost',
'type': 'OpenStack',
'username': '******',
'password': '******',
'tenant': 'tenant',
'host': 'https://server.com:5000'
}])
ost_cloud = self.get_ost_cloud()
inf = MagicMock()
inf.get_next_vm_id.return_value = 1
vm = VirtualMachine(inf, "1", ost_cloud.cloud, radl, radl, ost_cloud)
driver = MagicMock()
get_driver.return_value = driver
node = MagicMock()
node.id = "1"
node.state = "running"
node.extra = {
'flavorId': 'small',
'addresses': {
'os-lan': [{
'addr': '10.0.0.1',
'OS-EXT-IPS:type': 'fixed'
}]
}
}
node.public_ips = []
node.private_ips = ['10.0.0.1']
node.driver = driver
driver.list_nodes.return_value = [node]
node_size = MagicMock()
node_size.ram = 512
node_size.price = 1
node_size.disk = 1
node_size.vcpus = 1
node_size.name = "small"
driver.ex_get_size.return_value = node_size
volume = MagicMock()
volume.id = "vol1"
volume.attach.return_value = True
driver.create_volume.return_value = volume
pool = MagicMock()
pool.name = "pool1"
pool.list_floating_ips.return_value = []
pool.create_floating_ip.return_value = True
driver.ex_list_floating_ip_pools.return_value = [pool]
success, vm = ost_cloud.updateVMInfo(vm, auth)
self.assertTrue(success, msg="ERROR: updating VM info.")
self.assertNotIn("ERROR",
self.log.getvalue(),
msg="ERROR found in log: %s" % self.log.getvalue())
@patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver')
def test_40_stop(self, get_driver):
auth = Authentication([{
'id': 'ost',
'type': 'OpenStack',
'username': '******',
'password': '******',
'tenant': 'tenant',
'host': 'https://server.com:5000'
}])
ost_cloud = self.get_ost_cloud()
inf = MagicMock()
inf.get_next_vm_id.return_value = 1
vm = VirtualMachine(inf, "1", ost_cloud.cloud, "", "", ost_cloud)
driver = MagicMock()
get_driver.return_value = driver
node = MagicMock()
node.id = "1"
node.state = "running"
node.extra = {'flavorId': 'small'}
node.public_ips = ['158.42.1.1']
node.private_ips = ['10.0.0.1']
node.driver = driver
driver.list_nodes.return_value = [node]
driver.ex_stop_node.return_value = True
success, _ = ost_cloud.stop(vm, auth)
self.assertTrue(success, msg="ERROR: stopping VM info.")
self.assertNotIn("ERROR",
self.log.getvalue(),
msg="ERROR found in log: %s" % self.log.getvalue())
@patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver')
def test_50_start(self, get_driver):
auth = Authentication([{
'id': 'ost',
'type': 'OpenStack',
'username': '******',
'password': '******',
'tenant': 'tenant',
'host': 'https://server.com:5000'
}])
ost_cloud = self.get_ost_cloud()
inf = MagicMock()
inf.get_next_vm_id.return_value = 1
vm = VirtualMachine(inf, "1", ost_cloud.cloud, "", "", ost_cloud)
driver = MagicMock()
get_driver.return_value = driver
node = MagicMock()
node.id = "1"
node.state = "running"
node.extra = {'flavorId': 'small'}
node.public_ips = ['158.42.1.1']
node.private_ips = ['10.0.0.1']
node.driver = driver
driver.list_nodes.return_value = [node]
driver.ex_start_node.return_value = True
success, _ = ost_cloud.start(vm, auth)
self.assertTrue(success, msg="ERROR: stopping VM info.")
self.assertNotIn("ERROR",
self.log.getvalue(),
msg="ERROR found in log: %s" % self.log.getvalue())
@patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver')
def test_55_alter(self, get_driver):
radl_data = """
network net ()
system test (
cpu.arch='x86_64' and
cpu.count=1 and
memory.size=512m and
net_interface.0.connection = 'net' and
net_interface.0.dns_name = 'test' and
disk.0.os.name = 'linux' and
disk.0.image.url = 'one://server.com/1' and
disk.0.os.credentials.username = '******' and
disk.0.os.credentials.password = '******'
)"""
radl = radl_parse.parse_radl(radl_data)
new_radl_data = """
system test (
cpu.count>=2 and
memory.size>=2048m
)"""
new_radl = radl_parse.parse_radl(new_radl_data)
auth = Authentication([{
'id': 'ost',
'type': 'OpenStack',
'username': '******',
'password': '******',
'tenant': 'tenant',
'host': 'https://server.com:5000'
}])
ost_cloud = self.get_ost_cloud()
inf = MagicMock()
inf.get_next_vm_id.return_value = 1
vm = VirtualMachine(inf, "1", ost_cloud.cloud, radl, radl, ost_cloud)
driver = MagicMock()
get_driver.return_value = driver
node = MagicMock()
node.id = "1"
node.state = "running"
node.extra = {'flavorId': 'small'}
node.public_ips = ['158.42.1.1']
node.private_ips = ['10.0.0.1']
node.driver = driver
driver.list_nodes.return_value = [node]
node_size = MagicMock()
node_size.ram = 2048
node_size.price = 1
node_size.disk = 1
node_size.vcpus = 2
node_size.name = "small"
driver.list_sizes.return_value = [node_size]
driver.ex_resize.return_value = True
success, _ = ost_cloud.alterVM(vm, new_radl, auth)
self.assertTrue(success, msg="ERROR: modifying VM info.")
self.assertNotIn("ERROR",
self.log.getvalue(),
msg="ERROR found in log: %s" % self.log.getvalue())
@patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver')
@patch('time.sleep')
def test_60_finalize(self, sleep, get_driver):
auth = Authentication([{
'id': 'ost',
'type': 'OpenStack',
'username': '******',
'password': '******',
'tenant': 'tenant',
'host': 'https://server.com:5000'
}])
ost_cloud = self.get_ost_cloud()
radl_data = """
system test (
cpu.count>=2 and
memory.size>=2048m
)"""
radl = radl_parse.parse_radl(radl_data)
inf = MagicMock()
inf.get_next_vm_id.return_value = 1
vm = VirtualMachine(inf, "1", ost_cloud.cloud, radl, radl, ost_cloud)
driver = MagicMock()
driver.name = "OpenStack"
get_driver.return_value = driver
node = MagicMock()
node.id = "1"
node.state = "running"
node.extra = {'flavorId': 'small'}
node.public_ips = ['158.42.1.1']
node.private_ips = ['10.0.0.1']
node.driver = driver
node.destroy.return_value = True
driver.list_nodes.return_value = [node]
sg = MagicMock()
sg.id = sg.name = "sg1"
driver.ex_get_node_security_groups.return_value = [sg]
keypair = MagicMock()
driver.get_key_pair.return_value = keypair
vm.keypair = keypair
driver.delete_key_pair.return_value = True
driver.delete_security_group.return_value = True
driver.ex_list_floating_ips.return_value = []
success, _ = ost_cloud.finalize(vm, auth)
self.assertTrue(success, msg="ERROR: finalizing VM info.")
self.assertNotIn("ERROR",
self.log.getvalue(),
msg="ERROR found in log: %s" % self.log.getvalue())
def run_rounds(opts,args):
def get_cmd_wd(cmd, exec_rel_cwd=False):
''' get the proper working directory from a command line '''
new_cmd = []
wd = None
for i, part in reversed(list(enumerate(cmd.split()))):
if wd == None and os.path.exists(part):
wd = os.path.dirname(os.path.realpath(part))
basename = os.path.basename(part)
if i == 0:
if exec_rel_cwd:
new_cmd.insert(0, os.path.join(".", basename))
else:
new_cmd.insert(0, part)
else:
new_cmd.insert(0, basename)
else:
new_cmd.insert(0, part)
return wd, ' '.join(new_cmd)
def get_cmd_name(cmd):
''' get the name of a bot from the command line '''
for i, part in enumerate(reversed(cmd.split())):
if os.path.exists(part):
return part.split("/")[-2]
# this split of options is not needed, but left for documentation
game_options = {
"map": opts.map,
"attack": opts.attack,
"kill_points": opts.kill_points,
"food": opts.food,
"viewradius2": opts.viewradius2,
"attackradius2": opts.attackradius2,
"spawnradius2": opts.spawnradius2,
"loadtime": opts.loadtime,
"turntime": opts.turntime,
"turns": opts.turns,
"food_rate": opts.food_rate,
"food_turn": opts.food_turn,
"food_start": opts.food_start,
"food_visible": opts.food_visible,
"cutoff_turn": opts.cutoff_turn,
"cutoff_percent": opts.cutoff_percent,
"scenario": opts.scenario }
if opts.player_seed != None:
game_options['player_seed'] = opts.player_seed
if opts.engine_seed != None:
game_options['engine_seed'] = opts.engine_seed
engine_options = {
"loadtime": opts.loadtime,
"turntime": opts.turntime,
"map_file": opts.map,
"turns": opts.turns,
"log_replay": opts.log_replay,
"log_stream": opts.log_stream,
"log_input": opts.log_input,
"log_output": opts.log_output,
"log_error": opts.log_error,
"serial": opts.serial,
"strict": opts.strict,
"capture_errors": opts.capture_errors,
"secure_jail": opts.secure_jail,
"end_wait": opts.end_wait }
for round in range(opts.rounds):
# initialize game
game_id = round + opts.game_id
with open(opts.map, 'r') as map_file:
game_options['map'] = map_file.read()
if opts.engine_seed:
game_options['engine_seed'] = opts.engine_seed + round
game = Ants(game_options)
# initialize bots
bots = [get_cmd_wd(arg, exec_rel_cwd=opts.secure_jail) for arg in args]
bot_count = len(bots)
# insure correct number of bots, or fill in remaining positions
if game.num_players != len(bots):
if game.num_players > len(bots) and opts.fill:
extra = game.num_players - len(bots)
for _ in range(extra):
bots.append(bots[-1])
else:
print("Incorrect number of bots for map. Need {0}, got {1}"
.format(game.num_players, len(bots)), file=stderr)
for arg in args:
print("Bot Cmd: {0}".format(arg), file=stderr)
break
bot_count = len(bots)
# move position of first bot specified
if opts.position > 0 and opts.position <= len(bots):
first_bot = bots[0]
bots = bots[1:]
bots.insert(opts.position, first_bot)
# initialize file descriptors
if opts.log_dir and not os.path.exists(opts.log_dir):
os.mkdir(opts.log_dir)
if not opts.log_replay and not opts.log_stream and (opts.log_dir or opts.log_stdout):
opts.log_replay = True
replay_path = None # used for visualizer launch
if opts.log_replay:
if opts.log_dir:
replay_path = os.path.join(opts.log_dir, '{0}.replay'.format(game_id))
engine_options['replay_log'] = open(replay_path, 'w')
if opts.log_stdout:
if 'replay_log' in engine_options and engine_options['replay_log']:
engine_options['replay_log'] = Tee(sys.stdout, engine_options['replay_log'])
else:
engine_options['replay_log'] = sys.stdout
else:
engine_options['replay_log'] = None
if opts.log_stream:
if opts.log_dir:
engine_options['stream_log'] = open(os.path.join(opts.log_dir, '{0}.stream'.format(game_id)), 'w')
if opts.log_stdout:
if engine_options['stream_log']:
engine_options['stream_log'] = Tee(sys.stdout, engine_options['stream_log'])
else:
engine_options['stream_log'] = sys.stdout
else:
engine_options['stream_log'] = None
if opts.log_input and opts.log_dir:
engine_options['input_logs'] = [open(os.path.join(opts.log_dir, '{0}.bot{1}.input'.format(game_id, i)), 'w')
for i in range(bot_count)]
else:
engine_options['input_logs'] = None
if opts.log_output and opts.log_dir:
engine_options['output_logs'] = [open(os.path.join(opts.log_dir, '{0}.bot{1}.output'.format(game_id, i)), 'w')
for i in range(bot_count)]
else:
engine_options['output_logs'] = None
if opts.log_error and opts.log_dir:
if opts.log_stderr:
if opts.log_stdout:
engine_options['error_logs'] = [Tee(Comment(stderr), open(os.path.join(opts.log_dir, '{0}.bot{1}.error'.format(game_id, i)), 'w'))
for i in range(bot_count)]
else:
engine_options['error_logs'] = [Tee(stderr, open(os.path.join(opts.log_dir, '{0}.bot{1}.error'.format(game_id, i)), 'w'))
for i in range(bot_count)]
else:
engine_options['error_logs'] = [open(os.path.join(opts.log_dir, '{0}.bot{1}.error'.format(game_id, i)), 'w')
for i in range(bot_count)]
elif opts.log_stderr:
if opts.log_stdout:
engine_options['error_logs'] = [Comment(stderr)] * bot_count
else:
engine_options['error_logs'] = [stderr] * bot_count
else:
engine_options['error_logs'] = None
if opts.verbose:
if opts.log_stdout:
engine_options['verbose_log'] = Comment(sys.stdout)
else:
engine_options['verbose_log'] = sys.stdout
engine_options['game_id'] = game_id
if opts.rounds > 1:
print('# playgame round {0}, game id {1}'.format(round, game_id))
# intercept replay log so we can add player names
if opts.log_replay:
intcpt_replay_io = StringIO()
real_replay_io = engine_options['replay_log']
engine_options['replay_log'] = intcpt_replay_io
result = run_game(game, bots, engine_options)
# add player names, write to proper io, reset back to normal
if opts.log_replay:
replay_json = json.loads(intcpt_replay_io.getvalue())
replay_json['playernames'] = [get_cmd_name(arg) for arg in args]
real_replay_io.write(json.dumps(replay_json))
intcpt_replay_io.close()
engine_options['replay_log'] = real_replay_io
# close file descriptors
if engine_options['stream_log']:
engine_options['stream_log'].close()
if engine_options['replay_log']:
engine_options['replay_log'].close()
if engine_options['input_logs']:
for input_log in engine_options['input_logs']:
input_log.close()
if engine_options['output_logs']:
for output_log in engine_options['output_logs']:
output_log.close()
if engine_options['error_logs']:
for error_log in engine_options['error_logs']:
error_log.close()
if replay_path:
if opts.nolaunch:
if opts.html_file:
visualizer.visualize_locally.launch(replay_path, True, opts.html_file)
else:
if opts.html_file == None:
visualizer.visualize_locally.launch(replay_path,
generated_path="replay.{0}.html".format(game_id))
else:
visualizer.visualize_locally.launch(replay_path,
generated_path=opts.html_file)
class TestKubernetesConnector(unittest.TestCase):
"""
Class to test the IM connectors
"""
def setUp(self):
self.log = StringIO()
self.handler = logging.StreamHandler(self.log)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
self.handler.setFormatter(formatter)
logging.RootLogger.propagate = 0
logging.root.setLevel(logging.ERROR)
logger = logging.getLogger('CloudConnector')
logger.setLevel(logging.DEBUG)
logger.propagate = 0
for handler in logger.handlers:
logger.removeHandler(handler)
logger.addHandler(self.handler)
def tearDown(self):
self.handler.flush()
self.log.close()
self.log = StringIO()
self.handler.close()
@staticmethod
def get_kube_cloud():
cloud_info = CloudInfo()
cloud_info.type = "Kubernetes"
cloud_info.protocol = "http"
cloud_info.server = "server.com"
cloud_info.port = 8080
inf = MagicMock()
inf.id = "1"
cloud = KubernetesCloudConnector(cloud_info, inf)
return cloud
def test_10_concrete(self):
radl_data = """
network net ()
system test (
cpu.arch='x86_64' and
cpu.count>=1 and
memory.size>=512m and
net_interface.0.connection = 'net' and
net_interface.0.dns_name = 'test' and
disk.0.os.name = 'linux' and
disk.0.image.url = 'docker://someimage' and
disk.0.os.credentials.username = '******'
)"""
radl = radl_parse.parse_radl(radl_data)
radl_system = radl.systems[0]
auth = Authentication([{
'id': 'fogbow',
'type': 'Kubernetes',
'host': 'http://server.com:8080'
}])
kube_cloud = self.get_kube_cloud()
concrete = kube_cloud.concreteSystem(radl_system, auth)
self.assertEqual(len(concrete), 1)
self.assertNotIn("ERROR",
self.log.getvalue(),
msg="ERROR found in log: %s" % self.log.getvalue())
def get_response(self, method, url, verify, headers, data):
resp = MagicMock()
parts = uriparse(url)
url = parts[2]
if method == "GET":
if url == "/api/":
resp.status_code = 200
resp.text = '{"versions": "v1"}'
elif url.endswith("/pods/1"):
resp.status_code = 200
resp.text = (
'{"metadata": {"namespace":"namespace", "name": "name"}, "status": '
'{"phase":"Running", "hostIP": "158.42.1.1", "podIP": "10.0.0.1"}, '
'"spec": {"volumes": [{"persistentVolumeClaim": {"claimName" : "cname"}}]}}'
)
elif method == "POST":
if url.endswith("/pods"):
resp.status_code = 201
resp.text = '{"metadata": {"namespace":"namespace", "name": "name"}}'
if url.endswith("/namespaces"):
resp.status_code = 201
elif method == "DELETE":
if url.endswith("/pods/1"):
resp.status_code = 200
if url.endswith("/namespaces/namespace"):
resp.status_code = 200
elif "persistentvolumeclaims" in url:
resp.status_code = 200
elif method == "PATCH":
if url.endswith("/pods/1"):
resp.status_code = 201
return resp
@patch('requests.request')
@patch('IM.InfrastructureList.InfrastructureList.save_data')
def test_20_launch(self, save_data, requests):
radl_data = """
network net1 (outbound = 'yes' and outports = '8080')
network net2 ()
system test (
cpu.arch='x86_64' and
cpu.count>=1 and
memory.size>=512m and
net_interface.0.connection = 'net1' and
net_interface.0.dns_name = 'test' and
net_interface.1.connection = 'net2' and
disk.0.os.name = 'linux' and
disk.0.image.url = 'docker://someimage' and
disk.0.os.credentials.username = '******' and
disk.1.size=1GB and
disk.1.device='hdb' and
disk.1.mount_path='/mnt/path'
)"""
radl = radl_parse.parse_radl(radl_data)
radl.check()
auth = Authentication([{
'id': 'fogbow',
'type': 'Kubernetes',
'host': 'http://server.com:8080'
}])
kube_cloud = self.get_kube_cloud()
requests.side_effect = self.get_response
res = kube_cloud.launch(InfrastructureInfo(), radl, radl, 1, auth)
success, _ = res[0]
self.assertTrue(success, msg="ERROR: launching a VM.")
self.assertNotIn("ERROR",
self.log.getvalue(),
msg="ERROR found in log: %s" % self.log.getvalue())
@patch('requests.request')
def test_30_updateVMInfo(self, requests):
radl_data = """
network net (outbound = 'yes')
system test (
cpu.arch='x86_64' and
cpu.count=1 and
memory.size=512m and
net_interface.0.connection = 'net' and
net_interface.0.dns_name = 'test' and
disk.0.os.name = 'linux' and
disk.0.image.url = 'docker://someimage' and
disk.0.os.credentials.username = '******' and
disk.0.os.credentials.password = '******'
)"""
radl = radl_parse.parse_radl(radl_data)
radl.check()
auth = Authentication([{
'id': 'fogbow',
'type': 'Kubernetes',
'host': 'http://server.com:8080'
}])
kube_cloud = self.get_kube_cloud()
inf = MagicMock()
inf.id = "namespace"
vm = VirtualMachine(inf, "1", kube_cloud.cloud, radl, radl, kube_cloud,
1)
requests.side_effect = self.get_response
success, vm = kube_cloud.updateVMInfo(vm, auth)
self.assertTrue(success, msg="ERROR: updating VM info.")
self.assertNotIn("ERROR",
self.log.getvalue(),
msg="ERROR found in log: %s" % self.log.getvalue())
@patch('requests.request')
def test_55_alter(self, requests):
radl_data = """
network net ()
system test (
cpu.arch='x86_64' and
cpu.count=1 and
memory.size=512m and
net_interface.0.connection = 'net' and
net_interface.0.dns_name = 'test' and
disk.0.os.name = 'linux' and
disk.0.image.url = 'one://server.com/1' and
disk.0.os.credentials.username = '******' and
disk.0.os.credentials.password = '******'
)"""
radl = radl_parse.parse_radl(radl_data)
new_radl_data = """
system test (
cpu.count>=2 and
memory.size>=2048m
)"""
new_radl = radl_parse.parse_radl(new_radl_data)
auth = Authentication([{
'id': 'fogbow',
'type': 'Kubernetes',
'host': 'http://server.com:8080'
}])
kube_cloud = self.get_kube_cloud()
inf = MagicMock()
inf.id = "namespace"
vm = VirtualMachine(inf, "1", kube_cloud.cloud, radl, radl, kube_cloud,
1)
requests.side_effect = self.get_response
success, _ = kube_cloud.alterVM(vm, new_radl, auth)
self.assertTrue(success, msg="ERROR: modifying VM info.")
self.assertNotIn("ERROR",
self.log.getvalue(),
msg="ERROR found in log: %s" % self.log.getvalue())
@patch('requests.request')
def test_60_finalize(self, requests):
auth = Authentication([{
'id': 'fogbow',
'type': 'Kubernetes',
'host': 'http://server.com:8080'
}])
kube_cloud = self.get_kube_cloud()
inf = MagicMock()
inf.id = "namespace"
vm = VirtualMachine(inf, "1", kube_cloud.cloud, "", "", kube_cloud, 1)
requests.side_effect = self.get_response
success, _ = kube_cloud.finalize(vm, True, auth)
self.assertTrue(success, msg="ERROR: finalizing VM info.")
self.assertNotIn("ERROR",
self.log.getvalue(),
msg="ERROR found in log: %s" % self.log.getvalue())
class MavensMateTest(unittest.TestCase):
# redirects standard out to a new target
def redirectStdOut(self):
new_target = StringIO()
sys.stdout = new_target
return new_target
# runs an mm command, prints to command-specific stdout
def runCommand(self,
command_name_or_argv,
stdin,
as_json=True,
print_before_deserialization=True):
commandOut = self.redirectStdOut()
request.get_request_payload = mock.Mock(return_value=stdin)
if type(command_name_or_argv) is list:
sys.argv = command_name_or_argv
else:
sys.argv = ['mm.py', '-o', command_name_or_argv]
MavensMateRequestHandler().execute()
mm_response = commandOut.getvalue()
sys.stdout = self.saved_stdout
if print_before_deserialization:
if type(command_name_or_argv) is list:
print '[' + str(command_name_or_argv[2]) + '] ------->'
else:
print '[' + str(command_name_or_argv) + '] ------->'
print mm_response
if as_json:
mm_response = util.parse_mm_response(mm_response)
return mm_response
# runs before every test method
def setUp(self):
# self.commandStdOut = StringIO();
self.output = StringIO()
self.saved_stdout = sys.stdout
sys.stdout = self.output
# get settings from test client settings (modeled after ST3 settings)
# found in default_client_settings.json and user_client_settings.json
self.settings = util.get_plugin_client_settings()
# test suite runs for each supported version of the api
api_version = os.environ.get('SFDC_API_VERSION', 30.0)
self.settings['user']['mm_api_version'] = api_version
# set up CI-specific settings
is_ci = os.environ.get('CI') == 'true' or os.environ.get('CI') == True
if is_ci:
self.settings['user']['mm_workspace'] = os.path.join(
os.path.dirname(__file__), 'test_workspace')
self.settings['user']['mm_use_keyring'] = False
PluginConnection.get_plugin_client_settings = mock.Mock(
return_value=self.settings)
def tearDown(self):
self.output.close()
sys.stdout = self.saved_stdout
def getTestApiVersion(self):
test_api_version = os.environ.get('SFDC_API_VERSION', '30.0')
return int(float(test_api_version))
class TestOSTConnector(unittest.TestCase):
"""
Class to test the IM connectors
"""
def setUp(self):
self.error_in_create = True
self.log = StringIO()
self.handler = logging.StreamHandler(self.log)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
self.handler.setFormatter(formatter)
logging.RootLogger.propagate = 0
logging.root.setLevel(logging.ERROR)
logger = logging.getLogger('CloudConnector')
logger.setLevel(logging.DEBUG)
logger.propagate = 0
for handler in logger.handlers:
logger.removeHandler(handler)
logger.addHandler(self.handler)
def tearDown(self):
self.handler.flush()
self.log.close()
self.log = StringIO()
self.handler.close()
@staticmethod
def get_ost_cloud():
cloud_info = CloudInfo()
cloud_info.type = "OpenStack"
cloud_info.protocol = "https"
cloud_info.server = "server.com"
cloud_info.port = 5000
inf = MagicMock()
inf.id = "1"
one_cloud = OpenStackCloudConnector(cloud_info, inf)
return one_cloud
@patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver')
def test_10_concrete(self, get_driver):
radl_data = """
network net ()
system test (
cpu.arch='x86_64' and
cpu.count>=1 and
memory.size>=512m and
net_interface.0.connection = 'net' and
net_interface.0.dns_name = 'test' and
disk.0.os.name = 'linux' and
disk.0.image.url = 'ost://server.com/ami-id' and
disk.0.os.credentials.username = '******'
)"""
radl = radl_parse.parse_radl(radl_data)
radl_system = radl.systems[0]
auth = Authentication([{'id': 'ost', 'type': 'OpenStack', 'username': '******',
'password': '******', 'tenant': 'tenant', 'host': 'https://server.com:5000'}])
ost_cloud = self.get_ost_cloud()
driver = MagicMock()
get_driver.return_value = driver
node_size = MagicMock()
node_size.ram = 512
node_size.price = 1
node_size.disk = 1
node_size.vcpus = 1
node_size.name = "small"
driver.list_sizes.return_value = [node_size]
concrete = ost_cloud.concreteSystem(radl_system, auth)
self.assertEqual(len(concrete), 1)
self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
def create_node(self, **kwargs):
"""
Create VMs returning error only first time
"""
if self.error_in_create:
self.error_in_create = False
raise Exception("Error creating VM")
else:
node = MagicMock()
node.id = "ost1"
node.name = "ost1name"
return node
@patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver')
@patch('IM.InfrastructureList.InfrastructureList.save_data')
def test_20_launch(self, save_data, get_driver):
radl_data = """
network net1 (outbound = 'yes' and provider_id = 'public' and
outports = '8080,9000:9100' and sg_name= 'test')
network net2 ()
system test (
cpu.arch='x86_64' and
cpu.count=1 and
memory.size=512m and
net_interface.0.connection = 'net1' and
net_interface.0.dns_name = 'test' and
net_interface.1.connection = 'net2' and
disk.0.os.name = 'linux' and
disk.0.image.url = 'ost://server.com/ami-id' and
disk.0.os.credentials.username = '******' and
disk.1.size=1GB and
disk.1.device='hdb' and
disk.1.mount_path='/mnt/path'
)"""
radl = radl_parse.parse_radl(radl_data)
radl.check()
auth = Authentication([{'id': 'ost', 'type': 'OpenStack', 'username': '******',
'password': '******', 'tenant': 'tenant', 'host': 'https://server.com:5000'}])
ost_cloud = self.get_ost_cloud()
driver = MagicMock()
get_driver.return_value = driver
node_size = MagicMock()
node_size.ram = 512
node_size.price = 1
node_size.disk = 1
node_size.vcpus = 1
node_size.name = "small"
driver.list_sizes.return_value = [node_size]
net = MagicMock()
net.name = "public"
driver.ex_list_networks.return_value = [net]
sg = MagicMock()
sg.name = "sg"
driver.ex_create_security_group.return_value = sg
driver.ex_list_security_groups.return_value = []
driver.ex_create_security_group_rule.return_value = True
keypair = MagicMock()
keypair.public_key = "public"
keypair.private_key = "private"
driver.create_key_pair.return_value = keypair
driver.features = {'create_node': ['ssh_key']}
driver.create_node.side_effect = self.create_node
res = ost_cloud.launch_with_retry(InfrastructureInfo(), radl, radl, 1, auth, 2, 1)
success, _ = res[0]
self.assertTrue(success, msg="ERROR: launching a VM.")
self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
# test with proxy auth data
auth = Authentication([{'id': 'ost', 'type': 'OpenStack', 'proxy': 'proxy',
'tenant': 'tenant', 'host': 'https://server.com:5000'}])
res = ost_cloud.launch(InfrastructureInfo(), radl, radl, 1, auth)
success, _ = res[0]
self.assertTrue(success, msg="ERROR: launching a VM.")
@patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver')
def test_30_updateVMInfo(self, get_driver):
radl_data = """
network net (outbound = 'yes')
system test (
cpu.arch='x86_64' and
cpu.count=1 and
memory.size=512m and
net_interface.0.connection = 'net' and
net_interface.0.dns_name = 'test' and
disk.0.os.name = 'linux' and
disk.0.image.url = 'one://server.com/1' and
disk.0.os.credentials.username = '******' and
disk.0.os.credentials.password = '******'
)"""
radl = radl_parse.parse_radl(radl_data)
radl.check()
auth = Authentication([{'id': 'ost', 'type': 'OpenStack', 'username': '******',
'password': '******', 'tenant': 'tenant', 'host': 'https://server.com:5000'}])
ost_cloud = self.get_ost_cloud()
inf = MagicMock()
vm = VirtualMachine(inf, "1", ost_cloud.cloud, radl, radl, ost_cloud, 1)
driver = MagicMock()
get_driver.return_value = driver
node = MagicMock()
node.id = "1"
node.state = "running"
node.extra = {'flavorId': 'small',
'addresses': {'os-lan': [{'addr': '10.0.0.1', 'OS-EXT-IPS:type': 'fixed'}]}}
node.public_ips = []
node.private_ips = ['10.0.0.1']
node.driver = driver
driver.ex_get_node_details.return_value = node
node_size = MagicMock()
node_size.ram = 512
node_size.price = 1
node_size.disk = 1
node_size.vcpus = 1
node_size.name = "small"
driver.ex_get_size.return_value = node_size
volume = MagicMock()
volume.id = "vol1"
volume.attach.return_value = True
driver.create_volume.return_value = volume
pool = MagicMock()
pool.name = "pool1"
floating_ip = MagicMock()
floating_ip.ip_address = "8.8.8.8"
pool.list_floating_ips.return_value = []
pool.create_floating_ip.return_value = floating_ip
driver.ex_list_floating_ip_pools.return_value = [pool]
success, vm = ost_cloud.updateVMInfo(vm, auth)
self.assertTrue(success, msg="ERROR: updating VM info.")
self.assertEquals(vm.info.systems[0].getValue("net_interface.1.ip"), "10.0.0.1")
# In this case the Node has the float ip assigned
# node.public_ips = ['8.8.8.8']
floating_ip.node_id = node.id
pool.list_floating_ips.return_value = [floating_ip]
driver.ex_list_floating_ip_pools.return_value = [pool]
success, vm = ost_cloud.updateVMInfo(vm, auth)
self.assertTrue(success, msg="ERROR: updating VM info.")
self.assertEquals(vm.info.systems[0].getValue("net_interface.1.ip"), "10.0.0.1")
self.assertEquals(vm.info.systems[0].getValue("net_interface.0.ip"), "8.8.8.8")
# In this case the Node addresses are not available and it uses the old method
node.extra = {'flavorId': 'small'}
success, vm = ost_cloud.updateVMInfo(vm, auth)
self.assertEquals(vm.info.systems[0].getValue("net_interface.1.ip"), "10.0.0.1")
self.assertEquals(vm.info.systems[0].getValue("net_interface.0.ip"), "8.8.8.8")
self.assertTrue(success, msg="ERROR: updating VM info.")
# the node has a IPv6 IP
node = MagicMock()
node.id = "2"
node.state = "running"
node.extra = {'flavorId': 'small'}
node.public_ips = ['8.8.8.8', '2001:630:12:581:f816:3eff:fe92:2146']
node.private_ips = ['10.0.0.1']
node.driver = driver
driver.ex_get_node_details.return_value = node
success, vm = ost_cloud.updateVMInfo(vm, auth)
self.assertTrue(success, msg="ERROR: updating VM info.")
self.assertEquals(vm.info.systems[0].getValue("net_interface.0.ip"), "8.8.8.8")
self.assertEquals(vm.info.systems[0].getValue("net_interface.0.ipv6"), "2001:630:12:581:f816:3eff:fe92:2146")
self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
@patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver')
def test_40_stop(self, get_driver):
auth = Authentication([{'id': 'ost', 'type': 'OpenStack', 'username': '******',
'password': '******', 'tenant': 'tenant', 'host': 'https://server.com:5000'}])
ost_cloud = self.get_ost_cloud()
inf = MagicMock()
vm = VirtualMachine(inf, "1", ost_cloud.cloud, "", "", ost_cloud, 1)
driver = MagicMock()
get_driver.return_value = driver
node = MagicMock()
node.id = "1"
node.state = "running"
node.extra = {'flavorId': 'small'}
node.public_ips = ['158.42.1.1']
node.private_ips = ['10.0.0.1']
node.driver = driver
driver.ex_get_node_details.return_value = node
driver.ex_stop_node.return_value = True
success, _ = ost_cloud.stop(vm, auth)
self.assertTrue(success, msg="ERROR: stopping VM info.")
self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
@patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver')
def test_50_start(self, get_driver):
auth = Authentication([{'id': 'ost', 'type': 'OpenStack', 'username': '******',
'password': '******', 'tenant': 'tenant', 'host': 'https://server.com:5000'}])
ost_cloud = self.get_ost_cloud()
inf = MagicMock()
vm = VirtualMachine(inf, "1", ost_cloud.cloud, "", "", ost_cloud, 1)
driver = MagicMock()
get_driver.return_value = driver
node = MagicMock()
node.id = "1"
node.state = "running"
node.extra = {'flavorId': 'small'}
node.public_ips = ['158.42.1.1']
node.private_ips = ['10.0.0.1']
node.driver = driver
driver.ex_get_node_details.return_value = node
driver.ex_start_node.return_value = True
success, _ = ost_cloud.start(vm, auth)
self.assertTrue(success, msg="ERROR: stopping VM info.")
self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
@patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver')
def test_55_alter(self, get_driver):
radl_data = """
network net ()
system test (
cpu.arch='x86_64' and
cpu.count=1 and
memory.size=512m and
net_interface.0.connection = 'net' and
net_interface.0.dns_name = 'test' and
disk.0.os.name = 'linux' and
disk.0.image.url = 'one://server.com/1' and
disk.0.os.credentials.username = '******' and
disk.0.os.credentials.password = '******'
)"""
radl = radl_parse.parse_radl(radl_data)
new_radl_data = """
system test (
cpu.count>=2 and
memory.size>=2048m
)"""
new_radl = radl_parse.parse_radl(new_radl_data)
auth = Authentication([{'id': 'ost', 'type': 'OpenStack', 'username': '******',
'password': '******', 'tenant': 'tenant', 'host': 'https://server.com:5000'}])
ost_cloud = self.get_ost_cloud()
inf = MagicMock()
vm = VirtualMachine(inf, "1", ost_cloud.cloud, radl, radl, ost_cloud, 1)
driver = MagicMock()
get_driver.return_value = driver
node = MagicMock()
node.id = "1"
node.state = "running"
node.extra = {'flavorId': 'small'}
node.public_ips = ['158.42.1.1']
node.private_ips = ['10.0.0.1']
node.driver = driver
driver.ex_get_node_details.return_value = node
node_size = MagicMock()
node_size.ram = 2048
node_size.price = 1
node_size.disk = 1
node_size.vcpus = 2
node_size.name = "small"
driver.list_sizes.return_value = [node_size]
driver.ex_resize.return_value = True
success, _ = ost_cloud.alterVM(vm, new_radl, auth)
self.assertTrue(success, msg="ERROR: modifying VM info.")
self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
@patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver')
@patch('time.sleep')
def test_60_finalize(self, sleep, get_driver):
auth = Authentication([{'id': 'ost', 'type': 'OpenStack', 'username': '******',
'password': '******', 'tenant': 'tenant', 'host': 'https://server.com:5000'}])
ost_cloud = self.get_ost_cloud()
radl_data = """
network public (outboud = 'yes')
system test (
cpu.count>=2 and
memory.size>=2048m
)"""
radl = radl_parse.parse_radl(radl_data)
inf = MagicMock()
inf.id = "infid"
inf.radl = radl
vm = VirtualMachine(inf, "1", ost_cloud.cloud, radl, radl, ost_cloud, 1)
driver = MagicMock()
driver.name = "OpenStack"
get_driver.return_value = driver
node = MagicMock()
node.id = "1"
node.state = "running"
node.extra = {'flavorId': 'small'}
node.public_ips = ['158.42.1.1']
node.private_ips = ['10.0.0.1']
node.driver = driver
node.destroy.return_value = True
driver.ex_get_node_details.return_value = node
keypair = MagicMock()
driver.get_key_pair.return_value = keypair
vm.keypair = keypair
driver.delete_key_pair.return_value = True
driver.delete_security_group.return_value = True
driver.ex_list_floating_ips.return_value = []
success, _ = ost_cloud.finalize(vm, True, auth)
self.assertTrue(success, msg="ERROR: finalizing VM info.")
self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
@patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver')
def test_70_create_snapshot(self, get_driver):
auth = Authentication([{'id': 'ost', 'type': 'OpenStack', 'username': '******',
'password': '******', 'tenant': 'tenant', 'host': 'https://server.com:5000'}])
ost_cloud = self.get_ost_cloud()
inf = MagicMock()
vm = VirtualMachine(inf, "1", ost_cloud.cloud, "", "", ost_cloud, 1)
driver = MagicMock()
driver.name = "OpenStack"
get_driver.return_value = driver
node = MagicMock()
node.id = "1"
node.driver = driver
driver.ex_get_node_details.return_value = node
image = MagicMock()
image.id = "newimage"
driver.create_image.return_value = image
success, new_image = ost_cloud.create_snapshot(vm, 0, "image_name", True, auth)
self.assertTrue(success, msg="ERROR: creating snapshot: %s" % new_image)
self.assertEqual(new_image, "ost://server.com/newimage")
self.assertEqual(driver.create_image.call_args_list, [call(node, "image_name")])
self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
@patch('libcloud.compute.drivers.openstack.OpenStackNodeDriver')
def test_80_delete_image(self, get_driver):
auth = Authentication([{'id': 'ost', 'type': 'OpenStack', 'username': '******',
'password': '******', 'tenant': 'tenant', 'host': 'https://server.com:5000'}])
ost_cloud = self.get_ost_cloud()
driver = MagicMock()
driver.name = "OpenStack"
get_driver.return_value = driver
image = MagicMock()
image.id = "image"
driver.get_image.return_value = image
success, msg = ost_cloud.delete_image('ost://server.com/image', auth)
self.assertTrue(success, msg="ERROR: deleting image. %s" % msg)
self.assertEqual(driver.delete_image.call_args_list, [call(image)])
self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
>>> # f.write f.read f.seek f.close
>>>
>>> # We want it to store a string
>>> # f.getvalue()
>>>
>>>
>>> from StringIO import StringIO
>>> f = StringIO()
>>> dir(f)
['__doc__', '__init__', '__iter__', '__module__', 'buf', 'buflist', 'close', 'closed', 'flush', 'getvalue', 'isatty', 'len', 'next', 'pos', 'read', 'readline', 'readlines', 'seek', 'softspace', 'tell', 'truncate', 'write', 'writelines']
>>>
>>> f.write('Line one\n')
>>> f.write('Line two\n')
>>> f.getvalue()
'Line one\nLine two\n'
>>> f.close()
>>>
========== RESTART: /Users/raymond/Dropbox/Public/sj149/redirect.py ==========
The Starks Family
=================
* Eddard
* Catelyn
* Robb
* Sansa
* Arya
* Bram
* Rickon
* Jon Snow
Help on function show_family in module __main__:
def test_import_export(self):
filehandle = StringIO()
self.create_test_workbook(filehandle)
xls_file = filehandle.getvalue()
filehandle.close()
# import
browser = self.new_admin_browser()
browser.open('/import-agencies')
widget = browser.getControl(name='form.widgets.xls_file')
widget.add_file(xls_file, 'application/xls', 'import.xls')
browser.getControl('Import').click()
# export
browser.open('/export-agencies')
self.assertEquals(browser.headers['Content-disposition'], 'export.xls')
self.assertEquals(browser.headers['Content-Type'], 'application/xls')
self.assertTrue(int(browser.headers['Content-Length']) > 0)
# compare
workbook = xlrd.open_workbook(file_contents=browser.contents)
sheets = workbook.sheets()
self.assertEquals(len(sheets), 2)
self.assertEquals(sheets[0].name, u'Organizations')
self.assertEquals(sheets[0].ncols, len(TITLES_ORGANIZATION))
self.assertEquals(sheets[0].nrows, 5)
for row, content in enumerate(self.organizations):
self.assertEquals([cell.value for cell in sheets[0].row(row + 1)],
content)
self.assertEquals(sheets[1].name, u'People')
self.assertEquals(sheets[1].ncols, len(TITLES_REGISTER))
self.assertEquals(sheets[1].nrows, 4)
for row, content in enumerate(self.people):
self.assertEquals(
[cell.value for cell in sheets[1].row(row + 1)[:-1]],
content[:-1])
# roles are slightly different: position starts at zero, order is
# random
self.assertEquals(u'(0)(role)(2006)(*)(1)', sheets[1].cell(1,
-1).value)
self.assertTrue(u'(1)()()()(0)', sheets[1].cell(2, -1).value)
self.assertTrue(u'(3)()()()(0)', sheets[1].cell(2, -1).value)
self.assertTrue(u'(0)()()()(0)', sheets[1].cell(2, -1).value)
self.assertTrue(u'(2)(role)()()(0)', sheets[1].cell(3, -1).value)
self.assertTrue(u'(3)()()(*)(1)', sheets[1].cell(3, -1).value)
# browse imported items
browser.open('/organizations')
self.assertIn('des_0', browser.contents)
self.assertIn('port_0', browser.contents)
self.assertIn('org_1', browser.contents)
self.assertIn('org_2', browser.contents)
self.assertNotIn('org_3', browser.contents)
browser.open('/organizations/org_1')
self.assertIn('des_1', browser.contents)
self.assertIn('port_1', browser.contents)
self.assertNotIn('org_2', browser.contents)
self.assertNotIn('org_3', browser.contents)
browser.open('/organizations/org_2')
self.assertIn('des_2', browser.contents)
self.assertIn('port_2', browser.contents)
self.assertNotIn('org_1', browser.contents)
self.assertIn('org_3', browser.contents)
browser.open('/organizations/org_2/org_3')
self.assertIn('des_3', browser.contents)
self.assertIn('port_3', browser.contents)
which is faster.
In Python 3, these both modules are replaced with io module
"""
try:
from StringIO import StringIO # # for Python 2
except ImportError:
from io import StringIO # # for Python 3
# Arbitrary string
message = "This is just a normal string\n"
# Use StringIO method to set as file object
f = StringIO(message)
data = f.read()
print("Data:", data)
f.write("Second line written to file like object")
print(f"\n current cursor postion is {f.tell()}")
print("Again", f.read())
f.seek(0)
print(f"\n current cursor postion is {f.tell()}")
print("Again", f.read())
f.close()
# f.read()
# ValueError: I/O operation on closed file
def _html2text(html):
sio = StringIO()
html2text.html2text_file(html, sio.write)
text = sio.getvalue()
sio.close()
return text
