def test_NA10860_debug(monkeypatch, solver):
file = script_path("aldy.tests.resources/NA10860.bam")
with tmpfile(suffix=".tar.gz") as tmp:
with tmpfile(mode="w") as out, tmpfile(mode="w") as out_log:
out.close()
out_log.close()
assert_file(
monkeypatch,
file,
solver,
EXPECTED_NA10860 + "Preparing debug archive...",
{
"--debug": tmp.name[:-7],
"--log": out_log.name,
"--output": out.name
},
)
with open(script_path(
"aldy.tests.resources/NA10860.out.expected")) as f:
expected = f.read()
with open(out.name) as f:
produced = f.read()
assert produced == expected
# Check logs
with open(out_log.name) as f:
log = f.read()
s = "Major solver: MajorSol[1.33; sol=(1x*4, 1x*4.f, 1x*61); "
s += "cn=CNSol[6.73; sol=(2x*1,1x*61); "
s += "cn=3333333333333322222_2|333333333333334444444]"
assert s in log
out = subprocess.check_output(f"tar tzf {tmp.name}",
shell=True).decode("utf-8")
out = "\n".join(sorted(out.strip().split("\n")))
assert out == EXPECTED_NA10860_DEBUG_TAR.strip()
def test_NA10860_debug(monkeypatch, solver):
file = script_path("aldy.tests.resources/NA10860.bam")
with tmpfile(suffix=".tar.gz") as tmp:
with tmpfile(mode="w") as out, tmpfile(mode="w") as out_log:
out.close()
out_log.close()
assert_file(
monkeypatch,
file,
solver,
EXPECTED_NA10860 + "Preparing debug archive...",
{"--debug": tmp.name[:-7], "--log": out_log.name, "--output": out.name},
)
with open(script_path("aldy.tests.resources/NA10860.out.expected")) as f:
expected = f.read()
with open(out.name) as f:
produced = f.read()
assert produced == expected
# Check logs
with open(out_log.name) as f:
log = f.read()
s = " rs1058172 42523528.C>T 3267G>A "
s += "(cov= 21, cn= 0.9; impact=R365H)\n"
s = "[major] status= optimal; opt= 1.48; solution= 1x*4.021, 1x*4J, 1x*61\n"
assert s in log
out = subprocess.check_output(f"tar tzf {tmp.name}", shell=True).decode("utf-8")
out = "\n".join(sorted(out.strip().split("\n")))
assert out == EXPECTED_NA10860_DEBUG_TAR.strip()
def borderDiff((filename, blockSize, offset, index)):
print >> sys.stderr, 'Processing {0} ...'.format(index)
inputFile = open(fileName)
tmpInput = tmpfile('w+r', prefix='{1}-input-s{0}-'.format(index, MINEDEP), delete=False)
tmpOutput = tmpfile('w+r', prefix='{1}-output-s{0}-'.format(index, MINEDEP), delete=False)
# tmpOutput = tmpfile('w+r', prefix=MINEDEP) #In case the output is not required (just computing times) uncomment this and comment the prev
name = tmpOutput.name
myfiles.append(inputFile)
myfiles.append(tmpInput)
myfiles.append(tmpOutput)
try:
inputFile.seek(offset, 0)
copyBlock(inputFile, tmpInput, blockSize)
r = mineHypergraph(tmpInput, tmpOutput)
except:
raise
finally:
tmpInput.close()
inputFile.close()
tmpOutput.close()
myfiles.remove(inputFile)
myfiles.remove(tmpInput)
myfiles.remove(tmpOutput)
print >> sys.stderr, 'Done with {0}!'.format(index)
# return r #In case the output is not required (just computing times) uncomment this and comment the next
return name
def borderDiff((filename, blockSize, offset, index)):
print >> sys.stderr, 'Processing {0} ...'.format(index)
inputFile = open(fileName)
tmpInput = tmpfile('w+r',
prefix='{1}-input-s{0}-'.format(index, MINEDEP),
delete=False)
tmpOutput = tmpfile('w+r',
prefix='{1}-output-s{0}-'.format(index, MINEDEP),
delete=False)
# tmpOutput = tmpfile('w+r', prefix=MINEDEP) #In case the output is not required (just computing times) uncomment this and comment the prev
name = tmpOutput.name
myfiles.append(inputFile)
myfiles.append(tmpInput)
myfiles.append(tmpOutput)
try:
inputFile.seek(offset, 0)
copyBlock(inputFile, tmpInput, blockSize)
r = mineHypergraph(tmpInput, tmpOutput)
except:
raise
finally:
tmpInput.close()
inputFile.close()
tmpOutput.close()
myfiles.remove(inputFile)
myfiles.remove(tmpInput)
myfiles.remove(tmpOutput)
print >> sys.stderr, 'Done with {0}!'.format(index)
# return r #In case the output is not required (just computing times) uncomment this and comment the next
return name
def test_main(self, site_mock):
post_mock = site_mock.return_value.post
post_mock.return_value = {
'parse': {
'text': {
'*': 'parsing_output',
},
'modulestyles': [
'some_module',
'some_other_module',
],
},
}
expected_site = 'some_site'
expected_text = 'some_content'
expected_skin = 'some_skin'
expected_extra_modules = [
'some_extra_module', 'some_other_extra_module'
]
expected_modules = post_mock.return_value['parse'][
'modulestyles'] + expected_extra_modules
with tmpfile('w') as wikitext_file, tmpfile('r+') as out_file:
wikitext_file.write(expected_text)
wikitext_file.flush()
argv = [
'--site',
expected_site,
'--wikitext-file',
wikitext_file.name,
'--skin',
expected_skin,
'--extra-style-modules',
*expected_extra_modules,
'--output-file',
out_file.name,
]
expected_args = {
'text': expected_text,
'prop': 'text|modules|jsconfigvars',
'contentmodel': 'wikitext',
}
# Act
scripts.preview.main(argv)
# Assert
site_mock.assert_called_once_with(expected_site)
post_mock.assert_called_once_with('parse', **expected_args)
output_html = out_file.read()
self.assertRegex(output_html, 'parsing_output')
for module in expected_modules:
self.assertRegex(output_html, module)
def test_load_no_type_meta_post(self):
with tmpfile(suffix='.md', mode='w+t') as tmp:
tmp.write(md.no_type_markdown_text)
tmp.seek(0)
self.ssg.ifpath = tmp.name
post = self.ssg.load_posts()
self.assertTrue(self.ssg.default_post_type in post[0][1]['type'])
def runconsole(vmname, istty, api): """call: 'console VM' description: 'Log into the out-of-band management console on the given instance. .This is roughly equivilent to directly connecting to the serial port on the machine. .This command requires the programs "grep" and "ssh" be installed .and accessible in a folder listed in the PATH environment variable. .See NOTES section of the man page for extra notes.' args: 'VM: The instance to connect to.' errors: 'VM does not exist: Return exit code 3 .Fail to run ssh: Return exit code 4' """ console_ip = '49.156.16.12' # IP of OVM console server console_rsa = "%s ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA1ZKuRj8UkoCUvGKGSGd9vQAlP3uCmq+8vBdF/SeZ2sr0Gf9+ZTn5Di8EGBBwOArU791VTtTWTg2kSC6xbearWH9xxD8omnjaYyBmqBLZ0yimVuIQWh3QS5YglxdQoGZUJ7a7ddQDLvO11f4eirP6HcNYSfGT5070jqoEiETmdcoQsdsxdJFs6GBssMoMij1i4HRDbCDPMdViEOQ19LQBCd3LsTFcmZJ/LIO9BCxsSeyV5IPkUVVzVc29JOmqDbCTcHuOidrupVheSSkZjhB0Cq6L8tOaFP/5gj7Ab6PiZPC3hOoLFgPJ3zk50RfAT2/enKqwHQFnN1QzfBBMg1kJiw==" % console_ip # Check vmname exists. This also checks user/pswd is valid. if not getvmby(vmname, api, what='hostname'): if istty: print 'Cannot find vm "%s"' % vmname return 3 # Write a temporary known hosts file, which means we don't clutter up user's ~/.ssh/known_hosts: hostfile = tmpfile() hostfile.write(console_rsa) hostfile.flush() try: proc = Popen(['ssh', '-qqt', '-oGlobalKnownHostsFile=%s' % hostfile.name, 'console@%s' % console_ip, "%s %s %s" % (api.user, vmname, api.pswd)]) except OSError: if istty: print 'Failed to start remote shell. Is the program "ssh" installed and accessbile?' return 4
def runconsole(vmname, istty, api): console_ip = '49.156.16.12' console_rsa = "%s ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA1ZKuRj8UkoCUvGKGSGd9vQAlP3uCmq+8vBdF/SeZ2sr0Gf9+ZTn5Di8EGBBwOArU791VTtTWTg2kSC6xbearWH9xxD8omnjaYyBmqBLZ0yimVuIQWh3QS5YglxdQoGZUJ7a7ddQDLvO11f4eirP6HcNYSfGT5070jqoEiETmdcoQsdsxdJFs6GBssMoMij1i4HRDbCDPMdViEOQ19LQBCd3LsTFcmZJ/LIO9BCxsSeyV5IPkUVVzVc29JOmqDbCTcHuOidrupVheSSkZjhB0Cq6L8tOaFP/5gj7Ab6PiZPC3hOoLFgPJ3zk50RfAT2/enKqwHQFnN1QzfBBMg1kJiw==" % console_ip # Check vmname exists. This also checks user/pswd is valid. # Write a temporary known hosts file, which means we don't clutter up user's ~/.ssh/known_hosts: hostfile = tmpfile() hostfile.write(console_rsa) hostfile.flush() try: proc = Popen(['ssh', '-qqt', '-oGlobalKnownHostsFile=%s' % hostfile.name, 'console@%s' % console_ip, "%s %s " % (api.user, vmname)], stdin=PIPE, stdout=PIPE) # Extra two spaces are a workaround for a console-side error. except OSError: if istty: print 'Failed to start remote shell. Is the program "ssh" installed and accessbile?' return 3 proc.stdin.write(api.pswd + '\n') # Eat first two lines, which are a greeting and a password prompt. proc.stdout.readline() proc.stdout.readline() os.dup2(sys.stdin.fileno(), proc.stdin.fileno()) # Overwrite the proc PIPE fd with stdin
def upload():
if request.method == 'POST':
file = request.files['file']
parmname=request.form['filename']
session[parmname]=next(tmpfile())
if file:
filename_path = os.path.join(configdb.upload_folder, session[parmname])
file.save(filename_path)
return jsonify({"success":True})
def __init__(self, data):
headers = data.split('\r\n')
request = headers.pop(0)
self.method, self.path, self.version = request.split(' ')
self.headers = Headers(headers)
self.fd = tmpfile('a+')
self.data = None
self.path, sep, self.query = self.path.partition('?')
self.query = parse_qs(self.query)
def __init__(self, data):
headers = data.split('\r\n')
request = headers.pop(0)
self.method, self.path, self.version = request.split(' ')
self.headers = Headers(headers)
self.fd = tmpfile('a+')
self.data = None
self.path, sep, self.query = self.path.partition('?')
self.query = parse_qs(self.query)
def test_NA10860_vcf_out(monkeypatch, solver):
file = script_path("aldy.tests.resources/NA10860.bam")
with tmpfile(suffix=".vcf", mode="w") as out:
out.close()
assert_file(monkeypatch, file, solver, EXPECTED_NA10860, {"--output": out.name})
with open(script_path("aldy.tests.resources/NA10860.vcf.expected")) as f:
expected = f.read()
with open(out.name) as f:
produced = f.read()
assert produced == expected.replace("aldy-v3.1", f"aldy-v{__version__}")
def run_experiments(base, **kwargs):
"""Run all experiments found in base param.
and check that flambe executes without errors.
Before running the configs, it updates the save_path to
be a tempdir and updates (potentially) the iteration's
params (if found) to be 1.
"""
for fname in os.listdir(base):
full_f = os.path.join(base, fname)
if os.path.isfile(full_f) and fname.endswith('yaml'):
with tmpdir() as d, tmpfile() as f, tmpfile('w') as t:
content = open(full_f).read().format(**kwargs)
t.write(content)
t.flush()
new_exp = _preprocess_experiment(t.name, d)
if new_exp:
yaml.dump_all(new_exp, f)
ret = subprocess.run(['flambe', f.name, '-i'])
assert ret.returncode == 0
def edittext(self, text):
fd, path = tmpfile()
f = open(path, 'w')
f.write(text)
f.close()
subprocess.check_call([EDITOR, path])
f = open(path, 'r')
s = f.read()
f.close()
os.unlink(path)
if not s:
raise Exception("Must enter a query into the editor and save the file.")
return s.strip()
def edittext(self, text):
fd, path = tmpfile()
f = open(path, 'w')
f.write(text)
f.close()
subprocess.check_call([EDITOR, path])
f = open(path, 'r')
s = f.read()
f.close()
os.unlink(path)
if not s:
raise Exception(
"Must enter a query into the editor and save the file.")
return s.strip()
def convolve_row(row):
with tmpfile() as out_file:
slit_filename = path.join(
base_dir, slit_dir,
'{band}_row_{row:03d}.txt'.format(band=band, row=row))
doas_args = [
doas_cl, "--version=" + doas_version, "-c", config_filename,
"-o", out_file.name, "-xml", xml_option + slit_filename
]
subprocess.call(doas_args)
data = np.loadtxt(out_file, comments=';')
return data[:, 1]
def execute(scriptname):
import os
scripts = configScripts()
parameters = ns(request.form.items())
params_list = []
output_file = False
if 'parameters' in scripts[scriptname]:
for parm_name,parm_data in scripts[scriptname]['parameters'].items():
if parm_data.get('type', None) == 'FILE':
filename=os.path.join(configdb.upload_folder,session[parm_name])
parameters[parm_name] = filename
elif parm_data.get('type', None) == 'FILEDOWN':
session[parm_name]=next(tmpfile())
filename=os.path.join(configdb.download_folder,session[parm_name])
parameters[parm_name] = filename
output_file = parm_name
if not parameters.get(parm_name, None) and scripts[scriptname]['parameters'][parm_name].get('default',None):
parameters[parm_name] = scripts[scriptname]['parameters'][parm_name]['default']
script = scripts[scriptname].script
if type(script) is not list:
script = shlex.split(script)
commandline = [
piece.format(**parameters)
for piece in script
]
commandline = [cmd.replace('SOME_SRC',configdb.prefix) for cmd in commandline]
return_code=0
try:
output=subprocess.check_output(commandline,stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
output=e.output
return_code=e.returncode
try:
output_decoded=output.decode('utf-8')
except UnicodeDecodeError:
output_decoded=output.decode('latin-1')
return json.dumps(dict(
script_name=scriptname,
output_file=output_file,
return_code=return_code,
response=deansi.deansi(output_decoded),
commandline=commandline,
))
def runconsole(vmname, istty, api): """call: 'console VM' description: 'Log into the out-of-band management console on the given instance. .This is roughly equivilent to directly connecting to the serial port on the machine. .This command requires the programs "grep" and "ssh" be installed .and accessible in a folder listed in the PATH environment variable. .See NOTES section of the man page for extra notes.' args: 'VM: The instance to connect to.' errors: 'VM does not exist: Return exit code 3 .Fail to run ssh: Return exit code 4 .ssh exits with non-zero code (generic error): Return exit code 5 .ssh exits with code 255 (connection or protocol error): Return exit code 6' """ console_ip = '49.156.16.12' # IP of OVM console server console_rsa = "%s ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCzufEnOKEipxlbNqqWGy/yYMJi9g9/Co/dL4D+3RzhfH4aBDnIlrM8vfOAtIZ3UoXy7Bo9ZQdu+5XZVyZ4+ejQFJdjnphHEf82dJpwKhWajnCkqsakUBt9RoAVRKOJYxI9lCdW9xUlmr9CwpmWEnpy+fQwV4mDAa2q/eaVPk6cHvwCwYNbdyHahBrmP2EMsLShs4YQfY5hsufAN8KJPClPTn4qHYaHLHO6WwcB11uZJXEGLV4tvyfIpXVt8h6RTY4A7/b67eFFwz93Bca2/VElXgSNjW8IjF8dJdWrGNTj+2ndNE0ZdJYzO//lRyy/n7vpVUH7UmWLz4S/XfSYwZ9D" % console_ip # Check vmname exists. This also checks user/pswd is valid. if not getvmby(vmname, api, what='hostname'): if istty: print 'Cannot find vm "%s"' % vmname return 3 # Write a temporary known hosts file, which means we don't clutter up user's ~/.ssh/known_hosts: hostfile = tmpfile() hostfile.write(console_rsa) hostfile.flush() try: proc = Popen(['ssh', '-qqt', '-oGlobalKnownHostsFile=%s' % hostfile.name, 'console@%s' % console_ip, "%s %s %s" % (api.user, vmname, api.pswd)]) except OSError: if istty: print 'Failed to start remote shell. Is the program "ssh" installed and accessbile?' return 4 ret = proc.wait() if ret: if istty: print 'Remote shell failed with exit code %d' % ret if ret == 255: return 6 else: return 5
def wait_for_data(self):
try:
self.fd = tmpfile('a+')
except Exception:
debug()
return
try:
self.fd.write(self.data)
except Exception:
debug()
return
is_done = self.check_data_size()
if is_done:
return
xmap(self.spin, LOAD, self.get_data)
def request_file(url):
from tempfile import NamedTemporaryFile as tmpfile
from urllib.request import urlopen
u = urlopen(url)
f = tmpfile(delete=False)
block_size = 1024
while True:
buffer = u.read(block_size)
if not buffer:
break
f.write(buffer)
f.close()
return f.name
tempfile.tempdir = tempfile.mkdtemp(prefix=MINEDEP, dir="/tmp")
functions = [paramExecSHD_RS, paramExecSHD_DFS
] #[paramExecSHD_RS,paramExecMTMiner,paramExecSHD_DFS]
paramExec = functions[int(sys.argv[1])]
nbThreads = int(sys.argv[2])
inputFileName = sys.argv[3]
classfile = sys.argv[4]
outputName = sys.argv[5]
#print >>sys.stderr, sys.argv
nbPos, nbNeg = classDistribution(classfile)
tmpOutput = tmpfile('w+', prefix=MINEDEP, delete=False)
fileName = tmpOutput.name
tmpOutput.close()
try:
lowerBound(inputFileName, classfile, fileName)
except CalledProcessError as error:
print >> sys.stderr, 'LowerBound Error: file ({0}), error ({1})'.format(
inputFileName, error)
os.remove(fileName)
sys.exit(1)
outputFile = open(outputName, 'w')
offsets = computeOffset(fileName, nbPos, nbNeg)
proc = Pool(processes=nbThreads)
tempfile.tempdir = tempfile.mkdtemp(prefix=MINEDEP, dir="tmp")
functions = [paramExecSHD_RS, paramExecSHD_DFS] # [paramExecSHD_RS,paramExecMTMiner,paramExecSHD_DFS]
paramExec = functions[int(sys.argv[1])]
nbThreads = int(sys.argv[2])
inputFileName = sys.argv[3]
classfile = sys.argv[4]
outputName = sys.argv[5]
# print >>sys.stderr, sys.argv
nbPos, nbNeg = classDistribution(classfile)
tmpOutput = tmpfile('w+', prefix=MINEDEP, delete=False)
fileName = tmpOutput.name
tmpOutput.close()
try:
lowerBound(inputFileName, classfile, fileName)
except CalledProcessError as error:
print >> sys.stderr, 'LowerBound Error: file ({0}), error ({1})'.format(inputFileName, error)
os.remove(fileName)
sys.exit(1)
outputFile = open(outputName, 'w')
offsets = computeOffset(fileName, nbPos, nbNeg)
proc = Pool(processes=nbThreads)
inputs = [(fileName, nbNeg, offsets[i], i) for i in range(0, nbPos)]
# Restarting the kernel ensures that all modules and variables your code needs
# are actually generated and loaded in your code
# 8. You can use modules other than the ones mentioned below but do NOT use
# modules that are not part of the rsm-msba-spark docker container by default
import pandas as pd
import sqlite3
from datetime import date
import pyrsm as rsm
import urllib.request
from tempfile import NamedTemporaryFile as tmpfile
import os
import numpy as np
# load the original bbb.pkl data frame from a Dropbox link
bbb_file = tmpfile().name
urllib.request.urlretrieve(
"https://www.dropbox.com/s/6bulog0ij4pr52o/bbb.pkl?dl=1", bbb_file)
bbb = pd.read_pickle(bbb_file)
# view the data description of the original data to determine
# what needs to be re-created
rsm.describe(bbb)
# set the working directory to the location of this script
os.getcwd()
# os.chdir(os.path.dirname(os.path.abspath(__file__)))
# load demographics data from bbb_demographics.tsv
bbb_demographics = pd.read_csv('data/bbb_demographics.tsv', sep='\t')
bbb_demographics['zip'] = bbb_demographics['zip'].apply(
xs_out = args.o
else:
xs_out = "%s_%s_%s%s" % (name, args.binning, band, extension)
if args.I0 > 0.0:
convtype = 'iocorr'
conv_header = 'I0 correction, conc=%.4e' % args.I0
else:
convtype = 'std'
conv_header = 'standard'
time = datetime.now()
with open(path.join(base_dir, config_template), 'r') as f:
configstring = f.read()
with tmpfile() as config_file:
config_file.write(
configstring.format(convtype=convtype,
i0conc=args.I0,
input=xs_in,
calib=args.grid))
config_file.flush()
xs_data = map_parallel(
args.j, convolution(config_file.name, args.binning, band),
range(0, num_rows[args.binning][band]))
with open(xs_in) as f:
gen = (line for line in f if not line[0] in (';', '*'))
input_grid = np.genfromtxt(gen)[:, 0]
target_grid = np.loadtxt(args.grid)
def test_exporter_builder():
with tmpdir() as d, tmpdir() as d2, tmpfile(
mode="w", suffix=".yaml") as f, tmpfile(mode="w",
suffix=".yaml") as f2:
# First run an experiment
exp = """
!Experiment
name: exporter
save_path: {}
pipeline:
dataset: !SSTDataset
transform:
text: !TextField
label: !LabelField
model: !TextClassifier
embedder: !Embedder
embedding: !torch.Embedding
num_embeddings: !@ dataset.text.vocab_size
embedding_dim: 30
encoder: !PooledRNNEncoder
input_size: 30
rnn_type: lstm
n_layers: 1
hidden_size: 16
output_layer: !SoftmaxLayer
input_size: !@ model[embedder].encoder.rnn.hidden_size
output_size: !@ dataset.label.vocab_size
exporter: !Exporter
model: !@ model
text: !@ dataset.text
"""
exp = exp.format(d)
f.write(exp)
f.flush()
ret = subprocess.run(['flambe', f.name, '-i'])
assert ret.returncode == 0
# Then run a builder
builder = """
flambe_inference: tests/data/dummy_extensions/inference/
---
!Builder
destination: {0}
component: !flambe_inference.DummyInferenceEngine
model: !TextClassifier.load_from_path
path: {1}
"""
base = os.path.join(d, "output__exporter", "exporter")
path_aux = [
x for x in os.listdir(base) if os.path.isdir(os.path.join(base, x))
][0] # Should be only 1 folder bc of no variants
model_path = os.path.join(base, path_aux, "checkpoint",
"checkpoint.flambe", "model")
builder = builder.format(d2, model_path)
f2.write(builder)
f2.flush()
ret = subprocess.run(['flambe', f2.name, '-i'])
assert ret.returncode == 0
# The extensions needs to be imported using extensions.py module
extensions.import_modules(["flambe_inference"])
# Import the module after import_modules (which registered tags already)
from flambe_inference import DummyInferenceEngine
eng1 = flambe.load(d2)
assert type(eng1) is DummyInferenceEngine
assert type(eng1.model) is TextClassifier
extension_path = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
"tests/data/dummy_extensions/inference")
assert eng1._extensions == {"flambe_inference": extension_path}
eng2 = DummyInferenceEngine.load_from_path(d2)
assert type(eng2) is DummyInferenceEngine
assert type(eng2.model) is TextClassifier
assert eng2._extensions == {"flambe_inference": extension_path}
assert module_equals(eng1.model, eng2.model)
def test_load_invalid_markdown_file(self):
with tmpfile(suffix='.mdx') as tmp:
self.ssg.ifpath = tmp.name
self.assertFalse(self.ssg.load_posts())
continue
seen_hosts.append(node_name.lower())
if 'ec2' not in node.attributes:
print "Did not get ec2 for node", node.attributes.keys()
#print "Did not get ec2 for node", node.attributes['hostname']
continue
print "Adding node", node.name
include_file.write("%s\tIN\tCNAME\t%s.\n" % (node.name, node.attributes['ec2']['public_hostname']))
include_file.close()
# Update Serial in parent zone file
zone_file = open(domain_zonefile_path, 'r')
print "Zone file path is %s" % domain_zonefile_path
print "Include file path is %s" % hosts_include_file
tmp_zone = tmpfile(delete=False)
while True:
line = zone_file.readline()
serial_match = re.match(r'(\s*)(\d+)\s*;\s*serial', line)
if serial_match:
new_serial = int(serial_match.group(2)) + 1
tmp_zone.write("%s%s ; serial\n" % (serial_match.group(1), new_serial))
else:
tmp_zone.write(line)
if not line:
break
# Close and rename prior to BIND reload
zone_file.close()
from tempfile import NamedTemporaryFile as tmpfile
from csv2xls import xls
def decode(s, encodings=('gbk', 'utf8')):
for encoding in encodings:
try:
return s.decode(encoding)
except UnicodeDecodeError:
pass
return s.decode('ascii', 'ignore')
tempcsv = tmpfile(delete = False)
tempxls = tmpfile(delete = False)
tempcsv.write('''
11111,21111
11111,21111
''')
tempcsv.close()
tempxls.close()
print tempcsv.name
this_instance = xls()
f = open('/tmp/f**k.csv')
this_instance.options.infile_names = ['A']
this_instance.options.outfile_name = 'B'
this_instance.options.set_default_options()
this_instance.options.check_options()
