def dump_table(table, filename, con, std=None, delimiter=',', format=None, encoding='utf-8', inspector=None): from uliweb.utils.common import str_value from StringIO import StringIO import csv if not std: if isinstance(filename, (str, unicode)): std = open(filename, 'w') else: std = filename else: std = sys.stdout #add inspector table columns process, will not use model fields but database fields if inspector: meta = MetaData() table = Table(table.name, meta) inspector.reflecttable(table, None) result = do_(table.select()) fields = [x.name for x in table.c] if not format: print >>std, '#' + ' '.join(fields) elif format == 'txt': print >>std, '#' + ','.join(fields) for r in result: if not format: print >>std, r elif format == 'txt': buf = StringIO() fw = csv.writer(buf, delimiter=delimiter) fw.writerow([str_value(x, encoding=encoding) for x in r]) print >>std, buf.getvalue().rstrip() else: raise Exception, "Can't support the text format %s" % format
def render_POST(self, request): text = request.args.get("feedback") if text is None: raise FeedbackException("No text.") if len(text) > 50000: raise FeedbackException("Too much text.") text = text[0] # basic checksum to stop really lame kiddies spamming, see feedback.js for js version checksum = 0; text = text.decode("utf-8", "ignore") for x in text: checksum = ((checksum + 1) % 256) ^ (ord(x) % 256); sentchecksum = int(request.args.get("c", [0])[0]) if checksum != sentchecksum: raise FeedbackException("Bad checksum: %d vs. %d" % (sentchecksum, checksum)) msg = MIMEText(text.encode("utf-8"), _charset="utf-8") msg["Subject"] = "qwebirc feedback from %s" % request.getclientIP() msg["From"] = config.feedbackengine["from"] msg["To"] = config.feedbackengine["to"] email = StringIO(msg.as_string()) email.seek(0, 0) factorytype = SMTPSenderFactory factory = factorytype(fromEmail=config.feedbackengine["from"], toEmail=config.feedbackengine["to"], file=email, deferred=defer.Deferred()) reactor.connectTCP(config.feedbackengine["smtp_host"], config.feedbackengine["smtp_port"], factory) self.__hit() return "1"
def write_to_file(locations): """ locations = [ ('loc_type1', { 'headers': ['header1', 'header2', ...] 'rows': [ { 'header1': val1 'header2': val2 }, {...}, ] }) ] """ outfile = StringIO() writer = Excel2007ExportWriter() header_table = [(loc_type, [tab['headers']]) for loc_type, tab in locations] writer.open(header_table=header_table, file=outfile) for loc_type, tab in locations: headers = tab['headers'] tab_rows = [[row.get(header, '') for header in headers] for row in tab['rows']] writer.write([(loc_type, tab_rows)]) writer.close() return outfile.getvalue()
def test_getversion(self, server_proxy, requests): """ Test getversion operation """ proxy = MagicMock() proxy.GetVersion.return_value = (True, "1.0") server_proxy.return_value = proxy options = MagicMock() options.auth_file = get_abs_path("../../auth.dat") options.restapi = None parser = MagicMock() out = StringIO() oldstdout = sys.stdout sys.stdout = out res = main("getversion", options, [], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("1.0", output) out = StringIO() sys.stdout = out options.xmlrpc = None options.restapi = "https://localhost:8800" requests.side_effect = self.get_response res = main("getversion", options, [], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("1.0", output) sys.stdout = oldstdout
def test_sshvm_key(self, server_proxy): """ Test sshvm operation """ proxy = MagicMock() radl = open(get_abs_path("../files/test_priv.radl"), 'r').read() proxy.GetVMInfo.return_value = (True, radl) server_proxy.return_value = proxy options = MagicMock() options.auth_file = get_abs_path("../../auth.dat") options.restapi = None parser = MagicMock() out = StringIO() oldstdout = sys.stdout oldstderr = sys.stderr sys.stdout = out sys.stderr = out res = main("sshvm", options, ["infid", "vmid", "1"], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("ssh -p 1022 -i /tmp/", output) self.assertIn(" -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no [email protected]", output) sys.stdout = oldstdout sys.stderr = oldstderr
def test_getcontmsg(self, server_proxy, requests): """ Test getcontmsg operation """ proxy = MagicMock() proxy.GetInfrastructureContMsg.return_value = (True, "contmsg") server_proxy.return_value = proxy options = MagicMock() options.auth_file = get_abs_path("../../auth.dat") options.restapi = None parser = MagicMock() out = StringIO() oldstdout = sys.stdout sys.stdout = out res = main("getcontmsg", options, ["infid"], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("Msg Contextualizator: \n\ncontmsg", output) out = StringIO() sys.stdout = out options.xmlrpc = None options.restapi = "https://localhost:8800" requests.side_effect = self.get_response res = main("getcontmsg", options, ["infid"], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("Msg Contextualizator: \n\ncontmsg", output) sys.stdout = oldstdout
def test_getinfo(self, server_proxy, requests): """ Test getinfo operation """ proxy = MagicMock() proxy.GetVMInfo.return_value = (True, "radltest") proxy.GetInfrastructureInfo.return_value = (True, ["vm1"]) server_proxy.return_value = proxy options = MagicMock() options.auth_file = get_abs_path("../../auth.dat") options.restapi = None parser = MagicMock() out = StringIO() oldstdout = sys.stdout sys.stdout = out res = main("getinfo", options, ["infid"], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("Info about VM with ID: vm1\nradltest", output) out = StringIO() sys.stdout = out options.xmlrpc = None options.restapi = "https://localhost:8800" requests.side_effect = self.get_response res = main("getinfo", options, ["infid"], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("Info about VM with ID: vm1\nradltest", output) sys.stdout = oldstdout
def image(path): if '..' in path: abort(500) fd = open(join(app.root_path, "images", path)) data = fd.read() hsize = int(request.args.get("h", 0)) vsize = int(request.args.get("v", 0)) if hsize > 1000 or vsize > 1000: abort(500) if hsize: image = Image.open(StringIO(data)) x, y = image.size x1 = hsize y1 = int(1.0 * y * hsize / x) image.thumbnail((x1, y1), Image.ANTIALIAS) output = StringIO() image.save(output, "PNG") data = output.getvalue() if vsize: image = Image.open(StringIO(data)) x, y = image.size x1 = int(1.0 * x * vsize / y) y1 = vsize image.thumbnail((x1, y1), Image.ANTIALIAS) output = StringIO() image.save(output, "PNG") data = output.getvalue() response = make_response(data) response.headers['content-type'] = mimetypes.guess_type(path) return response
def _queryapi(self, method_url, get, post): c = pycurl.Curl() if bool(get): query_url = method_url + '?' + urlencode(get) else: query_url = method_url c.setopt(c.URL, query_url) if bool(post): # first clear all fields that are None post_cleared = {} for i in post: if post[i] is not None: post_cleared[i] = post[i] postfields = urlencode(post_cleared) c.setopt(c.POSTFIELDS, postfields) buffer = StringIO() c.setopt(c.WRITEFUNCTION, buffer.write) c.setopt(c.HTTPHEADER, ['PddToken: ' + self.token]) c.perform() http_response_code = c.getinfo(c.RESPONSE_CODE) http_response_data = json.loads(buffer.getvalue()) c.close() if 200 != http_response_code: self.module.fail_json(msg='Error querying yandex pdd api, HTTP status=' + c.getinfo(c.RESPONSE_CODE) + ' error=' + http_response_data.error) return (http_response_code, http_response_data)
def __call(self, method, endpoint, params={}, postfield={}, urlencode=True, timeout=True): c = pycurl.Curl() b = StringIO() # 24/4-2012 [email protected] avoid crash in libcurl "longjmp causes uninitialized stack frame" c.setopt(pycurl.NOSIGNAL, 1) c.setopt(pycurl.URL, self.server + endpoint.format(**params)) c.setopt(pycurl.FOLLOWLOCATION, False) c.setopt(pycurl.CONNECTTIMEOUT, 2) if timeout: c.setopt(pycurl.TIMEOUT, 2) c.setopt(pycurl.NOSIGNAL, 1) c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST) c.setopt(pycurl.USERPWD, self.user + ':' + self.password) c.setopt(pycurl.HTTPHEADER, ['X-Requested-Auth: Digest']) if (method == 'POST'): if urlencode: c.setopt(pycurl.POST, 1) c.setopt(pycurl.POSTFIELDS, urllib.urlencode(postfield)) else: c.setopt(pycurl.HTTPPOST, postfield) c.setopt(pycurl.WRITEFUNCTION, b.write) #c.setopt(pycurl.VERBOSE, True) try: c.perform() except: raise RuntimeError, 'connect timed out!' status_code = c.getinfo(pycurl.HTTP_CODE) c.close() if status_code != 200: logger.error('call error in %s, status code {%r}', self.server + endpoint.format(**params), status_code) raise IOError, 'Error in Matterhorn client' return b.getvalue()
def test_threading_easy_single(self): out = StringIO() threading_easy(self.func, self.myiter, 1, ',', out) benchmark = set(['mymy', 'namename', 'danieldaniel', 'isis', '']) results = set(out.getvalue().split(',')) self.assertEqual(results, benchmark)
def update_main_config(self, configobj): cfile = StringIO() configobj.write(cfile) cfile.seek(0) self.myconfig.readfp(cfile) self.save_main_config(self.mainconfigfilename) self.load_main_config()
def test_to_html(self): # big mixed biggie = DataFrame({'A' : randn(200), 'B' : tm.makeStringIndex(200)}, index=range(200)) biggie['A'][:20] = nan biggie['B'][:20] = nan s = biggie.to_html() buf = StringIO() retval = biggie.to_html(buf=buf) self.assert_(retval is None) self.assertEqual(buf.getvalue(), s) self.assert_(isinstance(s, basestring)) biggie.to_html(columns=['B', 'A'], col_space=17) biggie.to_html(columns=['B', 'A'], formatters={'A' : lambda x: '%.1f' % x}) biggie.to_html(columns=['B', 'A'], float_format=str) biggie.to_html(columns=['B', 'A'], col_space=12, float_format=str) frame = DataFrame(index=np.arange(200)) frame.to_html()
def test_to_string(self): buf = StringIO() s = self.ts.to_string() retval = self.ts.to_string(buf=buf) self.assert_(retval is None) self.assertEqual(buf.getvalue().strip(), s) # pass float_format format = '%.4f'.__mod__ result = self.ts.to_string(float_format=format) result = [x.split()[1] for x in result.split('\n')] expected = [format(x) for x in self.ts] self.assertEqual(result, expected) # empty string result = self.ts[:0].to_string() self.assertEqual(result, '') result = self.ts[:0].to_string(length=0) self.assertEqual(result, '') # name and length cp = self.ts.copy() cp.name = 'foo' result = cp.to_string(length=True, name=True) last_line = result.split('\n')[-1].strip() self.assertEqual(last_line, "Freq: B, Name: foo, Length: %d" % len(cp))
def test_create(self, server_proxy, requests): """ Test create operation """ proxy = MagicMock() proxy.CreateInfrastructure.return_value = (True, "inf1") server_proxy.return_value = proxy options = MagicMock() options.auth_file = get_abs_path("../../auth.dat") options.restapi = None parser = MagicMock() out = StringIO() oldstdout = sys.stdout sys.stdout = out res = main("create", options, [get_abs_path("../files/test.radl")], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("Infrastructure successfully created with ID: inf1", output) sys.stdout = oldstdout out = StringIO() sys.stdout = out options.xmlrpc = None options.restapi = "https://localhost:8800" requests.side_effect = self.get_response res = main("create", options, [get_abs_path("../files/test.radl")], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("Infrastructure successfully created with ID: inf1", output) sys.stdout = oldstdout
def _export_dashboard(request, dashboard_id): """ Exports dashboard as multi-sheet Excel workbook. This is a helper method for export_dashboard and export_shared_dashboard below. Renders an export without requiring login, should not be exposed directly via a URL pattern. """ dashboard = Dashboard.objects.get(id=dashboard_id) stream = StringIO() workbook = xlwt.Workbook() for metric in dashboard.metric_set.all(): metric.export(workbook) workbook.save(stream) response = HttpResponse( stream.getvalue(), mimetype='application/vnd.ms-excel' ) response['Content-Disposition'] = 'attachment; filename="%s-%s.xls"' \ % (slugify(dashboard.name), date.today()) stream.close() return response
def test_addresource(self, server_proxy, requests): """ Test addresource operation """ proxy = MagicMock() proxy.AddResource.return_value = (True, ["1"]) server_proxy.return_value = proxy options = MagicMock() options.auth_file = get_abs_path("../../auth.dat") options.restapi = None parser = MagicMock() out = StringIO() oldstdout = sys.stdout sys.stdout = out res = main("addresource", options, ["infid", get_abs_path("../files/test.radl")], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("Resources with IDs: 1 successfully added.", output) out = StringIO() sys.stdout = out options.xmlrpc = None options.restapi = "https://localhost:8800" requests.side_effect = self.get_response res = main("addresource", options, ["infid", get_abs_path("../files/test.radl")], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("Resources with IDs: 1 successfully added.", output) sys.stdout = oldstdout
def dump_fixtures(request): output = StringIO() fixture = request.GET.get('fixture', None) try: if fixture: call_command('dumpdata', fixture, '--indent=2', stdout=output) else: call_command('dumpdata', '--indent=2', stdout=output) data = output.getvalue() output.close() if fixture: file_label = 'fixtures_%s_%s' % (fixture, datetime.datetime.now().strftime('%d-%b-%Y_%H-%M')) else: file_label = 'fixtures_all_%s' % datetime.datetime.now().strftime('%d-%b-%Y_%H-%M') response = HttpResponse(data, content_type="application/json") response['Content-Disposition'] = 'attachment; filename=%s' % file_label return response except: dest = request.META.get('HTTP_REFERER', '/') messages.info(request, 'Fixture name not recognized: %s' % fixture) return HttpResponseRedirect(dest)
def test_getstate(self, server_proxy, requests): """ Test getstate operation """ proxy = MagicMock() proxy.GetInfrastructureState.return_value = (True, {"state": "running", "vm_states": {"vm1": "running"}}) server_proxy.return_value = proxy options = MagicMock() options.auth_file = get_abs_path("../../auth.dat") options.restapi = None parser = MagicMock() out = StringIO() oldstdout = sys.stdout sys.stdout = out res = main("getstate", options, ["infid"], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("The infrastructure is in state: running\nVM ID: vm1 is in state: running.", output) out = StringIO() sys.stdout = out options.xmlrpc = None options.restapi = "https://localhost:8800" requests.side_effect = self.get_response res = main("getstate", options, ["infid"], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("The infrastructure is in state: running\nVM ID: vm1 is in state: running.", output) sys.stdout = oldstdout
def deepCopy(obj): stream = StringIO() p = Pickler(stream, 1) p.dump(obj) stream.seek(0) u = Unpickler(stream) return u.load()
def test_rebootvm(self, server_proxy, requests): """ Test rebootvm operation """ proxy = MagicMock() proxy.RebootVM.return_value = (True, "") server_proxy.return_value = proxy options = MagicMock() options.auth_file = get_abs_path("../../auth.dat") options.restapi = None parser = MagicMock() out = StringIO() oldstdout = sys.stdout sys.stdout = out res = main("rebootvm", options, ["infid", "vmid"], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("VM successfully rebooted", output) out = StringIO() sys.stdout = out options.xmlrpc = None options.restapi = "https://localhost:8800" requests.side_effect = self.get_response res = main("rebootvm", options, ["infid", "vmid"], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("VM successfully rebooted", output) sys.stdout = oldstdout
def browse(self, max_lines=None, headers=None): """Try reading specified number of lines from the CSV object. Args: max_lines: max number of lines to read. If None, the whole file is read headers: a list of strings as column names. If None, it will use "col0, col1..." Returns: A pandas DataFrame with the schema inferred from the data. Raises: Exception if the csv object cannot be read or not enough lines to read, or the headers size does not match columns size. """ if self.path.startswith('gs://'): lines = Csv._read_gcs_lines(self.path, max_lines) else: lines = Csv._read_local_lines(self.path, max_lines) if len(lines) == 0: return pd.DataFrame(columns=headers) columns_size = len(next(csv.reader([lines[0]], delimiter=self._delimiter))) if headers is None: headers = ['col' + newstr(e) for e in range(columns_size)] if len(headers) != columns_size: raise Exception('Number of columns in CSV do not match number of headers') buf = StringIO() for line in lines: buf.write(line) buf.write('\n') buf.seek(0) df = pd.read_csv(buf, names=headers, delimiter=self._delimiter) for key, col in df.iteritems(): if self._is_probably_categorical(col): df[key] = df[key].astype('category') return df
def test_import(self, server_proxy, requests): """ Test import operation """ proxy = MagicMock() proxy.ImportInfrastructure.return_value = (True, "newinfid") server_proxy.return_value = proxy options = MagicMock() options.auth_file = get_abs_path("../../auth.dat") options.restapi = None parser = MagicMock() out = StringIO() oldstdout = sys.stdout sys.stdout = out res = main("import", options, [get_abs_path("../files/test.radl")], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("New Inf: newinfid", output) out = StringIO() sys.stdout = out options.xmlrpc = None options.restapi = "https://localhost:8800" requests.side_effect = self.get_response res = main("import", options, [get_abs_path("../files/test.radl")], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("New Inf: newinfid", output) sys.stdout = oldstdout
def command_response(self, cmd): """Return stdout produce by the specified CLI command""" buf = StringIO() with redirect_stdout(buf): gist.cli.main(argv=shlex.split(cmd), config=self.config) return buf.getvalue().splitlines()
def loop(self, filename, format): original_records = list(SeqIO.parse(open(filename, "rU"), format)) # now open a connection to load the database server = BioSeqDatabase.open_database(driver = DBDRIVER, user = DBUSER, passwd = DBPASSWD, host = DBHOST, db = TESTDB) db_name = "test_loop_%s" % filename # new namespace! db = server.new_database(db_name) count = db.load(original_records) self.assertEqual(count, len(original_records)) server.commit() #Now read them back... biosql_records = [db.lookup(name=rec.name) for rec in original_records] #And check they agree self.assertTrue(compare_records(original_records, biosql_records)) #Now write to a handle... handle = StringIO() SeqIO.write(biosql_records, handle, "gb") #Now read them back... handle.seek(0) new_records = list(SeqIO.parse(handle, "gb")) #And check they still agree self.assertEqual(len(new_records), len(original_records)) for old, new in zip(original_records, new_records): #TODO - remove this hack because we don't yet write these (yet): for key in ["comment", "references", "db_source"]: if key in old.annotations and key not in new.annotations: del old.annotations[key] self.assertTrue(compare_record(old, new)) #Done server.close()
def checkRoundtrip(self, t): s = StringIO() t.write(s) s.seek(0) t2 = xpt.Typelib.read(s) self.assert_(t2 is not None) self.assertEqualTypelibs(t, t2)
def getSavedFormInputForEdit(self, header=False, delimiter=','): """ returns saved as CSV text """ sbuf = StringIO() writer = csvwriter(sbuf, delimiter=delimiter) names = self.getColumnNames() titles = self.getColumnTitles() if header: encoded_titles = [] for t in titles: if isinstance(t, unicode): t = t.encode('utf-8') encoded_titles.append(t) writer.writerow(encoded_titles) for row in self.getSavedFormInput(): def get_data(row, i): data = row.get(i, '') if self._is_file_data(data): return data.filename if isinstance(data, unicode): return data.encode('utf-8') return data writer.writerow([get_data(row, i) for i in names]) res = sbuf.getvalue() sbuf.close() return res
def test_list(self, server_proxy, requests): """ Test list operation """ proxy = MagicMock() proxy.GetInfrastructureList.return_value = (True, ["inf1", "inf2"]) server_proxy.return_value = proxy options = MagicMock() options.auth_file = get_abs_path("../../auth.dat") options.xmlrpc = "https://localhost:8899" options.restapi = None options.verify = False parser = MagicMock() out = StringIO() oldstdout = sys.stdout sys.stdout = out res = main("list", options, [], parser) output = out.getvalue().strip() self.assertEquals(res, True) self.assertIn("IDs: \n inf1\n inf2", output) sys.stdout = oldstdout out = StringIO() sys.stdout = out options.xmlrpc = None options.restapi = "https://localhost:8800" requests.side_effect = self.get_response res = main("list", options, [], parser) self.assertEquals(res, True) output = out.getvalue().strip() self.assertIn("IDs: \n inf1\n inf2", output) sys.stdout = oldstdout
def request(url, expected_dict): buffer = StringIO() curl, result = get_curl(buffer, url) body = buffer.getvalue() rendering_time = "%s;%s;%s;%s;%s" % \ (curl.getinfo(curl.NAMELOOKUP_TIME), curl.getinfo(curl.CONNECT_TIME), curl.getinfo(curl.PRETRANSFER_TIME), curl.getinfo(curl.STARTTRANSFER_TIME), curl.getinfo(curl.TOTAL_TIME)) response_code = curl.getinfo(pycurl.HTTP_CODE) expected_response = expected_dict.get("expected_response", None) if expected_response is not None and \ expected_response != response_code: result = "UNEXPECTED (%s != %s)" % (expected_response, response_code) expected_text = expected_dict.get("expected_text", None) if expected_text is not None and \ str(expected_text) not in str(body): result = "UNEXPECTED (%s not in page content)" % (expected_text) curl.close() info_list = ('GET', url, response_code, rendering_time, result) return info_list
def update(filename='invenio.cfg', silent=True): """Update new config.py from conf options. The previous config.py is kept in a backup copy. """ d = get_instance_config_object(filename, silent) new_config = StringIO() keys = set(d.__dict__.keys()) | set(default_keys()) keys = list(keys) keys.sort() for key in keys: if key != key.upper(): continue value = d.__dict__.get(key, current_app.config[key]) type_ = type(value) prmt = key + ' (' + type_.__name__ + ') [' + pformat(value) + ']: ' new_value = raw_input(prmt) try: new_value = ast.literal_eval(new_value) except (SyntaxError, ValueError): pass print('>>>', key, '=', pformat(new_value)) print(key, '=', pformat(new_value), file=new_config) with current_app.open_instance_resource(filename, 'w') as config_file: config_file.write(new_config.getvalue())