def _format_commands_diff(commands_diff): """Format the commands differences into a human readable format.""" output = StringIO() for key, value in sorted(commands_diff.items()): if key == 'hammer': continue output.write('{}{}\n'.format( key, ' (new command)' if value['added_command'] else '' )) if value.get('added_subcommands'): output.write(' Added subcommands:\n') for subcommand in value.get('added_subcommands'): output.write(' * {}\n'.format(subcommand)) if value.get('added_options'): output.write(' Added options:\n') for option in value.get('added_options'): output.write(' * {}\n'.format(option)) if value.get('removed_subcommands'): output.write(' Removed subcommands:') for subcommand in value.get('removed_subcommands'): output.write(' * {}'.format(subcommand)) if value.get('removed_options'): output.write(' Removed options:\n') for option in value.get('removed_options'): output.write(' * {}\n'.format(option)) output.write('\n') output_value = output.getvalue() output.close() return output_value
def post(self, request): dataio = None success = False message = u"" data = u"" try: if self.check_basic_auth(): dataio = StringIO(request.read()) load_categories(get_text_from_data(dataio)) success = True message = u"Успешно загружены данные из 1С." return HttpResponse("success") else: error_message = u"Ошибка загрузки данных из 1С: неправильное имя пользователя или пароль." auth_message = u"HTTP_AUTHORIZATION: " + self.request.META.get('HTTP_AUTHORIZATION', ' ') success = False message = u"\n".join([error_message, auth_message]) return HttpResponse("failure", status=401) except Exception as e: if transaction.is_dirty(): transaction.rollback() success = False message = u"\n".join([u"Ошибка загрузки данных из 1С:", traceback.format_exc()]) if dataio: data = get_text_from_data(dataio) dataio.close() raise finally: with transaction.commit_on_success(): Log.objects.create(success=success, message=message, data=data)
def make_word(self, seed=None, min=3, max=30, tries=100): if seed is not None or not self._seeded: self._random.seed(seed) out = StringIO() tail = CircularBuffer(self.tail) tail.append(WORD_START) while True: c = self.choose_transition(self.transitions(tail.tuple()), self._random.random()) if c == WORD_STOP: break else: out.write(c) tail.append(c) result = out.getvalue() out.close() if min <= len(result) <= max: return result elif tries > 0: return self.make_word(seed, min, max, tries - 1) else: raise MatrixError
def do_POST(self): """Serve a POST request.""" r, info = self.deal_post_data() print(r, info, "by: ", self.client_address) f = StringIO() f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">') f.write("<html>\n<title>Upload Result Page</title>\n") f.write("<body>\n<h2>Upload Result Page</h2>\n") f.write("<hr>\n") if r: f.write("<strong>Success:</strong>") else: f.write("<strong>Failed:</strong>") f.write(info) f.write("<br><a href=\"%s\">back</a>" % self.headers['referer']) f.write("<hr><small>Powerd By: bones7456, check new version at ") f.write("<a href=\"http://li2z.cn/?s=SimpleHTTPServerWithUpload\">") f.write("here</a>.</small></body>\n</html>\n") length = f.tell() f.seek(0) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header("Content-Length", str(length)) self.end_headers() if f: self.copyfile(f, self.wfile) f.close()
def test_roundtrip_generators(self): # test that fasta and qual files can be streamed into memory and back # out to disk using generator reader and writer fps = list(map(lambda e: list(map(get_data_path, e)), [('empty', 'empty'), ('fasta_multi_seq_roundtrip', 'qual_multi_seq_roundtrip')])) for fasta_fp, qual_fp in fps: with open(fasta_fp, 'U') as fh: exp_fasta = fh.read() with open(qual_fp, 'U') as fh: exp_qual = fh.read() fasta_fh = StringIO() qual_fh = StringIO() _generator_to_fasta(_fasta_to_generator(fasta_fp, qual=qual_fp), fasta_fh, qual=qual_fh) obs_fasta = fasta_fh.getvalue() obs_qual = qual_fh.getvalue() fasta_fh.close() qual_fh.close() self.assertEqual(obs_fasta, exp_fasta) self.assertEqual(obs_qual, exp_qual)
def toXml(self, filename='', compress=False): xml = StringIO() xml.write("<?xml version='1.0' encoding='UTF-8'?>\n") xml.write( "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.0//EN\" \"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd \">\n") self.svg.toXml(0, xml) if not filename: if compress: import gzip f = StringIO() zf = gzip.GzipFile(fileobj=f, mode='wb') zf.write(xml.getvalue()) zf.close() f.seek(0) return f.read() else: return xml.getvalue() else: if filename[-4:] == 'svgz': import gzip f = gzip.GzipFile( filename=filename, mode="wb", compresslevel=9) f.write(xml.getvalue()) f.close() else: f = file(filename, 'w') f.write(xml.getvalue()) f.close()
def test_sequence_to_fastq_kwargs_passed(self): for constructor in [Sequence, DNA, RNA, Protein]: for components, kwargs_expected_fp in self.valid_files: for expected_kwargs, expected_fp in kwargs_expected_fp: observed_kwargs = {} # TODO: # some of the test files contain characters which are # invalid for RNA, so don't validate for now. Need to # fix this if constructor is RNA: observed_kwargs['validate'] = False # Can't use partials for this because the read # function below can't operate on partials if hasattr(constructor, 'lowercase'): expected_kwargs['lowercase'] = 'introns' observed_kwargs['lowercase'] = 'introns' fh = StringIO() for c in components: obj = constructor( c[2], metadata={'id': c[0], 'description': c[1]}, positional_metadata={'quality': c[3]}, **observed_kwargs) write(obj, into=fh, format='fastq', **expected_kwargs) observed = fh.getvalue() fh.close() with open(expected_fp, 'U') as f: expected = f.read() self.assertEqual(observed, expected)
def __call__(self, parser, namespace, values, option_string=None): output = StringIO() parser.print_help(output) text = output.getvalue() output.close() pydoc.pager(text) parser.exit()
def _in_process_execute(self, command): # from azure.cli import as cli_main from azure.cli.main import main as cli_main from six import StringIO from vcr.errors import CannotOverwriteExistingCassetteException if command.startswith('az '): command = command[3:] output_buffer = StringIO() try: # issue: stderr cannot be redirect in this form, as a result some failure information # is lost when command fails. self.exit_code = cli_main(shlex.split(command), file=output_buffer) or 0 self.output = output_buffer.getvalue() except CannotOverwriteExistingCassetteException as ex: raise AssertionError(ex) except CliExecutionError as ex: if ex.exception: raise ex.exception else: raise ex except Exception as ex: # pylint: disable=broad-except self.exit_code = 1 self.output = output_buffer.getvalue() self.process_error = ex finally: output_buffer.close()
def _save_data(self, key, hmmfile, stofile, colnames, x, cur=None): with open(hmmfile) as fh: hmmdata = fh.read() with open(stofile) as fh: stodata = fh.read() phylo = True if x.dtype == float else False coldata = json.dumps(colnames) xfile = StringIO() np.save(xfile, x) xdata = bz2.compress(xfile.getvalue(), 9) xfile.close() closable = False if cur is None: conn = sqlite3.connect(self.__filename) cur = conn.cursor() closable = True IdepiProjectData.__insert(cur, IdepiProjectData.__HMM, key, hmmdata) IdepiProjectData.__insert(cur, IdepiProjectData.__ALIGNMENT, key, stodata) IdepiProjectData.__insert(cur, IdepiProjectData.__COLUMNS, key, coldata) if phylo: IdepiProjectData.__insert(cur, IdepiProjectData.__PHYLO_DATA, key, xdata) else: IdepiProjectData.__insert(cur, IdepiProjectData.__DISCRETE_DATA, key, xdata) if closable: conn.close()
def open(self, filename=None): if filename is None: filename = self._base_uri else: if self._file_type == 's3': filename = urljoin(self._base_uri.replace( 's3://', 'http://'), filename.replace('\\', '/')).replace('http://', 's3://') elif self._file_type == 'http': filename = urljoin(self._base_uri, filename.replace('\\', '/')) else: filename = os.path.abspath(os.path.join(os.path.dirname( self._base_uri.replace('\\', '/')), filename.replace('\\', '/'))) f = None if self._file_type == 's3': uri_header, uri_body = filename.split('://', 1) us = uri_body.split('/') bucketname = us.pop(0) key = '/'.join(us) logger.info('Opening {}'.format(key)) f = StringIO(self._s3_bucket.Object(key).get()['Body'].read()) elif self._file_type == 'http': f = request.urlopen(filename) else: f = open(filename, 'rb') yield f f.close()
def _log_metadata(self, project): dependencies = self._extract_dependencies(project) output = StringIO() writer = csv.DictWriter(output, delimiter=',', quoting=csv.QUOTE_MINIMAL, lineterminator="\n", fieldnames=ordered_fieldnames) for dep in dependencies: license, homepage = self._get_pypi_license_homepage(**dep) if homepage and 'launchpad.net' in homepage: license = self._get_launchpad_license(homepage) if license == "UNKNOWN": license = "" if homepage == "UNKNOWN": homepage = "" info = dep info['license_info'] = license info['homepage'] = homepage info['project_name'] = project.name writer.writerow(info) self._project_deps[project.name] = output.getvalue() output.close()
def test_config_yaml_with_env(): #assert False, "Not Implemented" fileobj = StringIO(CONFIG) result = config_from_yaml(fileobj=fileobj, silent=True, upper_only=False, parse_env=True) fileobj.close() print(result) assert result['postfix']['home'] == HOME_ENV
def test_roundtrip_biological_sequences(self): fps = map(get_data_path, ['fasta_multi_seq_roundtrip', 'fasta_sequence_collection_different_type']) for reader, writer in ((_fasta_to_biological_sequence, _biological_sequence_to_fasta), (_fasta_to_nucleotide_sequence, _nucleotide_sequence_to_fasta), (_fasta_to_dna_sequence, _dna_sequence_to_fasta), (_fasta_to_rna_sequence, _rna_sequence_to_fasta), (_fasta_to_protein_sequence, _protein_sequence_to_fasta)): for fp in fps: # read obj1 = reader(fp) # write fh = StringIO() writer(obj1, fh) fh.seek(0) # read obj2 = reader(fh) fh.close() self.assertTrue(obj1.equals(obj2))
class Test_resource_validators(unittest.TestCase): def setUp(self): self.io = StringIO() def tearDown(self): self.io.close() def test_generate_deployment_name_from_file(self): #verify auto-gen from uri namespace = mock.MagicMock() namespace.template_uri = 'https://templates/template123.json?foo=bar' namespace.template_file = None namespace.deployment_name = None validate_deployment_name(namespace) self.assertEqual('template123', namespace.deployment_name) namespace = mock.MagicMock() namespace.template_file = __file__ namespace.template_uri = None namespace.deployment_name = None validate_deployment_name(namespace) self.assertEqual('test_validators', namespace.deployment_name) #verify use default if get a file content namespace = mock.MagicMock() namespace.template_file = '{"foo":"bar"}' namespace.template_uri = None namespace.deployment_name = None validate_deployment_name(namespace) self.assertEqual('deployment1', namespace.deployment_name)
def export_config(config=None, export_path=None, format='stdout'): if format == 'stdout': pprint.pprint(config) return fp = None stdout = False try: if export_path and export_path != 'stdout': fp = open(export_path, 'wt') else: stdout = True fp = StringIO() if format == 'yaml': yaml_dump(config, fp, Dumper=YAMLDumper, explicit_start=False, default_flow_style=False) elif format == 'json': jsonmod.dump(config, fp, indent=3) except: raise finally: if fp: if stdout: print(fp.getvalue()) fp.close()
def test_roundtrip_sequence_collections_and_alignments(self): fps = list(map(lambda e: list(map(get_data_path, e)), [('empty', 'empty'), ('fasta_sequence_collection_different_type', 'qual_sequence_collection_different_type')])) for reader, writer in ((_fasta_to_sequence_collection, _sequence_collection_to_fasta), (_fasta_to_alignment, _alignment_to_fasta)): for fasta_fp, qual_fp in fps: # read obj1 = reader(fasta_fp, qual=qual_fp) # write fasta_fh = StringIO() qual_fh = StringIO() writer(obj1, fasta_fh, qual=qual_fh) fasta_fh.seek(0) qual_fh.seek(0) # read obj2 = reader(fasta_fh, qual=qual_fh) fasta_fh.close() qual_fh.close() # TODO remove this custom equality testing code when # SequenceCollection has an equals method (part of #656). # We need this method to include IDs and description in the # comparison (not part of SequenceCollection.__eq__). self.assertEqual(obj1, obj2) for s1, s2 in zip(obj1, obj2): self.assertTrue(s1.equals(s2))
class TestCorpus(unittest.TestCase): def setUp(self): self.feature_table = FeatureTable.load(get_feature_table_fixture("feature_table.json")) self.corpus = Corpus.load(get_corpus_fixture("corpus.txt")) def test_number_of_words(self): self.assertEqual(len(self.corpus), 82) def test_get_item(self): self.assertEqual(self.corpus[3], "bab") self.assertEqual(self.corpus[5], "aab") def test_unicode_str(self): self.assertEqual(str(self.corpus), "Corpus with 82 words") if sys.version_info < (3, 0): self.assertEqual(unicode(self.corpus), "Corpus with 82 words") def test_get_list_corpus(self): corpus = Corpus.load(get_corpus_fixture("test_list_corpus.txt")) self.assertEqual(len(corpus), 5) def test_print_corpus(self): self.out = StringIO() self.saved_stdout = sys.stdout sys.stdout = self.out # temporarily take over sys.stdout self.corpus.print_corpus() self.assertEqual(len(self.out.getvalue().split("\n")), 9) self.out.close() sys.stdout = self.saved_stdout
def run_tests(modules, parallel, run_live, tests): if not modules and not tests: display('No tests set to run.') sys.exit(1) display(""" ============= Run Tests ============= """) if modules: display('Modules: {}'.format(', '.join(name for name, _, _ in modules))) # set environment variable if run_live: os.environ['AZURE_TEST_RUN_LIVE'] = 'True' test_paths = tests or [p for _, _, p in modules] display('Drive test by nosetests') from six import StringIO old_stderr = sys.stderr test_stderr = StringIO() sys.stderr = test_stderr runner = get_nose_runner(parallel=parallel, process_timeout=3600 if run_live else 600) results = runner([path for path in test_paths]) stderr_val = test_stderr.getvalue() sys.stderr = old_stderr test_stderr.close() failed_tests = summarize_tests(stderr_val) return results, failed_tests
def test_simple_cmd_with_redirected_output_fileobject(self): buff = StringIO() ret = ProcessHelper.run_subprocess(self.ECHO_COMMAND, output=buff) assert ret == 0 assert not os.path.exists(self.OUT_FILE) assert buff.readline().strip("\n") == self.PHRASE buff.close()
def _in_process_execute(self, cli_ctx, command, expect_failure=False): from six import StringIO from vcr.errors import CannotOverwriteExistingCassetteException if command.startswith('az '): command = command[3:] stdout_buf = StringIO() logging_buf = StringIO() try: # issue: stderr cannot be redirect in this form, as a result some failure information # is lost when command fails. self.exit_code = cli_ctx.invoke(shlex.split(command), out_file=stdout_buf) or 0 self.output = stdout_buf.getvalue() self.applog = logging_buf.getvalue() except CannotOverwriteExistingCassetteException as ex: raise AssertionError(ex) except CliExecutionError as ex: if expect_failure: self.exit_code = 1 self.output = stdout_buf.getvalue() self.applog = logging_buf.getvalue() elif ex.exception: raise ex.exception else: raise ex except Exception as ex: # pylint: disable=broad-except self.exit_code = 1 self.output = stdout_buf.getvalue() self.process_error = ex finally: stdout_buf.close() logging_buf.close()
def get_correct_indentation_diff(code, filename): """ Generate a diff to make code correctly indented. :param code: a string containing a file's worth of Python code :param filename: the filename being considered (used in diff generation only) :returns: a unified diff to make code correctly indented, or None if code is already correctedly indented """ code_buffer = StringIO(code) output_buffer = StringIO() reindenter = reindent.Reindenter(code_buffer) reindenter.run() reindenter.write(output_buffer) reindent_output = output_buffer.getvalue() output_buffer.close() if code != reindent_output: diff_generator = difflib.unified_diff(code.splitlines(True), reindent_output.splitlines(True), fromfile=filename, tofile=filename + " (reindented)") # work around http://bugs.python.org/issue2142 diff_tuple = map(clean_diff_line_for_python_bug_2142, diff_generator) diff = "".join(diff_tuple) return diff else: return None
def test_roundtrip_sequence_collections_and_alignments(self): fps = list(map(lambda e: list(map(get_data_path, e)), [('empty', 'empty'), ('fasta_sequence_collection_different_type', 'qual_sequence_collection_different_type')])) for reader, writer in ((_fasta_to_sequence_collection, _sequence_collection_to_fasta), (_fasta_to_alignment, _alignment_to_fasta)): for fasta_fp, qual_fp in fps: # read obj1 = reader(fasta_fp, qual=qual_fp) # write fasta_fh = StringIO() qual_fh = StringIO() writer(obj1, fasta_fh, qual=qual_fh) fasta_fh.seek(0) qual_fh.seek(0) # read obj2 = reader(fasta_fh, qual=qual_fh) fasta_fh.close() qual_fh.close() self.assertEqual(obj1, obj2)
def cmd(self, command, checks=None, allowed_exceptions=None, debug=False): #pylint: disable=no-self-use allowed_exceptions = allowed_exceptions or [] if not isinstance(allowed_exceptions, list): allowed_exceptions = [allowed_exceptions] if self._debug or debug: print('\n\tRUNNING: {}'.format(command)) command_list = shlex.split(command) output = StringIO() try: cli_main(command_list, file=output) except Exception as ex: # pylint: disable=broad-except ex_msg = str(ex) if not next((x for x in allowed_exceptions if x in ex_msg), None): raise ex self._track_executed_commands(command_list) result = output.getvalue().strip() output.close() if self._debug or debug: print('\tRESULT: {}\n'.format(result)) if checks: checks = [checks] if not isinstance(checks, list) else checks for check in checks: check.compare(result) result = result or '{}' try: return json.loads(result) except Exception: # pylint: disable=broad-except return result
def test_roundtrip_biological_sequences(self): fps = list(map(lambda e: list(map(get_data_path, e)), [('fasta_multi_seq_roundtrip', 'qual_multi_seq_roundtrip'), ('fasta_sequence_collection_different_type', 'qual_sequence_collection_different_type')])) for reader, writer in ((_fasta_to_biological_sequence, _biological_sequence_to_fasta), (partial(_fasta_to_dna_sequence, validate=False), _dna_sequence_to_fasta), (partial(_fasta_to_rna_sequence, validate=False), _rna_sequence_to_fasta), (partial(_fasta_to_protein_sequence, validate=False), _protein_sequence_to_fasta)): for fasta_fp, qual_fp in fps: # read obj1 = reader(fasta_fp, qual=qual_fp) # write fasta_fh = StringIO() qual_fh = StringIO() writer(obj1, fasta_fh, qual=qual_fh) fasta_fh.seek(0) qual_fh.seek(0) # read obj2 = reader(fasta_fh, qual=qual_fh) fasta_fh.close() qual_fh.close() self.assertEqual(obj1, obj2)
def fetch_data(self): # create a data frame directly from the full text of # the response from the returned file-descriptor. data = self.fetch_url(self.url) fd = StringIO() if isinstance(data, str): fd.write(data) else: for chunk in data: fd.write(chunk) self.fetch_size = fd.tell() fd.seek(0) try: # see if pandas can parse csv data frames = read_csv(fd, **self.pandas_kwargs) frames_hash = hashlib.md5(str(fd.getvalue()).encode('utf-8')) self.fetch_hash = frames_hash.hexdigest() except pd.parser.CParserError: # could not parse the data, raise exception raise Exception('Error parsing remote CSV data.') finally: fd.close() return frames
def write_to_db(self, db, transaction=None, commit=True): if transaction is None: transaction = db fp = StringIO() if len(self) < Timeseries.MAX_ALL_BOTTOM: top = '' middle = None self.write(fp) bottom = fp.getvalue() else: dates = sorted(self.keys()) self.write(fp, end=dates[Timeseries.ROWS_IN_TOP_BOTTOM - 1]) top = fp.getvalue() fp.truncate(0) fp.seek(0) self.write(fp, start=dates[Timeseries.ROWS_IN_TOP_BOTTOM], end=dates[-(Timeseries.ROWS_IN_TOP_BOTTOM + 1)]) middle = self.blob_create( zlib.compress(fp.getvalue().encode('ascii'))) fp.truncate(0) fp.seek(0) self.write(fp, start=dates[-Timeseries.ROWS_IN_TOP_BOTTOM]) bottom = fp.getvalue() fp.close() c = db.cursor() c.execute("DELETE FROM ts_records WHERE id=%d" % (self.id)) c.execute("""INSERT INTO ts_records (id, top, middle, bottom) VALUES (%s, %s, %s, %s)""", (self.id, top, middle, bottom)) c.close() if commit: transaction.commit()
class Test_resource_validators(unittest.TestCase): @classmethod def setUpClass(cls): pass @classmethod def tearDownClass(cls): pass def setUp(self): self.io = StringIO() def tearDown(self): self.io.close() def test_resource_type_valid(self): input = 'Test.Namespace/testtype' actual = validate_resource_type(input) self.assertEqual(actual.namespace, 'Test.Namespace') self.assertEqual(actual.type, 'testtype') def test_resource_type_invalid(self): pass def test_parent_valid(self): input = 'testtype/mytesttype' actual = validate_parent(input) self.assertEqual(actual.type, 'testtype') self.assertEqual(actual.name, 'mytesttype') def test_parent_invalid(self): pass
def qtree(q): q = optimize(q) f = StringIO() q.print_tree(out=f) f.seek(0) out = f.read() f.close() return out
def test_newick_to_tree_node_convert_underscores(self): fh = StringIO('(_:0.1, _a, _b)__;') tree = _newick_to_tree_node(fh, convert_underscores=False) fh2 = StringIO() _tree_node_to_newick(tree, fh2) self.assertEquals(fh2.getvalue(), "('_':0.1,'_a','_b')'__';\n") fh2.close() fh.close()
#!/usr/bin/env python from six import StringIO from six import print_ output = StringIO() output.write('First line.\n') print_('Second line.', file=output) # Retrieve file contents -- this will be # 'First line.\nSecond line.\n' contents = output.getvalue() print(contents) # Close object and discard memory buffer -- # .getvalue() will now raise an exception. output.close()
def __repr__(self): str_buffer = StringIO() self._pprint_impl(0, str_buffer) contents = str_buffer.getvalue() str_buffer.close() return contents
def toXml(self, filename='', compress=False): """drawing.toXml() ---->to the screen drawing.toXml(filename)---->to the file writes a svg drawing to the screen or to a file compresses if filename ends with svgz or if compress is true """ doctype = implementation.createDocumentType( 'svg', "-//W3C//DTD SVG 1.0//EN""", 'http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd ') global root # root is defined global so it can be used by the appender. Its also possible to use it as an arugument but # that is a bit messy. root = implementation.createDocument(None, None, doctype) # Create the xml document. global appender def appender(element, elementroot): """This recursive function appends elements to an element and sets the attributes and type. It stops when alle elements have been appended""" if element.namespace: e = root.createElementNS(element.namespace, element.type) else: e = root.createElement(element.type) if element.text: textnode = root.createTextNode(element.text) e.appendChild(textnode) # in element.attributes is supported from python 2.2 for attribute in list(element.attributes.keys()): e.setAttribute( attribute, str(element.attributes[attribute])) if element.elements: for el in element.elements: e = appender(el, e) elementroot.appendChild(e) return elementroot root = appender(self.svg, root) if not filename: xml = StringIO() PrettyPrint(root, xml) if compress: import gzip f = StringIO() zf = gzip.GzipFile(fileobj=f, mode='wb') zf.write(xml.getvalue()) zf.close() f.seek(0) return f.read() else: return xml.getvalue() else: try: if filename[-4:] == 'svgz': import gzip xml = StringIO() PrettyPrint(root, xml) f = gzip.GzipFile( filename=filename, mode='wb', compresslevel=9) f.write(xml.getvalue()) f.close() else: f = open(filename, 'w') PrettyPrint(root, f) f.close() except: print("Cannot write SVG file: " + filename)
def need_to_install_distro(remote): """ Installing kernels on rpm won't setup grub/boot into them. This installs the newest kernel package and checks its version and compares against the running kernel (uname -r). Similar check for deb. :returns: False if running the newest distro kernel. Returns the version of the newest if it is not running. """ dist_release = remote.os.name package_type = remote.os.package_type current = get_version_of_running_kernel(remote) log.info("Running kernel on {node}: {version}".format( node=remote.shortname, version=current)) installed_version = None if package_type == 'rpm': if dist_release in ['opensuse', 'sle']: install_stdout = remote.sh( 'sudo zypper --non-interactive install kernel-default') else: install_stdout = remote.sh('sudo yum install -y kernel') match = re.search("Package (.*) already installed", install_stdout, flags=re.MULTILINE) if 'Nothing to do' in install_stdout: installed_version = match.groups()[0] if match else '' err_mess = StringIO() err_mess.truncate(0) remote.run(args=[ 'echo', 'no', run.Raw('|'), 'sudo', 'yum', 'reinstall', 'kernel', run.Raw('||'), 'true' ], stderr=err_mess) reinstall_stderr = err_mess.getvalue() err_mess.close() if 'Skipping the running kernel' in reinstall_stderr: running_version = re.search( "Skipping the running kernel: (.*)", reinstall_stderr, flags=re.MULTILINE).groups()[0] if installed_version == running_version: log.info( 'Newest distro kernel already installed and running' ) return False else: remote.run(args=[ 'sudo', 'yum', 'reinstall', '-y', 'kernel', run.Raw('||'), 'true' ]) newest = get_latest_image_version_rpm(remote) if package_type == 'deb': newest = get_latest_image_version_deb(remote, dist_release) if current in newest or current.replace('-', '_') in newest: log.info('Newest distro kernel installed and running') return False log.info('Not newest distro kernel. Current: {cur} Expected: {new}'.format( cur=current, new=newest)) return newest
def get_latest_image_version_deb(remote, ostype): """ Get kernel image version of the newest kernel deb package. Used for distro case. Round-about way to get the newest kernel uname -r compliant version string from the virtual package which is the newest kenel for debian/ubuntu. """ remote.run(args=['sudo', 'apt-get', 'clean']) remote.run(args=['sudo', 'apt-get', 'update']) output = StringIO() newest = '' # Depend of virtual package has uname -r output in package name. Grab that. # Note that a dependency list may have multiple comma-separated entries, # but also each entry may be an alternative (pkg1 | pkg2) if 'debian' in ostype: remote.run( args=['sudo', 'apt-get', '-y', 'install', 'linux-image-amd64'], stdout=output) remote.run(args=['dpkg', '-s', 'linux-image-amd64'], stdout=output) for line in output.getvalue().split('\n'): if 'Depends:' in line: newest = line.split('linux-image-')[1] output.close() return newest # Ubuntu is a depend in a depend. if 'ubuntu' in ostype: try: remote.run(args=[ 'sudo', 'DEBIAN_FRONTEND=noninteractive', 'apt-get', '-y', 'install', 'linux-image-current-generic' ]) remote.run(args=['dpkg', '-s', 'linux-image-current-generic'], stdout=output) for line in output.getvalue().split('\n'): if 'Depends:' in line: depends = line.split('Depends: ')[1] remote.run( args=['sudo', 'apt-get', '-y', 'install', depends]) remote.run(args=['dpkg', '-s', depends], stdout=output) except run.CommandFailedError: # Non precise ubuntu machines (like trusty) don't have # linux-image-current-generic so use linux-image-generic instead. remote.run(args=[ 'sudo', 'DEBIAN_FRONTEND=noninteractive', 'apt-get', '-y', 'install', 'linux-image-generic' ], stdout=output) remote.run(args=['dpkg', '-s', 'linux-image-generic'], stdout=output) for line in output.getvalue().split('\n'): if 'Depends:' in line: newest = line.split('linux-image-')[1] if ',' in newest: newest = newest.split(',')[0] if '|' in newest: # not strictly correct, as any of the |-joined # packages may satisfy the dependency newest = newest.split('|')[0].strip() output.close() return newest
class TestParser(unittest.TestCase): def setUp(self): self.io = StringIO() def tearDown(self): self.io.close() def test_register_simple_commands(self): def test_handler1(): pass def test_handler2(): pass command = CliCommand('command the-name', test_handler1) command2 = CliCommand('sub-command the-second-name', test_handler2) cmd_table = { 'command the-name': command, 'sub-command the-second-name': command2 } parser = AzCliCommandParser() parser.load_command_table(cmd_table) args = parser.parse_args('command the-name'.split()) self.assertIs(args.func, test_handler1) args = parser.parse_args('sub-command the-second-name'.split()) self.assertIs(args.func, test_handler2) AzCliCommandParser.error = VerifyError(self, ) parser.parse_args('sub-command'.split()) self.assertTrue(AzCliCommandParser.error.called) def test_required_parameter(self): def test_handler(args): # pylint: disable=unused-argument pass command = CliCommand('test command', test_handler) command.add_argument('req', '--req', required=True) cmd_table = {'test command': command} parser = AzCliCommandParser() parser.load_command_table(cmd_table) args = parser.parse_args('test command --req yep'.split()) self.assertIs(args.func, test_handler) AzCliCommandParser.error = VerifyError(self) parser.parse_args('test command'.split()) self.assertTrue(AzCliCommandParser.error.called) def test_nargs_parameter(self): def test_handler(): pass command = CliCommand('test command', test_handler) command.add_argument('req', '--req', required=True, nargs=2) cmd_table = {'test command': command} parser = AzCliCommandParser() parser.load_command_table(cmd_table) args = parser.parse_args('test command --req yep nope'.split()) self.assertIs(args.func, test_handler) AzCliCommandParser.error = VerifyError(self) parser.parse_args('test command -req yep'.split()) self.assertTrue(AzCliCommandParser.error.called)
def from_directories(cls, directories, pattern=None, ignore=(), write=None, relative_to=None): """ convert directories to a simple manifest; returns ManifestParser instance pattern -- shell pattern (glob) or patterns of filenames to match ignore -- directory names to ignore write -- filename or file-like object of manifests to write; if `None` then a StringIO instance will be created relative_to -- write paths relative to this path; if false then the paths are absolute """ # determine output opened_manifest_file = None # name of opened manifest file absolute = not relative_to # whether to output absolute path names as names if isinstance(write, string_types): opened_manifest_file = write write = open(write, 'w') if write is None: write = StringIO() # walk the directories, generating manifests def callback(directory, dirpath, dirnames, filenames): # absolute paths filenames = [ os.path.join(dirpath, filename) for filename in filenames ] # ensure new manifest isn't added filenames = [ filename for filename in filenames if filename != opened_manifest_file ] # normalize paths if not absolute and relative_to: filenames = [ relpath(filename, relative_to) for filename in filenames ] # write to manifest write_content = '\n'.join([ '[{}]'.format(denormalize_path(filename)) for filename in filenames ]) print(write_content, file=write) cls._walk_directories(directories, callback, pattern=pattern, ignore=ignore) if opened_manifest_file: # close file write.close() manifests = [opened_manifest_file] else: # manifests/write is a file-like object; # rewind buffer write.flush() write.seek(0) manifests = [write] # make a ManifestParser instance return cls(manifests=manifests)
class Test_storage_validators(unittest.TestCase): @classmethod def setUpClass(cls): pass @classmethod def tearDownClass(cls): pass def setUp(self): self.io = StringIO() def tearDown(self): self.io.close() def test_permission_validator(self): from azure.storage.blob.models import ContainerPermissions from argparse import Namespace ns1 = Namespace(permission='rwdl') ns2 = Namespace(permission='abc') get_permission_validator(ContainerPermissions)(ns1) self.assertTrue(isinstance(ns1.permission, ContainerPermissions)) with self.assertRaises(ValueError): get_permission_validator(ContainerPermissions)(ns2) def test_datetime_string_type(self): input = "2017-01-01T12:30Z" actual = datetime_string_type(input) expected = "2017-01-01T12:30Z" self.assertEqual(actual, expected) input = "2017-01-01 12:30" with self.assertRaises(ValueError): actual = datetime_string_type(input) def test_datetime_type(self): input = "2017-01-01T12:30Z" actual = datetime_type(input) expected = datetime(2017, 1, 1, 12, 30, 0) self.assertEqual(actual, expected) input = "2017-01-01 12:30" with self.assertRaises(ValueError): actual = datetime_type(input) def test_ipv4_range_type(self): input = "111.22.3.111" actual = ipv4_range_type(input) expected = input self.assertEqual(actual, expected) input = "111.22.3.111-222.11.44.111" actual = ipv4_range_type(input) expected = input self.assertEqual(actual, expected) input = "111.22" with self.assertRaises(ValueError): actual = ipv4_range_type(input) input = "111.22.33.44-" with self.assertRaises(ValueError): actual = ipv4_range_type(input) def test_resource_types_type(self): input = "sso" actual = str(resource_type_type(input)) expected = "so" self.assertEqual(actual, expected) input = "blob" with self.assertRaises(ValueError): actual = resource_type_type(input) def test_services_type(self): input = "ttfqbqtf" actual = str(services_type(input)) expected = "bqtf" self.assertEqual(actual, expected) input = "everything" with self.assertRaises(ValueError): actual = services_type(input)
def table2string(table, out=None): """Given list of lists figure out their common widths and print to out Parameters ---------- table : list of lists of strings What is aimed to be printed out : None or stream Where to print. If None -- will print and return string Returns ------- string if out was None """ print2string = out is None if print2string: out = StringIO() # equalize number of elements in each row Nelements_max = len(table) \ and max(len(x) for x in table) for i, table_ in enumerate(table): table[i] += [''] * (Nelements_max - len(table_)) # figure out lengths within each column atable = np.asarray(table).astype(str) # eat whole entry while computing width for @w (for wide) markup_strip = re.compile('^@([lrc]|w.*)') col_width = [ max([len(markup_strip.sub('', x)) for x in column]) for column in atable.T ] string = "" for i, table_ in enumerate(table): string_ = "" for j, item in enumerate(table_): item = str(item) if item.startswith('@'): align = item[1] item = item[2:] if not align in ['l', 'r', 'c', 'w']: raise ValueError('Unknown alignment %s. Known are l,r,c' % align) else: align = 'c' NspacesL = max(ceil((col_width[j] - len(item)) / 2.0), 0) NspacesR = max(col_width[j] - NspacesL - len(item), 0) if align in ['w', 'c']: pass elif align == 'l': NspacesL, NspacesR = 0, NspacesL + NspacesR elif align == 'r': NspacesL, NspacesR = NspacesL + NspacesR, 0 else: raise RuntimeError('Should not get here with align=%s' % align) string_ += "%%%ds%%s%%%ds " \ % (NspacesL, NspacesR) % ('', item, '') string += string_.rstrip() + '\n' out.write(string) if print2string: value = out.getvalue() out.close() return value
def csv(self, number_of_events, event_type='random_event'): print("Generating CSV test data with {} events.".format( number_of_events)) event = eval('self.{}'.format(event_type)) csv = StringIO() header = [ schema.Required.event().id, schema.Required.seqno().id, schema.Required.timezone_sign().id, schema.Required.timezone_hour().id, schema.Required.timezone_min().id, schema.Required.locale().id, schema.Required.uid().id, schema.Required.message_id().id, schema.Required.build_id().id, schema.Required.platform_id().id, schema.Required.timestamp_utc().id, schema.Required.source().id, schema.Required.session_id().id ] custom = [] for attr in event["attributes"]: header.append(attr["id"]) custom.append(str(attr["value"])) header = self.__context[c.KEY_SEPERATOR_CSV].join(header) custom = self.__context[c.KEY_SEPERATOR_CSV].join(custom) csv.write(header + c.NEW_LINE) for i in range(0, number_of_events): csv.write("{}{}".format(event["id"], self.__context[c.KEY_SEPERATOR_CSV])) csv.write("{}{}".format(i, self.__context[c.KEY_SEPERATOR_CSV])) csv.write("-{}".format(self.__context[c.KEY_SEPERATOR_CSV])) csv.write("3{}".format(self.__context[c.KEY_SEPERATOR_CSV])) csv.write("0{}".format(self.__context[c.KEY_SEPERATOR_CSV])) csv.write("{}{}".format( self.__loc[randint(0, len(self.__loc) - 1)], self.__context[c.KEY_SEPERATOR_CSV])) csv.write("{}{}".format( self.__player_id[randint(0, len(self.__player_id) - 1)], self.__context[c.KEY_SEPERATOR_CSV])) csv.write("{}{}".format(str(self.uuid), self.__context[c.KEY_SEPERATOR_CSV])) csv.write("{}{}".format( self.__buildid[randint(0, len(self.__buildid) - 1)], self.__context[c.KEY_SEPERATOR_CSV])) csv.write("{}{}".format( self.__platforms[randint(0, len(self.__platforms) - 1)], self.__context[c.KEY_SEPERATOR_CSV])) csv.write("{}{}".format(time.time(), self.__context[c.KEY_SEPERATOR_CSV])) csv.write("{}{}".format(self.__source, self.__context[c.KEY_SEPERATOR_CSV])) if len(custom) == 0: csv.write("{}".format(self.__session_id[randint( 0, len(self.__session_id) - 1)])) else: csv.write("{}{}".format( self.__session_id[randint(0, len(self.__session_id) - 1)], self.__context[c.KEY_SEPERATOR_CSV])) csv.write(custom + c.NEW_LINE) result = csv.getvalue() csv.close() return result
def get_asos(time_window, lat_range=None, lon_range=None, station=None): """ Returns all of the station observations from the Iowa Mesonet from either a given latitude and longitude window or a given station code. Parameters ---------- time_window: tuple A 2 member list or tuple containing the start and end times. The times must be python datetimes. lat_range: tuple The latitude window to grab all of the ASOS observations from. lon_range: tuple The longitude window to grab all of the ASOS observations from. station: str The station ID to grab the ASOS observations from. Returns ------- asos_ds: dict of xarray datasets A dictionary of ACT datasets whose keys are the ASOS station IDs. Examples -------- If you want to obtain timeseries of ASOS observations for Chicago O'Hare Airport, simply do:: $ time_window = [datetime(2020, 2, 4, 2, 0), datetime(2020, 2, 10, 10, 0)] $ station = "KORD" $ my_asoses = act.discovery.get_asos(time_window, station="ORD") """ # First query the database for all of the JSON info for every station # Only add stations whose lat/lon are within the Grid's boundaries regions = """AF AL_ AI_ AQ_ AG_ AR_ AK AL AM_ AO_ AS_ AR AW_ AU_ AT_ AZ_ BA_ BE_ BB_ BG_ BO_ BR_ BF_ BT_ BS_ BI_ BM_ BB_ BY_ BZ_ BJ_ BW_ AZ CA CA_AB CA_BC CD_ CK_ CF_ CG_ CL_ CM_ CO CO_ CN_ CR_ CT CU_ CV_ CY_ CZ_ DE DK_ DJ_ DM_ DO_ DZ EE_ ET_ FK_ FM_ FJ_ FI_ FR_ GF_ PF_ GA_ GM_ GE_ DE_ GH_ GI_ KY_ GB_ GR_ GL_ GD_ GU_ GT_ GN_ GW_ GY_ HT_ HN_ HK_ HU_ IS_ IN_ ID_ IR_ IQ_ IE_ IL_ IT_ CI_ JM_ JP_ JO_ KZ_ KE_ KI_ KW_ LA_ LV_ LB_ LS_ LR_ LY_ LT_ LU_ MK_ MG_ MW_ MY_ MV_ ML_ CA_MB MH_ MR_ MU_ YT_ MX_ MD_ MC_ MA_ MZ_ MM_ NA_ NP_ AN_ NL_ CA_NB NC_ CA_NF NF_ NI_ NE_ NG_ MP_ KP_ CA_NT NO_ CA_NS CA_NU OM_ CA_ON PK_ PA_ PG_ PY_ PE_ PH_ PN_ PL_ PT_ CA_PE PR_ QA_ CA_QC RO_ RU_RW_ SH_ KN_ LC_ VC_ WS_ ST_ CA_SK SA_ SN_ RS_ SC_ SL_ SG_ SK_ SI_ SB_ SO_ ZA_ KR_ ES_ LK_ SD_ SR_ SZ_ SE_ CH_ SY_ TW_ TJ_ TZ_ TH_ TG_ TO_ TT_ TU TN_ TR_ TM_ UG_ UA_ AE_ UN_ UY_ UZ_ VU_ VE_ VN_ VI_ YE_ CA_YT ZM_ ZW_ EC_ EG_ FL GA GQ_ HI HR_ IA ID IL IO_ IN KS KH_ KY KM_ LA MA MD ME MI MN MO MS MT NC ND NE NH NJ NM NV NY OH OK OR PA RI SC SV_ SD TD_ TN TX UT VA VT VG_ WA WI WV WY""" networks = ["AWOS"] metadata_list = {} if lat_range is not None and lon_range is not None: lon_min, lon_max = lon_range lat_min, lat_max = lat_range for region in regions.split(): networks.append("%s_ASOS" % (region, )) site_list = [] for network in networks: # Get metadata uri = ("https://mesonet.agron.iastate.edu/" "geojson/network/%s.geojson") % (network, ) data = urlopen(uri) jdict = json.load(data) for site in jdict["features"]: lat = site["geometry"]["coordinates"][1] lon = site["geometry"]["coordinates"][0] if lat >= lat_min and lat <= lat_max: if lon >= lon_min and lon <= lon_max: station_metadata_dict = {} station_metadata_dict["site_latitude"] = lat station_metadata_dict["site_longitude"] = lat for my_keys in site["properties"]: station_metadata_dict[my_keys] = site[ "properties"][my_keys] metadata_list[site["properties"] ["sid"]] = station_metadata_dict site_list.append(site["properties"]["sid"]) elif station is not None: site_list = [station] for region in regions.split(): networks.append("%s_ASOS" % (region, )) for network in networks: # Get metadata uri = ("https://mesonet.agron.iastate.edu/" "geojson/network/%s.geojson") % (network, ) data = urlopen(uri) jdict = json.load(data) for site in jdict["features"]: lat = site["geometry"]["coordinates"][1] lon = site["geometry"]["coordinates"][0] if site["properties"]["sid"] == station: station_metadata_dict = {} station_metadata_dict["site_latitude"] = lat station_metadata_dict["site_longitude"] = lon for my_keys in site["properties"]: if my_keys == "elevation": station_metadata_dict["elevation"] = \ '%f meter' % site["properties"][my_keys] else: station_metadata_dict[my_keys] = \ site["properties"][my_keys] metadata_list[station] = station_metadata_dict # Get station metadata else: raise ValueError( "Either both lat_range and lon_range or station must " + "be specified!") # Get the timestamp for each request start_time = time_window[0] end_time = time_window[1] SERVICE = "http://mesonet.agron.iastate.edu/cgi-bin/request/asos.py?" service = SERVICE + "data=all&tz=Etc/UTC&format=comma&latlon=yes&" service += start_time.strftime( "year1=%Y&month1=%m&day1=%d&hour1=%H&minute1=%M&") service += end_time.strftime( "year2=%Y&month2=%m&day2=%d&hour2=%H&minute2=%M") station_obs = {} for stations in site_list: uri = "%s&station=%s" % (service, stations) print("Downloading: %s" % (stations, )) data = _download_data(uri) buf = StringIO() buf.write(data) buf.seek(0) my_df = pd.read_csv(buf, skiprows=5, na_values="M") if len(my_df['lat'].values) == 0: warnings.warn( "No data available at station %s between time %s and %s" % (stations, start_time.strftime('%Y-%m-%d %H:%M:%S'), end_time.strftime('%Y-%m-%d %H:%M:%S'))) else: def to_datetime(x): return datetime.strptime(x, "%Y-%m-%d %H:%M") my_df["time"] = my_df["valid"].apply(to_datetime) my_df = my_df.set_index("time") my_df = my_df.drop("valid", axis=1) my_df = my_df.drop("station", axis=1) my_df = my_df.to_xarray() my_df.attrs = metadata_list[stations] my_df["lon"].attrs["units"] = "degree" my_df["lon"].attrs["long_name"] = "Longitude" my_df["lat"].attrs["units"] = "degree" my_df["lat"].attrs["long_name"] = "Latitude" my_df["tmpf"].attrs["units"] = "degrees Fahrenheit" my_df["tmpf"].attrs[ "long_name"] = "Temperature in degrees Fahrenheit" # Fahrenheit to Celsius my_df["temp"] = (5. / 9. * my_df["tmpf"]) - 32.0 my_df["temp"].attrs["units"] = "degrees Celsius" my_df["temp"].attrs["long_name"] = "Temperature in degrees Celsius" my_df["dwpf"].attrs["units"] = "degrees Fahrenheit" my_df["dwpf"].attrs[ "long_name"] = "Dewpoint temperature in degrees Fahrenheit" # Fahrenheit to Celsius my_df["dwpc"] = (5. / 9. * my_df["tmpf"]) - 32.0 my_df["dwpc"].attrs["units"] = "degrees Celsius" my_df["dwpc"].attrs[ "long_name"] = "Dewpoint temperature in degrees Celsius" my_df["relh"].attrs["units"] = "percent" my_df["relh"].attrs["long_name"] = "Relative humidity" my_df["drct"].attrs["units"] = "degrees" my_df["drct"].attrs["long_name"] = "Wind speed in degrees" my_df["sknt"].attrs["units"] = "knots" my_df["sknt"].attrs["long_name"] = "Wind speed in knots" my_df["spdms"] = my_df["sknt"] * 0.514444 my_df["spdms"].attrs["units"] = "m s-1" my_df["spdms"].attrs[ "long_name"] = "Wind speed in meters per second" my_df['u'] = -np.sin(np.deg2rad(my_df["drct"])) * my_df["spdms"] my_df['u'].attrs["units"] = "m s-1" my_df['u'].attrs["long_name"] = "Zonal component of surface wind" my_df['v'] = -np.cos(np.deg2rad(my_df["drct"])) * my_df["spdms"] my_df['v'].attrs["units"] = "m s-1" my_df['v'].attrs[ "long_name"] = "Meridional component of surface wind" my_df["mslp"].attrs["units"] = "mb" my_df["mslp"].attrs["long_name"] = "Mean Sea Level Pressure" my_df["alti"].attrs["units"] = "in Hg" my_df["alti"].attrs[ "long_name"] = "Atmospheric pressure in inches of Mercury" my_df["vsby"].attrs["units"] = "mi" my_df["vsby"].attrs["long_name"] = "Visibility" my_df["vsbykm"] = my_df["vsby"] * 1.60934 my_df["vsbykm"].attrs["units"] = 'km' my_df["vsbykm"].attrs["long_name"] = "Visibility" my_df["gust"] = my_df["gust"] * 0.514444 my_df["gust"].attrs["units"] = 'm s-1' my_df["gust"].attrs["long_name"] = "Wind gust speed" my_df["skyc1"].attrs["long_name"] = "Sky level 1 coverage" my_df["skyc2"].attrs["long_name"] = "Sky level 2 coverage" my_df["skyc3"].attrs["long_name"] = "Sky level 3 coverage" my_df["skyc4"].attrs["long_name"] = "Sky level 4 coverage" my_df["skyl1"] = my_df["skyl1"] * 0.3048 my_df["skyl2"] = my_df["skyl2"] * 0.3048 my_df["skyl3"] = my_df["skyl3"] * 0.3048 my_df["skyl4"] = my_df["skyl4"] * 0.3048 my_df["skyl1"].attrs["long_name"] = "Sky level 1 altitude" my_df["skyl2"].attrs["long_name"] = "Sky level 2 altitude" my_df["skyl3"].attrs["long_name"] = "Sky level 3 altitude" my_df["skyl4"].attrs["long_name"] = "Sky level 4 altitude" my_df["skyl1"].attrs["long_name"] = "meter" my_df["skyl2"].attrs["long_name"] = "meter" my_df["skyl3"].attrs["long_name"] = "meter" my_df["skyl4"].attrs["long_name"] = "meter" my_df["wxcodes"].attrs["long_name"] = "Weather code" my_df["ice_accretion_1hr"] = my_df["ice_accretion_1hr"] * 2.54 my_df["ice_accretion_1hr"].attrs["units"] = "cm" my_df["ice_accretion_1hr"].attrs[ "long_name"] = "1 hour ice accretion" my_df["ice_accretion_3hr"] = my_df["ice_accretion_3hr"] * 2.54 my_df["ice_accretion_3hr"].attrs["units"] = "cm" my_df["ice_accretion_3hr"].attrs[ "long_name"] = "3 hour ice accretion" my_df["ice_accretion_6hr"] = my_df["ice_accretion_3hr"] * 2.54 my_df["ice_accretion_6hr"].attrs["units"] = "cm" my_df["ice_accretion_6hr"].attrs[ "long_name"] = "6 hour ice accretion" my_df["peak_wind_gust"] = my_df["peak_wind_gust"] * 0.514444 my_df["peak_wind_gust"].attrs["units"] = 'm s-1' my_df["peak_wind_gust"].attrs["long_name"] = "Peak wind gust speed" my_df["peak_wind_drct"].attrs["drct"] = 'degree' my_df["peak_wind_drct"].attrs[ "long_name"] = "Peak wind gust direction" my_df['u_peak'] = -np.sin(np.deg2rad( my_df["peak_wind_drct"])) * my_df["peak_wind_gust"] my_df['u_peak'].attrs["units"] = "m s-1" my_df['u_peak'].attrs[ "long_name"] = "Zonal component of surface wind" my_df['v_peak'] = -np.cos(np.deg2rad( my_df["peak_wind_drct"])) * my_df["peak_wind_gust"] my_df['v_peak'].attrs["units"] = "m s-1" my_df['v_peak'].attrs[ "long_name"] = "Meridional component of surface wind" my_df["metar"].attrs["long_name"] = "Raw METAR code" my_df.attrs['_datastream'] = stations buf.close() station_obs[stations] = my_df return station_obs
class log_output(object): """Context manager that logs its output to a file. In the simplest case, the usage looks like this:: with log_output('logfile.txt'): # do things ... output will be logged Any output from the with block will be redirected to ``logfile.txt``. If you also want the output to be echoed to ``stdout``, use the ``echo`` parameter:: with log_output('logfile.txt', echo=True): # do things ... output will be logged and printed out And, if you just want to echo *some* stuff from the parent, use ``force_echo``:: with log_output('logfile.txt', echo=False) as logger: # do things ... output will be logged with logger.force_echo(): # things here will be echoed *and* logged Under the hood, we spawn a daemon and set up a pipe between this process and the daemon. The daemon writes our output to both the file and to stdout (if echoing). The parent process can communicate with the daemon to tell it when and when not to echo; this is what force_echo does. You can also enable/disable echoing by typing 'v'. We try to use OS-level file descriptors to do the redirection, but if stdout or stderr has been set to some Python-level file object, we use Python-level redirection instead. This allows the redirection to work within test frameworks like nose and pytest. """ def __init__(self, file_like=None, echo=False, debug=False, buffer=False): """Create a new output log context manager. Args: file_like (str or stream): open file object or name of file where output should be logged echo (bool): whether to echo output in addition to logging it debug (bool): whether to enable tty debug mode during logging buffer (bool): pass buffer=True to skip unbuffering output; note this doesn't set up any *new* buffering log_output can take either a file object or a filename. If a filename is passed, the file will be opened and closed entirely within ``__enter__`` and ``__exit__``. If a file object is passed, this assumes the caller owns it and will close it. By default, we unbuffer sys.stdout and sys.stderr because the logger will include output from executed programs and from python calls. If stdout and stderr are buffered, their output won't be printed in the right place w.r.t. output from commands. Logger daemon is not started until ``__enter__()``. """ self.file_like = file_like self.echo = echo self.debug = debug self.buffer = buffer self._active = False # used to prevent re-entry def __call__(self, file_like=None, echo=None, debug=None, buffer=None): """Thie behaves the same as init. It allows a logger to be reused. Arguments are the same as for ``__init__()``. Args here take precedence over those passed to ``__init__()``. With the ``__call__`` function, you can save state between uses of a single logger. This is useful if you want to remember, e.g., the echo settings for a prior ``with log_output()``:: logger = log_output() with logger('foo.txt'): # log things; user can change echo settings with 'v' with logger('bar.txt'): # log things; logger remembers prior echo settings. """ if file_like is not None: self.file_like = file_like if echo is not None: self.echo = echo if debug is not None: self.debug = debug if buffer is not None: self.buffer = buffer return self def __enter__(self): if self._active: raise RuntimeError("Can't re-enter the same log_output!") if self.file_like is None: raise RuntimeError( "file argument must be set by either __init__ or __call__") # set up a stream for the daemon to write to self.close_log_in_parent = True self.write_log_in_parent = False if isinstance(self.file_like, string_types): self.log_file = open(self.file_like, 'w') elif _file_descriptors_work(self.file_like): self.log_file = self.file_like self.close_log_in_parent = False else: self.log_file = StringIO() self.write_log_in_parent = True # record parent color settings before redirecting. We do this # because color output depends on whether the *original* stdout # is a TTY. New stdout won't be a TTY so we force colorization. self._saved_color = tty.color._force_color forced_color = tty.color.get_color_when() # also record parent debug settings -- in case the logger is # forcing debug output. self._saved_debug = tty._debug # OS-level pipe for redirecting output to logger self.read_fd, self.write_fd = os.pipe() # Multiprocessing pipe for communication back from the daemon # Currently only used to save echo value between uses self.parent, self.child = multiprocessing.Pipe() # Sets a daemon that writes to file what it reads from a pipe try: # need to pass this b/c multiprocessing closes stdin in child. try: input_stream = os.fdopen(os.dup(sys.stdin.fileno())) except BaseException: input_stream = None # just don't forward input if this fails self.process = multiprocessing.Process(target=self._writer_daemon, args=(input_stream, )) self.process.daemon = True # must set before start() self.process.start() os.close(self.read_fd) # close in the parent process finally: if input_stream: input_stream.close() # Flush immediately before redirecting so that anything buffered # goes to the original stream sys.stdout.flush() sys.stderr.flush() # Now do the actual output rediction. self.use_fds = _file_descriptors_work(sys.stdout, sys.stderr) if self.use_fds: # We try first to use OS-level file descriptors, as this # redirects output for subprocesses and system calls. # Save old stdout and stderr file descriptors self._saved_stdout = os.dup(sys.stdout.fileno()) self._saved_stderr = os.dup(sys.stderr.fileno()) # redirect to the pipe we created above os.dup2(self.write_fd, sys.stdout.fileno()) os.dup2(self.write_fd, sys.stderr.fileno()) os.close(self.write_fd) else: # Handle I/O the Python way. This won't redirect lower-level # output, but it's the best we can do, and the caller # shouldn't expect any better, since *they* have apparently # redirected I/O the Python way. # Save old stdout and stderr file objects self._saved_stdout = sys.stdout self._saved_stderr = sys.stderr # create a file object for the pipe; redirect to it. pipe_fd_out = os.fdopen(self.write_fd, 'w') sys.stdout = pipe_fd_out sys.stderr = pipe_fd_out # Unbuffer stdout and stderr at the Python level if not self.buffer: sys.stdout = Unbuffered(sys.stdout) sys.stderr = Unbuffered(sys.stderr) # Force color and debug settings now that we have redirected. tty.color.set_color_when(forced_color) tty._debug = self.debug # track whether we're currently inside this log_output self._active = True # return this log_output object so that the user can do things # like temporarily echo some ouptut. return self def __exit__(self, exc_type, exc_val, exc_tb): # Flush any buffered output to the logger daemon. sys.stdout.flush() sys.stderr.flush() # restore previous output settings, either the low-level way or # the python way if self.use_fds: os.dup2(self._saved_stdout, sys.stdout.fileno()) os.close(self._saved_stdout) os.dup2(self._saved_stderr, sys.stderr.fileno()) os.close(self._saved_stderr) else: sys.stdout = self._saved_stdout sys.stderr = self._saved_stderr # print log contents in parent if needed. if self.write_log_in_parent: string = self.parent.recv() self.file_like.write(string) if self.close_log_in_parent: self.log_file.close() # recover and store echo settings from the child before it dies self.echo = self.parent.recv() # join the daemon process. The daemon will quit automatically # when the write pipe is closed; we just wait for it here. self.process.join() # restore old color and debug settings tty.color._force_color = self._saved_color tty._debug = self._saved_debug self._active = False # safe to enter again @contextmanager def force_echo(self): """Context manager to force local echo, even if echo is off.""" if not self._active: raise RuntimeError( "Can't call force_echo() outside log_output region!") # This uses the xon/xoff to highlight regions to be echoed in the # output. We us these control characters rather than, say, a # separate pipe, because they're in-band and assured to appear # exactly before and after the text we want to echo. sys.stdout.write(xon) sys.stdout.flush() yield sys.stdout.write(xoff) sys.stdout.flush() def _writer_daemon(self, stdin): """Daemon that writes output to the log file and stdout.""" # Use line buffering (3rd param = 1) since Python 3 has a bug # that prevents unbuffered text I/O. in_pipe = os.fdopen(self.read_fd, 'r', 1) os.close(self.write_fd) echo = self.echo # initial echo setting, user-controllable force_echo = False # parent can force echo for certain output # list of streams to select from istreams = [in_pipe, stdin] if stdin else [in_pipe] log_file = self.log_file try: with keyboard_input(stdin): while True: # No need to set any timeout for select.select # Wait until a key press or an event on in_pipe. rlist, _, _ = select.select(istreams, [], []) # Allow user to toggle echo with 'v' key. # Currently ignores other chars. if stdin in rlist: if stdin.read(1) == 'v': echo = not echo # Handle output from the with block process. if in_pipe in rlist: # If we arrive here it means that in_pipe was # ready for reading : it should never happen that # line is false-ish line = in_pipe.readline() if not line: break # EOF # find control characters and strip them. controls = control.findall(line) line = re.sub(control, '', line) # Echo to stdout if requested or forced if echo or force_echo: sys.stdout.write(line) sys.stdout.flush() # Stripped output to log file. log_file.write(_strip(line)) log_file.flush() if xon in controls: force_echo = True if xoff in controls: force_echo = False except BaseException: tty.error("Exception occurred in writer daemon!") traceback.print_exc() finally: # send written data back to parent if we used a StringIO if self.write_log_in_parent: self.child.send(log_file.getvalue()) log_file.close() # send echo value back to the parent so it can be preserved. self.child.send(echo)
class TestStorageValidators(unittest.TestCase): def setUp(self): self.io = StringIO() def tearDown(self): self.io.close() def test_permission_validator(self): ContainerPermissions = get_sdk(ResourceType.DATA_STORAGE, 'blob.models#ContainerPermissions') ns1 = Namespace(permission='rwdl') ns2 = Namespace(permission='abc') get_permission_validator(ContainerPermissions)(ns1) self.assertTrue(isinstance(ns1.permission, ContainerPermissions)) with self.assertRaises(ValueError): get_permission_validator(ContainerPermissions)(ns2) def test_datetime_string_type(self): input = "2017-01-01T12:30Z" actual = get_datetime_type(True)(input) expected = "2017-01-01T12:30Z" self.assertEqual(actual, expected) input = "2017-01-01 12:30" with self.assertRaises(ValueError): actual = get_datetime_type(True)(input) def test_datetime_type(self): input = "2017-01-01T12:30Z" actual = get_datetime_type(False)(input) expected = datetime(2017, 1, 1, 12, 30, 0) self.assertEqual(actual, expected) input = "2017-01-01 12:30" with self.assertRaises(ValueError): actual = get_datetime_type(False)(input) def test_ipv4_range_type(self): input = "111.22.3.111" actual = ipv4_range_type(input) expected = input self.assertEqual(actual, expected) input = "111.22.3.111-222.11.44.111" actual = ipv4_range_type(input) expected = input self.assertEqual(actual, expected) input = "111.22" with self.assertRaises(ValueError): actual = ipv4_range_type(input) input = "111.22.33.44-" with self.assertRaises(ValueError): actual = ipv4_range_type(input) def test_resource_types_type(self): input = "sso" actual = str(resource_type_type(input)) expected = "so" self.assertEqual(actual, expected) input = "blob" with self.assertRaises(ValueError): actual = resource_type_type(input) def test_services_type(self): input = "ttfqbqtf" actual = str(services_type(input)) expected = "bqtf" self.assertEqual(actual, expected) input = "everything" with self.assertRaises(ValueError): actual = services_type(input) def test_storage_process_blob_source_uri_redundent_parameter(self): with self.assertRaises(ValueError): process_blob_source_uri( Namespace(copy_source='https://example.com', source_sas='some_sas')) with self.assertRaises(ValueError): process_blob_source_uri( Namespace(copy_source='https://example.com', source_account_name='account_name'))
class S3HandlerTestUpload(unittest.TestCase): """ This class tests the ability to upload objects into an S3 bucket as well as multipart uploads """ def setUp(self): self.session = botocore.session.get_session(EnvironmentVariables) self.service = self.session.get_service('s3') self.endpoint = self.service.get_endpoint('us-east-1') params = {'region': 'us-east-1', 'acl': ['private']} self.s3_handler = S3Handler(self.session, params) self.s3_handler_multi = S3Handler(self.session, multi_threshold=10, chunksize=2, params=params) self.bucket = create_bucket(self.session) self.loc_files = make_loc_files() self.s3_files = [ self.bucket + '/text1.txt', self.bucket + '/another_directory/text2.txt' ] self.output = StringIO() self.saved_stderr = sys.stderr sys.stderr = self.output def tearDown(self): self.output.close() sys.stderr = self.saved_stderr clean_loc_files(self.loc_files) s3_cleanup(self.bucket, self.session) def test_upload(self): # Confirm there are no objects in the bucket. self.assertEqual(len(list_contents(self.bucket, self.session)), 0) # Create file info objects to perform upload. files = [self.loc_files[0], self.loc_files[1]] tasks = [] for i in range(len(files)): tasks.append( FileInfo( src=self.loc_files[i], dest=self.s3_files[i], operation_name='upload', size=0, service=self.service, endpoint=self.endpoint, )) # Perform the upload. self.s3_handler.call(tasks) # Confirm the files were uploaded. self.assertEqual(len(list_contents(self.bucket, self.session)), 2) def test_multi_upload(self): files = [self.loc_files[0], self.loc_files[1]] tasks = [] for i in range(len(files)): tasks.append( FileInfo( src=self.loc_files[i], dest=self.s3_files[i], size=15, operation_name='upload', service=self.service, endpoint=self.endpoint, )) # Note nothing is uploaded because the file is too small # a print statement will show up if it fails. self.s3_handler_multi.call(tasks) print_op = "Error: Your proposed upload is smaller than the minimum" self.assertIn(print_op, self.output.getvalue())
class TestStorageValidators(unittest.TestCase): def setUp(self): self.io = StringIO() self.cli = MockCLI() self.loader = MockLoader(self.cli) def tearDown(self): self.io.close() def test_permission_validator(self): t_container_permissions = get_sdk(self.cli, CUSTOM_DATA_STORAGE, 'blob.models#ContainerPermissions') ns1 = Namespace(permission='rwdl') ns2 = Namespace(permission='abc') get_permission_validator(t_container_permissions)(ns1) self.assertTrue(isinstance(ns1.permission, t_container_permissions)) with self.assertRaises(ValueError): get_permission_validator(t_container_permissions)(ns2) def test_datetime_string_type(self): input = "2017-01-01T12:30Z" actual = get_datetime_type(True)(input) expected = "2017-01-01T12:30Z" self.assertEqual(actual, expected) input = "2017-01-01 12:30" with self.assertRaises(ValueError): get_datetime_type(True)(input) def test_datetime_type(self): import datetime input = "2017-01-01T12:30Z" actual = get_datetime_type(False)(input) expected = datetime.datetime(2017, 1, 1, 12, 30, 0) self.assertEqual(actual, expected) input = "2017-01-01 12:30" with self.assertRaises(ValueError): actual = get_datetime_type(False)(input) def test_ipv4_range_type(self): input = "111.22.3.111" actual = ipv4_range_type(input) expected = input self.assertEqual(actual, expected) input = "111.22.3.111-222.11.44.111" actual = ipv4_range_type(input) expected = input self.assertEqual(actual, expected) input = "111.22" with self.assertRaises(ValueError): actual = ipv4_range_type(input) input = "111.22.33.44-" with self.assertRaises(ValueError): actual = ipv4_range_type(input) def test_resource_types_type(self): input = "sso" actual = str(resource_type_type(self.loader)(input)) expected = "so" self.assertEqual(actual, expected) input = "blob" with self.assertRaises(ValueError): actual = resource_type_type(self.loader)(input) def test_services_type(self): input = "ttfqbqtf" actual = str(services_type(self.loader)(input)) if supported_api_version(self.cli, CUSTOM_DATA_STORAGE, max_api='2016-05-31') or \ supported_api_version(self.cli, CUSTOM_DATA_STORAGE, min_api='2017-07-29'): expected = "bqtf" else: expected = "bqf" self.assertEqual(actual, expected) input = "everything" with self.assertRaises(ValueError): services_type(self.loader)(input) def test_storage_process_blob_source_uri_redundent_parameter(self): with self.assertRaises(ValueError): process_blob_source_uri(MockCmd(self.cli), Namespace(copy_source='https://example.com', source_sas='some_sas')) with self.assertRaises(ValueError): process_blob_source_uri(MockCmd(self.cli), Namespace(copy_source='https://example.com', source_account_name='account_name')) def test_storage_get_char_options_validator(self): with self.assertRaises(ValueError) as cm: get_char_options_validator('abc', 'no_such_property')(object()) self.assertEqual('Missing options --no-such-property.', str(cm.exception)) ns = Namespace(services='bcd') with self.assertRaises(ValueError) as cm: get_char_options_validator('abc', 'services')(ns) self.assertEqual('--services: only valid values are: a, b, c.', str(cm.exception)) ns = Namespace(services='ab') get_char_options_validator('abc', 'services')(ns) result = getattr(ns, 'services') self.assertIs(type(result), set) self.assertEqual(result, set('ab'))
def main(): print_env_vars() time_str = datetime.utcnow().strftime('%Y%m%d%H%M%S') az(["login"], _out=sys.stdout, _err=sys.stdout) resource_group = 'azurecli-release-debian-' + time_str vm_name = 'vm-debian-' + time_str print_status('Creating resource group.') az(['group', 'create', '-l', 'westus', '-n', resource_group], _out=sys.stdout, _err=sys.stdout) print_status('Creating VM.') az([ 'vm', 'create', '-g', resource_group, '-n', vm_name, '--generate-ssh-keys', '--authentication-type', 'ssh', '--image', 'Canonical:UbuntuServer:14.04.4-LTS:latest', '--admin-username', 'ubuntu' ], _out=sys.stdout, _err=sys.stdout) io = StringIO() print_status('Getting VM IP address.') az([ 'vm', 'list-ip-addresses', '--resource-group', resource_group, '--name', vm_name, '--query', '[0].virtualMachine.network.publicIpAddresses[0].ipAddress' ], _out=io) ip_address = io.getvalue().strip().replace('"', '') print_status('VM IP address is {}'.format(ip_address)) io.close() vm_connect_str = "ubuntu@{}".format(ip_address) my_vm = ssh.bake(['-oStrictHostKeyChecking=no', vm_connect_str]) print_status('Installing git.') my_vm([ 'sudo', 'apt-get', 'update', '&&', 'sudo', 'apt-get', 'install', '-y', 'git' ], _out=sys.stdout, _err=sys.stdout) io = StringIO() my_vm(['mktemp', '-d'], _out=io) repo_dir = io.getvalue().strip() io.close() print_status('Cloning repo.') my_vm([ 'git', 'clone', 'https://github.com/{}'.format( script_env.get('REPO_NAME')), repo_dir ], _out=sys.stdout, _err=sys.stdout) path_to_debian_build_script = os.path.join(repo_dir, 'packaged_releases', 'debian', 'debian_build.sh') path_to_dir_creator = os.path.join(repo_dir, 'packaged_releases', 'debian', 'debian_dir_creator.sh') io = StringIO() my_vm(['mktemp', '-d'], _out=io) build_artifact_dir = io.getvalue().strip() io.close() print_status('Running debian build scripts.') my_vm(['chmod', '+x', path_to_debian_build_script, path_to_dir_creator], _out=sys.stdout, _err=sys.stdout) my_vm([ 'export', 'CLI_VERSION={}'.format(script_env.get('CLI_VERSION')), '&&', 'export', 'CLI_DOWNLOAD_SHA256={}'.format( script_env.get('CLI_DOWNLOAD_SHA256')), '&&', 'export', 'BUILD_ARTIFACT_DIR={}'.format(build_artifact_dir), '&&', path_to_debian_build_script, path_to_dir_creator ], _out=sys.stdout, _err=sys.stdout) print_status('Debian build complete.') io = StringIO() my_vm(['ls', build_artifact_dir], _out=io) artifact_name = io.getvalue().strip() io.close() deb_file_path = os.path.join(build_artifact_dir, artifact_name) print_status('Installing the .deb on the build machine') my_vm(['sudo', 'dpkg', '-i', deb_file_path], _out=sys.stdout, _err=sys.stdout) # Upload to Azure Storage print_status('Uploading .deb to Azure storage.') my_vm([ 'az', 'storage', 'container', 'create', '--name', 'repos', '--public-access', 'blob', '--connection-string', '"{}"'.format( script_env.get('AZURE_STORAGE_CONNECTION_STRING')) ], _out=sys.stdout, _err=sys.stdout) my_vm([ 'az', 'storage', 'blob', 'upload', '-f', deb_file_path, '-n', artifact_name, '-c', 'repos', '--connection-string', '"{}"'.format( script_env.get('AZURE_STORAGE_CONNECTION_STRING')) ], _out=sys.stdout, _err=sys.stdout) io = StringIO() my_vm([ 'az', 'storage', 'blob', 'url', '-n', artifact_name, '-c', 'repos', '--output', 'tsv', '--connection-string', '"{}"'.format( script_env.get('AZURE_STORAGE_CONNECTION_STRING')) ], _out=io) deb_url = io.getvalue().strip() io.close() print_status('Debian file uploaded to the following URL.') print_status(deb_url) # Publish to apt service my_vm(['wget', '-q', 'https://bootstrap.pypa.io/get-pip.py'], _out=sys.stdout, _err=sys.stdout) my_vm(['sudo', 'python', 'get-pip.py'], _out=sys.stdout, _err=sys.stdout) my_vm(['sudo', 'pip', 'install', '--upgrade', 'requests'], _out=sys.stdout, _err=sys.stdout) upload_script = REPO_UPLOAD_SCRIPT_TMPL.format( cli_version=script_env.get('CLI_VERSION'), repo_id=script_env.get('DEBIAN_REPO_ID'), source_url=deb_url, repo_package_url=script_env.get('DEBIAN_REPO_URL'), repo_user=script_env.get('DEBIAN_REPO_USERNAME'), repo_pass=script_env.get('DEBIAN_REPO_PASSWORD')) my_vm(['echo', '-e', '"{}"'.format(upload_script), '>>', 'repo_upload.py'], _out=sys.stdout, _err=sys.stdout) my_vm(['python', 'repo_upload.py'], _out=sys.stdout, _err=sys.stdout) print_status('Done. :)') give_chance_to_cancel('Delete resource group (in background)') az(['group', 'delete', '--name', resource_group, '--yes', '--no-wait'], _out=sys.stdout, _err=sys.stdout) print_status('Finished. :)')
class TestApplication(unittest.TestCase): @classmethod def setUpClass(cls): pass @classmethod def tearDownClass(cls): pass def setUp(self): self.io = StringIO() def tearDown(self): self.io.close() def test_application_register_and_call_handlers(self): handler_called = [False] def handler(**kwargs): kwargs['args'][0] = True def other_handler(**kwargs): # pylint: disable=unused-variable self.assertEqual(kwargs['args'], 'secret sauce') config = Configuration([]) app = Application(config) app.raise_event('was_handler_called', args=handler_called) self.assertFalse(handler_called[0], "Raising event with no handlers registered somehow failed...") app.register('was_handler_called', handler) self.assertFalse(handler_called[0]) # Registered handler won't get called if event with different name # is raised... app.raise_event('other_handler_called', args=handler_called) self.assertFalse(handler_called[0], 'Wrong handler called!') app.raise_event('was_handler_called', args=handler_called) self.assertTrue(handler_called[0], "Handler didn't get called") app.raise_event('other_handler_called', args='secret sauce') def test_list_value_parameter(self): hellos = [] def handler(args): hellos.append(args) command = CliCommand('test command', handler) command.add_argument('hello', '--hello', nargs='+', action=IterateAction) command.add_argument('something', '--something') cmd_table = {'test command': command} argv = 'az test command --hello world sir --something else'.split() config = Configuration(argv) config.get_command_table = lambda: cmd_table application = Application(config) application.execute(argv[1:]) self.assertEqual(2, len(hellos)) self.assertEqual(hellos[0]['hello'], 'world') self.assertEqual(hellos[0]['something'], 'else') self.assertEqual(hellos[1]['hello'], 'sir') self.assertEqual(hellos[1]['something'], 'else')
def make_html_slider(drawing, keyframes, obs_div, other, div_timeseries, visualize_controls): nkeyframes = len(keyframes.timestamps) # language=html controls_html = """\ <div id="slidecontainer"> <div id='fixedui'> Select time: <input autofocus type="range" min="0" max="%s" value="0" class="slider" id="time-range" onchange="showVal(this.value)" oninput="showVal(this.value)"/> <span id="time-display"></span> </div> </div> <style type='text/css'> #slidecontainer { height: 3em; } #time-range { width: 50%%; } #fixedui { position: fixed; width: 100%%; height: 3em; background-color: white; } .keyframe[visualize="hide"] { display: none; } .keyframe[visualize="show"] { display: inherit; } td#obs { padding: 1em; vertical-align: top; } #observation_sequence { width: 320px; } td#obs img { width: 90%%;} </style> <script type='text/javascript'> function showVal(newVal) { elements = document.querySelectorAll('.keyframe'); elements.forEach(_ => _.setAttribute('visualize', 'hide')); elements_show = document.querySelectorAll('.keyframe' + newVal ); elements_show.forEach(_ => _.setAttribute('visualize', 'show')); } document.addEventListener("DOMContentLoaded", function(event) { showVal(0); }); </script> """ % (nkeyframes - 1) if nkeyframes <= 1: controls_html += """ <style> .slidecontainer { display: none; } </style> """ controls = bs(controls_html) valbox = controls.find("span", id="time-display") assert valbox is not None for i, timestamp in keyframes: t = Tag(name="span") t.attrs["class"] = "keyframe keyframe%d" % i t.attrs["visualize"] = "hide" t.append("t = %.2f" % timestamp) valbox.append(t) from six import StringIO f = StringIO() drawing.write(f, pretty=True) drawing_svg = f.getvalue() f.close() # drawing_svg = drawing.tostring(pretty=True) # language=html doc = """\ <html lang='en'> <head></head> <body> <style> /*svg {{ background-color: #eee;}}*/ body {{ font-family: system-ui, sans-serif; }} </style> {controls} <table> <tr> <td style="width: 640px; vertical-align:top;"> {drawing} </td> <td id="obs" > {visualize_controls} <div id="observation_sequence"> {obs_div} </div> </td> </tr> </table> <script type='text/javascript'> showVal(0); </script> {div_timeseries} {other} </body> </html> """.format( controls=str(controls), drawing=drawing_svg, obs_div=obs_div, other=other, div_timeseries=div_timeseries, visualize_controls=visualize_controls, ) return doc
class TestParser(unittest.TestCase): def setUp(self): self.io = StringIO() def tearDown(self): self.io.close() def test_register_simple_commands(self): def test_handler1(): pass def test_handler2(): pass cli = DummyCli() cli.loader = mock.MagicMock() cli.loader.cli_ctx = cli command = AzCliCommand(cli.loader, 'command the-name', test_handler1) command2 = AzCliCommand(cli.loader, 'sub-command the-second-name', test_handler2) cmd_table = {'command the-name': command, 'sub-command the-second-name': command2} cli.commands_loader.command_table = cmd_table parser = AzCliCommandParser(cli) parser.load_command_table(cli.commands_loader) args = parser.parse_args('command the-name'.split()) self.assertIs(args.func, command) args = parser.parse_args('sub-command the-second-name'.split()) self.assertIs(args.func, command2) AzCliCommandParser.error = VerifyError(self,) parser.parse_args('sub-command'.split()) self.assertTrue(AzCliCommandParser.error.called) def test_required_parameter(self): def test_handler(args): # pylint: disable=unused-argument pass cli = DummyCli() cli.loader = mock.MagicMock() cli.loader.cli_ctx = cli command = AzCliCommand(cli.loader, 'test command', test_handler) command.add_argument('req', '--req', required=True) cmd_table = {'test command': command} cli.commands_loader.command_table = cmd_table parser = AzCliCommandParser(cli) parser.load_command_table(cli.commands_loader) args = parser.parse_args('test command --req yep'.split()) self.assertIs(args.func, command) AzCliCommandParser.error = VerifyError(self) parser.parse_args('test command'.split()) self.assertTrue(AzCliCommandParser.error.called) def test_nargs_parameter(self): def test_handler(): pass cli = DummyCli() cli.loader = mock.MagicMock() cli.loader.cli_ctx = cli command = AzCliCommand(cli.loader, 'test command', test_handler) command.add_argument('req', '--req', required=True, nargs=2) cmd_table = {'test command': command} cli.commands_loader.command_table = cmd_table parser = AzCliCommandParser(cli) parser.load_command_table(cli.commands_loader) args = parser.parse_args('test command --req yep nope'.split()) self.assertIs(args.func, command) AzCliCommandParser.error = VerifyError(self) parser.parse_args('test command -req yep'.split()) self.assertTrue(AzCliCommandParser.error.called) def test_case_insensitive_enum_choices(self): from enum import Enum class TestEnum(Enum): # pylint: disable=too-few-public-methods opt1 = "ALL_CAPS" opt2 = "camelCase" opt3 = "snake_case" def test_handler(): pass cli = DummyCli() cli.loader = mock.MagicMock() cli.loader.cli_ctx = cli command = AzCliCommand(cli.loader, 'test command', test_handler) command.add_argument('opt', '--opt', required=True, **enum_choice_list(TestEnum)) cmd_table = {'test command': command} cli.commands_loader.command_table = cmd_table parser = AzCliCommandParser(cli) parser.load_command_table(cli.commands_loader) args = parser.parse_args('test command --opt alL_cAps'.split()) self.assertEqual(args.opt, 'ALL_CAPS') args = parser.parse_args('test command --opt CAMELCASE'.split()) self.assertEqual(args.opt, 'camelCase') args = parser.parse_args('test command --opt sNake_CASE'.split()) self.assertEqual(args.opt, 'snake_case') def _mock_import_lib(_): mock_obj = mock.MagicMock() mock_obj.__path__ = __name__ return mock_obj def _mock_iter_modules(_): return [(None, __name__, None)] def _mock_extension_modname(ext_name, ext_dir): return ext_name def _mock_get_extensions(): MockExtension = namedtuple('Extension', ['name', 'preview', 'experimental', 'path', 'get_metadata']) return [MockExtension(name=__name__ + '.ExtCommandsLoader', preview=False, experimental=False, path=None, get_metadata=lambda: {}), MockExtension(name=__name__ + '.Ext2CommandsLoader', preview=False, experimental=False, path=None, get_metadata=lambda: {})] def _mock_load_command_loader(loader, args, name, prefix): from enum import Enum class TestEnum(Enum): # pylint: disable=too-few-public-methods enum_1 = 'enum_1' enum_2 = 'enum_2' def test_handler(): pass class TestCommandsLoader(AzCommandsLoader): def load_command_table(self, args): super(TestCommandsLoader, self).load_command_table(args) command = AzCliCommand(loader, 'test module', test_handler) command.add_argument('opt', '--opt', required=True, **enum_choice_list(TestEnum)) self.command_table['test module'] = command return self.command_table # A command from an extension class ExtCommandsLoader(AzCommandsLoader): def load_command_table(self, args): super(ExtCommandsLoader, self).load_command_table(args) command = AzCliCommand(loader, 'test extension', test_handler) command.add_argument('opt', '--opt', required=True, **enum_choice_list(TestEnum)) self.command_table['test extension'] = command return self.command_table if prefix == 'azure.cli.command_modules.': command_loaders = {'TestCommandsLoader': TestCommandsLoader} else: command_loaders = {'ExtCommandsLoader': ExtCommandsLoader} module_command_table = {} for _, loader_cls in command_loaders.items(): command_loader = loader_cls(cli_ctx=loader.cli_ctx) command_table = command_loader.load_command_table(args) if command_table: module_command_table.update(command_table) loader.loaders.append(command_loader) # this will be used later by the load_arguments method return module_command_table, command_loader.command_group_table @mock.patch('importlib.import_module', _mock_import_lib) @mock.patch('pkgutil.iter_modules', _mock_iter_modules) @mock.patch('azure.cli.core.commands._load_command_loader', _mock_load_command_loader) @mock.patch('azure.cli.core.extension.get_extension_modname', _mock_extension_modname) @mock.patch('azure.cli.core.extension.get_extensions', _mock_get_extensions) def test_parser_error_spellchecker(self): cli = DummyCli() main_loader = MainCommandsLoader(cli) cli.loader = main_loader cli.loader.load_command_table(None) parser = cli.parser_cls(cli) parser.load_command_table(cli.loader) logger_msgs = [] choice_lists = [] original_get_close_matches = difflib.get_close_matches def mock_log_error(_, msg): logger_msgs.append(msg) def mock_get_close_matches(*args, **kwargs): choice_lists.append(original_get_close_matches(*args, **kwargs)) # run multiple faulty commands and save error logs, as well as close matches with mock.patch('logging.Logger.error', mock_log_error), \ mock.patch('difflib.get_close_matches', mock_get_close_matches): faulty_cmd_args = [ 'test module1 --opt enum_1', 'test extension1 --opt enum_1', 'test foo_bar --opt enum_3', 'test module --opt enum_3', 'test extension --opt enum_3' ] for text in faulty_cmd_args: with self.assertRaises(SystemExit): parser.parse_args(text.split()) parser.parse_args('test module --opt enum_1'.split()) # assert the right type of error msg is logged for command vs argument parsing self.assertEqual(len(logger_msgs), 5) for msg in logger_msgs[:3]: self.assertIn("not in the", msg) self.assertIn("command group", msg) for msg in logger_msgs[3:]: self.assertIn("not a valid value for '--opt'.", msg) # assert the right choices are matched as "close". # If these don't hold, matching algorithm should be deemed flawed. for choices in choice_lists[:2]: self.assertEqual(len(choices), 1) self.assertEqual(len(choice_lists[2]), 0) for choices in choice_lists[3:]: self.assertEqual(len(choices), 2) for choice in ['enum_1', 'enum_2']: self.assertIn(choice, choices)
def _mol_writer(data, fmt='sdf', filepath_or_buffer=None, update_properties=True, molecule_column=None, columns=None): """Universal writing function for private use. .. versionadded:: 0.3 Parameters ---------- fmt : string The format of molecular file filepath_or_buffer : string or None File path update_properties : bool, optional (default=True) Switch to update properties from the DataFrames to the molecules while writting. molecule_column : string or None, optional (default='mol') Name of molecule column. If None the molecules will be skipped. columns : list or None, optional (default=None) A list of columns to write to file. If None then all available fields are written. """ if filepath_or_buffer is None: out = StringIO() elif hasattr(filepath_or_buffer, 'write'): out = filepath_or_buffer else: out = oddt.toolkit.Outputfile(fmt, filepath_or_buffer, overwrite=True) if isinstance(data, pd.DataFrame): molecule_column = molecule_column or data._molecule_column for ix, row in data.iterrows(): mol = row[molecule_column].clone if update_properties: new_data = row.to_dict() del new_data[molecule_column] mol.data.update(new_data) if columns: for k in mol.data.keys(): if k not in columns: del mol.data[k] if filepath_or_buffer is None or hasattr(filepath_or_buffer, 'write'): out.write(mol.write(fmt)) else: out.write(mol) elif isinstance(data, pd.Series): for mol in data: if filepath_or_buffer is None or hasattr(filepath_or_buffer, 'write'): out.write(mol.write(fmt)) else: out.write(mol) if filepath_or_buffer is None: return out.getvalue() elif not hasattr(filepath_or_buffer, 'write'): # dont close foreign buffer out.close()
class Test_resource_validators(unittest.TestCase): def setUp(self): self.io = StringIO() def tearDown(self): self.io.close() def test_validate_lock_params(self): valid = [{ 'test': 'just name', 'name': 'foo' }, { 'test': 'name and group', 'name': 'foo', 'resource_group_name': 'foo', }, { 'test': 'name, group, type [compact]', 'name': 'foo', 'resource_group_name': 'bar', 'resource_name': 'baz', 'resource_type': 'Microsoft.Compute/VirtualMachines' }, { 'test': 'name, group, type, namespace', 'name': 'foo', 'resource_group_name': 'bar', 'resource_name': 'baz', 'resource_type': 'VirtualMachines', 'resource_provider_namespace': 'Microsoft.Compute', }, { 'test': 'name, group, type, namespace, parent', 'name': 'foo', 'resource_group_name': 'bar', 'resource_name': 'baz', 'resource_type': 'VirtualMachines', 'resource_provider_namespace': 'Microsoft.Compute', 'parent_resource_path': 'Foo.Bar/baz', }] for valid_namespace in valid: namespace_obj = NamespaceObject() for key in valid_namespace: setattr(namespace_obj, key, valid_namespace[key]) try: # If unexpected invalid, this throws, so no need for asserts validate_lock_parameters(namespace_obj) except CLIError as ex: self.fail('Test {} failed. {}'.format(valid_namespace['test'], ex)) def test_validate_lock_params_invalid(self): invalid = [{ 'test': 'just name and type', 'name': 'foo', 'resource_type': 'baz' }, { 'test': 'name and group and type', 'name': 'foo', 'resource_group_name': 'foo', 'resource_type': 'bar', }, { 'test': 'name, group, no type', 'name': 'foo', 'resource_group_name': 'bar', 'resource_name': 'baz', }, { 'test': 'name, group, type, namespace', 'name': 'foo', 'resource_group_name': 'bar', 'resource_name': 'baz', 'resource_provider_namespace': 'Microsoft.Compute', }, { 'test': 'name, group, type, namespace, parent', 'name': 'foo', 'resource_group_name': 'bar', 'resource_type': 'VirtualMachines', 'resource_provider_namespace': 'Microsoft.Compute', 'parent_resource_path': 'Foo.Bar/baz', }] for invalid_namespace in invalid: with self.assertRaises(CLIError): namespace_obj = NamespaceObject() for key in invalid_namespace: setattr(namespace_obj, key, invalid_namespace[key]) validate_lock_parameters(namespace_obj) def test_generate_deployment_name_from_file(self): # verify auto-gen from uri namespace = mock.MagicMock() namespace.template_uri = 'https://templates/template123.json?foo=bar' namespace.template_file = None namespace.deployment_name = None _validate_deployment_name(namespace) self.assertEqual('template123', namespace.deployment_name) namespace = mock.MagicMock() namespace.template_file = __file__ namespace.template_uri = None namespace.deployment_name = None _validate_deployment_name(namespace) file_base_name = os.path.basename(__file__) file_base_name = file_base_name[:str.find(file_base_name, '.')] self.assertEqual(file_base_name, namespace.deployment_name) # verify use default if get a file content namespace = mock.MagicMock() namespace.template_file = '{"foo":"bar"}' namespace.template_uri = None namespace.deployment_name = None _validate_deployment_name(namespace) self.assertEqual('deployment1', namespace.deployment_name)
def run_pipe_sequence(self, pipe_sequence, final_ins=None, final_outs=None, final_errs=None, environ={}, cwd=None): if self.debug: self.logger.debug(str(pipe_sequence)) _, current_state = self.get_current_worker_and_state() n_simple_commands = len(pipe_sequence.lst) prev_outs = None for idx, simple_command in enumerate(pipe_sequence.lst): # The temporary_environ needs to be reset for each simple command # i.e. A=42 script1 | script2 # The value of A should not be carried to script2 current_state.temporary_environ = {} for assignment in simple_command.assignments: current_state.temporary_environ[ assignment.identifier] = assignment.value # Only update the worker's env for pure assignments if simple_command.cmd_word == '' and idx == 0 and n_simple_commands == 1: current_state.environ.update(current_state.temporary_environ) current_state.temporary_environ = {} if prev_outs: # If previous output has gone to a file, we use a dummy empty string as ins ins = StringIO() if type(prev_outs) == file else prev_outs else: ins = final_ins or current_state.sys_stdin__ outs = current_state.sys_stdout__ errs = current_state.sys_stderr__ if simple_command.io_redirect: # Truncate file or append to file mode = 'w' if simple_command.io_redirect.operator == '>' else 'a' # For simplicity, stdout redirect works for stderr as well. # Note this is different from a real shell. if simple_command.io_redirect.filename == '&3': outs = _SYS_STDOUT errs = _SYS_STDERR else: errs = outs = open(simple_command.io_redirect.filename, mode) elif idx < n_simple_commands - 1: # before the last piped command outs = StringIO() else: if final_outs: outs = final_outs if final_errs: errs = final_errs if self.debug: self.logger.debug('io %s %s\n' % (ins, outs)) try: if simple_command.cmd_word != '': script_file = self.find_script_file( simple_command.cmd_word) if self.debug: self.logger.debug('script is %s\n' % script_file) if self.input_encoding_utf8: # Python 2 is not fully unicode compatible. Some modules (e.g. runpy) # insist for ASCII arguments. The encoding here helps eliminates possible # errors caused by unicode arguments. simple_command_args = [ arg.encode('utf-8') for arg in simple_command.args ] else: simple_command_args = simple_command.args if script_file.endswith('.py'): self.exec_py_file(script_file, simple_command_args, ins, outs, errs) elif is_binary_file(script_file): raise ShNotExecutable(script_file) else: self.exec_sh_file(script_file, simple_command_args, ins, outs, errs) else: current_state.return_value = 0 if current_state.return_value != 0: break # break out of the pipe_sequence, but NOT pipe_sequence list if isinstance(outs, StringIO): outs.seek( 0) # rewind for next command in the pipe sequence prev_outs = outs # This catch all exception is for when the exception is raised # outside of the actual command execution, i.e. exec_py_file # exec_sh_file, e.g. command not found, not executable etc. except ShFileNotFound as e: err_msg = '%s\n' % e.args[0] if self.debug: self.logger.debug(err_msg) self.stash.write_message(err_msg) # set exit code to 127 current_state.return_value = 127 break # break out of the pipe_sequence, but NOT pipe_sequence list except Exception as e: err_msg = '%s\n' % e.args[0] if self.debug: self.logger.debug(err_msg) self.stash.write_message(err_msg) break # break out of the pipe_sequence, but NOT pipe_sequence list finally: if isinstance(outs, file) and not isinstance(outs, StringIO): # StringIO is subclass of IOBase in py3 but not in py2 outs.close() if isinstance(ins, StringIO): # release the string buffer ins.close()
class TestOutput(unittest.TestCase): @classmethod def setUpClass(cls): pass @classmethod def tearDownClass(cls): pass def setUp(self): self.io = StringIO() def tearDown(self): self.io.close() def test_out_json_valid(self): """ The JSON output when the input is a dict should be the dict serialized to JSON """ output_producer = OutputProducer(formatter=format_json, file=self.io) output_producer.out( CommandResultItem({ 'active': True, 'id': '0b1f6472' })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""{ "active": true, "id": "0b1f6472" } """)) def test_out_json_from_ordered_dict(self): """ The JSON output when the input is OrderedDict should be serialized to JSON """ output_producer = OutputProducer(formatter=format_json, file=self.io) output_producer.out( CommandResultItem(OrderedDict({ 'active': True, 'id': '0b1f6472' }))) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""{ "active": true, "id": "0b1f6472" } """)) def test_out_json_byte(self): output_producer = OutputProducer(formatter=format_json, file=self.io) output_producer.out( CommandResultItem({ 'active': True, 'contents': b'0b1f6472' })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""{ "active": true, "contents": "0b1f6472" } """)) def test_out_json_byte_empty(self): output_producer = OutputProducer(formatter=format_json, file=self.io) output_producer.out( CommandResultItem({ 'active': True, 'contents': b'' })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""{ "active": true, "contents": "" } """)) # TABLE output tests def test_out_table(self): output_producer = OutputProducer(formatter=format_table, file=self.io) obj = OrderedDict() obj['active'] = True obj['val'] = '0b1f6472' output_producer.out(CommandResultItem(obj)) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines(""" Active Val -------- -------- 1 0b1f6472 """)) def test_out_table_list_of_lists(self): output_producer = OutputProducer(formatter=format_table, file=self.io) obj = [['a', 'b'], ['c', 'd']] output_producer.out(CommandResultItem(obj)) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""Column1 Column2 --------- --------- a b c d """)) def test_out_table_complex_obj(self): output_producer = OutputProducer(formatter=format_table, file=self.io) obj = OrderedDict() obj['name'] = 'qwerty' obj['val'] = '0b1f6472qwerty' obj['sub'] = {'1'} result_item = CommandResultItem(obj) output_producer.out(result_item) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""Name Val ------ -------------- qwerty 0b1f6472qwerty """)) def test_out_table_no_query_no_transformer_order(self): output_producer = OutputProducer(formatter=format_table, file=self.io) obj = { 'name': 'qwerty', 'val': '0b1f6472qwerty', 'active': True, 'sub': '0b1f6472' } result_item = CommandResultItem(obj, table_transformer=None, is_query_active=False) output_producer.out(result_item) # Should be alphabetical order as no table transformer and query is not active. self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines(""" Active Name Sub Val -------- ------ -------- -------------- 1 qwerty 0b1f6472 0b1f6472qwerty """)) def test_out_table_no_query_yes_transformer_order(self): output_producer = OutputProducer(formatter=format_table, file=self.io) obj = { 'name': 'qwerty', 'val': '0b1f6472qwerty', 'active': True, 'sub': '0b1f6472' } def transformer(r): return OrderedDict([('Name', r['name']), ('Val', r['val']), ('Active', r['active']), ('Sub', r['sub'])]) result_item = CommandResultItem(obj, table_transformer=transformer, is_query_active=False) output_producer.out(result_item) # Should be table transformer order self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""Name Val Active Sub ------ -------------- -------- -------- qwerty 0b1f6472qwerty 1 0b1f6472 """)) # TSV output tests def test_output_format_dict(self): obj = {} obj['A'] = 1 obj['B'] = 2 result = format_tsv(CommandResultItem(obj)) self.assertEqual(result, '1\t2\n') def test_output_format_dict_sort(self): obj = {} obj['B'] = 1 obj['A'] = 2 result = format_tsv(CommandResultItem(obj)) self.assertEqual(result, '2\t1\n') def test_output_format_ordereddict_not_sorted(self): obj = OrderedDict() obj['B'] = 1 obj['A'] = 2 result = format_tsv(CommandResultItem(obj)) self.assertEqual(result, '1\t2\n') def test_output_format_ordereddict_list_not_sorted(self): obj1 = OrderedDict() obj1['B'] = 1 obj1['A'] = 2 obj2 = OrderedDict() obj2['A'] = 3 obj2['B'] = 4 result = format_tsv(CommandResultItem([obj1, obj2])) self.assertEqual(result, '1\t2\n3\t4\n')
class CommandArchitectureTest(S3HandlerBaseTest): def setUp(self): super(CommandArchitectureTest, self).setUp() self.session = FakeSession() self.bucket = make_s3_files(self.session) self.loc_files = make_loc_files() self.output = StringIO() self.saved_stdout = sys.stdout sys.stdout = self.output def tearDown(self): self.output.close() sys.stdout = self.saved_stdout super(CommandArchitectureTest, self).setUp() clean_loc_files(self.loc_files) s3_cleanup(self.bucket, self.session) def test_create_instructions(self): """ This tests to make sure the instructions for any command is generated properly. """ cmds = ['cp', 'mv', 'rm', 'sync', 'mb', 'rb'] instructions = { 'cp': ['file_generator', 's3_handler'], 'mv': ['file_generator', 's3_handler'], 'rm': ['file_generator', 's3_handler'], 'sync': ['file_generator', 'comparator', 's3_handler'], 'mb': ['s3_handler'], 'rb': ['s3_handler'] } params = { 'filters': True, 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None } for cmd in cmds: cmd_arc = CommandArchitecture(self.session, cmd, { 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None }) cmd_arc.create_instructions() self.assertEqual(cmd_arc.instructions, instructions[cmd]) # Test if there is a filter. cmd_arc = CommandArchitecture(self.session, 'cp', params) cmd_arc.create_instructions() self.assertEqual(cmd_arc.instructions, ['file_generator', 'filters', 's3_handler']) def test_run_cp_put(self): # This ensures that the architecture sets up correctly for a ``cp`` put # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_file = 's3://' + self.bucket + '/' + 'text1.txt' local_file = self.loc_files[0] rel_local_file = os.path.relpath(local_file) filters = [['--include', '*']] params = { 'dir_op': False, 'dryrun': True, 'quiet': False, 'src': local_file, 'dest': s3_file, 'filters': filters, 'paths_type': 'locals3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True } cmd_arc = CommandArchitecture(self.session, 'cp', params) cmd_arc.create_instructions() cmd_arc.run() output_str = "(dryrun) upload: %s to %s" % (rel_local_file, s3_file) self.assertIn(output_str, self.output.getvalue()) def test_error_on_same_line_as_status(self): s3_file = 's3://' + 'bucket-does-not-exist' + '/' + 'text1.txt' local_file = self.loc_files[0] rel_local_file = os.path.relpath(local_file) filters = [['--include', '*']] params = { 'dir_op': False, 'dryrun': False, 'quiet': False, 'src': local_file, 'dest': s3_file, 'filters': filters, 'paths_type': 'locals3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True } cmd_arc = CommandArchitecture(self.session, 'cp', params) cmd_arc.create_instructions() cmd_arc.run() # Also, we need to verify that the error message is on the *same* line # as the upload failed line, to make it easier to track. output_str = ( "upload failed: %s to %s Error: Bucket does not exist\n" % (rel_local_file, s3_file)) self.assertIn(output_str, self.output.getvalue()) def test_run_cp_get(self): # This ensures that the architecture sets up correctly for a ``cp`` get # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_file = 's3://' + self.bucket + '/' + 'text1.txt' local_file = self.loc_files[0] rel_local_file = os.path.relpath(local_file) filters = [['--include', '*']] params = { 'dir_op': False, 'dryrun': True, 'quiet': False, 'src': s3_file, 'dest': local_file, 'filters': filters, 'paths_type': 's3local', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True } cmd_arc = CommandArchitecture(self.session, 'cp', params) cmd_arc.create_instructions() cmd_arc.run() output_str = "(dryrun) download: %s to %s" % (s3_file, rel_local_file) self.assertIn(output_str, self.output.getvalue()) def test_run_cp_copy(self): # This ensures that the architecture sets up correctly for a ``cp`` copy # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_file = 's3://' + self.bucket + '/' + 'text1.txt' filters = [['--include', '*']] params = { 'dir_op': False, 'dryrun': True, 'quiet': False, 'src': s3_file, 'dest': s3_file, 'filters': filters, 'paths_type': 's3s3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True } cmd_arc = CommandArchitecture(self.session, 'cp', params) cmd_arc.create_instructions() cmd_arc.run() output_str = "(dryrun) copy: %s to %s" % (s3_file, s3_file) self.assertIn(output_str, self.output.getvalue()) def test_run_mv(self): # This ensures that the architecture sets up correctly for a ``mv`` # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_file = 's3://' + self.bucket + '/' + 'text1.txt' filters = [['--include', '*']] params = { 'dir_op': False, 'dryrun': True, 'quiet': False, 'src': s3_file, 'dest': s3_file, 'filters': filters, 'paths_type': 's3s3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True } cmd_arc = CommandArchitecture(self.session, 'mv', params) cmd_arc.create_instructions() cmd_arc.run() output_str = "(dryrun) move: %s to %s" % (s3_file, s3_file) self.assertIn(output_str, self.output.getvalue()) def test_run_remove(self): # This ensures that the architecture sets up correctly for a ``rm`` # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_file = 's3://' + self.bucket + '/' + 'text1.txt' filters = [['--include', '*']] params = { 'dir_op': False, 'dryrun': True, 'quiet': False, 'src': s3_file, 'dest': s3_file, 'filters': filters, 'paths_type': 's3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True } cmd_arc = CommandArchitecture(self.session, 'rm', params) cmd_arc.create_instructions() cmd_arc.run() output_str = "(dryrun) delete: %s" % s3_file self.assertIn(output_str, self.output.getvalue()) def test_run_sync(self): # This ensures that the architecture sets up correctly for a ``sync`` # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_file = 's3://' + self.bucket + '/' + 'text1.txt' local_file = self.loc_files[0] s3_prefix = 's3://' + self.bucket + '/' local_dir = self.loc_files[3] rel_local_file = os.path.relpath(local_file) filters = [['--include', '*']] params = { 'dir_op': True, 'dryrun': True, 'quiet': False, 'src': local_dir, 'dest': s3_prefix, 'filters': filters, 'paths_type': 'locals3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True } cmd_arc = CommandArchitecture(self.session, 'sync', params) cmd_arc.create_instructions() cmd_arc.run() output_str = "(dryrun) upload: %s to %s" % (rel_local_file, s3_file) self.assertIn(output_str, self.output.getvalue()) def test_run_mb(self): # This ensures that the architecture sets up correctly for a ``rb`` # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_prefix = 's3://' + self.bucket + '/' params = { 'dir_op': True, 'dryrun': True, 'quiet': False, 'src': s3_prefix, 'dest': s3_prefix, 'paths_type': 's3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True } cmd_arc = CommandArchitecture(self.session, 'mb', params) cmd_arc.create_instructions() cmd_arc.run() output_str = "(dryrun) make_bucket: %s" % s3_prefix self.assertIn(output_str, self.output.getvalue()) def test_run_rb(self): # This ensures that the architecture sets up correctly for a ``rb`` # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_prefix = 's3://' + self.bucket + '/' params = { 'dir_op': True, 'dryrun': True, 'quiet': False, 'src': s3_prefix, 'dest': s3_prefix, 'paths_type': 's3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True } cmd_arc = CommandArchitecture(self.session, 'rb', params) cmd_arc.create_instructions() rc = cmd_arc.run() output_str = "(dryrun) remove_bucket: %s" % s3_prefix self.assertIn(output_str, self.output.getvalue()) self.assertEqual(rc, 0) def test_run_rb_nonzero_rc(self): # This ensures that the architecture sets up correctly for a ``rb`` # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_prefix = 's3://' + self.bucket + '/' params = { 'dir_op': True, 'dryrun': False, 'quiet': False, 'src': s3_prefix, 'dest': s3_prefix, 'paths_type': 's3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True } cmd_arc = CommandArchitecture(self.session, 'rb', params) cmd_arc.create_instructions() rc = cmd_arc.run() output_str = "remove_bucket failed: %s" % s3_prefix self.assertIn(output_str, self.output.getvalue()) self.assertEqual(rc, 1)
def get_iem_obs(Grid, window=60.): """ Returns all of the station observations from the Iowa Mesonet for a given Grid in the format needed for PyDDA. Parameters ---------- Grid: pyART Grid The Grid to retrieve the station data for. window: float The window (in minutes) to look for the nearest observation in time. Returns ------- station_data: list of dicts A list of dictionaries containing the following entries as keys: *lat* - Latitude of the site (float) *lon* - Longitude of the site (float) *u* - u wind at the site (float) *v* - v wind at the site (float) *w* - w wind at the site (assumed to be 0) (float) *site_id* - Station ID (string) *x*, *y*, *z* - The (x, y, z) coordinates of the site in the Grid. (floats) """ # First query the database for all of the JSON info for every station # Only add stations whose lat/lon are within the Grid's boundaries regions = """AF AL_ AI_ AQ_ AG_ AR_ AK AL AM_ AO_ AS_ AR AW_ AU_ AT_ AZ_ BA_ BE_ BB_ BG_ BO_ BR_ BF_ BT_ BS_ BI_ BM_ BB_ BY_ BZ_ BJ_ BW_ AZ CA CA_AB CA_BC CD_ CK_ CF_ CG_ CL_ CM_ CO CO_ CN_ CR_ CT CU_ CV_ CY_ CZ_ DE DK_ DJ_ DM_ DO_ DZ EE_ ET_ FK_ FM_ FJ_ FI_ FR_ GF_ PF_ GA_ GM_ GE_ DE_ GH_ GI_ KY_ GB_ GR_ GL_ GD_ GU_ GT_ GN_ GW_ GY_ HT_ HN_ HK_ HU_ IS_ IN_ ID_ IR_ IQ_ IE_ IL_ IT_ CI_ JM_ JP_ JO_ KZ_ KE_ KI_ KW_ LA_ LV_ LB_ LS_ LR_ LY_ LT_ LU_ MK_ MG_ MW_ MY_ MV_ ML_ CA_MB MH_ MR_ MU_ YT_ MX_ MD_ MC_ MA_ MZ_ MM_ NA_ NP_ AN_ NL_ CA_NB NC_ CA_NF NF_ NI_ NE_ NG_ MP_ KP_ CA_NT NO_ CA_NS CA_NU OM_ CA_ON PK_ PA_ PG_ PY_ PE_ PH_ PN_ PL_ PT_ CA_PE PR_ QA_ CA_QC RO_ RU_RW_ SH_ KN_ LC_ VC_ WS_ ST_ CA_SK SA_ SN_ RS_ SC_ SL_ SG_ SK_ SI_ SB_ SO_ ZA_ KR_ ES_ LK_ SD_ SR_ SZ_ SE_ CH_ SY_ TW_ TJ_ TZ_ TH_ TG_ TO_ TT_ TU TN_ TR_ TM_ UG_ UA_ AE_ UN_ UY_ UZ_ VU_ VE_ VN_ VI_ YE_ CA_YT ZM_ ZW_ EC_ EG_ FL GA GQ_ HI HR_ IA ID IL IO_ IN KS KH_ KY KM_ LA MA MD ME MI MN MO MS MT NC ND NE NH NJ NM NV NY OH OK OR PA RI SC SV_ SD TD_ TN TX UT VA VT VG_ WA WI WV WY""" networks = ["AWOS"] grid_lon_min = Grid.point_longitude["data"].min() grid_lon_max = Grid.point_longitude["data"].max() grid_lat_min = Grid.point_latitude["data"].min() grid_lat_max = Grid.point_latitude["data"].max() for region in regions.split(): networks.append("%s_ASOS" % (region,)) site_list = [] elevations = [] for network in networks: # Get metadata uri = ("https://mesonet.agron.iastate.edu/" "geojson/network/%s.geojson" ) % (network,) data = urlopen(uri) jdict = json.load(data) for site in jdict["features"]: lat = site["geometry"]["coordinates"][1] lon = site["geometry"]["coordinates"][0] if lat >= grid_lat_min and lat <= grid_lat_max and lon >= grid_lon_min and lon <= grid_lon_max: site_list.append((site["properties"]["sid"], site["properties"]["elevation"])) # Get the timestamp for each request grid_time = datetime.strptime(Grid.time["units"], "seconds since %Y-%m-%dT%H:%M:%SZ") start_time = grid_time - timedelta(minutes=window / 2.) end_time = grid_time + timedelta(minutes=window / 2.) SERVICE = "http://mesonet.agron.iastate.edu/cgi-bin/request/asos.py?" service = SERVICE + "data=all&tz=Etc/UTC&format=comma&latlon=yes&" service += start_time.strftime("year1=%Y&month1=%m&day1=%d&") service += end_time.strftime("year2=%Y&month2=%m&day2=%d&") station_obs = [] for stations, elevations in site_list: uri = "%s&station=%s" % (service, stations) print("Downloading: %s" % (stations,)) data = _download_data(uri) buf = StringIO() buf.write(data) buf.seek(0) my_df = pd.read_csv(buf, skiprows=5) stat_dict = {} if len(my_df['lat'].values) == 0: warnings.warn( "No data available at station %s between time %s and %s" % (stations, start_time.strftime('%Y-%m-%d %H:%M:%S'), end_time.strftime('%Y-%m-%d %H:%M:%S'))) else: stat_dict['lat'] = my_df['lat'].values[0] stat_dict['lon'] = my_df['lon'].values[0] stat_dict['x'], stat_dict['y'] = pyart.core.geographic_to_cartesian( stat_dict['lon'], stat_dict['lat'], Grid.get_projparams()) stat_dict['x'] = stat_dict['x'][0] stat_dict['y'] = stat_dict['y'][0] stat_dict['z'] = elevations - Grid.origin_altitude["data"][0] if my_df['drct'].values[0] == 'M': continue drct = float(my_df['drct'].values[0]) s_ms = float(my_df['sknt'].values[0]) * 0.514444 stat_dict['u'] = -np.sin(np.deg2rad(drct)) * s_ms stat_dict['v'] = -np.cos(np.deg2rad(drct)) * s_ms stat_dict['site_id'] = stations station_obs.append(stat_dict) buf.close() return station_obs
class TestOutput(unittest.TestCase): @classmethod def setUpClass(cls): pass @classmethod def tearDownClass(cls): pass def setUp(self): self.io = StringIO() def tearDown(self): self.io.close() def test_out_json_valid(self): """ The JSON output when the input is a dict should be the dict serialized to JSON """ output_producer = OutputProducer(formatter=format_json, file=self.io) output_producer.out( CommandResultItem({ 'active': True, 'id': '0b1f6472' })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""{ "active": true, "id": "0b1f6472" } """)) def test_out_json_from_ordered_dict(self): """ The JSON output when the input is OrderedDict should be serialized to JSON """ output_producer = OutputProducer(formatter=format_json, file=self.io) output_producer.out( CommandResultItem(OrderedDict({ 'active': True, 'id': '0b1f6472' }))) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""{ "active": true, "id": "0b1f6472" } """)) def test_out_json_byte(self): output_producer = OutputProducer(formatter=format_json, file=self.io) output_producer.out( CommandResultItem({ 'active': True, 'contents': b'0b1f6472' })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""{ "active": true, "contents": "0b1f6472" } """)) def test_out_json_byte_empty(self): output_producer = OutputProducer(formatter=format_json, file=self.io) output_producer.out( CommandResultItem({ 'active': True, 'contents': b'' })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""{ "active": true, "contents": "" } """)) def test_out_boolean_valid(self): output_producer = OutputProducer(formatter=format_list, file=self.io) output_producer.out(CommandResultItem(True)) self.assertEqual(util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""True\n\n\n""")) # TABLE output tests def test_out_table(self): output_producer = OutputProducer(formatter=format_table, file=self.io) obj = OrderedDict() obj['active'] = True obj['val'] = '0b1f6472' output_producer.out(CommandResultItem(obj)) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines(""" Active Val -------- -------- 1 0b1f6472 """)) def test_out_table_list_of_lists(self): output_producer = OutputProducer(formatter=format_table, file=self.io) obj = [['a', 'b'], ['c', 'd']] output_producer.out(CommandResultItem(obj)) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""Column1 Column2 --------- --------- a b c d """)) def test_out_table_complex_obj(self): output_producer = OutputProducer(formatter=format_table, file=self.io) obj = OrderedDict() obj['name'] = 'qwerty' obj['val'] = '0b1f6472qwerty' obj['sub'] = {'1'} result_item = CommandResultItem(obj) output_producer.out(result_item) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""Name Val ------ -------------- qwerty 0b1f6472qwerty """)) def test_out_table_no_query_no_transformer_order(self): output_producer = OutputProducer(formatter=format_table, file=self.io) obj = { 'name': 'qwerty', 'val': '0b1f6472qwerty', 'active': True, 'sub': '0b1f6472' } result_item = CommandResultItem(obj, table_transformer=None, is_query_active=False) output_producer.out(result_item) # Should be alphabetical order as no table transformer and query is not active. self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines(""" Active Name Sub Val -------- ------ -------- -------------- 1 qwerty 0b1f6472 0b1f6472qwerty """)) def test_out_table_no_query_yes_transformer_order(self): output_producer = OutputProducer(formatter=format_table, file=self.io) obj = { 'name': 'qwerty', 'val': '0b1f6472qwerty', 'active': True, 'sub': '0b1f6472' } def transformer(r): return OrderedDict([('Name', r['name']), ('Val', r['val']), ('Active', r['active']), ('Sub', r['sub'])]) result_item = CommandResultItem(obj, table_transformer=transformer, is_query_active=False) output_producer.out(result_item) # Should be table transformer order self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""Name Val Active Sub ------ -------------- -------- -------- qwerty 0b1f6472qwerty 1 0b1f6472 """)) # LIST output tests def test_out_list_valid(self): output_producer = OutputProducer(formatter=format_list, file=self.io) output_producer.out( CommandResultItem({ 'active': True, 'id': '0b1f6472' })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""Active : True Id : 0b1f6472 """)) def test_out_list_valid_caps(self): output_producer = OutputProducer(formatter=format_list, file=self.io) output_producer.out( CommandResultItem({ 'active': True, 'TESTStuff': 'blah' })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""Test Stuff : blah Active : True """)) def test_out_list_valid_none_val(self): output_producer = OutputProducer(formatter=format_list, file=self.io) output_producer.out( CommandResultItem({ 'active': None, 'id': '0b1f6472' })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""Active : None Id : 0b1f6472 """)) def test_out_list_valid_empty_array(self): output_producer = OutputProducer(formatter=format_list, file=self.io) output_producer.out( CommandResultItem({ 'active': None, 'id': '0b1f6472', 'hosts': [] })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""Active : None Id : 0b1f6472 Hosts : None """)) def test_out_list_valid_array_complex(self): output_producer = OutputProducer(formatter=format_list, file=self.io) output_producer.out( CommandResultItem([{ 'active': True, 'id': '783yesdf' }, { 'active': False, 'id': '3hjnme32' }, { 'active': False, 'id': '23hiujbs' }])) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""Active : True Id : 783yesdf Active : False Id : 3hjnme32 Active : False Id : 23hiujbs """)) def test_out_list_valid_str_array(self): output_producer = OutputProducer(formatter=format_list, file=self.io) output_producer.out( CommandResultItem(['location', 'id', 'host', 'server'])) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""location id host server """)) def test_out_list_valid_complex_array(self): output_producer = OutputProducer(formatter=format_list, file=self.io) output_producer.out( CommandResultItem({ 'active': True, 'id': '0b1f6472', 'myarray': ['1', '2', '3', '4'] })) self.assertEqual( util.normalize_newlines(self.io.getvalue()), util.normalize_newlines("""Active : True Id : 0b1f6472 Myarray : 1 2 3 4 """)) def test_out_list_format_key_simple(self): lo = ListOutput() self.assertEqual(lo._formatted_keys_cache, {}) lo._get_formatted_key('locationId') self.assertEqual(lo._formatted_keys_cache, {'locationId': 'Location Id'}) def test_out_list_format_key_single(self): lo = ListOutput() self.assertEqual(lo._formatted_keys_cache, {}) lo._get_formatted_key('location') self.assertEqual(lo._formatted_keys_cache, {'location': 'Location'}) def test_out_list_format_key_multiple_caps(self): lo = ListOutput() self.assertEqual(lo._formatted_keys_cache, {}) lo._get_formatted_key('fooIDS') self.assertEqual(lo._formatted_keys_cache, {'fooIDS': 'Foo Ids'}) def test_out_list_format_key_multiple_words(self): lo = ListOutput() self.assertEqual(lo._formatted_keys_cache, {}) lo._get_formatted_key('locationIdState') self.assertEqual(lo._formatted_keys_cache, {'locationIdState': 'Location Id State'}) # TSV output tests def test_output_format_dict(self): obj = {} obj['A'] = 1 obj['B'] = 2 result = format_tsv(CommandResultItem(obj)) self.assertEqual(result, '1\t2\n') def test_output_format_dict_sort(self): obj = {} obj['B'] = 1 obj['A'] = 2 result = format_tsv(CommandResultItem(obj)) self.assertEqual(result, '2\t1\n') def test_output_format_ordereddict_not_sorted(self): obj = OrderedDict() obj['B'] = 1 obj['A'] = 2 result = format_tsv(CommandResultItem(obj)) self.assertEqual(result, '1\t2\n') def test_output_format_ordereddict_list_not_sorted(self): obj1 = OrderedDict() obj1['B'] = 1 obj1['A'] = 2 obj2 = OrderedDict() obj2['A'] = 3 obj2['B'] = 4 result = format_tsv(CommandResultItem([obj1, obj2])) self.assertEqual(result, '1\t2\n3\t4\n')
class CommandArchitectureTest(S3HandlerBaseTest): def setUp(self): super(CommandArchitectureTest, self).setUp() self.session = FakeSession() self.bucket = make_s3_files(self.session) self.loc_files = make_loc_files() self.output = StringIO() self.err_output = StringIO() self.saved_stdout = sys.stdout self.saved_stderr = sys.stderr sys.stdout = self.output sys.stderr = self.err_output def tearDown(self): self.output.close() self.err_output.close() sys.stdout = self.saved_stdout sys.stderr = self.saved_stderr super(CommandArchitectureTest, self).tearDown() clean_loc_files(self.loc_files) s3_cleanup(self.bucket, self.session) def test_set_endpoint_no_source(self): cmd_arc = CommandArchitecture(self.session, 'sync', {'region': 'us-west-1', 'endpoint_url': None, 'verify_ssl': None, 'source_region': None}) cmd_arc.set_endpoints() endpoint = cmd_arc._endpoint source_endpoint = cmd_arc._source_endpoint self.assertEqual(endpoint.region_name, 'us-west-1') self.assertEqual(source_endpoint.region_name, 'us-west-1') def test_set_endpoint_with_source(self): cmd_arc = CommandArchitecture(self.session, 'sync', {'region': 'us-west-1', 'endpoint_url': None, 'verify_ssl': None, 'paths_type': 's3s3', 'source_region': ['us-west-2']}) cmd_arc.set_endpoints() endpoint = cmd_arc._endpoint source_endpoint = cmd_arc._source_endpoint self.assertEqual(endpoint.region_name, 'us-west-1') self.assertEqual(source_endpoint.region_name, 'us-west-2') def test_create_instructions(self): """ This tests to make sure the instructions for any command is generated properly. """ cmds = ['cp', 'mv', 'rm', 'sync', 'mb', 'rb'] instructions = {'cp': ['file_generator', 'file_info_builder', 's3_handler'], 'mv': ['file_generator', 'file_info_builder', 's3_handler'], 'rm': ['file_generator', 'file_info_builder', 's3_handler'], 'sync': ['file_generator', 'comparator', 'file_info_builder', 's3_handler'], 'mb': ['s3_handler'], 'rb': ['s3_handler']} params = {'filters': True, 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'is_stream': False} for cmd in cmds: cmd_arc = CommandArchitecture(self.session, cmd, {'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'is_stream': False}) cmd_arc.create_instructions() self.assertEqual(cmd_arc.instructions, instructions[cmd]) # Test if there is a filter. cmd_arc = CommandArchitecture(self.session, 'cp', params) cmd_arc.create_instructions() self.assertEqual(cmd_arc.instructions, ['file_generator', 'filters', 'file_info_builder', 's3_handler']) def test_choose_sync_strategy_default(self): session = Mock() cmd_arc = CommandArchitecture(session, 'sync', {'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None}) # Check if no plugins return their sync strategy. Should # result in the default strategies session.emit.return_value = None sync_strategies = cmd_arc.choose_sync_strategies() self.assertEqual( sync_strategies['file_at_src_and_dest_sync_strategy'].__class__, SizeAndLastModifiedSync ) self.assertEqual( sync_strategies['file_not_at_dest_sync_strategy'].__class__, MissingFileSync ) self.assertEqual( sync_strategies['file_not_at_src_sync_strategy'].__class__, NeverSync ) def test_choose_sync_strategy_overwrite(self): session = Mock() cmd_arc = CommandArchitecture(session, 'sync', {'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None}) # Check that the default sync strategy is overwritted if a plugin # returns its sync strategy. mock_strategy = Mock() mock_strategy.sync_type = 'file_at_src_and_dest' mock_not_at_dest_sync_strategy = Mock() mock_not_at_dest_sync_strategy.sync_type = 'file_not_at_dest' mock_not_at_src_sync_strategy = Mock() mock_not_at_src_sync_strategy.sync_type = 'file_not_at_src' responses = [(None, mock_strategy), (None, mock_not_at_dest_sync_strategy), (None, mock_not_at_src_sync_strategy)] session.emit.return_value = responses sync_strategies = cmd_arc.choose_sync_strategies() self.assertEqual( sync_strategies['file_at_src_and_dest_sync_strategy'], mock_strategy ) self.assertEqual( sync_strategies['file_not_at_dest_sync_strategy'], mock_not_at_dest_sync_strategy ) self.assertEqual( sync_strategies['file_not_at_src_sync_strategy'], mock_not_at_src_sync_strategy ) def test_run_cp_put(self): # This ensures that the architecture sets up correctly for a ``cp`` put # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_file = 's3://' + self.bucket + '/' + 'text1.txt' local_file = self.loc_files[0] rel_local_file = os.path.relpath(local_file) filters = [['--include', '*']] params = {'dir_op': False, 'dryrun': True, 'quiet': False, 'src': local_file, 'dest': s3_file, 'filters': filters, 'paths_type': 'locals3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True, 'page_size': None, 'is_stream': False} cmd_arc = CommandArchitecture(self.session, 'cp', params) cmd_arc.create_instructions() cmd_arc.run() output_str = "(dryrun) upload: %s to %s" % (rel_local_file, s3_file) self.assertIn(output_str, self.output.getvalue()) def test_error_on_same_line_as_status(self): s3_file = 's3://' + 'bucket-does-not-exist' + '/' + 'text1.txt' local_file = self.loc_files[0] rel_local_file = os.path.relpath(local_file) filters = [['--include', '*']] params = {'dir_op': False, 'dryrun': False, 'quiet': False, 'src': local_file, 'dest': s3_file, 'filters': filters, 'paths_type': 'locals3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True, 'page_size': None, 'is_stream': False} cmd_arc = CommandArchitecture(self.session, 'cp', params) cmd_arc.create_instructions() cmd_arc.run() # Also, we need to verify that the error message is on the *same* line # as the upload failed line, to make it easier to track. output_str = ( "upload failed: %s to %s Error: Bucket does not exist\n" % ( rel_local_file, s3_file)) self.assertIn(output_str, self.err_output.getvalue()) def test_run_cp_get(self): # This ensures that the architecture sets up correctly for a ``cp`` get # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_file = 's3://' + self.bucket + '/' + 'text1.txt' local_file = self.loc_files[0] rel_local_file = os.path.relpath(local_file) filters = [['--include', '*']] params = {'dir_op': False, 'dryrun': True, 'quiet': False, 'src': s3_file, 'dest': local_file, 'filters': filters, 'paths_type': 's3local', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True, 'page_size': None, 'is_stream': False} cmd_arc = CommandArchitecture(self.session, 'cp', params) cmd_arc.create_instructions() cmd_arc.run() output_str = "(dryrun) download: %s to %s" % (s3_file, rel_local_file) self.assertIn(output_str, self.output.getvalue()) def test_run_cp_copy(self): # This ensures that the architecture sets up correctly for a ``cp`` copy # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_file = 's3://' + self.bucket + '/' + 'text1.txt' filters = [['--include', '*']] params = {'dir_op': False, 'dryrun': True, 'quiet': False, 'src': s3_file, 'dest': s3_file, 'filters': filters, 'paths_type': 's3s3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True, 'page_size': None, 'is_stream': False} cmd_arc = CommandArchitecture(self.session, 'cp', params) cmd_arc.create_instructions() cmd_arc.run() output_str = "(dryrun) copy: %s to %s" % (s3_file, s3_file) self.assertIn(output_str, self.output.getvalue()) def test_run_mv(self): # This ensures that the architecture sets up correctly for a ``mv`` # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_file = 's3://' + self.bucket + '/' + 'text1.txt' filters = [['--include', '*']] params = {'dir_op': False, 'dryrun': True, 'quiet': False, 'src': s3_file, 'dest': s3_file, 'filters': filters, 'paths_type': 's3s3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True, 'page_size': None, 'is_stream': False} cmd_arc = CommandArchitecture(self.session, 'mv', params) cmd_arc.create_instructions() cmd_arc.run() output_str = "(dryrun) move: %s to %s" % (s3_file, s3_file) self.assertIn(output_str, self.output.getvalue()) def test_run_remove(self): # This ensures that the architecture sets up correctly for a ``rm`` # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_file = 's3://' + self.bucket + '/' + 'text1.txt' filters = [['--include', '*']] params = {'dir_op': False, 'dryrun': True, 'quiet': False, 'src': s3_file, 'dest': s3_file, 'filters': filters, 'paths_type': 's3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True, 'page_size': None, 'is_stream': False} cmd_arc = CommandArchitecture(self.session, 'rm', params) cmd_arc.create_instructions() cmd_arc.run() output_str = "(dryrun) delete: %s" % s3_file self.assertIn(output_str, self.output.getvalue()) def test_run_sync(self): # This ensures that the architecture sets up correctly for a ``sync`` # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_file = 's3://' + self.bucket + '/' + 'text1.txt' local_file = self.loc_files[0] s3_prefix = 's3://' + self.bucket + '/' local_dir = self.loc_files[3] rel_local_file = os.path.relpath(local_file) filters = [['--include', '*']] params = {'dir_op': True, 'dryrun': True, 'quiet': False, 'src': local_dir, 'dest': s3_prefix, 'filters': filters, 'paths_type': 'locals3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True, 'page_size': None, 'is_stream': False} cmd_arc = CommandArchitecture(self.session, 'sync', params) cmd_arc.create_instructions() cmd_arc.run() output_str = "(dryrun) upload: %s to %s" % (rel_local_file, s3_file) self.assertIn(output_str, self.output.getvalue()) def test_run_mb(self): # This ensures that the architecture sets up correctly for a ``rb`` # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_prefix = 's3://' + self.bucket + '/' params = {'dir_op': True, 'dryrun': True, 'quiet': False, 'src': s3_prefix, 'dest': s3_prefix, 'paths_type': 's3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True, 'page_size': None, 'is_stream': False} cmd_arc = CommandArchitecture(self.session, 'mb', params) cmd_arc.create_instructions() cmd_arc.run() output_str = "(dryrun) make_bucket: %s" % s3_prefix self.assertIn(output_str, self.output.getvalue()) def test_run_rb(self): # This ensures that the architecture sets up correctly for a ``rb`` # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_prefix = 's3://' + self.bucket + '/' params = {'dir_op': True, 'dryrun': True, 'quiet': False, 'src': s3_prefix, 'dest': s3_prefix, 'paths_type': 's3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True, 'page_size': None, 'is_stream': False} cmd_arc = CommandArchitecture(self.session, 'rb', params) cmd_arc.create_instructions() rc = cmd_arc.run() output_str = "(dryrun) remove_bucket: %s" % s3_prefix self.assertIn(output_str, self.output.getvalue()) self.assertEqual(rc, 0) def test_run_rb_nonzero_rc(self): # This ensures that the architecture sets up correctly for a ``rb`` # command. It is just just a dry run, but all of the components need # to be wired correctly for it to work. s3_prefix = 's3://' + self.bucket + '/' params = {'dir_op': True, 'dryrun': False, 'quiet': False, 'src': s3_prefix, 'dest': s3_prefix, 'paths_type': 's3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True, 'page_size': None, 'is_stream': False} cmd_arc = CommandArchitecture(self.session, 'rb', params) cmd_arc.create_instructions() rc = cmd_arc.run() output_str = "remove_bucket failed: %s" % s3_prefix self.assertIn(output_str, self.err_output.getvalue()) self.assertEqual(rc, 1)
class TestParser(unittest.TestCase): def setUp(self): self.io = StringIO() def tearDown(self): self.io.close() def test_register_simple_commands(self): def test_handler1(): pass def test_handler2(): pass cli = TestCli() cli.loader = mock.MagicMock() cli.loader.cli_ctx = cli command = AzCliCommand(cli.loader, 'command the-name', test_handler1) command2 = AzCliCommand(cli.loader, 'sub-command the-second-name', test_handler2) cmd_table = { 'command the-name': command, 'sub-command the-second-name': command2 } parser = AzCliCommandParser(cli) parser.load_command_table(cmd_table) args = parser.parse_args('command the-name'.split()) self.assertIs(args.func, command) args = parser.parse_args('sub-command the-second-name'.split()) self.assertIs(args.func, command2) AzCliCommandParser.error = VerifyError(self, ) parser.parse_args('sub-command'.split()) self.assertTrue(AzCliCommandParser.error.called) def test_required_parameter(self): def test_handler(args): # pylint: disable=unused-argument pass cli = TestCli() cli.loader = mock.MagicMock() cli.loader.cli_ctx = cli command = AzCliCommand(cli.loader, 'test command', test_handler) command.add_argument('req', '--req', required=True) cmd_table = {'test command': command} parser = AzCliCommandParser(cli) parser.load_command_table(cmd_table) args = parser.parse_args('test command --req yep'.split()) self.assertIs(args.func, command) AzCliCommandParser.error = VerifyError(self) parser.parse_args('test command'.split()) self.assertTrue(AzCliCommandParser.error.called) def test_nargs_parameter(self): def test_handler(): pass cli = TestCli() cli.loader = mock.MagicMock() cli.loader.cli_ctx = cli command = AzCliCommand(cli.loader, 'test command', test_handler) command.add_argument('req', '--req', required=True, nargs=2) cmd_table = {'test command': command} parser = AzCliCommandParser(cli) parser.load_command_table(cmd_table) args = parser.parse_args('test command --req yep nope'.split()) self.assertIs(args.func, command) AzCliCommandParser.error = VerifyError(self) parser.parse_args('test command -req yep'.split()) self.assertTrue(AzCliCommandParser.error.called) def test_case_insensitive_enum_choices(self): from enum import Enum class TestEnum(Enum): # pylint: disable=too-few-public-methods opt1 = "ALL_CAPS" opt2 = "camelCase" opt3 = "snake_case" def test_handler(): pass cli = TestCli() cli.loader = mock.MagicMock() cli.loader.cli_ctx = cli command = AzCliCommand(cli.loader, 'test command', test_handler) command.add_argument('opt', '--opt', required=True, **enum_choice_list(TestEnum)) cmd_table = {'test command': command} parser = AzCliCommandParser(cli) parser.load_command_table(cmd_table) args = parser.parse_args('test command --opt alL_cAps'.split()) self.assertEqual(args.opt, 'ALL_CAPS') args = parser.parse_args('test command --opt CAMELCASE'.split()) self.assertEqual(args.opt, 'camelCase') args = parser.parse_args('test command --opt sNake_CASE'.split()) self.assertEqual(args.opt, 'snake_case')
def detailed_traceback(): buf = StringIO() cgitb_hook(format="text", file=buf) tb_txt = buf.getvalue() buf.close() return tb_txt
class TestApplication(unittest.TestCase): @classmethod def setUpClass(cls): pass @classmethod def tearDownClass(cls): pass def setUp(self): self.io = StringIO() def tearDown(self): self.io.close() def test_application_register_and_call_handlers(self): handler_called = [False] def handler(**kwargs): kwargs['args'][0] = True def other_handler(**kwargs): # pylint: disable=unused-variable self.assertEqual(kwargs['args'], 'secret sauce') config = Configuration([]) app = Application(config) app.raise_event('was_handler_called', args=handler_called) self.assertFalse( handler_called[0], "Raising event with no handlers registered somehow failed...") app.register('was_handler_called', handler) self.assertFalse(handler_called[0]) # Registered handler won't get called if event with different name # is raised... app.raise_event('other_handler_called', args=handler_called) self.assertFalse(handler_called[0], 'Wrong handler called!') app.raise_event('was_handler_called', args=handler_called) self.assertTrue(handler_called[0], "Handler didn't get called") app.raise_event('other_handler_called', args='secret sauce') def test_list_value_parameter(self): hellos = [] def handler(args): hellos.append(args) command = CliCommand('test command', handler) command.add_argument('hello', '--hello', nargs='+', action=IterateAction) command.add_argument('something', '--something') cmd_table = {'test command': command} argv = 'az test command --hello world sir --something else'.split() config = Configuration(argv) config.get_command_table = lambda: cmd_table application = Application(config) application.execute(argv[1:]) self.assertEqual(2, len(hellos)) self.assertEqual(hellos[0]['hello'], 'world') self.assertEqual(hellos[0]['something'], 'else') self.assertEqual(hellos[1]['hello'], 'sir') self.assertEqual(hellos[1]['something'], 'else') def test_expand_file_prefixed_files(self): f = tempfile.NamedTemporaryFile(delete=False) f.close() f_with_bom = tempfile.NamedTemporaryFile(delete=False) f_with_bom.close() with open(f.name, 'w+') as stream: stream.write('foo') from codecs import open as codecs_open with codecs_open(f_with_bom.name, encoding='utf-8-sig', mode='w+') as stream: stream.write('foo') cases = [[['bar=baz'], ['bar=baz']], [['bar', 'baz'], ['bar', 'baz']], [['bar=@{}'.format(f.name)], ['bar=foo']], [['bar=@{}'.format(f_with_bom.name)], ['bar=foo']], [['bar', '@{}'.format(f.name)], ['bar', 'foo']], [['bar', f.name], ['bar', f.name]], [['[email protected]'], ['[email protected]']], [['bar', '*****@*****.**'], ['bar', '*****@*****.**']], [['bar=mymongo=@connectionstring'], ['bar=mymongo=@connectionstring']]] for test_case in cases: try: args = Application._expand_file_prefixed_files(test_case[0]) # pylint: disable=protected-access self.assertEqual(args, test_case[1], 'Failed for: {}'.format(test_case[0])) except CLIError as ex: self.fail('Unexpected error for {} ({}): {}'.format( test_case[0], args, ex)) os.remove(f.name)