def check_file_list(any_format,ignore_warnings=True, warnings=[]): """ get the folder for the job or a batch job a batch job can be either a readable text file name: file_name.any_extension - contents must be file with list of folders, one per line or it can be a glob-like search pattern for folders//* the folders may or may not contain valid tif data with well-named frames if any are invalid, the invalid ones are added to warnings. the valid ones are returned in a list a caller can decide what to do about the warnings which will be listed (ignore/quit) If print_info==True: the following folders can be processed: folder range #gaps the following cannot be processed: folder issue """ import os from glob import glob folders = [any_format] if os.is_file(any_format): with open(any_format) as f: folders = [_f for _f in f] if any_format[-1]=="*": folders = list(glob(any_format)) valid_folders = [] for f in folders: c = context.folder_context(f) if not c.meta_key["valid"]: warnings.append(f) else: valid_folders.append(c.meta_key) #list the goods and the bads by simply listing the info for the folder but then say warnings in tabs if req if ignore_warnings == False and len(warnings) > 0: #some folders cannot be processed. Do you want to continue(y) or quit(n)? pass return valid_folders
def _get_all_warning_lines(self, data): warnings = [] for row in data: splitrow = row.split() if len(splitrow) > 0: if splitrow[0] == 'WARNING:': warnings.append(" ".join(splitrow[1:])) return warnings
def load_notebook(resources=None, verbose=False, hide_banner=False): ''' Prepare the IPython notebook for displaying Bokeh plots. Args: resources (Resource, optional) : how and where to load BokehJS from verbose (bool, optional) : whether to report detailed settings (default: False) hide_banner (bool, optional): whether to hide the Bokeh banner (default: False) Returns: None ''' global _notebook_loaded from .resources import INLINE from .templates import NOTEBOOK_LOAD, RESOURCES if resources is None: resources = INLINE plot_resources = RESOURCES.render( js_raw = resources.js_raw, css_raw = resources.css_raw, js_files = resources.js_files, css_files = resources.css_files, ) if resources.mode == 'inline': js_info = 'inline' css_info = 'inline' else: js_info = resources.js_files[0] if len(resources.js_files) == 1 else resources.js_files css_info = resources.css_files[0] if len(resources.css_files) == 1 else resources.css_files warnings = ["Warning: " + msg['text'] for msg in resources.messages if msg['type'] == 'warn'] if _notebook_loaded: warnings.append('Warning: BokehJS previously loaded') _notebook_loaded = resources html = NOTEBOOK_LOAD.render( plot_resources = plot_resources, logo_url = resources.logo_url, verbose = verbose, js_info = js_info, css_info = css_info, bokeh_version = __version__, warnings = warnings, hide_banner = hide_banner, ) utils.publish_display_data({'text/html': html})
def validate_nbextension_python(spec, full_dest, logger=None): """Assess the health of an installed nbextension Returns a list of warnings. Parameters ---------- spec : dict A single entry of _jupyter_nbextension_paths(): [{ 'section': 'notebook', 'src': 'mockextension', 'dest': '_mockdestination', 'require': '_mockdestination/index' }] full_dest : str The on-disk location of the installed nbextension: this should end with `nbextensions/<dest>` logger : Jupyter logger [optional] Logger instance to use """ infos = [] warnings = [] section = spec.get("section", None) if section in NBCONFIG_SECTIONS: infos.append(u" {} section: {}".format(GREEN_OK, section)) else: warnings.append(u" {} section: {}".format(RED_X, section)) require = spec.get("require", None) if require is not None: require_path = os.path.join( full_dest[0:-len(spec["dest"])], u"{}.js".format(require)) if os.path.exists(require_path): infos.append(u" {} require: {}".format(GREEN_OK, require_path)) else: warnings.append(u" {} require: {}".format(RED_X, require_path)) if logger: if warnings: logger.warning("- Validating: problems found:") for msg in warnings: logger.warning(msg) for msg in infos: logger.info(msg) logger.warning(u"Full spec: {}".format(spec)) else: logger.info(u"- Validating: {}".format(GREEN_OK)) return warnings
def autosummary_directive(dirname, arguments, options, content, lineno, content_offset, block_text, state, state_machine): """ Pretty table containing short signatures and summaries of functions etc. autosummary also generates a (hidden) toctree:: node. """ names = [] names += [ x.strip().split()[0] for x in content if x.strip() and re.search(r'^[a-zA-Z_]', x.strip()[0]) ] table, warnings, real_names = get_autosummary(names, state, 'nosignatures' in options) node = table env = state.document.settings.env suffix = env.config.source_suffix all_docnames = env.found_docs.copy() dirname = posixpath.dirname(env.docname) if 'toctree' in options: tree_prefix = options['toctree'].strip() docnames = [] for name in names: name = real_names.get(name, name) docname = tree_prefix + name if docname.endswith(suffix): docname = docname[:-len(suffix)] docname = posixpath.normpath(posixpath.join(dirname, docname)) if docname not in env.found_docs: warnings.append( state.document.reporter.warning( 'toctree references unknown document %r' % docname, line=lineno)) docnames.append(docname) tocnode = sphinx.addnodes.toctree() tocnode['includefiles'] = docnames tocnode['maxdepth'] = -1 tocnode['glob'] = None tocnode['entries'] = [(None, docname) for docname in docnames] tocnode = autosummary_toc('', '', tocnode) return warnings + [node] + [tocnode] else: return warnings + [node]
def warnings(self): """Returns all the work's warnings Returns: list: List of warnings """ html = self._soup.find("dd", {"class": "warning tags"}) warnings = [] if html is not None: for warning in html.find_all("li"): warnings.append(warning.a.string) return warnings
def add_location(tracker, item): if 'track' not in item: # We can only add location for 'track' items. return item tracker_item = tracker.track(artist_name=item['artist'], track_name=item['track']) if tracker_item: item.update(tracker_item) else: warnings = item.get('warnings', []) warnings.append('tracker: Unknown track') item['warnings'] = warnings return item
def autosummary_directive(dirname, arguments, options, content, lineno, content_offset, block_text, state, state_machine): """ Pretty table containing short signatures and summaries of functions etc. autosummary also generates a (hidden) toctree:: node. """ names = [] names += [x.strip().split()[0] for x in content if x.strip() and re.search(r'^[a-zA-Z_]', x.strip()[0])] table, warnings, real_names = get_autosummary(names, state, 'nosignatures' in options) node = table env = state.document.settings.env suffix = env.config.source_suffix all_docnames = env.found_docs.copy() dirname = posixpath.dirname(env.docname) if 'toctree' in options: tree_prefix = options['toctree'].strip() docnames = [] for name in names: name = real_names.get(name, name) docname = tree_prefix + name if docname.endswith(suffix): docname = docname[:-len(suffix)] docname = posixpath.normpath(posixpath.join(dirname, docname)) if docname not in env.found_docs: warnings.append(state.document.reporter.warning( 'toctree references unknown document %r' % docname, line=lineno)) docnames.append(docname) tocnode = sphinx.addnodes.toctree() tocnode['includefiles'] = docnames tocnode['maxdepth'] = -1 tocnode['glob'] = None tocnode['entries'] = [(None, docname) for docname in docnames] tocnode = autosummary_toc('', '', tocnode) return warnings + [node] + [tocnode] else: return warnings + [node]
def _get_empty_line_warnings(self): if len(self.empty_lines) == 0: return [] warnings = [] empty_line = (self.empty_lines[0], self.empty_lines[0]) for line in self.empty_lines[1:]: if line == empty_line[1] + 1: empty_line = (empty_line[0], line) else: warnings.append(Block._get_empty_line_msg(empty_line)) empty_line = (line, line) warnings.append(Block._get_empty_line_msg(empty_line)) return warnings
def validate_nbextension(require, logger=None): """Validate a named nbextension. Looks across all of the nbextension directories. Returns a list of warnings. require : str require.js path used to load the extension logger : Jupyter logger [optional] Logger instance to use """ warnings = [] infos = [] js_exists = False for exts in _nbextension_dirs(): # Does the Javascript entrypoint actually exist on disk? js = u"{}.js".format(os.path.join(exts, *require.split("/"))) js_exists = os.path.exists(js) if js_exists: break require_tmpl = u" - require? {} {}" if js_exists: infos.append(require_tmpl.format(GREEN_OK, require)) else: warnings.append(require_tmpl.format(RED_X, require)) if logger: if warnings: logger.warning(u" - Validating: problems found:") for msg in warnings: logger.warning(msg) for msg in infos: logger.info(msg) else: logger.info(u" - Validating: {}".format(GREEN_OK)) return warnings
def validate_nbextension(require, logger=None): """Validate a named nbextension. Looks across all of the nbextension directories. Returns a list of warnings. require : str require.js path used to load the extension logger : Jupyter logger [optional] Logger instance to use """ warnings = [] infos = [] js_exists = False for exts in jupyter_path('nbextensions'): # Does the Javascript entrypoint actually exist on disk? js = u"{}.js".format(os.path.join(exts, *require.split("/"))) js_exists = os.path.exists(js) if js_exists: break require_tmpl = u" - require? {} {}" if js_exists: infos.append(require_tmpl.format(GREEN_OK, require)) else: warnings.append(require_tmpl.format(RED_X, require)) if logger: if warnings: logger.warning(u" - Validating: problems found:") for msg in warnings: logger.warning(msg) for msg in infos: logger.info(msg) else: logger.info(u" - Validating: {}".format(GREEN_OK)) return warnings
def _read_warnings(self): """Poll all the warning loggers and extract any new warnings that have been logged. If the warnings belong to a category that is currently disabled, this method will discard them and they will no longer be retrievable. Returns a list of (timestamp, message) tuples, where timestamp is an integer epoch timestamp.""" warnings = [] while True: # pull in a line of output from every logger that has # output ready to be read loggers, _, _ = select.select(self.warning_loggers, [], [], 0) closed_loggers = set() for logger in loggers: line = logger.readline() # record any broken pipes (aka line == empty) if len(line) == 0: closed_loggers.add(logger) continue # parse out the warning timestamp, msgtype, msg = line.split('\t', 2) timestamp = int(timestamp) # if the warning is valid, add it to the results if self.warning_manager.is_valid(timestamp, msgtype): warnings.append((timestamp, msg.strip())) # stop listening to loggers that are closed self.warning_loggers -= closed_loggers # stop if none of the loggers have any output left if not loggers: break # sort into timestamp order warnings.sort() return warnings
def process_errors_and_warnings(messages): errors, warnings = [], [] for message in messages: # By default, script templates include an # 'upstream = None' line in the parameters cell but users may delete # it. If that happens and they reference 'upstream' in the body of the # notebook but the task does not have upstream dependencies, the # injected cell won't add the upstream variable, causing an undefined # name error. To provide more context, we modify the original error and # add a hint. Note that this only happens when using the Python API # directly, since the Spec API requires an upstream = None placeholder if (isinstance(message, UndefinedName) and message.message_args == ('upstream', )): message.message = ( message.message + '. Did you forget to declare upstream dependencies?') if isinstance(message, _ERRORS): errors.append(message) else: warnings.append(message) return _process_messages(errors), _process_messages(warnings)
def load_filenames(args): global gl warnings = [] errors = [] for arg in args: #{ for sampfname in glob.glob(arg): if True: try: sampf = file(sampfname, "rb") except IOError, msg: errors.append(msg) if sampf: sampf.close() samp = Samp() # strip directory basename = sampfname.replace("\\", "/") basename = sampfname.split("/")[-1] # strip ".wav" extension basename = basename.split(".") basename = ".".join(basename[0:-1]) basename = jtrans.tr(basename, DELIMS, " ") # get layer name parts = basename.split(" ") if len(parts) <= abs(LAYER_LOC): loc = LAYER_LOC if loc >= 0: loc += 1 print >> sys.stderr, ( "After splitting filename '%s' delimiters," % (basename)) print >> sys.stderr, ( "there aren't enough parts to find part number %d." % loc) sys.exit(1) layername = parts[LAYER_LOC] # get note: might be MIDI number or note name if len(parts) <= abs(NOTE_LOC): loc = NOTE_LOC if loc >= 0: loc += 1 print >> sys.stderr, ( "After splitting filename '%s' at delimiters, there aren't" % (basename)) print >> sys.stderr, ( "there aren't enough parts to find part number %d." % loc) sys.exit(1) notespec = parts[NOTE_LOC] mnote = jmidi.notenum(notespec) if mnote == None: print >> sys.stderr, ( "Invalid MIDI note designation '%s' in '%s'" % (notespec, basename)) sys.exit(1) # print sampfname, mnote, layername, jmidi.mnote_name(mnote)[0] samp.fname = sampfname samp.mnote = mnote samp.notename = jmidi.mnote_name(mnote, pad=None) samp.layername = layername if layername not in gl.layernum: warnings.append("Sample for unconfigured layer '%s': %s" % (samp.layername, samp.fname)) continue samp.layer = gl.layernum[layername] if samp.layer == None: warnings.append("Sample for missing layer '%s': %s" % (samp.layername, samp.fname)) continue x = LO_KEY - MAX_NOTE_SHIFT if (samp.mnote < max(0, LO_KEY - MAX_NOTE_SHIFT) or samp.mnote > HI_KEY + MAX_NOTE_SHIFT): warnings.append("Sample outside useful note range (%s): %s" % (samp.notename, samp.fname)) continue samp.char = None gl.samps[sampfname] = samp gl.grid[samp.layer][mnote] = samp
def DoPresubmitChecks(change, committing, verbose, output_stream, input_stream, default_presubmit, may_prompt): """Runs all presubmit checks that apply to the files in the change. This finds all PRESUBMIT.py files in directories enclosing the files in the change (up to the repository root) and calls the relevant entrypoint function depending on whether the change is being committed or uploaded. Prints errors, warnings and notifications. Prompts the user for warnings when needed. Args: change: The Change object. committing: True if 'gcl commit' is running, False if 'gcl upload' is. verbose: Prints debug info. output_stream: A stream to write output from presubmit tests to. input_stream: A stream to read input from the user. default_presubmit: A default presubmit script to execute in any case. may_prompt: Enable (y/n) questions on warning or error. Warning: If may_prompt is true, output_stream SHOULD be sys.stdout and input_stream SHOULD be sys.stdin. Return: True if execution can continue, False if not. """ start_time = time.time() presubmit_files = ListRelevantPresubmitFiles(change.AbsoluteLocalPaths(True), change.RepositoryRoot()) if not presubmit_files and verbose: output_stream.write("Warning, no presubmit.py found.\n") results = [] executer = PresubmitExecuter(change, committing) if default_presubmit: if verbose: output_stream.write("Running default presubmit script.\n") fake_path = os.path.join(change.RepositoryRoot(), "PRESUBMIT.py") results += executer.ExecPresubmitScript(default_presubmit, fake_path) for filename in presubmit_files: filename = os.path.abspath(filename) if verbose: output_stream.write("Running %s\n" % filename) # Accept CRLF presubmit script. presubmit_script = gclient_utils.FileRead(filename, "rU") results += executer.ExecPresubmitScript(presubmit_script, filename) errors = [] notifications = [] warnings = [] for result in results: if not result.IsFatal() and not result.ShouldPrompt(): notifications.append(result) elif result.ShouldPrompt(): warnings.append(result) else: errors.append(result) error_count = 0 for name, items in (("Messages", notifications), ("Warnings", warnings), ("ERRORS", errors)): if items: output_stream.write("** Presubmit %s **\n" % name) for item in items: if not item._Handle(output_stream, input_stream, may_prompt=False): error_count += 1 output_stream.write("\n") total_time = time.time() - start_time if total_time > 1.0: print "Presubmit checks took %.1fs to calculate." % total_time if not errors and warnings and may_prompt: if not PromptYesNo( input_stream, output_stream, "There were presubmit warnings. " "Are you sure you wish to continue? (y/N): " ): error_count += 1 global _ASKED_FOR_FEEDBACK # Ask for feedback one time out of 5. if len(results) and random.randint(0, 4) == 0 and not _ASKED_FOR_FEEDBACK: output_stream.write( "Was the presubmit check useful? Please send feedback " "& hate mail to [email protected]!\n" ) _ASKED_FOR_FEEDBACK = True return error_count == 0
def infer_degeneracy(parameterObj, transcriptObjs, zstore): #degeneracy_arrays_by_sample = collections.defaultdict(list) warnings = [] beds_rejected = [] # needs to know how many sites in output total_sites = 0 transcriptObjs_by_sequence_id = collections.defaultdict(list) transcriptObjs_valid = 0 length_by_sequence_id = collections.Counter() for transcriptObj in tqdm(transcriptObjs, total=len(transcriptObjs), desc="[%] Checking for ORFs... ", ncols=150, position=0, leave=True): if not transcriptObj.is_orf(): warnings.append( "[-] Transcript %s has no ORF: START=%s, STOP=%s, DIVISIBLE_BY_3=%s (will be skipped)" % (transcriptObj.transcript_id, transcriptObj.has_start(), transcriptObj.has_stop(), transcriptObj.is_divisible_by_three())) beds_rejected.append(transcriptObj.bed) else: total_sites += transcriptObj.positions.shape[0] transcriptObjs_by_sequence_id[transcriptObj.sequence_id].append( transcriptObj) length_by_sequence_id[ transcriptObj.sequence_id] += transcriptObj.positions.shape[0] transcriptObjs_valid += 1 samples = zstore.attrs['samples'] degeneracy_chars = "U%s" % (len(samples) * 6 ) # could be improved with ploidy? #data = np.zeros(total_sites, dtype={'names':('sequence_id', 'start', 'end', 'degeneracy', 'codon_pos', 'orientation'),'formats':('U16', 'i8', 'i8', degeneracy_chars, 'i1', 'U1')}) if warnings: with open("%s.cds.rejected_transcripts.bed" % parameterObj.outprefix, 'w') as fh: fh.write("\n".join(beds_rejected) + "\n") print("\n".join(warnings)) dfs = [] with tqdm(total=transcriptObjs_valid, ncols=150, desc="[%] Inferring degeneracy... ", position=0, leave=True) as pbar: for sequence_id, transcriptObjs in transcriptObjs_by_sequence_id.items( ): offset = 0 data = np.zeros(length_by_sequence_id[sequence_id], dtype={ 'names': ('sequence_id', 'start', 'end', 'degeneracy', 'codon_pos', 'orientation'), 'formats': ('U16', 'i8', 'i8', degeneracy_chars, 'i1', 'U1') }) if sequence_id in zstore['seqs']: # sequence has variants pos = np.array(zstore["seqs/%s/variants/pos" % sequence_id]) gts = np.array(zstore["seqs/%s/variants/gts" % sequence_id]) alt = np.array(zstore["seqs/%s/variants/alt" % sequence_id]) ref = np.array(zstore["seqs/%s/variants/ref" % sequence_id]) alleles_raw = np.column_stack([ref, alt]) # initiate boolean mask with False mask = np.zeros(alleles_raw.shape, dtype=bool) acs = allel.GenotypeArray(gts).count_alleles() # overwrite with True those alleles_raw that occur in gts mask[:, 0:acs.shape[1]] = acs alleles = np.where(mask, alleles_raw, '') for transcriptObj in transcriptObjs: start, end = offset, offset + transcriptObj.positions.shape[0] data[start:end]['sequence_id'] = transcriptObj.sequence_id data[start:end]['start'] = transcriptObj.positions data[start:end]['end'] = transcriptObj.positions + 1 data[start:end]['codon_pos'][0::3] = 1 data[start:end]['codon_pos'][1::3] = 2 data[start:end]['codon_pos'][2::3] = 3 data[start:end]['orientation'] = transcriptObj.orientation if not sequence_id in zstore['seqs']: #print("transcriptObj.degeneracy", type(transcriptObj.degeneracy), transcriptObj.degeneracy.shape) data[start:end]['degeneracy'] = transcriptObj.degeneracy else: #pos_in_cds_mask = np.isin(pos, transcriptObj.positions, assume_unique=True) # will crash if non-unique pos cds_in_pos_mask = np.isin( transcriptObj.positions, pos, assume_unique=True) # will crash if non-unique pos for i in range(0, len(transcriptObj.sequence), 3): codon_start = start + i if not np.any( cds_in_pos_mask[codon_start:codon_start + 3]): data[codon_start:codon_start + 3][ 'degeneracy'] = transcriptObj.degeneracy[i:i + 3] else: codon_list = list( filter(lambda codon: len(codon) == 3, [ "".join(x) for x in itertools.product( *alleles[codon_start:codon_start + 3]) ])) data[codon_start:codon_start + 3]['degeneracy'] = degeneracy( codon_list) if codon_list else 3 * ['NA'] offset = end pbar.update() dfs.append( pd.DataFrame(data=data, columns=[ 'sequence_id', 'start', 'end', 'degeneracy', 'codon_pos', 'orientation' ])) shutil.rmtree(parameterObj.tmp_dir) #df = pd.DataFrame(data=data, columns=['sequence_id', 'start', 'end', 'degeneracy', 'codon_pos', 'orientation']) #write_df(df.sort_values(['sequence_id', 'start'], ascending=[True, True]), out_f="%s.cds.bed" % (parameterObj.outprefix), sep='\t', header=False, status=False) write_df(pd.concat(dfs), out_f="%s.cds.bed" % (parameterObj.outprefix), sep='\t', header=False, status=False)
def cog_validate( # noqa: C901 src_path: Union[str, pathlib.PurePath], strict: bool = False, quiet: bool = False) -> Tuple[bool, List[str], List[str]]: """ Validate Cloud Optimized Geotiff. This script is the rasterio equivalent of https://svn.osgeo.org/gdal/trunk/gdal/swig/python/samples/validate_cloud_optimized_geotiff.py Parameters ---------- src_path: str or PathLike object A dataset path or URL. Will be opened in "r" mode. strict: bool Treat warnings as errors quiet: bool Remove standard outputs Returns ------- is_valid: bool True is src_path is a valid COG. errors: list List of validation errors. warnings: list List of validation warnings. """ if isinstance(src_path, str): src_path = pathlib.Path(src_path) errors = [] warnings = [] details: Dict[str, Any] = {} if not GDALVersion.runtime().at_least("2.2"): raise Exception("GDAL 2.2 or above required") config = dict(GDAL_DISABLE_READDIR_ON_OPEN="FALSE") with rasterio.Env(**config): with rasterio.open(src_path) as src: if not src.driver == "GTiff": raise Exception("The file is not a GeoTIFF") filelist = [pathlib.Path(f).name for f in src.files] if len(filelist) > 1 and f"{src_path.name}.ovr" in filelist: errors.append( "Overviews found in external .ovr file. They should be internal" ) overviews = src.overviews(1) if src.width > 512 or src.height > 512: if not src.is_tiled: errors.append( "The file is greater than 512xH or 512xW, but is not tiled" ) if not overviews: warnings.append( "The file is greater than 512xH or 512xW, it is recommended " "to include internal overviews") ifd_offset = int(src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1)) # Starting from GDAL 3.1, GeoTIFF and COG have ghost headers # e.g: # """ # GDAL_STRUCTURAL_METADATA_SIZE=000140 bytes # LAYOUT=IFDS_BEFORE_DATA # BLOCK_ORDER=ROW_MAJOR # BLOCK_LEADER=SIZE_AS_UINT4 # BLOCK_TRAILER=LAST_4_BYTES_REPEATED # KNOWN_INCOMPATIBLE_EDITION=NO # """ # # This header should be < 200bytes if ifd_offset > 300: errors.append( f"The offset of the main IFD should be < 300. It is {ifd_offset} instead" ) ifd_offsets = [ifd_offset] details["ifd_offsets"] = {} details["ifd_offsets"]["main"] = ifd_offset if overviews and overviews != sorted(overviews): errors.append("Overviews should be sorted") for ix, dec in enumerate(overviews): # NOTE: Size check is handled in rasterio `src.overviews` methods # https://github.com/mapbox/rasterio/blob/4ebdaa08cdcc65b141ed3fe95cf8bbdd9117bc0b/rasterio/_base.pyx # We just need to make sure the decimation level is > 1 if not dec > 1: errors.append( "Invalid Decimation {} for overview level {}".format( dec, ix)) # Check that the IFD of descending overviews are sorted by increasing # offsets ifd_offset = int( src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1, ovr=ix)) ifd_offsets.append(ifd_offset) details["ifd_offsets"]["overview_{}".format(ix)] = ifd_offset if ifd_offsets[-1] < ifd_offsets[-2]: if ix == 0: errors.append( "The offset of the IFD for overview of index {} is {}, " "whereas it should be greater than the one of the main " "image, which is at byte {}".format( ix, ifd_offsets[-1], ifd_offsets[-2])) else: errors.append( "The offset of the IFD for overview of index {} is {}, " "whereas it should be greater than the one of index {}, " "which is at byte {}".format( ix, ifd_offsets[-1], ix - 1, ifd_offsets[-2])) block_offset = int( src.get_tag_item("BLOCK_OFFSET_0_0", "TIFF", bidx=1)) if not block_offset: errors.append("Missing BLOCK_OFFSET_0_0") data_offset = int(block_offset) if block_offset else None data_offsets = [data_offset] details["data_offsets"] = {} details["data_offsets"]["main"] = data_offset for ix, dec in enumerate(overviews): data_offset = int( src.get_tag_item("BLOCK_OFFSET_0_0", "TIFF", bidx=1, ovr=ix)) data_offsets.append(data_offset) details["data_offsets"]["overview_{}".format(ix)] = data_offset if data_offsets[-1] < ifd_offsets[-1]: if len(overviews) > 0: errors.append( "The offset of the first block of the smallest overview " "should be after its IFD") else: errors.append( "The offset of the first block of the image should " "be after its IFD") for i in range(len(data_offsets) - 2, 0, -1): if data_offsets[i] < data_offsets[i + 1]: errors.append( "The offset of the first block of overview of index {} should " "be after the one of the overview of index {}".format( i - 1, i)) if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]: errors.append( "The offset of the first block of the main resolution image " "should be after the one of the overview of index {}". format(len(overviews) - 1)) for ix, dec in enumerate(overviews): with rasterio.open(src_path, OVERVIEW_LEVEL=ix) as ovr_dst: if ovr_dst.width >= 512 or ovr_dst.height >= 512: if not ovr_dst.is_tiled: errors.append( "Overview of index {} is not tiled".format(ix)) if warnings and not quiet: click.secho("The following warnings were found:", fg="yellow", err=True) for w in warnings: click.echo("- " + w, err=True) click.echo(err=True) if errors and not quiet: click.secho("The following errors were found:", fg="red", err=True) for e in errors: click.echo("- " + e, err=True) is_valid = False if errors or (warnings and strict) else True return is_valid, errors, warnings
def process_item(self, item): docs, material_dict = item grouped = group_by_material_id(material_dict, docs, 'input_structure') formula = docs[0]['pretty_formula'] if not grouped: formula = Structure.from_dict(list( material_dict.values())[0]).composition.reduced_formula logger.debug("No material match for {}".format(formula)) # For now just do the most recent one that's not failed # TODO: better sorting of docs all_docs = [] for task_id, elastic_docs in grouped.items(): elastic_docs = sorted(elastic_docs, key=lambda x: (x['order'], x['state'], x['completed_at'])) grouped_by_order = { k: list(v) for k, v in groupby(elastic_docs, key=lambda x: x['order']) } soec_docs = grouped_by_order.get(2) toec_docs = grouped_by_order.get(3) if soec_docs: final_doc = soec_docs[-1] else: final_doc = toec_docs[-1] structure = Structure.from_dict(final_doc['optimized_structure']) formula = structure.composition.reduced_formula elements = [s.symbol for s in structure.composition.elements] chemsys = '-'.join(elements) # Issue warning if relaxed structure differs warnings = final_doc.get('warnings') or [] opt = Structure.from_dict(final_doc['optimized_structure']) init = Structure.from_dict(final_doc['input_structure']) # TODO: are these the right params? if not StructureMatcher().fit(init, opt): warnings.append("Inequivalent optimization structure") material_mag = CollinearMagneticStructureAnalyzer( opt).ordering.value material_mag = mag_types[material_mag] if final_doc['magnetic_type'] != material_mag: warnings.append("Elastic magnetic phase is {}".format( final_doc['magnetic_type'])) warnings = warnings or None # Filter for failure and warnings k_vrh = final_doc['k_vrh'] if k_vrh < 0 or k_vrh > 600: state = 'failed' elif warnings is not None: state = 'warning' else: state = 'successful' final_doc.update({"warnings": warnings}) elastic_summary = { 'task_id': task_id, 'all_elastic_fits': elastic_docs, 'elasticity': final_doc, 'spacegroup': init.get_space_group_info()[0], 'magnetic_type': final_doc['magnetic_type'], 'pretty_formula': formula, 'chemsys': chemsys, 'elements': elements, 'last_updated': self.elasticity.lu_field, 'state': state } if toec_docs: # TODO: this should be a bit more refined final_toec_doc = deepcopy(toec_docs[-1]) et_exp = ElasticTensorExpansion.from_voigt( final_toec_doc['elastic_tensor_expansion']) symbol_dict = et_exp[1].zeroed(1e-2).get_symbol_dict() final_toec_doc.update({"symbol_dict": symbol_dict}) set_(elastic_summary, "elasticity.third_order", final_toec_doc) all_docs.append(jsanitize(elastic_summary)) # elastic_summary.update(final_doc) return all_docs
def DoReviewChecks(change, committing, verbose, output_stream, input_stream, default_review, may_prompt): """Runs all review checks that apply to the files in the change. This finds all .review.py files in directories enclosing the files in the change (up to the repository root) and calls the relevant entrypoint function depending on whether the change is being committed or uploaded. Prints errors, warnings and notifications. Prompts the user for warnings when needed. Args: change: The Change object. committing: True if 'gcl commit' is running, False if 'gcl upload' is. verbose: Prints debug info. output_stream: A stream to write output from review tests to. input_stream: A stream to read input from the user. default_review: A default review script to execute in any case. may_prompt: Enable (y/n) questions on warning or error. Warning: If may_prompt is true, output_stream SHOULD be sys.stdout and input_stream SHOULD be sys.stdin. Return: True if execution can continue, False if not. """ start_time = time.time() review_files = ListRelevantReviewFiles(change.AbsoluteLocalPaths(True), change.RepositoryRoot()) if not review_files and verbose: output_stream.write("Warning, no .review.py found.\n") results = [] executer = ReviewExecuter(change, committing) if default_review: if verbose: output_stream.write("Running default review script.\n") fake_path = os.path.join(change.RepositoryRoot(), '.review.py') results += executer.ExecReviewScript(default_review, fake_path) for filename in review_files: filename = os.path.abspath(filename) if verbose: output_stream.write("Running %s\n" % filename) # Accept CRLF review script. review_script = read_file(filename, 'rU') results += executer.ExecReviewScript(review_script, filename) errors = [] notifications = [] warnings = [] for result in results: if not result.IsFatal() and not result.ShouldPrompt(): notifications.append(result) elif result.ShouldPrompt(): warnings.append(result) else: errors.append(result) error_count = 0 for name, items in (('Messages', notifications), ('Warnings', warnings), ('ERRORS', errors)): if items: output_stream.write('** Review %s **\n' % name) for item in items: if not item._Handle( output_stream, input_stream, may_prompt=False): error_count += 1 output_stream.write('\n') total_time = time.time() - start_time if total_time > 1.0: print "Review checks took %.1fs to calculate." % total_time if not errors and warnings and may_prompt: if not PromptYesNo( input_stream, output_stream, 'There were review warnings. ' 'Are you sure you wish to continue? (y/N): '): error_count += 1 return (error_count == 0)
#save to json file with open(dir + 'results/'+ logs["header"]["exp_name"] +'_raw.json', 'w') as outfile: json.dump(avgarray, outfile) #save to dat type file with open(dir + 'results/'+ logs["header"]["exp_name"]+"_raw.dat", "w") as data_file: for i in range(len(frequencyA2[si:ei])): data_file.write("%f %f %f\n" % (frequencyA2[si:ei][i], left[i], right[i])) #Stops the logging timer endtime = time.localtime() #Extra information for logs warnings = list() if(outlier_count != 0): warnings.append("Damaged scans detected!") if(files != int(logs["header"]["N_scans,N_cal"][0])*8): warnings.append("File count does not match predicted!") #Logging data json dictionary data rez_logs = {} rez_logs["exp_name"]= logs["header"]["exp_name"] rez_logs["start_time"]= time.strftime("%Y-%m-%dT%H:%M:%S", starttime) rez_logs["end_time"] = time.strftime("%Y-%m-%dT%H:%M:%S", endtime) rez_logs["cores"] = comm.size rez_logs["file_count"] = files rez_logs["scan_count"] = logs["header"]["N_scans,N_cal"][0] rez_logs["exp_start"] = logs["1s0"]["date"] rez_logs["exp_end"] = logs[str(list(logs.keys())[-1])]["date"] rez_logs["anomalies"] = outlier_count
def check_calib(self, frame_type="ALL"): warnings = [] if frame_type=="BIAS": if "data" not in self.cal_data["BIAS"]: warnings.append("Warning: No bias image found - can't do bias subtraction. Continue anyway?") elif frame_type=="DARK": if len(self.cal_data["DARK"])==0: warnings.append("Warning: No dark images found - can't account for dark current. Continue anyway?") else: for tag in self.cal_data["DARK"]: if not self.cal_data["DARK"][tag]["master"]: warnings.append("Warning: No bias frame applied to dark frame with exposure {}. Continue anyway?".format(tag)) elif frame_type=="Flat Field": if len(self.cal_data["Flat Field"])==0: warnings.append("Warning: No flat images found - can't account for detector sensitivity. Continue anyway?") else: for tag in self.cal_data["Flat Field"]: if not self.cal_data["Flat Field"][tag]["master"]: warnings.append("Warning: No bias frame applied to flat frame with filter \"{}\". Continue anyway?".format(tag)) elif frame_type=="ALL": warnings.extend(self.check_calib("BIAS")) warnings.extend(self.check_calib("DARK")) warnings.extend(self.check_calib("Flat Field")) else: warnings.append("Warning: Unknown frame type. Continue anyway?") return warnings
def load_filenames(args): global gl warnings = [] errors = [] for arg in args: #{ for sampfname in glob.glob(arg): if True: try: sampf = file(sampfname, "rb") except IOError, msg: errors.append(msg) if sampf: sampf.close() samp = Samp() # strip directory basename = sampfname.replace("\\", "/") basename = sampfname.split("/")[-1] # strip ".wav" extension basename = basename.split(".") basename = ".".join(basename[0:-1]) basename = jtrans.tr(basename, DELIMS, " ") # get layer name parts = basename.split(" ") if len(parts) <= abs(LAYER_LOC): loc = LAYER_LOC if loc >= 0: loc += 1 print >>sys.stderr, ( "After splitting filename '%s' delimiters," % (basename)) print >>sys.stderr, ( "there aren't enough parts to find part number %d." % loc) sys.exit(1) layername = parts[LAYER_LOC] # get note: might be MIDI number or note name if len(parts) <= abs(NOTE_LOC): loc = NOTE_LOC if loc >= 0: loc += 1 print >>sys.stderr, ( "After splitting filename '%s' at delimiters, there aren't" % (basename)) print >>sys.stderr, ( "there aren't enough parts to find part number %d." % loc) sys.exit(1) notespec = parts[NOTE_LOC] mnote = jmidi.notenum(notespec) if mnote == None: print >>sys.stderr, ( "Invalid MIDI note designation '%s' in '%s'" % (notespec, basename)) sys.exit(1) # print sampfname, mnote, layername, jmidi.mnote_name(mnote)[0] samp.fname = sampfname samp.mnote = mnote samp.notename = jmidi.mnote_name(mnote, pad=None) samp.layername = layername if layername not in gl.layernum: warnings.append("Sample for unconfigured layer '%s': %s" % (samp.layername, samp.fname)) continue samp.layer = gl.layernum[layername] if samp.layer == None: warnings.append("Sample for missing layer '%s': %s" % (samp.layername, samp.fname)) continue x = LO_KEY - MAX_NOTE_SHIFT if (samp.mnote < max(0, LO_KEY - MAX_NOTE_SHIFT) or samp.mnote > HI_KEY + MAX_NOTE_SHIFT): warnings.append("Sample outside useful note range (%s): %s" % (samp.notename, samp.fname)) continue samp.char = None gl.samps[sampfname] = samp gl.grid[samp.layer][mnote] = samp
def wsdl_validate_params(self, struct, value): """Validate the arguments (actual values) for the parameters structure. Fail for any invalid arguments or type mismatches.""" errors = [] warnings = [] valid = True # Determine parameter type if type(struct) == type(value): typematch = True if not isinstance(struct, dict) and isinstance(value, dict): typematch = True # struct can be a dict or derived (Struct) else: typematch = False if struct == str: struct = unicode # fix for py2 vs py3 string handling if not isinstance(struct, (list, dict, tuple)) and struct in TYPE_MAP.keys(): if not type(value) == struct: try: struct(value) # attempt to cast input to parameter type except: valid = False errors.append('Type mismatch for argument value. parameter(%s): %s, value(%s): %s' % (type(struct), struct, type(value), value)) elif isinstance(struct, list) and len(struct) == 1 and not isinstance(value, list): # parameter can have a dict in a list: [{}] indicating a list is allowed, but not needed if only one argument. next_valid, next_errors, next_warnings = self.wsdl_validate_params(struct[0], value) if not next_valid: valid = False errors.extend(next_errors) warnings.extend(next_warnings) # traverse tree elif isinstance(struct, dict): if struct and value: for key in value: if key not in struct: valid = False errors.append('Argument key %s not in parameter. parameter: %s, args: %s' % (key, struct, value)) else: next_valid, next_errors, next_warnings = self.wsdl_validate_params(struct[key], value[key]) if not next_valid: valid = False errors.extend(next_errors) warnings.extend(next_warnings) for key in struct: if key not in value: warnings.append('Parameter key %s not in args. parameter: %s, value: %s' % (key, struct, value)) elif struct and not value: warnings.append('parameter keys not in args. parameter: %s, args: %s' % (struct, value)) elif not struct and value: valid = False errors.append('Args keys not in parameter. parameter: %s, args: %s' % (struct, value)) else: pass elif isinstance(struct, list): struct_list_value = struct[0] for item in value: next_valid, next_errors, next_warnings = self.wsdl_validate_params(struct_list_value, item) if not next_valid: valid = False errors.extend(next_errors) warnings.extend(next_warnings) elif not typematch: valid = False errors.append('Type mismatch. parameter(%s): %s, value(%s): %s' % (type(struct), struct, type(value), value)) return (valid, errors, warnings)
def first_parse(lines): """ first parse of file to fin start/end of sections and get metadata Parameters ---------- lines: List Returns ------- """ scf_start_first = None scf_end_last = None opt_start = None opt_end = None steps_num = None steps = [] warnings = [] non_terminating_errors = [] found_job_done = False start_time, end_time, elapsed_time, nprocs = None, None, None, None for i, line in enumerate(lines): line = line.strip() if line.startswith("JOB DONE."): found_job_done = True # convergence NOT achieved after *** iterations: stopping if "convergence NOT achieved" in line: non_terminating_errors.append(line) if "Error in routine cdiaghg" in line: non_terminating_errors.append(line) if "DEPRECATED" in line: warnings.append(line) if "Geometry Optimization" in line: if "End of" in line: if not opt_start: raise IOError( "found end of geometry optimization before start in line #{0}: {1}" .format(i, line)) if opt_end: raise IOError( "found multiple geometry optimization ends in line #{0} and #{1}" .format(opt_end, i)) opt_end = i else: if opt_start: raise IOError( "found multiple geometry optimization starts in " "line #{0} and #{1}".format(opt_start, i)) opt_start = i if fnmatch(line, "number of * steps *=*") and opt_start and not opt_end: new_step = split_numbers(line)[0] if not steps_num is None: if new_step != steps_num + 1: pass # NOTE: this isn't strictly true since the step history can be reset # TODO maybe use the line above "number of scf cycles = " # raise IOError("expecting geometry optimization step {0} at " # "line #{1}: {2}".format(steps_num+1, i, line)) steps[-1] = (steps[-1][0], i) steps_num += 1 else: steps_num = 0 # steps_num = new_step steps.append((i, None)) # Can get: # lsda relaxation : a final configuration with zero # absolute magnetization has been found # # the program is checking if it is really the minimum energy structure # by performing a new scf iteration without any "electronic" history # # TODO should this be a separate step? if fnmatch( line, '*performing a new scf iteration without any "electronic" history' ): steps[-1] = (steps[-1][0], i) steps_num += 1 steps.append((i, None)) if "Self-consistent Calculation" in line and not scf_start_first: scf_start_first = i if "End of self-consistent calculation" in line: scf_end_last = i if fnmatch(line, "Parallel version *, running on * processors"): if nprocs is not None: raise IOError("found second nprocs on line {0}: {1}".format( i, line)) nprocs = int(line.split()[-2]) elif fnmatch(line, "Parallel version *, running on * processor cores"): if nprocs is not None: raise IOError("found second nprocs on line {0}: {1}".format( i, line)) nprocs = int(line.split()[-3]) # NB: time uses spaces instead of 0, e.g. This run was terminated on: 12:43: 3 6Sep2017 if fnmatch(line, "Program*starts on * at *"): if start_time is not None: raise IOError( "found second start time on line {0}: {1}".format(i, line)) line_segs = line.split("at") atime = line_segs[-1].replace(" ", "") adate = line_segs[-2].split()[-1] start_time = datetime.strptime(atime + ' ' + adate, '%H:%M:%S %d%b%Y') if line.startswith("This run was terminated on:"): if end_time is not None: raise IOError("found second end time on line {0}: {1}".format( i, line)) line_segs = line.split() adate = line_segs[-1] atime = ":".join(" ".join(line_segs[:-1]).split(":")[-3:]).replace( " ", "") end_time = datetime.strptime(atime + ' ' + adate, '%H:%M:%S %d%b%Y') if not found_job_done: non_terminating_errors.append("Did not find indicator: JOB DONE.") if opt_end and steps: steps[-1] = (steps[-1][0], opt_end) if opt_start and not opt_end: raise IOError("found start of geometry optimization but no end") if scf_start_first and not scf_end_last: raise IOError("found start of SCF but no end") if start_time and end_time: delta_time = end_time - start_time m, s = divmod(delta_time.total_seconds(), 60) h, m = divmod(m, 60) elapsed_time = "%d:%02d:%02d" % (h, m, s) meta = {"elapsed_time": elapsed_time, "nprocs": nprocs} return opt_start, opt_end, steps, scf_start_first, scf_end_last, warnings, non_terminating_errors, meta
def DoPresubmitChecks(change, committing, verbose, output_stream, input_stream, default_presubmit, may_prompt): """Runs all presubmit checks that apply to the files in the change. This finds all PRESUBMIT.py files in directories enclosing the files in the change (up to the repository root) and calls the relevant entrypoint function depending on whether the change is being committed or uploaded. Prints errors, warnings and notifications. Prompts the user for warnings when needed. Args: change: The Change object. committing: True if 'gcl commit' is running, False if 'gcl upload' is. verbose: Prints debug info. output_stream: A stream to write output from presubmit tests to. input_stream: A stream to read input from the user. default_presubmit: A default presubmit script to execute in any case. may_prompt: Enable (y/n) questions on warning or error. Warning: If may_prompt is true, output_stream SHOULD be sys.stdout and input_stream SHOULD be sys.stdin. Return: True if execution can continue, False if not. """ start_time = time.time() presubmit_files = ListRelevantPresubmitFiles( change.AbsoluteLocalPaths(True), change.RepositoryRoot()) if not presubmit_files and verbose: output_stream.write("Warning, no presubmit.py found.\n") results = [] executer = PresubmitExecuter(change, committing) if default_presubmit: if verbose: output_stream.write("Running default presubmit script.\n") fake_path = os.path.join(change.RepositoryRoot(), 'PRESUBMIT.py') results += executer.ExecPresubmitScript(default_presubmit, fake_path) for filename in presubmit_files: filename = os.path.abspath(filename) if verbose: output_stream.write("Running %s\n" % filename) # Accept CRLF presubmit script. presubmit_script = gclient_utils.FileRead(filename, 'rU') results += executer.ExecPresubmitScript(presubmit_script, filename) errors = [] notifications = [] warnings = [] for result in results: if not result.IsFatal() and not result.ShouldPrompt(): notifications.append(result) elif result.ShouldPrompt(): warnings.append(result) else: errors.append(result) error_count = 0 for name, items in (('Messages', notifications), ('Warnings', warnings), ('ERRORS', errors)): if items: output_stream.write('** Presubmit %s **\n' % name) for item in items: if not item._Handle( output_stream, input_stream, may_prompt=False): error_count += 1 output_stream.write('\n') total_time = time.time() - start_time if total_time > 1.0: print "Presubmit checks took %.1fs to calculate." % total_time if not errors and warnings and may_prompt: if not PromptYesNo( input_stream, output_stream, 'There were presubmit warnings. ' 'Are you sure you wish to continue? (y/N): '): error_count += 1 global _ASKED_FOR_FEEDBACK # Ask for feedback one time out of 5. if (len(results) and random.randint(0, 4) == 0 and not _ASKED_FOR_FEEDBACK): output_stream.write( "Was the presubmit check useful? Please send feedback " "& hate mail to [email protected]!\n") _ASKED_FOR_FEEDBACK = True return (error_count == 0)
error = None warnings = [] with catch_warnings(record=True) as w: try: run_test(test, debug=debug) except UnheardNoteError, e: unheard = e except MissingRecordingError, e: missing_rec = e except Exception: error = sys.exc_info() for warning in w: if warning.category is ExtraNoteWarning: warnings.append(warning.message) if unheard is not None or error is not None or missing_rec is not None: color_name = 'red!' results['errors'] += 1 elif warnings: color_name = 'yellow!' results['warnings'] += 1 else: color_name = 'green!' results['ok'] += 1 print color(color_name, test.name) for warning in warnings: print ' %2d: Extra notes:' % warning.expectation, \
def load_notebook(resources=None, verbose=False, force=False, skip=False): ''' Prepare the IPython notebook for displaying Bokeh plots. Args: resources (Resource, optional) : how and where to load BokehJS from verbose (bool, optional) : whether to report detailed settings (default: False) force (bool, optional) : whether to skip IPython notebook check (default: False) Returns: None ''' global _notebook_loaded # It's possible the IPython folks will chance things in the future, `force` parameter # provides an escape hatch as long as `displaypub` works if not force: notebook = False try: notebook = 'notebook' in get_ipython( ).config.IPKernelApp.parent_appname except Exception: pass if not notebook: raise RuntimeError('load_notebook only works inside an ' 'IPython notebook, try using force=True.') from .resources import INLINE from .templates import NOTEBOOK_LOAD, RESOURCES if resources is None: resources = INLINE plot_resources = RESOURCES.render( js_raw=resources.js_raw, css_raw=resources.css_raw, js_files=resources.js_files, css_files=resources.css_files, ) if resources.mode == 'inline': js_info = 'inline' css_info = 'inline' else: js_info = resources.js_files[0] if len( resources.js_files) == 1 else resources.js_files css_info = resources.css_files[0] if len( resources.css_files) == 1 else resources.css_files warnings = [ "Warning: " + msg['text'] for msg in resources.messages if msg['type'] == 'warn' ] if _notebook_loaded: warnings.append('Warning: BokehJS previously loaded') _notebook_loaded = resources html = NOTEBOOK_LOAD.render( plot_resources=plot_resources, logo_url=resources.logo_url, verbose=verbose, js_info=js_info, css_info=css_info, bokeh_version=__version__, warnings=warnings, skip=skip, ) utils.publish_display_data({'text/html': html})
def load_notebook(resources=None, verbose=False, force=False, skip=False): ''' Prepare the IPython notebook for displaying Bokeh plots. Args: resources (Resource, optional) : a resource object describing how and where to load BokehJS from verbose (bool, optional) : whether to report detailed settings (default: False) force (bool, optional) : whether to skip IPython notebook check (default: False) Returns: None ''' global _notebook_loaded # It's possible the IPython folks will chance things in the future, `force` parameter # provides an escape hatch as long as `displaypub` works if not force: notebook = False try: notebook = 'notebook' in get_ipython().config.IPKernelApp.parent_appname except Exception: pass if not notebook: raise RuntimeError('load_notebook only works inside an ' 'IPython notebook, try using force=True.') from .resources import INLINE from .templates import NOTEBOOK_LOAD, RESOURCES if resources is None: resources = INLINE plot_resources = RESOURCES.render( js_raw = resources.js_raw, css_raw = resources.css_raw, js_files = resources.js_files, css_files = resources.css_files, ) if resources.mode == 'inline': js_info = 'inline' css_info = 'inline' else: js_info = resources.js_files[0] if len(resources.js_files) == 1 else resources.js_files css_info = resources.css_files[0] if len(resources.css_files) == 1 else resources.css_files warnings = ["Warning: " + msg['text'] for msg in resources.messages if msg['type'] == 'warn'] if _notebook_loaded: warnings.append('Warning: BokehJS previously loaded') _notebook_loaded = resources html = NOTEBOOK_LOAD.render( plot_resources = plot_resources, logo_url = resources.logo_url, verbose = verbose, js_info = js_info, css_info = css_info, bokeh_version = __version__, warnings = warnings, skip = skip, ) utils.publish_display_data({'text/html': html})
def get_autosummary(names, state, no_signatures=False): """ Generate a proper table node for autosummary:: directive. Parameters ---------- names : list of str Names of Python objects to be imported and added to the table. document : document Docutils document object """ document = state.document real_names = {} warnings = [] prefixes = [''] prefixes.insert(0, document.settings.env.currmodule) table = nodes.table('') group = nodes.tgroup('', cols=2) table.append(group) group.append(nodes.colspec('', colwidth=10)) group.append(nodes.colspec('', colwidth=90)) body = nodes.tbody('') group.append(body) def append_row(*column_texts): row = nodes.row('') for text in column_texts: node = nodes.paragraph('') vl = ViewList() vl.append(text, '<autosummary>') state.nested_parse(vl, 0, node) try: if isinstance(node[0], nodes.paragraph): node = node[0] except IndexError: pass row.append(nodes.entry('', node)) body.append(row) for name in names: try: obj, real_name = import_by_name(name, prefixes=prefixes) except ImportError: warnings.append(document.reporter.warning( 'failed to import %s' % name)) append_row(":obj:`%s`" % name, "") continue real_names[name] = real_name doc = get_doc_object(obj) if doc['Summary']: title = " ".join(doc['Summary']) else: title = "" col1 = u":obj:`%s <%s>`" % (name, real_name) if doc['Signature']: sig = re.sub('^[^(\[]*', '', doc['Signature'].strip()) if '=' in sig: # abbreviate optional arguments sig = re.sub(r', ([a-zA-Z0-9_]+)=', r'[, \1=', sig, count=1) sig = re.sub(r'\(([a-zA-Z0-9_]+)=', r'([\1=', sig, count=1) sig = re.sub(r'=[^,)]+,', ',', sig) sig = re.sub(r'=[^,)]+\)$', '])', sig) # shorten long strings sig = re.sub(r'(\[.{16,16}[^,]*?),.*?\]\)', r'\1, ...])', sig) else: sig = re.sub(r'(\(.{16,16}[^,]*?),.*?\)', r'\1, ...)', sig) # make signature contain non-breaking spaces col1 += u"\\ \u00a0" + unicode(sig).replace(u" ", u"\u00a0") col2 = title append_row(col1, col2) return table, warnings, real_names
def get_warnings(elastic_tensor, structure): """ Generates all warnings that apply to a fitted elastic tensor Args: elastic_tensor (ElasticTensor): elastic tensor for which to determine warnings structure (Structure): structure for which elastic tensor is determined Returns: list of warnings """ warnings = [] if any([s.is_rare_earth_metal for s in structure.species]): warnings.append("Contains a rare earth element") eigs, eigvecs = np.linalg.eig(elastic_tensor.voigt) if np.any(eigs < 0.0): warnings.append("Elastic tensor has a negative eigenvalue") c11, c12, c13 = elastic_tensor.voigt[0, 0:3] c23 = elastic_tensor.voigt[1, 2] # TODO: these should be revisited at some point, are they complete? # I think they might only apply to cubic systems if (abs((c11 - c12) / c11) < 0.05 or c11 < c12): warnings.append("c11 and c12 are within 5% or c12 is greater than c11") if (abs((c11 - c13) / c11) < 0.05 or c11 < c13): warnings.append("c11 and c13 are within 5% or c13 is greater than c11") if (abs((c11 - c23) / c11) < 0.05 or c11 < c23): warnings.append("c11 and c23 are within 5% or c23 is greater than c11") moduli = ["k_voigt", "k_reuss", "k_vrh", "g_voigt", "g_reuss", "g_vrh"] moduli_array = np.array([getattr(elastic_tensor, m) for m in moduli]) if np.any(moduli_array < 2): warnings.append("One or more K, G below 2 GPa") return warnings
def get_state_and_warnings(elastic_doc): """ Generates all warnings that apply to a fitted elastic tensor Args: elastic_tensor (ElasticTensor): elastic tensor for which to determine warnings structure (Structure): structure for which elastic tensor is determined Returns: list of warnings """ structure = elastic_doc['optimized_structure'] warnings = [] if any([s.is_rare_earth_metal for s in structure.species]): warnings.append("Structure contains a rare earth element") vet = np.array(elastic_doc['elastic_tensor']) eigs, eigvecs = np.linalg.eig(vet) if np.any(eigs < 0.0): warnings.append("Elastic tensor has a negative eigenvalue") c11, c12, c13 = vet[0, 0:3] c23 = vet[1, 2] # TODO: these should be revisited at some point, are they complete? # I think they might only apply to cubic systems if (abs((c11 - c12) / c11) < 0.05 or c11 < c12): warnings.append("c11 and c12 are within 5% or c12 is greater than c11") if (abs((c11 - c13) / c11) < 0.05 or c11 < c13): warnings.append("c11 and c13 are within 5% or c13 is greater than c11") if (abs((c11 - c23) / c11) < 0.05 or c11 < c23): warnings.append("c11 and c23 are within 5% or c23 is greater than c11") moduli = ["k_voigt", "k_reuss", "k_vrh", "g_voigt", "g_reuss", "g_vrh"] moduli_array = np.array([get(elastic_doc, m) for m in moduli]) if np.any(moduli_array < 2): warnings.append("One or more K, G below 2 GPa") if np.any(moduli_array > 1000): warnings.append("One or more K, G above 1000 GPa") if elastic_doc['order'] == 3: if elastic_doc.get("average_linear_thermal_expansion", 0) < -0.1: warnings.append("Negative thermal expansion") if len(elastic_doc['strains']) < 80: warnings.append("Fewer than 80 strains, TOEC may be deficient") failure_states = [moduli_array[2] < 0] if any(failure_states): state = "failed" elif warnings: state = "warning" else: state = "successful" warnings = None return state, warnings
def infer_degeneracy(transcriptObjs, samples, variant_arrays_by_seq_id): sequence_id_arrays = [] start_arrays = [] end_arrays = [] degeneracy_arrays_by_sample = collections.defaultdict(list) warnings = [] for transcriptObj in tqdm(transcriptObjs, total=len(transcriptObjs), desc="[%] Inferring degeneracy... ", ncols=150, position=0, leave=True): if not transcriptObj.is_orf(): warnings.append("[-] Transcript %s has no ORF: START=%s, STOP=%s, DIVISIBLE_BY_3=%s (will be skipped)" % (transcriptObj.transcript_id, transcriptObj.has_start(), transcriptObj.has_stop(), transcriptObj.is_divisible_by_three())) else: if not transcriptObj.sequence_id in variant_arrays_by_seq_id: cds_pos_mask = np.array([]) else: cds_pos_mask = np.isin(transcriptObj.positions, variant_arrays_by_seq_id[transcriptObj.sequence_id]['POS'], assume_unique=True) # will crash if non-unique pos if np.any(cds_pos_mask): sequence_id_arrays.append(np.full(cds_pos_mask.shape[0], transcriptObj.sequence_id)) start_arrays.append(transcriptObj.positions) end_arrays.append(transcriptObj.positions + 1) gt_pos_mask = np.isin(variant_arrays_by_seq_id[transcriptObj.sequence_id]['POS'], transcriptObj.positions, assume_unique=True) # will crash if non-unique pos alleles = np.column_stack( [ variant_arrays_by_seq_id[transcriptObj.sequence_id]['REF'][gt_pos_mask], variant_arrays_by_seq_id[transcriptObj.sequence_id]['ALT'][gt_pos_mask], np.full(gt_pos_mask[gt_pos_mask==True].shape, '') ]) for idx, sample in enumerate(samples): # at least one variant pos in VCF (even if all variants are HOMREF) gt_sample = variant_arrays_by_seq_id[transcriptObj.sequence_id]['GT'][gt_pos_mask, idx] idx0 = np.array([np.arange(alleles.shape[0]), gt_sample[:,0]]) idx1 = np.array([np.arange(alleles.shape[0]), gt_sample[:,1]]) variants = np.vstack( [ alleles[tuple(idx0)], alleles[tuple(idx1)] ]).T cds_sample_array = np.full((transcriptObj.positions.shape[0], variants.shape[1]), '') # has shape (gt_sample, ploidy) cds_sample_array[cds_pos_mask] = variants cds_sample_array[~cds_pos_mask,0] = transcriptObj.sequence[~cds_pos_mask] degeneracies = [] for i in range(0, len(transcriptObj.sequence), 3): codon_list = list(filter(lambda codon: len (codon) == 3, ["".join(x) for x in itertools.product(*cds_sample_array[i:i+3])])) _degeneracy = degeneracy(codon_list) if codon_list else 3 * ['NA'] degeneracies.append(_degeneracy) _deg = np.concatenate(degeneracies) #print("#", transcriptObj.transcript_id, transcriptObj.positions.shape, _deg.shape, _deg[0:3]) degeneracy_arrays_by_sample[sample].append(_deg) else: sequence_id_arrays.append(np.full(transcriptObj.positions.shape[0], transcriptObj.sequence_id)) start_arrays.append(transcriptObj.positions) end_arrays.append(transcriptObj.positions + 1) for idx, sample in enumerate(samples): degeneracy_arrays_by_sample[sample].append(transcriptObj.degeneracy) print("\n".join(warnings)) #for sample in samples: # print('degeneracy_arrays_by_sample[%s].shape' % sample, degeneracy_arrays_by_sample[sample].shape) for sample in tqdm(samples, total=len(samples), desc="[%] Writing output... ", ncols=150): #a = np.concatenate(sequence_id_arrays) #b = np.concatenate(start_arrays) #c = np.concatenate(end_arrays) #d = np.concatenate(degeneracy_arrays_by_sample[sample]) #print('a.shape', a.shape) #print('type(b)', type(b), 'b.shape', b.shape) #print('c.shape', c.shape) #print('d.shape', d.shape, d[0:3]) data = np.vstack([ np.concatenate(sequence_id_arrays), np.concatenate(start_arrays), np.concatenate(end_arrays), np.concatenate(degeneracy_arrays_by_sample[sample]), ]).T df = pd.DataFrame(data=data, columns=['sequence_id', 'start', 'end', 'degeneracy']) write_df(df, out_f="%s.tsv" % sample, sep='\t', header=False, status=False)
def run_once2( repo: Repo, environment: Environment, env_vars: Mapping[str, str], script: Path, limits: Mapping[str, int], memoize: bool, trace: bool, trace_imports: bool, tmp_dir: Path, ) -> ExecutionProfile: warnings = [] perf_log = tmp_dir / "perf.log" trace_log = tmp_dir / "trace.log" coverage_file = tmp_dir / ".coverage" command, runexec_stats, stdout, stderr = run_exec_cmd( environment, [ *([ "coverage", "run", str(script), "--data-file", str(coverage_file), "--rcfile=/dev/null" ] if trace else [ "python", str(ROOT / "trace_imports.py"), str(script), str(trace_log) ] if trace_imports else ["python", str(script)]), ], { "CHARMONIUM_CACHE_DISABLE": "0" if memoize else "1", "CHARMONIUM_CACHE_PERF_LOG": str(perf_log), **env_vars, }, tmp_dir, time_limit=limits["time"], mem_limit=limits["mem"], dir_modes={ "/": container.DIR_READ_ONLY, # TODO: Ideally, I would use `--overlay-dir` with tmp_dir instead of --full--access-dir. # See https://github.com/sosy-lab/benchexec/issues/815 "/home": container.DIR_OVERLAY, str(tmp_dir): container.DIR_FULL_ACCESS, str(repo.dir): container.DIR_READ_ONLY, }, tmp_dir=tmp_dir, ) trace_data = None if trace_imports: if runexec_stats.exitcode != 0: print(stdout) print(stderr) print(command) raise ValueError assert trace_log.exists() trace_data = sorted( list( set([ str(unpyc(Path(line))) for line in trace_log.read_text().split("\n") if line.startswith("/") and Path(line).is_relative_to(repo.dir) ]))) trace_log.unlink() if trace: if runexec_stats.exitcode != 0: print(stdout) print(stderr) print(command) raise ValueError assert coverage_file.exists() coverage_json = tmp_dir / "coverage.json" subprocess.run([ "coverage", "json", "--data-file", str(coverage_file), "-o", str(coverage_json), "--rcfile=/dev/null" ], cwd=tmp_dir) coverage_file.unlink() coverage_obj = json.loads(coverage_json.read_text()) coverage_json.unlink() trace_data = [] for file in sorted(coverage_obj["files"]): file_text = Path(file).read_text().split("\n") for line in coverage_obj["files"][file]["executed_lines"]: print(file, line) trace_data.append(file_text[line - 1]) if runexec_stats.termination_reason: warnings.append( f"Terminated for {runexec_stats.termination_reason} out with {limits['time']=} {limits['mem']=}" ) if perf_log.exists(): calls, kwargs = parse_memoized_log(perf_log) else: if memoize: warnings.append( "No perf log produced, despite enabling memoization.") # no profiling information calls = [] kwargs = {} return ExecutionProfile( output=stdout, command=command, log=stderr, success=runexec_stats.exitcode == 0, func_calls=calls, internal_stats=kwargs, runexec=runexec_stats, warnings=warnings, trace_data=trace_data, )
def accumulateDeprecations(message, category, stacklevel): self.assertEqual(DeprecationWarning, category) self.assertEqual(stacklevel, 2) warnings.append(message)
def migrate(customer_id, username, password, key): warnings = [] customer = masterdb.Customer.get(customer_id) h = Pingdom(username, password, key) # create the contacts contacts = {} for cid, contact in h.contacts.iteritems(): try: panopta_contact = masterdb.Contact.selectBy(customer=customer, fullname=contact.name)[0] except: timezone = "America/Chicago" panopta_contact = masterdb.Contact( customer=customer, fullname=contact.name, timezone=timezone ) warnings.append("contact '%s' created with default timezone of %s" % ( contact.name, timezone )) channels = [] contacts[cid] = (panopta_contact, channels) phone = getattr(contact, "cellphone", None) if phone: phone = "+"+phone sms_type = masterdb.ContactType.selectBy(textkey="sms")[0] try: masterdb.ContactChannel.selectBy(contact_info=phone, contact_type=sms_type)[0] except: channel = masterdb.ContactChannel( contact_info=phone, contact_type=sms_type, contact=panopta_contact, label="" ) channels.append(channel) email = getattr(contact, "email", None) if email: email_type = masterdb.ContactType.selectBy(textkey="email.html")[0] try: masterdb.ContactChannel.selectBy(contact_info=email, contact_type=email_type)[0] except: channel = masterdb.ContactChannel( contact_info=email, contact_type=email_type, contact=panopta_contact, label="" ) channels.append(channel) # migrate the servers try: server_group = masterdb.ServerGroup.selectBy(name="Pingdom")[0] except: server_group = masterdb.ServerGroup( name="Pingdom", server_group=None, customer=customer, notification_schedule=None ) for server in h.servers.values(): servers = masterdb.Server.selectBy(customer=customer, fqdn=server.hostname) if not servers.count(): logging.info("creating server \"%s\"" % server.hostname) panopta_server = masterdb.Server( name = server.hostname, fqdn = server.hostname, last_known_ip = None, customer = customer, primary_node = masterdb.MonitorNode.get_closest_node(server.ip), status = 'active', notification_schedule=None, server_group=server_group ) else: panopta_server = servers[0] # create the different notification schedules from the checks schedules = {} for check in server.checks: if not check.notifications: continue sk = [contact.id for contact in check.notifications] sname = [contact.name for contact in check.notifications] sname.sort() sk.sort() sk = [str(i) for i in sk] if len(sname) > 3: sname = sname[:3] sname.append("and %d more" % len(sname) - 3) sk = ",".join(sk) sname = ", ".join(sname) # create the notification schedule if it doesn't exist schedule = schedules.get(sk, None) if not schedule: logging.info("creating notification schedule \"%s\"" % sname) schedule = masterdb.NotificationSchedule( customer = customer, name = sname, description = "", ) schedules[sk] = schedule action = masterdb.NotificationAction( schedule = schedule, trigger_delay = 0, ) for contact in check.notifications: panopta_contact, channels = contacts[contact.id] for channel in channels: action.addContactChannel(channel) # save the applicable schedule for the next step, when we actually # create the monitor points check.schedule = schedule # add the service checks for check in server.checks: service_type = masterdb.ServiceType.selectBy(textkey=check.textkey)[0] try: # does a check like this already exist on the server? masterdb.MonitorPoint.selectBy( server=panopta_server, name=check.name, service_type=service_type, frequency=check.frequency, port=check.port, status='active' )[0] continue except: logging.info("creating monitor point \"%s\"" % check.name) mp = masterdb.MonitorPoint( server=panopta_server, name=check.name, service_type=service_type, frequency=check.frequency, port=check.port, metadata=None, status='active', notification_schedule=check.schedule ) mp.storeMetadata(check.metadata)
def get_autosummary(names, state, no_signatures=False): """ Generate a proper table node for autosummary:: directive. Parameters ---------- names : list of str Names of Python objects to be imported and added to the table. document : document Docutils document object """ document = state.document real_names = {} warnings = [] prefixes = [''] prefixes.insert(0, document.settings.env.currmodule) table = nodes.table('') group = nodes.tgroup('', cols=2) table.append(group) group.append(nodes.colspec('', colwidth=10)) group.append(nodes.colspec('', colwidth=90)) body = nodes.tbody('') group.append(body) def append_row(*column_texts): row = nodes.row('') for text in column_texts: node = nodes.paragraph('') vl = ViewList() vl.append(text, '<autosummary>') state.nested_parse(vl, 0, node) try: if isinstance(node[0], nodes.paragraph): node = node[0] except IndexError: pass row.append(nodes.entry('', node)) body.append(row) for name in names: try: obj, real_name = import_by_name(name, prefixes=prefixes) except ImportError: warnings.append( document.reporter.warning('failed to import %s' % name)) append_row(":obj:`%s`" % name, "") continue real_names[name] = real_name doc = get_doc_object(obj) if doc['Summary']: title = " ".join(doc['Summary']) else: title = "" col1 = u":obj:`%s <%s>`" % (name, real_name) if doc['Signature']: sig = re.sub('^[^(\[]*', '', doc['Signature'].strip()) if '=' in sig: # abbreviate optional arguments sig = re.sub(r', ([a-zA-Z0-9_]+)=', r'[, \1=', sig, count=1) sig = re.sub(r'\(([a-zA-Z0-9_]+)=', r'([\1=', sig, count=1) sig = re.sub(r'=[^,)]+,', ',', sig) sig = re.sub(r'=[^,)]+\)$', '])', sig) # shorten long strings sig = re.sub(r'(\[.{16,16}[^,]*?),.*?\]\)', r'\1, ...])', sig) else: sig = re.sub(r'(\(.{16,16}[^,]*?),.*?\)', r'\1, ...)', sig) # make signature contain non-breaking spaces col1 += u"\\ \u00a0" + unicode(sig).replace(u" ", u"\u00a0") col2 = title append_row(col1, col2) return table, warnings, real_names
def wsdl_validate_params(self, struct, value): """Validate the arguments (actual values) for the parameters structure. Fail for any invalid arguments or type mismatches.""" errors = [] warnings = [] valid = True # Determine parameter type if type(struct) == type(value): typematch = True if not isinstance(struct, dict) and isinstance(value, dict): typematch = True # struct can be a dict or derived (Struct) else: typematch = False if struct == str: struct = unicode # fix for py2 vs py3 string handling if not isinstance(struct, (list, dict, tuple)) and struct in TYPE_MAP.keys(): if not type(value) == struct and value is not None: try: struct(value) # attempt to cast input to parameter type except: valid = False errors.append( 'Type mismatch for argument value. parameter(%s): %s, value(%s): %s' % (type(struct), struct, type(value), value)) elif isinstance(struct, list) and len(struct) == 1 and not isinstance( value, list): # parameter can have a dict in a list: [{}] indicating a list is allowed, but not needed if only one argument. next_valid, next_errors, next_warnings = self.wsdl_validate_params( struct[0], value) if not next_valid: valid = False errors.extend(next_errors) warnings.extend(next_warnings) # traverse tree elif isinstance(struct, dict): if struct and value: for key in value: if key not in struct: valid = False errors.append( 'Argument key %s not in parameter. parameter: %s, args: %s' % (key, struct, value)) else: next_valid, next_errors, next_warnings = self.wsdl_validate_params( struct[key], value[key]) if not next_valid: valid = False errors.extend(next_errors) warnings.extend(next_warnings) for key in struct: if key not in value: warnings.append( 'Parameter key %s not in args. parameter: %s, value: %s' % (key, struct, value)) elif struct and not value: warnings.append( 'parameter keys not in args. parameter: %s, args: %s' % (struct, value)) elif not struct and value: valid = False errors.append( 'Args keys not in parameter. parameter: %s, args: %s' % (struct, value)) else: pass elif isinstance(struct, list): struct_list_value = struct[0] for item in value: next_valid, next_errors, next_warnings = self.wsdl_validate_params( struct_list_value, item) if not next_valid: valid = False errors.extend(next_errors) warnings.extend(next_warnings) elif not typematch: valid = False errors.append('Type mismatch. parameter(%s): %s, value(%s): %s' % (type(struct), struct, type(value), value)) return (valid, errors, warnings)
validate_required(row, 'Closing Quarter', specials, msg)) else: errors.extend(validate_values(row, 'Closing Fiscal Year')) errors.extend(validate_values(row, 'Closing Quarter')) ### # Data validation rules. This should catch any bad data. ### if (row.get('record validity', '').lower() == 'valid facility' and row.get('closing stage', '').lower() != 'closed' and row.get('ownership type', '').lower() == 'agency owned' and row.get('data center tier', '').lower() not in map( str.lower, VALID_VALUES['Data Center Tier'])): warnings.append( 'Only tiered data centers need to be reported, marked as "{}"'. format(row.get('data center tier'))) # Impossible PUEs # PUE = 1.0: if (row.get('avg electricity usage') and row.get('avg it electricity usage') and row.get('avg electricity usage') == row.get('avg it electricity usage')): warnings.append( 'Avg Electricity Usage ({}) for a facility should never be equal to Avg IT Electricity Usage ({})' .format(row.get('avg electricity usage'), row.get('avg it electricity usage')))
def cog_validate(src_path): """ Validate Cloud Optimized Geotiff. Parameters ---------- src_path : str or PathLike object A dataset path or URL. Will be opened in "r" mode. This script is the rasterio equivalent of https://svn.osgeo.org/gdal/trunk/gdal/swig/python/samples/validate_cloud_optimized_geotiff.py """ errors = [] warnings = [] details = {} if not GDALVersion.runtime().at_least("2.2"): raise Exception("GDAL 2.2 or above required") config = dict(GDAL_DISABLE_READDIR_ON_OPEN="FALSE") with rasterio.Env(**config): with rasterio.open(src_path) as src: if not src.driver == "GTiff": raise Exception("The file is not a GeoTIFF") filelist = [os.path.basename(f) for f in src.files] src_bname = os.path.basename(src_path) if len(filelist) > 1 and src_bname + ".ovr" in filelist: errors.append( "Overviews found in external .ovr file. They should be internal" ) overviews = src.overviews(1) if src.width > 512 or src.height > 512: if not src.is_tiled: errors.append( "The file is greater than 512xH or 512xW, but is not tiled" ) if not overviews: warnings.append( "The file is greater than 512xH or 512xW, it is recommended " "to include internal overviews") ifd_offset = int(src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1)) ifd_offsets = [ifd_offset] if ifd_offset not in (8, 16): errors.append( "The offset of the main IFD should be 8 for ClassicTIFF " "or 16 for BigTIFF. It is {} instead".format(ifd_offset)) details["ifd_offsets"] = {} details["ifd_offsets"]["main"] = ifd_offset if overviews and overviews != sorted(overviews): errors.append("Overviews should be sorted") for ix, dec in enumerate(overviews): # NOTE: Size check is handled in rasterio `src.overviews` methods # https://github.com/mapbox/rasterio/blob/4ebdaa08cdcc65b141ed3fe95cf8bbdd9117bc0b/rasterio/_base.pyx # We just need to make sure the decimation level is > 1 if not dec > 1: errors.append( "Invalid Decimation {} for overview level {}".format( dec, ix)) # Check that the IFD of descending overviews are sorted by increasing # offsets ifd_offset = int( src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1, ovr=ix)) ifd_offsets.append(ifd_offset) details["ifd_offsets"]["overview_{}".format(ix)] = ifd_offset if ifd_offsets[-1] < ifd_offsets[-2]: if ix == 0: errors.append( "The offset of the IFD for overview of index {} is {}, " "whereas it should be greater than the one of the main " "image, which is at byte {}".format( ix, ifd_offsets[-1], ifd_offsets[-2])) else: errors.append( "The offset of the IFD for overview of index {} is {}, " "whereas it should be greater than the one of index {}, " "which is at byte {}".format( ix, ifd_offsets[-1], ix - 1, ifd_offsets[-2])) block_offset = int( src.get_tag_item("BLOCK_OFFSET_0_0", "TIFF", bidx=1)) if not block_offset: errors.append("Missing BLOCK_OFFSET_0_0") data_offset = int(block_offset) if block_offset else None data_offsets = [data_offset] details["data_offsets"] = {} details["data_offsets"]["main"] = data_offset for ix, dec in enumerate(overviews): data_offset = int( src.get_tag_item("BLOCK_OFFSET_0_0", "TIFF", bidx=1, ovr=ix)) data_offsets.append(data_offset) details["data_offsets"]["overview_{}".format(ix)] = data_offset if data_offsets[-1] < ifd_offsets[-1]: if len(overviews) > 0: errors.append( "The offset of the first block of the smallest overview " "should be after its IFD") else: errors.append( "The offset of the first block of the image should " "be after its IFD") for i in range(len(data_offsets) - 2, 0, -1): if data_offsets[i] < data_offsets[i + 1]: errors.append( "The offset of the first block of overview of index {} should " "be after the one of the overview of index {}".format( i - 1, i)) if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]: errors.append( "The offset of the first block of the main resolution image " "should be after the one of the overview of index {}". format(len(overviews) - 1)) for ix, dec in enumerate(overviews): with rasterio.open(src_path, OVERVIEW_LEVEL=ix) as ovr_dst: if ovr_dst.width >= 512 or ovr_dst.height >= 512: if not ovr_dst.is_tiled: errors.append( "Overview of index {} is not tiled".format(ix)) if warnings: click.secho("The following warnings were found:", fg="yellow", err=True) for w in warnings: click.echo("- " + w, err=True) click.echo(err=True) if errors: click.secho("The following errors were found:", fg="red", err=True) for e in errors: click.echo("- " + e, err=True) return False return True
def DoReviewChecks(change, committing, verbose, output_stream, input_stream, default_review, may_prompt): """Runs all review checks that apply to the files in the change. This finds all .review.py files in directories enclosing the files in the change (up to the repository root) and calls the relevant entrypoint function depending on whether the change is being committed or uploaded. Prints errors, warnings and notifications. Prompts the user for warnings when needed. Args: change: The Change object. committing: True if 'gcl commit' is running, False if 'gcl upload' is. verbose: Prints debug info. output_stream: A stream to write output from review tests to. input_stream: A stream to read input from the user. default_review: A default review script to execute in any case. may_prompt: Enable (y/n) questions on warning or error. Warning: If may_prompt is true, output_stream SHOULD be sys.stdout and input_stream SHOULD be sys.stdin. Return: True if execution can continue, False if not. """ start_time = time.time() review_files = ListRelevantReviewFiles(change.AbsoluteLocalPaths(True), change.RepositoryRoot()) if not review_files and verbose: output_stream.write("Warning, no .review.py found.\n") results = [] executer = ReviewExecuter(change, committing) if default_review: if verbose: output_stream.write("Running default review script.\n") fake_path = os.path.join(change.RepositoryRoot(), '.review.py') results += executer.ExecReviewScript(default_review, fake_path) for filename in review_files: filename = os.path.abspath(filename) if verbose: output_stream.write("Running %s\n" % filename) # Accept CRLF review script. review_script = read_file(filename, 'rU') results += executer.ExecReviewScript(review_script, filename) errors = [] notifications = [] warnings = [] for result in results: if not result.IsFatal() and not result.ShouldPrompt(): notifications.append(result) elif result.ShouldPrompt(): warnings.append(result) else: errors.append(result) error_count = 0 for name, items in (('Messages', notifications), ('Warnings', warnings), ('ERRORS', errors)): if items: output_stream.write('** Review %s **\n' % name) for item in items: if not item._Handle(output_stream, input_stream, may_prompt=False): error_count += 1 output_stream.write('\n') total_time = time.time() - start_time if total_time > 1.0: print "Review checks took %.1fs to calculate." % total_time if not errors and warnings and may_prompt: if not PromptYesNo(input_stream, output_stream, 'There were review warnings. ' 'Are you sure you wish to continue? (y/N): '): error_count += 1 return (error_count == 0)