def upload(): """Receives an image, assigns a unique id and stores it""" if 'file' not in request.files: raise ApiException('Expected file upload', 422) filestream = request.files['file'] name, extension = os.path.splitext(filestream.filename) extension = extension.lstrip(".") name = name.strip() maxlen = current_app.config['MAXIMUM_FILENAME_LENGTH'] formats = current_app.config["ALLOWED_FORMATS"] if len(name) > maxlen: raise ApiException(f'Maximum filename length is {maxlen} characters', 400) if extension not in formats: raise ApiException(f'Unsupported extension {extension}. Expected one of {formats}', 422) if not validate_image(filestream, valid_formats=formats): raise ApiException(f"Unsupported file type", 422) filestream.seek(0) file = filestream.read() digest = hashfunc(file).hexdigest() key = f"{digest}.{extension}" success = current_app.storage.set(key, file) if success: return ApiResult({"id": digest}) raise ApiException("Error saving file", 500)
def test_cache_integrate_fall_through_integrate_false(): """See also test_cache_integrate_fall_through_no_integrate.""" mdl = DoNotUseModel('notme') x = numpy.asarray([2, 3, 7, 100]) y = mdl(x, integrate=False) expected = [1, 1, 1, 1] assert y == pytest.approx(expected) # A simplified version of check_cache (as we have no # integrate setting). # cache = mdl._cache assert len(cache) == 1 pars = [] data = [ numpy.asarray(pars).tobytes(), b'0', # not integrated x.tobytes() ] token = b''.join(data) digest = hashfunc(token).digest() assert digest in cache assert cache[digest] == pytest.approx(expected)
def checksum(self): ''' Checksum the file contents, used as a proxy for comparing the actual content. ''' csum = self._checksum if csum is None: path = self.path U = Upd() pathspace = U.columns - 64 label = "scan " + ( path if len(path) < pathspace else '...' + path[-(pathspace - 3):] # pylint: disable=unsubscriptable-object ) with Pfx("checksum %r", path): csum = hashfunc() with open(path, 'rb') as fp: length = os.fstat(fp.fileno()).st_size read_len = 0 for data in progressbar( read_from(fp, rsize=1024*1024), label=label, total=length, units_scale=BINARY_BYTES_SCALE, itemlenfunc=len, update_frequency=128, upd=U, ): csum.update(data) read_len += len(data) assert read_len == self.size csum = csum.digest() self._checksum = csum return csum
def sig( self ): """Returns this site signature as a string. The signature uniquely identifies this site. Each instance has a different signature.""" sig = self.pages() + self.output() + self.templatesDir try: s=hashfunc.new(sig).hexdigest() except: s=hashfunc(sig).hexdigest() return s
def get_hash(source): f = open(source, 'r', encoding='utf-8') hsh = hashfunc() for l in f: hsh.update(l.encode('utf-8')) f.close() return hsh.hexdigest()
def caching_json_request(method, url, query_args=None, body=None): key = hashfunc(repr((method, url, query_args, body))).hexdigest() if not key in cache: cache[key] = json_request(method=method, url=url, query_args=query_args, body=body) return cache[key]
def sig(self): """Returns this site signature as a string. The signature uniquely identifies this site. Each instance has a different signature.""" sig = self.pages() + self.output() + self.templatesDir try: s = hashfunc.new(sig).hexdigest() except: s = hashfunc(sig).hexdigest() return s
def index_project(name): try: proj = Project.objects.get(name=name) except Project.DoesNotExist: print("No such project. Abort.") return if not os.path.isdir(proj.root): print("Directory {} does not exist. Abort.".format(proj.root)) return files_to_index = [] for root, dirs, files in os.walk(proj.root): for fp in files: abspath = os.path.join(root, fp) with open(abspath, "rb") as f: hashval = hashfunc(f.read()).hexdigest() relpath = abspath.replace(proj.root, "") try: blob = Blob.objects.get(project=proj, path=relpath) if hashval != blob.checksum: # TODO: remove all refs and defs related to this blob files_to_index.append(blob) blob.checksum = hashval blob.save() except Blob.DoesNotExist: blob = Blob(project=proj, path=relpath, checksum=hashval) blob.save() files_to_index.append(blob) # index definitions for blob in files_to_index: abspath = proj.root + blob.path print("processing defs in {}".format(blob.path)) for ident, itype, line in parse_defs(abspath): ident = get_or_create_ident(ident, proj) def_entry = Def(ident=ident, blob=blob, line=line, type=itype) def_entry.save() # index references for blob in files_to_index: abspath = proj.root + blob.path print("processing refs in {}".format(blob.path)) for token, tokentype, line in tokenize(abspath): try: ident = Ident.objects.get(value=token, project=proj) ref_entry = Ref(ident=ident, blob=blob, line=line) ref_entry.save() except Ident.DoesNotExist: # ignore tokens that is not in ident pass
def check_cache(mdl, expected, x, xhi=None): """Check the cache contents. We assume only one value is being cached at a time. The code matches that in sherpa.models.model.modelCacher1d, so all it does is check we are using this method. """ cache = mdl._cache assert len(cache) == 1 pars = [p.val for p in mdl.pars] data = [ numpy.asarray(pars).tobytes(), b'1' if mdl.integrate else b'0', x.tobytes() ] if xhi is not None: data.append(xhi.tobytes()) token = b''.join(data) digest = hashfunc(token).digest() assert digest in cache assert cache[digest] == pytest.approx(expected)
def get_urn(key): """Generate a unique identifier from the key.""" urn = UUID(hashfunc(key).hexdigest()).get_urn() urn = urn.split(':')[-1] return urn
def hasChanged(self, path): """Tells wether the given resource has changed since last build for this website or not. The path is converted to an absolute location, so moving the website directory will cause a rebuild.""" # Maybe we already know if the path has changed path = os.path.abspath(path) res = self.changed.get(path) if not self.site.isTemplate(path) and res != None: return res data = None def load_data(path): fd = file(path, 'r') res = fd.read() fd.close() return res # Is the page a template ? template_has_changed = res or False if self.site.isTemplate(path): data = load_data(path) # If so, we look for the extends defintion template = None for line in data.split("\n"): line = line.strip() if line and not line.startswith("##") and not line.startswith( "#extends"): break if line.startswith("#extends"): template = line.strip()[len("#extends"):].strip() # If the template was flagged with ALWAYS_REBUILD, then we force # the build if RE_ALWAYS_REBUILD.match(line): template_has_changed = True break depends = RE_DEPENDS.match(line) # Handles dependencies if depends: dep_path = depends.group(1).strip() dep_path = os.path.expanduser(dep_path) dep_abspath = os.path.abspath(dep_path) # The path may be relative to the current path if dep_abspath != dep_path: dep_abspath = os.path.abspath( os.path.dirname(path) + "/" + dep_path) dep_path = dep_abspath for dependency in glob.glob(dep_path): if self.hasChanged(dependency): template_has_changed = True break if template_has_changed: break # If there was a template extended, we check if it is present in the # templates directory if template and template.startswith("Templates"): template_path = apply(os.path.join, template.split(".")[1:]) template_path = os.path.join(self.site.templatesDir, template_path + ".tmpl") # And if this template has changed, then this one too if self.hasChanged(template_path): template_has_changed = True # There is a SHA1 mode for real checksum change detection if self.site.changeDetectionMethod() == CHANGE_CHECKSUM: try: # sha1 chksum = hashfunc.new(data or load_data(path)).hexdigest() except: chksum = hashfunc(data or load_data(path)).hexdigest() # hashlib # Default is modification time (faster) else: if os.path.exists(path): chksum = os.stat(path)[stat.ST_MTIME] else: warn("Path does not exists: " + path) chksum = "0" # We get the previous checksum checksums = self.checksums.get(self.site.sig()) if checksums: old_checksum = checksums.get(path) else: old_checksum = None # Then we compare to registered checksums # If the checksum has changed if template_has_changed or old_checksum != chksum: # We take care of the mode if not self.checksums.get(self.site.sig()): self.checksums[self.site.sig()] = {} self.checksums[self.site.sig()][path] = chksum self.changed[path] = True return True else: self.changed[path] = False return False
def md5(path): return hashfunc(open(path, 'rb').read()).hexdigest()
def hasChanged( self, path ): """Tells wether the given resource has changed since last build for this website or not. The path is converted to an absolute location, so moving the website directory will cause a rebuild.""" # Maybe we already know if the path has changed path = os.path.abspath(path) res = self.changed.get(path) if not self.site.isTemplate(path) and res != None: return res data = None def load_data(path): fd = file(path, 'r') res = fd.read() fd.close() return res # Is the page a template ? template_has_changed = res or False if self.site.isTemplate(path): data = load_data(path) # If so, we look for the extends defintion template = None for line in data.split("\n"): line = line.strip() if line and not line.startswith("##") and not line.startswith("#extends"): break if line.startswith("#extends"): template = line.strip()[len("#extends"):].strip() # If the template was flagged with ALWAYS_REBUILD, then we force # the build if RE_ALWAYS_REBUILD.match(line): template_has_changed = True break depends = RE_DEPENDS.match(line) # Handles dependencies if depends: dep_path = depends.group(1).strip() dep_path = os.path.expanduser(dep_path) dep_abspath = os.path.abspath(dep_path) # The path may be relative to the current path if dep_abspath != dep_path: dep_abspath = os.path.abspath(os.path.dirname(path) + "/" + dep_path) dep_path = dep_abspath for dependency in glob.glob(dep_path): if self.hasChanged(dependency): template_has_changed = True break if template_has_changed: break # If there was a template extended, we check if it is present in the # templates directory if template and template.startswith("Templates"): template_path = apply(os.path.join, template.split(".")[1:]) template_path = os.path.join(self.site.templatesDir, template_path + ".tmpl") # And if this template has changed, then this one too if self.hasChanged(template_path): template_has_changed = True # There is a SHA1 mode for real checksum change detection if self.site.changeDetectionMethod() == CHANGE_CHECKSUM: try: # sha1 chksum = hashfunc.new(data or load_data(path)).hexdigest() except: chksum = hashfunc(data or load_data(path)).hexdigest() # hashlib # Default is modification time (faster) else: if os.path.exists(path): chksum = os.stat(path)[stat.ST_MTIME] else: warn("Path does not exists: " + path) chksum = "0" # We get the previous checksum checksums = self.checksums.get(self.site.sig()) if checksums: old_checksum = checksums.get(path) else: old_checksum = None # Then we compare to registered checksums # If the checksum has changed if template_has_changed or old_checksum != chksum: # We take care of the mode if not self.checksums.get(self.site.sig()): self.checksums[self.site.sig()] = {} self.checksums[self.site.sig()][path] = chksum self.changed[path] = True return True else: self.changed[path] = False return False