Example #1
0
def upload():
    """Receives an image, assigns a unique id and stores it"""

    if 'file' not in request.files:
        raise ApiException('Expected file upload', 422)

    filestream = request.files['file']
    name, extension = os.path.splitext(filestream.filename)
    extension = extension.lstrip(".")
    name = name.strip()
    maxlen = current_app.config['MAXIMUM_FILENAME_LENGTH']
    formats = current_app.config["ALLOWED_FORMATS"]

    if len(name) > maxlen:
        raise ApiException(f'Maximum filename length is {maxlen} characters', 400)

    if extension not in formats:
        raise ApiException(f'Unsupported extension {extension}. Expected one of {formats}', 422)

    if not validate_image(filestream, valid_formats=formats):
        raise ApiException(f"Unsupported file type", 422)

    filestream.seek(0)
    file = filestream.read()
    digest = hashfunc(file).hexdigest()
    key = f"{digest}.{extension}"
    success = current_app.storage.set(key, file)

    if success:
        return ApiResult({"id": digest})
    raise ApiException("Error saving file", 500)
Example #2
0
def test_cache_integrate_fall_through_integrate_false():
    """See also test_cache_integrate_fall_through_no_integrate."""

    mdl = DoNotUseModel('notme')
    x = numpy.asarray([2, 3, 7, 100])
    y = mdl(x, integrate=False)

    expected = [1, 1, 1, 1]
    assert y == pytest.approx(expected)

    # A simplified version of check_cache (as we have no
    # integrate setting).
    #
    cache = mdl._cache
    assert len(cache) == 1

    pars = []
    data = [
        numpy.asarray(pars).tobytes(),
        b'0',  # not integrated
        x.tobytes()
    ]

    token = b''.join(data)
    digest = hashfunc(token).digest()
    assert digest in cache
    assert cache[digest] == pytest.approx(expected)
Example #3
0
 def checksum(self):
   ''' Checksum the file contents, used as a proxy for comparing the actual content.
   '''
   csum = self._checksum
   if csum is None:
     path = self.path
     U = Upd()
     pathspace = U.columns - 64
     label = "scan " + (
         path if len(path) < pathspace else '...' + path[-(pathspace - 3):]  # pylint: disable=unsubscriptable-object
     )
     with Pfx("checksum %r", path):
       csum = hashfunc()
       with open(path, 'rb') as fp:
         length = os.fstat(fp.fileno()).st_size
         read_len = 0
         for data in progressbar(
             read_from(fp, rsize=1024*1024),
             label=label,
             total=length,
             units_scale=BINARY_BYTES_SCALE,
             itemlenfunc=len,
             update_frequency=128,
             upd=U,
         ):
           csum.update(data)
           read_len += len(data)
         assert read_len == self.size
     csum = csum.digest()
     self._checksum = csum
   return csum
Example #4
0
	def sig( self ):
		"""Returns this site signature as a string. The signature uniquely
		identifies this site. Each instance has a different signature."""
		sig =  self.pages() + self.output() + self.templatesDir
		try:      s=hashfunc.new(sig).hexdigest()
		except:   s=hashfunc(sig).hexdigest()
		return s
Example #5
0
def get_hash(source):
    f = open(source, 'r', encoding='utf-8')
    hsh = hashfunc()
    for l in f:
        hsh.update(l.encode('utf-8'))

    f.close()
    return hsh.hexdigest()
Example #6
0
 def caching_json_request(method, url, query_args=None, body=None):
     key = hashfunc(repr((method, url, query_args, body))).hexdigest()
     if not key in cache:
         cache[key] = json_request(method=method,
                                   url=url,
                                   query_args=query_args,
                                   body=body)
     return cache[key]
Example #7
0
    def sig(self):
        """Returns this site signature as a string. The signature uniquely
		identifies this site. Each instance has a different signature."""
        sig = self.pages() + self.output() + self.templatesDir
        try:
            s = hashfunc.new(sig).hexdigest()
        except:
            s = hashfunc(sig).hexdigest()
        return s
Example #8
0
def index_project(name):

    try:
        proj = Project.objects.get(name=name)
    except Project.DoesNotExist:
        print("No such project. Abort.")
        return

    if not os.path.isdir(proj.root):
        print("Directory {} does not exist. Abort.".format(proj.root))
        return

    files_to_index = []

    for root, dirs, files in os.walk(proj.root):
        for fp in files:
            abspath = os.path.join(root, fp)
            with open(abspath, "rb") as f:
                hashval = hashfunc(f.read()).hexdigest()
            relpath = abspath.replace(proj.root, "")
            try:
                blob = Blob.objects.get(project=proj, path=relpath)
                if hashval != blob.checksum:
                    # TODO: remove all refs and defs related to this blob
                    files_to_index.append(blob)
                    blob.checksum = hashval
                    blob.save()
            except Blob.DoesNotExist:
                blob = Blob(project=proj, path=relpath, checksum=hashval)
                blob.save()
                files_to_index.append(blob)

    # index definitions

    for blob in files_to_index:
        abspath = proj.root + blob.path
        print("processing defs in {}".format(blob.path))
        for ident, itype, line in parse_defs(abspath):
            ident = get_or_create_ident(ident, proj)
            def_entry = Def(ident=ident, blob=blob, line=line, type=itype)
            def_entry.save()

    # index references

    for blob in files_to_index:
        abspath = proj.root + blob.path
        print("processing refs in {}".format(blob.path))
        for token, tokentype, line in tokenize(abspath):
            try:
                ident = Ident.objects.get(value=token, project=proj)
                ref_entry = Ref(ident=ident, blob=blob, line=line)
                ref_entry.save()
            except Ident.DoesNotExist:
                # ignore tokens that is not in ident
                pass
Example #9
0
def check_cache(mdl, expected, x, xhi=None):
    """Check the cache contents.

    We assume only one value is being cached at a time. The
    code matches that in sherpa.models.model.modelCacher1d,
    so all it does is check we are using this method.
    """

    cache = mdl._cache
    assert len(cache) == 1

    pars = [p.val for p in mdl.pars]
    data = [
        numpy.asarray(pars).tobytes(), b'1' if mdl.integrate else b'0',
        x.tobytes()
    ]
    if xhi is not None:
        data.append(xhi.tobytes())

    token = b''.join(data)
    digest = hashfunc(token).digest()
    assert digest in cache
    assert cache[digest] == pytest.approx(expected)
Example #10
0
def get_urn(key):
    """Generate a unique identifier from the key."""
    urn = UUID(hashfunc(key).hexdigest()).get_urn()
    urn = urn.split(':')[-1]
    return urn
Example #11
0
    def hasChanged(self, path):
        """Tells wether the given resource has changed since last build for this
		website or not. The path is converted to an absolute location, so
		moving the website directory will cause a rebuild."""
        # Maybe we already know if the path has changed
        path = os.path.abspath(path)
        res = self.changed.get(path)
        if not self.site.isTemplate(path) and res != None:
            return res
        data = None

        def load_data(path):
            fd = file(path, 'r')
            res = fd.read()
            fd.close()
            return res

        # Is the page a template ?
        template_has_changed = res or False
        if self.site.isTemplate(path):
            data = load_data(path)
            # If so, we look for the extends defintion
            template = None
            for line in data.split("\n"):
                line = line.strip()
                if line and not line.startswith("##") and not line.startswith(
                        "#extends"):
                    break
                if line.startswith("#extends"):
                    template = line.strip()[len("#extends"):].strip()
                # If the template was flagged with ALWAYS_REBUILD, then we force
                # the build
                if RE_ALWAYS_REBUILD.match(line):
                    template_has_changed = True
                    break
                depends = RE_DEPENDS.match(line)
                # Handles dependencies
                if depends:
                    dep_path = depends.group(1).strip()
                    dep_path = os.path.expanduser(dep_path)
                    dep_abspath = os.path.abspath(dep_path)
                    # The path may be relative to the current path
                    if dep_abspath != dep_path:
                        dep_abspath = os.path.abspath(
                            os.path.dirname(path) + "/" + dep_path)
                    dep_path = dep_abspath
                    for dependency in glob.glob(dep_path):
                        if self.hasChanged(dependency):
                            template_has_changed = True
                            break
                    if template_has_changed: break
            # If there was a template extended, we check if it is present in the
            # templates directory
            if template and template.startswith("Templates"):
                template_path = apply(os.path.join, template.split(".")[1:])
                template_path = os.path.join(self.site.templatesDir,
                                             template_path + ".tmpl")
                # And if this template has changed, then this one too
                if self.hasChanged(template_path):
                    template_has_changed = True
        # There is a SHA1 mode for real checksum change detection
        if self.site.changeDetectionMethod() == CHANGE_CHECKSUM:
            try:  # sha1
                chksum = hashfunc.new(data or load_data(path)).hexdigest()
            except:
                chksum = hashfunc(data
                                  or load_data(path)).hexdigest()  # hashlib
        # Default is modification time (faster)
        else:
            if os.path.exists(path):
                chksum = os.stat(path)[stat.ST_MTIME]
            else:
                warn("Path does not exists: " + path)
                chksum = "0"
        # We get the previous checksum
        checksums = self.checksums.get(self.site.sig())
        if checksums:
            old_checksum = checksums.get(path)
        else:
            old_checksum = None
        # Then we compare to registered checksums
        # If the checksum has changed
        if template_has_changed or old_checksum != chksum:
            # We take care of the mode
            if not self.checksums.get(self.site.sig()):
                self.checksums[self.site.sig()] = {}
            self.checksums[self.site.sig()][path] = chksum
            self.changed[path] = True
            return True
        else:
            self.changed[path] = False
            return False
Example #12
0
def md5(path):
    return hashfunc(open(path, 'rb').read()).hexdigest()
Example #13
0
	def hasChanged( self, path ):
		"""Tells wether the given resource has changed since last build for this
		website or not. The path is converted to an absolute location, so
		moving the website directory will cause a rebuild."""
		# Maybe we already know if the path has changed
		path = os.path.abspath(path)
		res  = self.changed.get(path) 
		if not self.site.isTemplate(path) and res != None:
			return res
		data = None
		def load_data(path):
			fd  = file(path, 'r')
			res = fd.read()
			fd.close()
			return res
		# Is the page a template ?
		template_has_changed = res or False
		if self.site.isTemplate(path):
			data = load_data(path)
			# If so, we look for the extends defintion
			template = None
			for line in data.split("\n"):
				line = line.strip()
				if line and not line.startswith("##") and not line.startswith("#extends"): break
				if line.startswith("#extends"):
					template = line.strip()[len("#extends"):].strip()
				# If the template was flagged with ALWAYS_REBUILD, then we force
				# the build
				if RE_ALWAYS_REBUILD.match(line):
					template_has_changed = True
					break
				depends = RE_DEPENDS.match(line)
				# Handles dependencies
				if depends:
					dep_path = depends.group(1).strip()
					dep_path = os.path.expanduser(dep_path)
					dep_abspath = os.path.abspath(dep_path)
					# The path may be relative to the current path
					if dep_abspath != dep_path:
						dep_abspath = os.path.abspath(os.path.dirname(path) + "/" + dep_path)
					dep_path = dep_abspath
					for dependency in glob.glob(dep_path):
						if self.hasChanged(dependency):
							template_has_changed = True
							break
					if template_has_changed: break
			# If there was a template extended, we check if it is present in the
			# templates directory
			if template and template.startswith("Templates"):
				template_path = apply(os.path.join, template.split(".")[1:])
				template_path = os.path.join(self.site.templatesDir, template_path + ".tmpl")
				# And if this template has changed, then this one too
				if self.hasChanged(template_path):
					template_has_changed = True
		# There is a SHA1 mode for real checksum change detection
		if self.site.changeDetectionMethod() == CHANGE_CHECKSUM:
			try:															# sha1
				chksum = hashfunc.new(data or load_data(path)).hexdigest()
			except:
				chksum = hashfunc(data or load_data(path)).hexdigest() # hashlib
		# Default is modification time (faster)
		else:
			if os.path.exists(path):
				chksum  = os.stat(path)[stat.ST_MTIME]
			else:
				warn("Path does not exists: " + path)
				chksum  = "0"
		# We get the previous checksum
		checksums = self.checksums.get(self.site.sig())
		if checksums:
			old_checksum = checksums.get(path)
		else:
			old_checksum = None
		# Then we compare to registered checksums
		# If the checksum has changed
		if template_has_changed or old_checksum != chksum:
			# We take care of the mode
			if not self.checksums.get(self.site.sig()):
				self.checksums[self.site.sig()] = {}
			self.checksums[self.site.sig()][path] = chksum
			self.changed[path] = True
			return True
		else:
			self.changed[path] = False
			return False