Exemplo n.º 1
0
 def save(self, *args, **kwargs):
     if not self.created:
         self.created = datetime.datetime.utcnow().replace(tzinfo=timezone.utc)
     self.md5 = md5hash(self.url).hexdigest()
     parsed = urlparse(self.url)
     self.netloc = parsed.netloc
     super(Url, self).save(*args, **kwargs)
Exemplo n.º 2
0
    def create(cls, mime, data, filename=""):
        bin = Binary(cls._core)
        
        filename = md5hash(data).hexdigest()[:6]+"_"+filename

        bin.set_filename(filename)
        bin.set_mime(mime)
        bin.set_data(data)

        return bin
Exemplo n.º 3
0
    def create(cls, mime, data, filename=""):
        bin = Binary(cls._core)

        filename = md5hash(data).hexdigest()[:6] + "_" + filename

        bin.set_filename(filename)
        bin.set_mime(mime)
        bin.set_data(data)

        return bin
Exemplo n.º 4
0
    def get_by_filename(cls, filename):
        data_fetched = False
        configuration = Configuration()
        binary_cache_path = os.path.join(configuration.get_entry("global.binary_cache"),
                                         configuration.get_entry("core.instance_id"))

        if not os.path.exists(binary_cache_path):
            os.mkdir(binary_cache_path,True)

        db = Database()
        if os.path.exists(os.path.join(binary_cache_path, filename)):
            cachefile = open(os.path.join(binary_cache_path, filename),"rb")
            data = cachefile.read()
            cachefile.close()

            md5 = md5hash(data).hexdigest()
            sha256 = sha256hash(data).hexdigest()

            stmnt = "SELECT BIN_ID, BIN_MIME,   \
                     (SELECT BIN_DATA FROM BINARIES WHERE BIN_FILENAME = ? AND BIN_MD5 != ? AND BIN_SHA256 != ?) AS BIN_DATA \
                     FROM BINARIES WHERE BIN_FILENAME = ?  ;"
            cur = db.query(stmnt, (filename, md5, sha256, filename))
            row = cur.fetchonemap()
            if row is not None:
                data_fetched = True
                if row["BIN_DATA"] is None:
                    bin = Binary()
                    bin.set_id(row["BIN_ID"])
                    bin.set_filename(filename)
                    bin.set_mime(row["BIN_MIME"])
                    bin.set_data(data)
                    return bin
            else:
                raise BinaryException(BinaryException.get_msg(0, filename))
        if not data_fetched:
            stmnt = "SELECT BIN_ID, BIN_MIME, BIN_DATA FROM BINARIES WHERE BIN_FILENAME = ? ;"
            cur = db.query(stmnt, (filename,))
            row = cur.fetchonemap()
        if row is not None:
            bin = Binary()
            bin.set_id(row["BIN_ID"])
            bin.set_filename(filename)
            bin.set_mime(row["BIN_MIME"])
            bin.set_data(base64.b64decode(row["BIN_DATA"]))

            cachefile = open(os.path.join(binary_cache_path, filename),"wb")
            cachefile.write(bin.get_data())
            cachefile.close()

            return bin
        else:
            raise BinaryException(BinaryException.get_msg(0, filename))
Exemplo n.º 5
0
 def _get_resources(self):
     logger.info("Ingesting Sepsis md5 file information from {0}".format(self.path))
     resources = []
     for md5_file in glob(self.path + '/*.md5'):
         logger.info("Processing md5 file {0}".format(md5_file))
         for filename, md5, file_info in self.parse_md5file(md5_file):
             resource = {}
             resource['md5'] = md5
             xlsx_info = self.metadata_info[os.path.basename(md5_file)]
             # analysed data has duplicate PNG images in it -- we need to keep the ID unique
             resource['id'] = 'u-' + md5hash((self.ckan_data_type + xlsx_info['ticket'] + md5).encode('utf8')).hexdigest()
             resource['name'] = filename
             legacy_url = urljoin(xlsx_info['base_url'], filename)
             resources.append(((xlsx_info['ticket'],), legacy_url, resource))
     return resources
Exemplo n.º 6
0
    def store(self):
        """
        Stores this binary into the database
        """
        db = Database()
        data_io = base64.b64encode(self._data)
        md5 = md5hash(self._data).hexdigest()
        sha256 = sha256hash(self._data).hexdigest()
        if self._id is None:
            self.set_id(db.get_seq_next("BIN_GEN"))

        user_id = Session.get_current_session_user().get_id()
        stmnt = "UPDATE OR INSERT INTO BINARIES (BIN_ID, BIN_FILENAME, BIN_MIME, BIN_USR_OWNER, \
                   BIN_USR_LASTCHANGE, \
                   BIN_DATE_LASTCHANGE, BIN_SHA256, BIN_MD5, BIN_DATA) \
                 VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ? ) MATCHING (BIN_ID) ;"
        db.query(stmnt, (self._id, self._filename, self._mime, user_id, user_id, sha256, md5, data_io),commit=True)
Exemplo n.º 7
0
def md5sum(fileName):
    hashvalue = md5hash()

    try:
        f = open(fileName, "r")
    except:
        return ""

    fData = f.read()
    hashvalue.update(fData)
    del fData
    f.close()

    hexvalue = string.hexdigits
    md5res = ""
    for c in hashvalue.digest():
        i = ord(c)
        md5res = md5res + hexvalue[(i >> 4) & 0xF] + hexvalue[i & 0xF]

    return md5res
Exemplo n.º 8
0
def md5sum(fileName):
    hashvalue = md5hash()

    try:
        f = open(fileName, "r")
    except:
        return ""

    fData = f.read()
    hashvalue.update(fData)
    del fData
    f.close()

    hexvalue = string.hexdigits
    md5res = ""
    for c in hashvalue.digest():
        i = ord(c)
        md5res = md5res + hexvalue[(i >> 4) & 0xF] + hexvalue[i & 0xf]

    return md5res
Exemplo n.º 9
0
    def store(self):
        """
        Stores this binary into the database
        """
        db = self._core.get_db()
        data_io = StringIO(base64.b64encode(self._data))
        md5 = md5hash(self._data).hexdigest()
        sha256 = sha256hash(self._data).hexdigest()
        if self._id is None:
            self.set_id(db.get_seq_next("BIN_GEN"))

        user_id = self._core.get_session_manager().get_current_session_user(
        ).get_id()
        stmnt = "UPDATE OR INSERT INTO BINARIES (BIN_ID, BIN_FILENAME, BIN_MIME, BIN_USR_OWNER, \
                   BIN_USR_LASTCHANGE, \
                   BIN_DATE_LASTCHANGE, BIN_SHA256, BIN_MD5, BIN_DATA) \
                 VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ? ) MATCHING (BIN_ID) ;"

        db.query(self._core,
                 stmnt, (self._id, self._filename, self._mime, user_id,
                         user_id, sha256, md5, data_io),
                 commit=True)
        data_io.close()
Exemplo n.º 10
0
 def getHash(self):
     return md5hash(self.url_data.encode()).hexdigest()
Exemplo n.º 11
0
def md5(token):
    """
    Returns an md5 hash of a token passed as a string, performing an internal 
    conversion of the token to bytes if run in Python 3
    """
    return md5hash(token.encode('utf-8'))