def save(self, *args, **kwargs):
if not self.created:
self.created = datetime.datetime.utcnow().replace(tzinfo=timezone.utc)
self.md5 = md5hash(self.url).hexdigest()
parsed = urlparse(self.url)
self.netloc = parsed.netloc
super(Url, self).save(*args, **kwargs)
def create(cls, mime, data, filename=""):
bin = Binary(cls._core)
filename = md5hash(data).hexdigest()[:6]+"_"+filename
bin.set_filename(filename)
bin.set_mime(mime)
bin.set_data(data)
return bin
def create(cls, mime, data, filename=""):
bin = Binary(cls._core)
filename = md5hash(data).hexdigest()[:6] + "_" + filename
bin.set_filename(filename)
bin.set_mime(mime)
bin.set_data(data)
return bin
def get_by_filename(cls, filename):
data_fetched = False
configuration = Configuration()
binary_cache_path = os.path.join(configuration.get_entry("global.binary_cache"),
configuration.get_entry("core.instance_id"))
if not os.path.exists(binary_cache_path):
os.mkdir(binary_cache_path,True)
db = Database()
if os.path.exists(os.path.join(binary_cache_path, filename)):
cachefile = open(os.path.join(binary_cache_path, filename),"rb")
data = cachefile.read()
cachefile.close()
md5 = md5hash(data).hexdigest()
sha256 = sha256hash(data).hexdigest()
stmnt = "SELECT BIN_ID, BIN_MIME, \
(SELECT BIN_DATA FROM BINARIES WHERE BIN_FILENAME = ? AND BIN_MD5 != ? AND BIN_SHA256 != ?) AS BIN_DATA \
FROM BINARIES WHERE BIN_FILENAME = ? ;"
cur = db.query(stmnt, (filename, md5, sha256, filename))
row = cur.fetchonemap()
if row is not None:
data_fetched = True
if row["BIN_DATA"] is None:
bin = Binary()
bin.set_id(row["BIN_ID"])
bin.set_filename(filename)
bin.set_mime(row["BIN_MIME"])
bin.set_data(data)
return bin
else:
raise BinaryException(BinaryException.get_msg(0, filename))
if not data_fetched:
stmnt = "SELECT BIN_ID, BIN_MIME, BIN_DATA FROM BINARIES WHERE BIN_FILENAME = ? ;"
cur = db.query(stmnt, (filename,))
row = cur.fetchonemap()
if row is not None:
bin = Binary()
bin.set_id(row["BIN_ID"])
bin.set_filename(filename)
bin.set_mime(row["BIN_MIME"])
bin.set_data(base64.b64decode(row["BIN_DATA"]))
cachefile = open(os.path.join(binary_cache_path, filename),"wb")
cachefile.write(bin.get_data())
cachefile.close()
return bin
else:
raise BinaryException(BinaryException.get_msg(0, filename))
def _get_resources(self):
logger.info("Ingesting Sepsis md5 file information from {0}".format(self.path))
resources = []
for md5_file in glob(self.path + '/*.md5'):
logger.info("Processing md5 file {0}".format(md5_file))
for filename, md5, file_info in self.parse_md5file(md5_file):
resource = {}
resource['md5'] = md5
xlsx_info = self.metadata_info[os.path.basename(md5_file)]
# analysed data has duplicate PNG images in it -- we need to keep the ID unique
resource['id'] = 'u-' + md5hash((self.ckan_data_type + xlsx_info['ticket'] + md5).encode('utf8')).hexdigest()
resource['name'] = filename
legacy_url = urljoin(xlsx_info['base_url'], filename)
resources.append(((xlsx_info['ticket'],), legacy_url, resource))
return resources
def store(self):
"""
Stores this binary into the database
"""
db = Database()
data_io = base64.b64encode(self._data)
md5 = md5hash(self._data).hexdigest()
sha256 = sha256hash(self._data).hexdigest()
if self._id is None:
self.set_id(db.get_seq_next("BIN_GEN"))
user_id = Session.get_current_session_user().get_id()
stmnt = "UPDATE OR INSERT INTO BINARIES (BIN_ID, BIN_FILENAME, BIN_MIME, BIN_USR_OWNER, \
BIN_USR_LASTCHANGE, \
BIN_DATE_LASTCHANGE, BIN_SHA256, BIN_MD5, BIN_DATA) \
VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ? ) MATCHING (BIN_ID) ;"
db.query(stmnt, (self._id, self._filename, self._mime, user_id, user_id, sha256, md5, data_io),commit=True)
def md5sum(fileName):
hashvalue = md5hash()
try:
f = open(fileName, "r")
except:
return ""
fData = f.read()
hashvalue.update(fData)
del fData
f.close()
hexvalue = string.hexdigits
md5res = ""
for c in hashvalue.digest():
i = ord(c)
md5res = md5res + hexvalue[(i >> 4) & 0xF] + hexvalue[i & 0xF]
return md5res
def md5sum(fileName):
hashvalue = md5hash()
try:
f = open(fileName, "r")
except:
return ""
fData = f.read()
hashvalue.update(fData)
del fData
f.close()
hexvalue = string.hexdigits
md5res = ""
for c in hashvalue.digest():
i = ord(c)
md5res = md5res + hexvalue[(i >> 4) & 0xF] + hexvalue[i & 0xf]
return md5res
def store(self):
"""
Stores this binary into the database
"""
db = self._core.get_db()
data_io = StringIO(base64.b64encode(self._data))
md5 = md5hash(self._data).hexdigest()
sha256 = sha256hash(self._data).hexdigest()
if self._id is None:
self.set_id(db.get_seq_next("BIN_GEN"))
user_id = self._core.get_session_manager().get_current_session_user(
).get_id()
stmnt = "UPDATE OR INSERT INTO BINARIES (BIN_ID, BIN_FILENAME, BIN_MIME, BIN_USR_OWNER, \
BIN_USR_LASTCHANGE, \
BIN_DATE_LASTCHANGE, BIN_SHA256, BIN_MD5, BIN_DATA) \
VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ? ) MATCHING (BIN_ID) ;"
db.query(self._core,
stmnt, (self._id, self._filename, self._mime, user_id,
user_id, sha256, md5, data_io),
commit=True)
data_io.close()
def getHash(self):
return md5hash(self.url_data.encode()).hexdigest()
def md5(token):
"""
Returns an md5 hash of a token passed as a string, performing an internal
conversion of the token to bytes if run in Python 3
"""
return md5hash(token.encode('utf-8'))