def do_import(self):
"""Take care of creating the database structure, delegating
the loading of the contest data and putting them on the
database.
"""
logger.info("Creating database structure.")
if self.drop:
try:
with SessionGen() as session:
FSObject.delete_all(session)
session.commit()
metadata.drop_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
try:
metadata.create_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
contest = Contest.import_from_dict(self.loader.import_contest(self.path))
logger.info("Creating contest on the database.")
with SessionGen() as session:
session.add(contest)
logger.info("Analyzing database.")
session.commit()
contest_id = contest.id
analyze_all_tables(session)
logger.info("Import finished (new contest id: %s)." % contest_id)
return True
def put_file(self, digest, origin, description=""):
"""See FileCacherBackend.put_file().
"""
with SessionGen() as session:
# Check digest uniqueness
if FSObject.get_from_digest(digest, session) is not None:
logger.debug("File %s already on database, "
"dropping this one." % digest)
session.rollback()
# If it is not already present, copy the file into the
# lobject
else:
fso = FSObject(description=description)
logger.debug("Sending file %s to the database." % digest)
with open(origin, 'rb') as temp_file:
with fso.get_lobject(session, mode='wb') as lobject:
logger.debug("Large object created.")
buf = temp_file.read(self.CHUNK_SIZE)
while buf != '':
while len(buf) > 0:
written = lobject.write(buf)
buf = buf[written:]
if self.service is not None:
self.service._step()
buf = temp_file.read(self.CHUNK_SIZE)
fso.digest = digest
session.add(fso)
session.commit()
logger.debug("File %s sent to the database." % digest)
def put_file(self, digest, origin, description=""):
"""See FileCacherBackend.put_file().
"""
with SessionGen() as session:
# Check digest uniqueness
if FSObject.get_from_digest(digest, session) is not None:
logger.debug("File %s already on database, "
"dropping this one." % digest)
session.rollback()
# If it is not already present, copy the file into the
# lobject
else:
fso = FSObject(description=description)
logger.debug("Sending file %s to the database." % digest)
with open(origin, 'rb') as temp_file:
with fso.get_lobject(session, mode='wb') as lobject:
logger.debug("Large object created.")
buf = temp_file.read(self.CHUNK_SIZE)
while buf != '':
while len(buf) > 0:
written = lobject.write(buf)
buf = buf[written:]
if self.service is not None:
self.service._step()
buf = temp_file.read(self.CHUNK_SIZE)
fso.digest = digest
session.add(fso)
session.commit()
logger.debug("File %s sent to the database." % digest)
def _prepare_db(self):
logger.info("Creating database structure.")
if self.drop:
try:
with SessionGen() as session:
FSObject.delete_all(session)
session.commit()
metadata.drop_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
try:
metadata.create_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
def delete(self, digest):
"""See FileCacherBackend.delete().
"""
with SessionGen() as session:
fso = FSObject.get_from_digest(digest, session)
fso.delete()
session.commit()
def delete(self, digest):
"""See FileCacherBackend.delete().
"""
with SessionGen() as session:
fso = FSObject.get_from_digest(digest, session)
fso.delete()
session.commit()
def describe(self, digest):
"""See FileCacherBackend.describe().
"""
with SessionGen() as session:
fso = FSObject.get_from_digest(digest, session)
if fso is not None:
return fso.description
else:
return None
def describe(self, digest):
"""See FileCacherBackend.describe().
"""
with SessionGen() as session:
fso = FSObject.get_from_digest(digest, session)
if fso is not None:
return fso.description
else:
return None
def get_size(self, digest):
"""See FileCacherBackend.get_size().
"""
with SessionGen() as session:
fso = FSObject.get_from_digest(digest, session)
if fso is not None:
with fso.get_lobject(session, mode='rb') as lobject:
return lobject.seek(0, os.SEEK_END)
else:
return None
def get_size(self, digest):
"""See FileCacherBackend.get_size().
"""
with SessionGen() as session:
fso = FSObject.get_from_digest(digest, session)
if fso is not None:
with fso.get_lobject(session, mode='rb') as lobject:
return lobject.seek(0, os.SEEK_END)
else:
return None
def delete(self, digest):
"""Delete from cache and FS the file with that digest.
digest (string): the file to delete.
"""
self.delete_from_cache(digest)
with SessionGen() as session:
fso = FSObject.get_from_digest(digest, session)
fso.delete()
session.commit()
def put_file(self, digest, origin, description=""):
"""See FileCacherBackend.put_file().
"""
try:
with SessionGen() as session:
# Check digest uniqueness
if FSObject.get_from_digest(digest, session) is not None:
logger.debug("File %s already on database, "
"dropping this one." % digest)
session.rollback()
# If it is not already present, copy the file into the
# lobject
else:
fso = FSObject(description=description)
logger.debug("Sending file %s to the database." % digest)
with open(origin, 'rb') as temp_file:
with fso.get_lobject(session, mode='wb') \
as lobject:
logger.debug("Large object created.")
buf = temp_file.read(self.CHUNK_SIZE)
while buf != '':
while len(buf) > 0:
written = lobject.write(buf)
buf = buf[written:]
# Cooperative yield
gevent.sleep(0)
buf = temp_file.read(self.CHUNK_SIZE)
fso.digest = digest
session.add(fso)
session.commit()
logger.debug("File %s sent to the database." % digest)
except IntegrityError:
logger.warning("File %s caused an IntegrityError, ignoring..."
% digest)
def describe(digest):
"""Return the description of a file given its digest.
digest (string): the digest to describe.
return (string): the description associated.
"""
with SessionGen() as session:
fso = FSObject.get_from_digest(digest, session)
if fso is not None:
return fso.description
else:
return None
def get_size(self, digest):
"""See FileCacherBackend.get_size().
"""
# TODO - The business logic may be moved in FSObject, for
# better generality
with SessionGen() as session:
fso = FSObject.get_from_digest(digest, session)
if fso is not None:
with fso.get_lobject(session, mode='rb') as lobject:
return lobject.seek(0, os.SEEK_END)
else:
return None
def do_import(self):
"""Take care of creating the database structure, delegating
the loading of the contest data and putting them on the
database.
"""
logger.info("Creating database structure.")
if self.drop:
try:
with SessionGen() as session:
FSObject.delete_all(session)
session.commit()
metadata.drop_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
try:
metadata.create_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
contest = Contest.import_from_dict(
self.loader.import_contest(self.path))
logger.info("Creating contest on the database.")
with SessionGen() as session:
session.add(contest)
logger.info("Analyzing database.")
session.commit()
contest_id = contest.id
analyze_all_tables(session)
logger.info("Import finished (new contest id: %s)." % contest_id)
return True
def get_file(self, digest, dest):
"""See FileCacherBackend.get_file().
"""
with open(dest, 'wb') as temp_file:
# hasher = hashlib.sha1()
with SessionGen() as session:
fso = FSObject.get_from_digest(digest, session)
# Copy the file into the lobject
with fso.get_lobject(mode='rb') as lobject:
buf = lobject.read(self.CHUNK_SIZE)
while buf != '':
# hasher.update(buf)
temp_file.write(buf)
if self.service is not None:
self.service._step()
buf = lobject.read(self.CHUNK_SIZE)
def get_file(self, digest, dest):
"""See FileCacherBackend.get_file().
"""
with open(dest, 'wb') as temp_file:
# hasher = hashlib.sha1()
with SessionGen() as session:
fso = FSObject.get_from_digest(digest, session)
# Copy the file into the lobject
with fso.get_lobject(mode='rb') as lobject:
buf = lobject.read(self.CHUNK_SIZE)
while buf != '':
# hasher.update(buf)
temp_file.write(buf)
if self.service is not None:
self.service._step()
buf = lobject.read(self.CHUNK_SIZE)
def put_file(self, description="", binary_data=None,
file_obj=None, path=None):
"""Put a file in the storage, and keep a copy locally. The
caller has to provide exactly one among binary_data, file_obj
and path.
description (string): a human-readable description of the
content.
binary_data (string): the content of the file to send.
file_obj (file): the file-like object to send.
path (string): the file to send.
"""
temp_fd, temp_path = tempfile.mkstemp(dir=self.tmp_dir)
os.close(temp_fd)
# Input checking
if [binary_data, file_obj, path].count(None) != 2:
error_string = "No content (or too many) specified in put_file."
logger.error(error_string)
raise ValueError(error_string)
logger.debug("Reading input file to store on the database.")
# Copy the file content, whatever forms it arrives, into the
# temporary file
# TODO - This could be long lasting: probably it would be wise
# to call self.service._step() periodically, but this would
# require reimplementing of shutil functions
if path is not None:
shutil.copy(path, temp_path)
elif binary_data is not None:
with open(temp_path, 'wb') as temp_file:
temp_file.write(binary_data)
else: # file_obj is not None.
with open(temp_path, 'wb') as temp_file:
shutil.copyfileobj(file_obj, temp_file)
hasher = hashlib.sha1()
fso = FSObject(description=description)
# Calculate the file SHA1 digest
with open(temp_path, 'rb') as temp_file:
buf = temp_file.read(self.CHUNK_SIZE)
while buf != '':
hasher.update(buf)
buf = temp_file.read(self.CHUNK_SIZE)
digest = hasher.hexdigest()
logger.debug("File has digest %s." % digest)
# Check the digest uniqueness
with SessionGen() as session:
if FSObject.get_from_digest(digest, session) is not None:
logger.debug("File %s already on database, "
"dropping this one." % digest)
session.rollback()
# If it is not already present, copy the file into the
# lobject
else:
logger.debug("Sending file %s to the database." % digest)
with open(temp_path, 'rb') as temp_file:
with fso.get_lobject(session, mode='wb') as lobject:
logger.debug("Large object created.")
buf = temp_file.read(self.CHUNK_SIZE)
while buf != '':
while len(buf) > 0:
written = lobject.write(buf)
buf = buf[written:]
if self.service is not None:
self.service._step()
buf = temp_file.read(self.CHUNK_SIZE)
fso.digest = digest
session.add(fso)
session.commit()
logger.debug("File %s sent to the database." % digest)
# Move the temporary file in the cache
shutil.move(temp_path,
os.path.join(self.obj_dir, digest))
return digest
def get_file(self, digest, path=None, file_obj=None,
string=False, temp_path=False, temp_file_obj=False):
"""Get a file from the storage, possibly using the cache if
the file is available there.
digest (string): the sha1 sum of the file.
path (string): a path where to save the file.
file_obj (file): a handler where to save the file (that is not
closed at return).
string (bool): True to return content as a string.
temp_path (bool): True to return path of a temporary file with
that content. The file is reserved to the
caller, who has the duty to unlink it.
temp_file-obj (bool): True to return a file object opened to a
temporary file with that content. The
file is reserved to the caller. Use this
method only for debugging purpose, as it
leave a file lying in the temporary
directory of FileCacher.
"""
if [string, temp_path, temp_file_obj].count(True) > 1:
raise ValueError("Ask for at most one amongst content, "
"temp path and temp file obj.")
cache_path = os.path.join(self.obj_dir, digest)
cache_exists = os.path.exists(cache_path)
logger.debug("Getting file %s" % (digest))
if not cache_exists:
logger.debug("File %s not in cache, downloading "
"from database." % digest)
temp_file, temp_filename = tempfile.mkstemp(dir=self.tmp_dir)
temp_file = os.fdopen(temp_file, "wb")
# Receives the file from the database
with open(temp_filename, 'wb') as temp_file:
# hasher = hashlib.sha1()
with SessionGen() as session:
fso = FSObject.get_from_digest(digest, session)
# Copy the file into the lobject
with fso.get_lobject(mode='rb') as lobject:
buf = lobject.read(self.CHUNK_SIZE)
while buf != '':
# hasher.update(buf)
temp_file.write(buf)
if self.service is not None:
self.service._step()
buf = lobject.read(self.CHUNK_SIZE)
# And move it in the cache
shutil.move(temp_filename, cache_path)
logger.debug("File %s downloaded." % digest)
# Saving to path
if path is not None:
shutil.copy(cache_path, path)
# Saving to file object
if file_obj is not None:
with open(cache_path, "rb") as file_:
shutil.copyfileobj(file_, file_obj)
# Returning string?
if string:
with open(cache_path, "rb") as cache_file:
return cache_file.read()
# Returning temporary file?
elif temp_path:
temp_file, temp_filename = tempfile.mkstemp(dir=self.tmp_dir)
os.close(temp_file)
shutil.copy(cache_path, temp_filename)
return temp_filename
# Returning temporary file object?
elif temp_file_obj:
temp_file, temp_filename = tempfile.mkstemp(dir=self.tmp_dir)
os.close(temp_file)
shutil.copy(cache_path, temp_filename)
temp_file = open(temp_filename, "rb")
return temp_file
