def do_import(self): """Take care of creating the database structure, delegating the loading of the contest data and putting them on the database. """ logger.info("Creating database structure.") if self.drop: try: with SessionGen() as session: FSObject.delete_all(session) session.commit() metadata.drop_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False try: metadata.create_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False contest = Contest.import_from_dict(self.loader.import_contest(self.path)) logger.info("Creating contest on the database.") with SessionGen() as session: session.add(contest) logger.info("Analyzing database.") session.commit() contest_id = contest.id analyze_all_tables(session) logger.info("Import finished (new contest id: %s)." % contest_id) return True
def put_file(self, digest, origin, description=""): """See FileCacherBackend.put_file(). """ with SessionGen() as session: # Check digest uniqueness if FSObject.get_from_digest(digest, session) is not None: logger.debug("File %s already on database, " "dropping this one." % digest) session.rollback() # If it is not already present, copy the file into the # lobject else: fso = FSObject(description=description) logger.debug("Sending file %s to the database." % digest) with open(origin, 'rb') as temp_file: with fso.get_lobject(session, mode='wb') as lobject: logger.debug("Large object created.") buf = temp_file.read(self.CHUNK_SIZE) while buf != '': while len(buf) > 0: written = lobject.write(buf) buf = buf[written:] if self.service is not None: self.service._step() buf = temp_file.read(self.CHUNK_SIZE) fso.digest = digest session.add(fso) session.commit() logger.debug("File %s sent to the database." % digest)
def _prepare_db(self): logger.info("Creating database structure.") if self.drop: try: with SessionGen() as session: FSObject.delete_all(session) session.commit() metadata.drop_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False try: metadata.create_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False
def delete(self, digest): """See FileCacherBackend.delete(). """ with SessionGen() as session: fso = FSObject.get_from_digest(digest, session) fso.delete() session.commit()
def describe(self, digest): """See FileCacherBackend.describe(). """ with SessionGen() as session: fso = FSObject.get_from_digest(digest, session) if fso is not None: return fso.description else: return None
def get_size(self, digest): """See FileCacherBackend.get_size(). """ with SessionGen() as session: fso = FSObject.get_from_digest(digest, session) if fso is not None: with fso.get_lobject(session, mode='rb') as lobject: return lobject.seek(0, os.SEEK_END) else: return None
def delete(self, digest): """Delete from cache and FS the file with that digest. digest (string): the file to delete. """ self.delete_from_cache(digest) with SessionGen() as session: fso = FSObject.get_from_digest(digest, session) fso.delete() session.commit()
def put_file(self, digest, origin, description=""): """See FileCacherBackend.put_file(). """ try: with SessionGen() as session: # Check digest uniqueness if FSObject.get_from_digest(digest, session) is not None: logger.debug("File %s already on database, " "dropping this one." % digest) session.rollback() # If it is not already present, copy the file into the # lobject else: fso = FSObject(description=description) logger.debug("Sending file %s to the database." % digest) with open(origin, 'rb') as temp_file: with fso.get_lobject(session, mode='wb') \ as lobject: logger.debug("Large object created.") buf = temp_file.read(self.CHUNK_SIZE) while buf != '': while len(buf) > 0: written = lobject.write(buf) buf = buf[written:] # Cooperative yield gevent.sleep(0) buf = temp_file.read(self.CHUNK_SIZE) fso.digest = digest session.add(fso) session.commit() logger.debug("File %s sent to the database." % digest) except IntegrityError: logger.warning("File %s caused an IntegrityError, ignoring..." % digest)
def describe(digest): """Return the description of a file given its digest. digest (string): the digest to describe. return (string): the description associated. """ with SessionGen() as session: fso = FSObject.get_from_digest(digest, session) if fso is not None: return fso.description else: return None
def get_size(self, digest): """See FileCacherBackend.get_size(). """ # TODO - The business logic may be moved in FSObject, for # better generality with SessionGen() as session: fso = FSObject.get_from_digest(digest, session) if fso is not None: with fso.get_lobject(session, mode='rb') as lobject: return lobject.seek(0, os.SEEK_END) else: return None
def do_import(self): """Take care of creating the database structure, delegating the loading of the contest data and putting them on the database. """ logger.info("Creating database structure.") if self.drop: try: with SessionGen() as session: FSObject.delete_all(session) session.commit() metadata.drop_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False try: metadata.create_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False contest = Contest.import_from_dict( self.loader.import_contest(self.path)) logger.info("Creating contest on the database.") with SessionGen() as session: session.add(contest) logger.info("Analyzing database.") session.commit() contest_id = contest.id analyze_all_tables(session) logger.info("Import finished (new contest id: %s)." % contest_id) return True
def get_file(self, digest, dest): """See FileCacherBackend.get_file(). """ with open(dest, 'wb') as temp_file: # hasher = hashlib.sha1() with SessionGen() as session: fso = FSObject.get_from_digest(digest, session) # Copy the file into the lobject with fso.get_lobject(mode='rb') as lobject: buf = lobject.read(self.CHUNK_SIZE) while buf != '': # hasher.update(buf) temp_file.write(buf) if self.service is not None: self.service._step() buf = lobject.read(self.CHUNK_SIZE)
def put_file(self, description="", binary_data=None, file_obj=None, path=None): """Put a file in the storage, and keep a copy locally. The caller has to provide exactly one among binary_data, file_obj and path. description (string): a human-readable description of the content. binary_data (string): the content of the file to send. file_obj (file): the file-like object to send. path (string): the file to send. """ temp_fd, temp_path = tempfile.mkstemp(dir=self.tmp_dir) os.close(temp_fd) # Input checking if [binary_data, file_obj, path].count(None) != 2: error_string = "No content (or too many) specified in put_file." logger.error(error_string) raise ValueError(error_string) logger.debug("Reading input file to store on the database.") # Copy the file content, whatever forms it arrives, into the # temporary file # TODO - This could be long lasting: probably it would be wise # to call self.service._step() periodically, but this would # require reimplementing of shutil functions if path is not None: shutil.copy(path, temp_path) elif binary_data is not None: with open(temp_path, 'wb') as temp_file: temp_file.write(binary_data) else: # file_obj is not None. with open(temp_path, 'wb') as temp_file: shutil.copyfileobj(file_obj, temp_file) hasher = hashlib.sha1() fso = FSObject(description=description) # Calculate the file SHA1 digest with open(temp_path, 'rb') as temp_file: buf = temp_file.read(self.CHUNK_SIZE) while buf != '': hasher.update(buf) buf = temp_file.read(self.CHUNK_SIZE) digest = hasher.hexdigest() logger.debug("File has digest %s." % digest) # Check the digest uniqueness with SessionGen() as session: if FSObject.get_from_digest(digest, session) is not None: logger.debug("File %s already on database, " "dropping this one." % digest) session.rollback() # If it is not already present, copy the file into the # lobject else: logger.debug("Sending file %s to the database." % digest) with open(temp_path, 'rb') as temp_file: with fso.get_lobject(session, mode='wb') as lobject: logger.debug("Large object created.") buf = temp_file.read(self.CHUNK_SIZE) while buf != '': while len(buf) > 0: written = lobject.write(buf) buf = buf[written:] if self.service is not None: self.service._step() buf = temp_file.read(self.CHUNK_SIZE) fso.digest = digest session.add(fso) session.commit() logger.debug("File %s sent to the database." % digest) # Move the temporary file in the cache shutil.move(temp_path, os.path.join(self.obj_dir, digest)) return digest
def get_file(self, digest, path=None, file_obj=None, string=False, temp_path=False, temp_file_obj=False): """Get a file from the storage, possibly using the cache if the file is available there. digest (string): the sha1 sum of the file. path (string): a path where to save the file. file_obj (file): a handler where to save the file (that is not closed at return). string (bool): True to return content as a string. temp_path (bool): True to return path of a temporary file with that content. The file is reserved to the caller, who has the duty to unlink it. temp_file-obj (bool): True to return a file object opened to a temporary file with that content. The file is reserved to the caller. Use this method only for debugging purpose, as it leave a file lying in the temporary directory of FileCacher. """ if [string, temp_path, temp_file_obj].count(True) > 1: raise ValueError("Ask for at most one amongst content, " "temp path and temp file obj.") cache_path = os.path.join(self.obj_dir, digest) cache_exists = os.path.exists(cache_path) logger.debug("Getting file %s" % (digest)) if not cache_exists: logger.debug("File %s not in cache, downloading " "from database." % digest) temp_file, temp_filename = tempfile.mkstemp(dir=self.tmp_dir) temp_file = os.fdopen(temp_file, "wb") # Receives the file from the database with open(temp_filename, 'wb') as temp_file: # hasher = hashlib.sha1() with SessionGen() as session: fso = FSObject.get_from_digest(digest, session) # Copy the file into the lobject with fso.get_lobject(mode='rb') as lobject: buf = lobject.read(self.CHUNK_SIZE) while buf != '': # hasher.update(buf) temp_file.write(buf) if self.service is not None: self.service._step() buf = lobject.read(self.CHUNK_SIZE) # And move it in the cache shutil.move(temp_filename, cache_path) logger.debug("File %s downloaded." % digest) # Saving to path if path is not None: shutil.copy(cache_path, path) # Saving to file object if file_obj is not None: with open(cache_path, "rb") as file_: shutil.copyfileobj(file_, file_obj) # Returning string? if string: with open(cache_path, "rb") as cache_file: return cache_file.read() # Returning temporary file? elif temp_path: temp_file, temp_filename = tempfile.mkstemp(dir=self.tmp_dir) os.close(temp_file) shutil.copy(cache_path, temp_filename) return temp_filename # Returning temporary file object? elif temp_file_obj: temp_file, temp_filename = tempfile.mkstemp(dir=self.tmp_dir) os.close(temp_file) shutil.copy(cache_path, temp_filename) temp_file = open(temp_filename, "rb") return temp_file