def __init__(self, version, credentials, pid, debug=False, debug_threshold=255):
"""Initialize connection and collect file_list of files to dump.
:type version: int
:type credentials: string
:type pid: basestring
:type debug: bool
:type debug_threshold: int
"""
self.pid = pid
self.version = version
self.debug = Debug(self.pid, debug=debug, debug_threshold=debug_threshold)
self.status_code = StatusCode
self.paperdb_state_code = PaperDBStateCode
self.paperdb_state = self.paperdb_state_code.initialize
self.connection_timeout = 90
self.connection_time = timedelta()
self.check_credentials_file(credentials)
self.credentials = credentials
self.connect = ''
self.cur = ''
self.db_connect('init', credentials)
self.file_list = []
self.file_md5_dict = {}
self.claimed_files = []
self.claimed_state = 0
def __init__(self,
pid,
drive_select=1,
rewrite_path=None,
debug=False,
debug_threshold=128):
"""initialize"""
self.pid = pid
self.debug = Debug(self.pid,
debug=debug,
debug_threshold=debug_threshold)
self.drive_select = drive_select
self.rewrite_path = rewrite_path
## if we're not using disk queuing we open the drives differently;
## we need to track different states
## for faster archiving we keep some data in memory instead of queuing to disk
self.archive_bytes = BytesIO()
self.archive_tar = tarfile.open(mode='w:', fileobj=self.archive_bytes)
self.archive_info = tarfile.TarInfo()
## tape opened with tar
## this is a dictionary where we will do:
## self.tape_drive[drive_int] = tarfile.open(mode='w:')
self.tape_filehandle = {}
self.tape_drive = {}
## if we use tarfile, we need to track the state
self.drive_states = RamTarStateCode
self.drive_state = self.ramtar_tape_drive(drive_select,
self.drive_states.drive_init)
def __init__(self,
pid,
drive_select=2,
debug=False,
disk_queue=True,
debug_threshold=128):
"""initialize debugging and pid"""
self.pid = pid
self.debug = Debug(pid, debug=debug, debug_threshold=debug_threshold)
self.drive_select = drive_select
def __init__(self,
version,
pid,
tape_size,
disk_queue=True,
drive_select=2,
debug=False,
debug_threshold=255):
"""init with debugging
:type drive_select: int
:param drive_select: 0 = nst0, 1 = nst1, 2 = nst{1,2}
:type disk_queue: bool
:param disk_queue: write archives to a disk queue first?
"""
self.version = version
self.pid = pid
self.debug = Debug(self.pid,
debug=debug,
debug_threshold=debug_threshold)
self.tape_size = tape_size
self._tape_dev = '/dev/changer'
self.status_code = StatusCode
self.drive_select = drive_select
self.archive_tar = ''
self.drive_ids = []
self.tape_ids = []
self.label_in_drive = [] ## return label in given drive
self.check_inventory()
self.tape_drives = Drives(self.pid,
drive_select=drive_select,
debug=debug,
debug_threshold=debug_threshold)
self.disk_queue = disk_queue
if not self.disk_queue:
## we need to use Ramtar
self.ramtar = FastTar(pid,
drive_select=drive_select,
rewrite_path=None,
debug=debug,
debug_threshold=debug_threshold)
## TODO(dconover): implement a lock on the changer to prevent overlapping requests
self.changer_state = 0
def __init__(self,
version,
pid,
debug=False,
debug_threshold=255,
local_transfer=True):
"""Archive file and tar management
:type version: int
:type pid: basestring
:type local_transfer: bool
:type debug_threshold: int
:type debug: bool
:type self: object
"""
self.pid = pid
self.debug = Debug(self.pid,
debug=debug,
debug_threshold=debug_threshold)
self.version = version
#self.transfer = LocalTransfer() if local_transfer else Transfer()
self.transfer = LocalTransfer() if local_transfer else None
dir_status, self.archive_copy_dir = self.ensure_dir(
'/papertape/shm/%s/' % self.pid)
dir_status, self.queue_dir = self.ensure_dir('/papertape/queue/%s/' %
self.pid)
if dir_status is not True:
self.debug.output('data dir init failed')
raise Exception
self.catalog_name = "{0:s}/paper.{1:s}.file_list".format(
self.queue_dir, self.pid)
self.tape_ids_filename = "{0:s}/paper.{1:s}.tape_ids.file_list".format(
self.queue_dir, self.pid)
self.archive_list = [] ## working file_list of files to write
self.tape_list = [] ## cumulative file_list of written files
self.item_index = 0 ## number of file path index (human readable line numbers in catalog)
self.archive_state = 0 ## current archive state
def __init__(self,
version,
credentials,
pid,
debug=False,
debug_threshold=255):
"""Initialize connection and collect file_list of tape_ids."""
self.version = version
self.pid = pid
self.debug = Debug(self.pid,
debug=debug,
debug_threshold=debug_threshold)
self.status_code = StatusCode
## database variables
self.connection_timeout = 90
self.connection_time = datetime.timedelta()
self.credentials = credentials
self.connect = ''
self.cur = ''
self.db_connect('init', credentials)
self.mtxdb_state = 0 ## current dump state
def __init__(self,
credentials,
debug=False,
pid=None,
disk_queue=True,
drive_select=2,
debug_threshold=255):
"""initialize"""
self.version = __version__
self.pid = "%0.6d%0.3d" % (getpid(),
randint(1, 999)) if pid is None else pid
self.debug = Debug(self.pid,
debug=debug,
debug_threshold=debug_threshold)
self.status_code = StatusCode
self.mtx_creds = '~/.my.mtx.cnf'
self.debug.output(credentials)
self.paper_creds = credentials
self.tape_ids = ''
## each dump process 6gb to /dev/shm (two at a time)
self.batch_size_mb = 12000
## (1.5Tb -1 batch)
self.tape_size = (1.5 * 1000 * 1000) - self.batch_size_mb
#self.tape_size = 13000
## setup PaperDB connection
#self.paperdb = PaperDB(self.version, self.paper_creds, self.pid, debug=True, debug_threshold=debug_threshold)
## test database
self.paperdb = TestPaperDB(self.version,
self.paper_creds,
self.pid,
debug=True,
debug_threshold=debug_threshold)
## reload test data
#self.paperdb.load_sample_data()
## setup tape library
self.labeldb = MtxDB(self.version,
self.mtx_creds,
self.pid,
debug=debug,
debug_threshold=debug_threshold)
## setup file access
self.files = Archive(self.version,
self.pid,
debug=debug,
debug_threshold=debug_threshold)
## use the pid here to lock changer
self.drive_select = drive_select
self.tape = Changer(self.version,
self.pid,
self.tape_size,
debug=True,
drive_select=drive_select,
disk_queue=disk_queue,
debug_threshold=debug_threshold)
self.dump_list = []
self.tape_index = 0
self.tape_used_size = 0 ## each dump process should write one tape worth of data
self.dump_state_code = DumpStateCode
self.dump_state = self.dump_state_code.initialize