def main():
copier = Copier()
keyboard = Keyboard()
scanner = Scanner()
printer = Printer()
database = Database()
copier.copy(scanner, database)
def go():
"""
Requests from users go here
:return:
"""
if request.method == 'POST':
# extract all details from user request
json = request.get_json()
public = json['public']
links = json['links']
folder_id = json['folderId']
cred = Credentials(session[ACCESS_TOKEN])
service = build('drive', 'v3', credentials=cred)
# extract ids from links
# match pattern is 25 words or dash characters, might change this later
pattern = '[-\w]{25,}'
file_ids = extractIds(pattern, links)
response_body = dict()
response_body['response'] = []
c = Copier(service)
response_body['response'] = c.make_copies(file_ids,
public=public,
destination=folder_id)
return jsonify(response_body), 200
else:
# fun
message = {'greeting': 'Hello boss'}
return jsonify(message)
def __init__(self, filesystem, logger=None):
self.filesystem = cache_directory(read_only(filesystem))
self.filesystem.desc('/')
self.caps_list = {}
self.results = []
self.logger = logger
self.copier = Copier()
def copy_origin(self, origins, destination, link = False):
copier = Copier(link=link)
for origin in origins:
self._logger.info(
"Copying media from '%s' to '%s'" % (origin, destination)
)
try:
components = origin.split(':')
mod, directory = components[:2]
except ValueError:
raise zc.buildout.UserError(
"Error in '%s': media_origin must be in the form "
"'custom.module:directory'" % self.name
)
try:
__, ws = self.rws
mod = dotted_import(mod, [d.location for d in ws])
except ImportError:
raise zc.buildout.UserError(
"Error in '%s': media_origin is '%s' "
"but we cannot find module '%s'" % (self.name, origin, mod)
)
orig_directory = os.path.join(
os.path.dirname(mod.__file__),
directory
)
if not os.path.isdir(orig_directory):
raise zc.buildout.UserError(
"Error in '%s': media_origin is '%s' "
"but '%s' does not seem to be a directory" % (
self.name, origin, directory
)
)
if len(components) > 2:
target = os.path.join(destination, components[2])
else:
target = destination
copier.copy(orig_directory, target)
try:
copier.execute()
except OSError, e:
raise zc.buildout.UserError(
"Failed to copy %s into '%s': %s" % (
', '.join([ "'%s'" % o for o in origins ]),
destination,
e
)
)
def run(self, window):
if not self.meta_info.finished:
messagebox.showinfo(message="Modification is already happening.", title="Error")
window.after(50, lambda: self.listen_for_result(window))
return
c = Copier(self.meta_info)
self.meta_info.finished = False
self.new_thread = threading.Thread(target=c.run)
self.new_thread.start()
window.after(50, lambda: self.listen_for_result(window))
def main():
utils.print_header("MUSIC BACKUP PROGRAM")
configs = Configs.from_json("config.json")
analyzer = Analyzer(configs)
dst_files = analyzer.get_backup_files()
src_files = analyzer.get_source_files()
analyzer.compare_directories()
summary = ""
if analyzer.files_to_backup > 0 and configs.backup_enabled:
utils.print_header("COPYING TO BACKUP")
print("Starting copying process...\n")
copier = Copier(configs.source_path, configs.backup_path, src_files,
dst_files, analyzer.files_to_backup)
backed_up_count = copier.copy()
summary += "Backed up a total of {} files!".format(backed_up_count)
if analyzer.files_to_backcopy > 0 and configs.backcopy_enabled:
utils.print_header("COPYING TO LOCAL")
print("Starting copying process...")
copier = Copier(configs.backup_path, configs.backcopy_path, dst_files,
src_files, analyzer.files_to_backcopy)
backcopied_count = copier.copy()
summary += "Copied a total of {} files to your local!".format(
backcopied_count)
if summary and (configs.backcopy_enabled or configs.backup_enabled):
utils.print_header("SUMMARY")
print(summary)
print("\nComplete!")
return
def copy(self, S_DIR: str, test_num: str, setting: List[int]):
# Tests
Tk()
meta_info = MetaInformation()
meta_info.set_dirs(S_DIR + "_original/" + test_num,
S_DIR + "_replacement", S_DIR + "_copy/" + test_num)
meta_info.finished = False
# Set the meta_info data
meta_info.air_pockets.set(setting[0])
meta_info.water_blocks.set(setting[1])
meta_info.repl_blocks.set(setting[2])
meta_info.apocket_size.set(setting[3])
meta_info.wpocket_size.set(setting[4])
meta_info.repl_area.set(setting[5])
# Run the copy process
c = Copier(meta_info)
c.run()
while not meta_info.text_queue.empty():
print(meta_info.text_queue.get(0))
def main():
copier = Copier()
keyboard = Keyboard()
# scanner = Scanner()
copier.copy(keyboard)
def start_work(self):
if self.mode != SLAVE_MODE:
self.log.critical(f'Function only works in {SLAVE_MODE} mode!')
return False
while True:
try:
self.log.info(
f'Waiting for jobs on queue: {self.job_queue_name}, {self.files_processed} files have been processed so far'
)
for msg in self.job_queue.receiveMsgs(self.VISIBILITY_TIMEOUT):
self.log.info(f'Received a job!')
extender = None
data = None
try:
data = json.loads(msg.body)
self.log.debug(data)
# Make sure job is in correct format
if (self.ADAPTER_CONF in data and self.BUCKET in data
and self.INFO in data and self.TTL in data
and self.OVERWRITE in data
and self.PREFIX in data
and self.DRY_RUN in data
and self.VERIFY_MD5 in data):
extender = VisibilityExtender(
msg, self.VISIBILITY_TIMEOUT)
dryrun = data[self.DRY_RUN]
verify_md5 = data[self.VERIFY_MD5]
adapter_config = data[self.ADAPTER_CONF]
bucket_name = data[self.BUCKET]
prefix = data[self.PREFIX]
if self.adapter_config != adapter_config:
self.adapter_config = adapter_config
self._init_adapter(
adapter_module=adapter_config[
self.ADAPTER_MODULE],
adapter_class=adapter_config[
self.ADAPTER_CLASS],
params=adapter_config[self.ADAPTER_PARAMS])
self.bucket_name = bucket_name
self.prefix = prefix
self.copier = Copier(bucket_name, prefix,
self.adapter)
if bucket_name != self.bucket_name:
self.bucket_name = bucket_name
self.copier.set_bucket(bucket_name)
if prefix != self.prefix:
self.prefix = prefix
self.copier.set_prefix(prefix)
result = self.copier.copy_file(
data[self.INFO], data[self.OVERWRITE], dryrun
or self.dryrun, verify_md5)
if result[Copier.STATUS]:
self.result_queue.sendMsgToQueue(
result,
f'{result[Copier.NAME]}_{get_time_stamp()}'
)
else:
self._deal_with_failed_file_sqs(data)
extender.stop()
extender = None
self.files_processed += 1
self.log.info(f'Copying file finished!')
msg.delete()
else:
self.log.error(f'Wrong message type!')
self.log.error(data)
msg.delete()
except Exception as e:
self.log.debug(e)
self.log.critical(
f'Something wrong happened while processing file! Check debug log for details.'
)
if data:
self._deal_with_failed_file_sqs(data)
finally:
if extender:
extender.stop()
extender = None
except KeyboardInterrupt:
self.log.info('Good bye!')
return
def copy_all(self):
"""
Read file information from pre-manifest and copy them all to destination bucket
:return:
"""
if self.mode != SOLO_MODE:
self.log.critical(f'Function only works in {SOLO_MODE} mode!')
return False
self.copier = Copier(self.bucket_name, self.prefix, self.adapter)
file_queue = deque(self._read_pre_manifest())
indexd_manifest = self.get_indexd_manifest_name(self.pre_manifest)
neo4j_manifest = self.get_neo4j_manifest_name(self.pre_manifest)
with open(indexd_manifest, 'w', newline='\n') as indexd_f:
indexd_writer = csv.DictWriter(indexd_f,
delimiter='\t',
fieldnames=self.MANIFEST_FIELDS)
indexd_writer.writeheader()
with open(neo4j_manifest, 'w', newline='\n') as neo4j_f:
fieldnames = self.DATA_FIELDS
for field in self.adapter.filter_fields(self.field_names):
if field not in fieldnames:
fieldnames.append(field)
neo4j_writer = csv.DictWriter(neo4j_f,
delimiter='\t',
fieldnames=fieldnames)
neo4j_writer.writeheader()
while file_queue:
job = file_queue.popleft()
job[self.TTL] -= 1
file_info = job[self.INFO]
try:
result = self.copier.copy_file(file_info,
self.overwrite,
self.dryrun,
self.verify_md5)
if result[Copier.STATUS]:
indexd_record = {}
self.populate_indexd_record(indexd_record, result)
indexd_writer.writerow(indexd_record)
neo4j_record = result[Copier.FIELDS]
self.populate_neo4j_record(neo4j_record, result)
neo4j_writer.writerow(neo4j_record)
else:
self._deal_with_failed_file(job, file_queue)
except Exception as e:
self.log.debug(e)
self._deal_with_failed_file(job, file_queue)
if self.files_skipped > 0:
self.log.info(f'Files skipped: {self.files_skipped}')
self.log.info(f'Files processed: {self.files_processed}')
self.log.info(
f'Files not found: {len(self.copier.files_not_found)}')
self.log.info(f'Files copied: {self.copier.files_copied}')
self.log.info(
f'Files exist at destination: {self.copier.files_exist_at_dest}'
)
self.log.info(f'Files failed: {self.files_failed}')
class FileLoader:
GUID = 'GUID'
MD5 = 'md5'
SIZE = 'size'
URL = 'url'
MANIFEST_FIELDS = [GUID, MD5, SIZE, Copier.ACL, URL]
NODE_TYPE = 'type'
FILE_NAME = 'file_name'
FILE_SIZE = "file_size"
MD5_SUM = 'md5sum'
FILE_STAT = 'file_status'
FILE_LOC = 'file_location'
FILE_FORMAT = 'file_format'
DATA_FIELDS = [
NODE_TYPE, FILE_NAME, UUID, FILE_SIZE, MD5_SUM, FILE_STAT, FILE_LOC,
FILE_FORMAT, Copier.ACL
]
DEFAULT_NODE_TYPE = 'file'
DEFAULT_STAT = 'uploaded'
INDEXD_GUID_PREFIX = 'dg.4DFC/'
INDEXD_MANIFEST_EXT = '.tsv'
VISIBILITY_TIMEOUT = 30
ADAPTER_MODULE = 'adapter_module'
ADAPTER_CLASS = 'adapter_class'
ADAPTER_PARAMS = 'adapter_params'
# keys in job dict
ADAPTER_CONF = 'adapter_config'
TTL = 'ttl'
INFO = 'file_info'
LINE = 'line_num'
OVERWRITE = 'overwrite'
DRY_RUN = 'dry_run'
BUCKET = 'bucket'
PREFIX = 'prefix'
VERIFY_MD5 = 'verify_md5'
def __init__(self,
mode,
adapter_module=None,
adapter_class=None,
adapter_params=None,
domain=None,
bucket=None,
prefix=None,
pre_manifest=None,
first=1,
count=-1,
job_queue=None,
result_queue=None,
retry=3,
overwrite=False,
dryrun=False,
verify_md5=False):
""""
:param bucket: string type
:param pre_manifest: string type, holds path to pre-manifest
:param first: first file of files to process, file 1 is in line 2 of pre-manifest
:param count: number of files to process
:param adapter: any object that has following methods/properties defined in adapter_attrs
"""
if mode not in Config.valid_modes:
raise ValueError(f'Invalid loading mode: {mode}')
self.mode = mode
if mode != SOLO_MODE:
if not job_queue:
raise ValueError(
f'Job queue name is required in {self.mode} mode!')
self.job_queue_name = job_queue
self.job_queue = Queue(job_queue)
if not result_queue:
raise ValueError(
f'Result queue name is required in {self.mode} mode!')
self.result_queue_name = result_queue
self.result_queue = Queue(result_queue)
if self.mode != SLAVE_MODE:
if not bucket:
raise ValueError('Empty destination bucket name')
self.bucket_name = bucket
if prefix and isinstance(prefix, str):
self.prefix = removeTrailingSlash(prefix)
else:
raise ValueError(f'Invalid prefix: "{prefix}"')
if not pre_manifest or not os.path.isfile(pre_manifest):
raise ValueError(
f'Pre-manifest: "{pre_manifest}" dosen\'t exist')
self.pre_manifest = pre_manifest
if not domain:
raise ValueError(f'Empty domain!')
self.domain = domain
self.adapter_config = {
self.ADAPTER_PARAMS: adapter_params,
self.ADAPTER_CLASS: adapter_class,
self.ADAPTER_MODULE: adapter_module
}
self._init_adapter(adapter_module, adapter_class, adapter_params)
else:
self.adapter = None
self.adapter_config = {}
self.copier = None
if not first > 0 or count == 0:
raise ValueError(f'Invalid first ({first}) or count ({count})')
self.skip = first - 1
self.count = count
if not isinstance(retry, int) and retry > 0:
raise ValueError(f'Invalid retry value: {retry}')
self.retry = retry
if not isinstance(overwrite, bool):
raise TypeError(f'Invalid overwrite value: {overwrite}')
self.overwrite = overwrite
if not isinstance(dryrun, bool):
raise TypeError(f'Invalid dryrun value: {dryrun}')
self.dryrun = dryrun
self.verify_md5 = verify_md5
self.log = get_logger('FileLoader')
# Statistics
self.files_processed = 0
self.files_skipped = 0
self.files_failed = 0
def _init_adapter(self, adapter_module, adapter_class, params):
"""
Initialize different adapters base on given adapter_name
:param adapter_name:
:return:
"""
self.adapter = load_plugin(adapter_module, adapter_class, params)
if not hasattr(self.adapter, 'filter_fields'):
raise TypeError(
f'Adapter "{adapter_class}" does not have a "filter_fields" method'
)
def get_indexd_manifest_name(self, file_name):
folder = os.path.dirname(file_name)
base_name = os.path.basename(file_name)
name, _ = os.path.splitext(base_name)
new_name = '{}_indexd{}'.format(name, self.INDEXD_MANIFEST_EXT)
return os.path.join(folder, new_name)
@staticmethod
def get_s3_location(bucket, key):
return "s3://{}/{}".format(bucket, key)
@staticmethod
def get_neo4j_manifest_name(file_name):
folder = os.path.dirname(file_name)
base_name = os.path.basename(file_name)
name, ext = os.path.splitext(base_name)
new_name = '{}_neo4j{}'.format(name, ext)
return os.path.join(folder, new_name)
def populate_indexd_record(self, record, result):
record[self.SIZE] = result[Copier.SIZE]
record[self.MD5] = result[Copier.MD5]
record[Copier.ACL] = result[Copier.ACL]
record[self.URL] = self.get_s3_location(self.bucket_name,
result[Copier.KEY])
record[self.GUID] = '{}{}'.format(
self.INDEXD_GUID_PREFIX,
get_uuid(self.domain, "file", record[self.URL]))
return record
def populate_neo4j_record(self, record, result):
if self.NODE_TYPE not in record:
record[self.NODE_TYPE] = self.DEFAULT_NODE_TYPE
record[self.FILE_NAME] = result[Copier.NAME]
record[self.FILE_SIZE] = result[Copier.SIZE]
record[self.FILE_LOC] = self.get_s3_location(self.bucket_name,
result[Copier.KEY])
file_name = result[Copier.NAME]
record[self.MD5_SUM] = result[Copier.MD5]
record[self.FILE_FORMAT] = (
os.path.splitext(file_name)[1]).split('.')[1].lower()
record[UUID] = get_uuid(self.domain, "file", record[self.FILE_LOC])
record[self.FILE_STAT] = self.DEFAULT_STAT
record[Copier.ACL] = result[Copier.ACL]
return record
@staticmethod
def _clean_up_field_names(headers):
'''
Removes leading and trailing spaces from header names
:param headers:
:return:
'''
return [header.strip() for header in headers]
@staticmethod
def _clean_up_record(record):
'''
Removes leading and trailing spaces from keys in org_record
:param record:
:return:
'''
return {key.strip(): value for key, value in record.items()}
def _read_pre_manifest(self):
files = []
with open(self.pre_manifest) as pre_m:
reader = csv.DictReader(pre_m, delimiter='\t')
self.field_names = self._clean_up_field_names(reader.fieldnames)
for _ in range(self.skip):
next(reader)
self.files_skipped += 1
line_num = self.files_skipped + 1
for info in reader:
self.files_processed += 1
line_num += 1
files.append({
self.ADAPTER_CONF: self.adapter_config,
self.LINE: line_num,
self.TTL: self.retry,
self.OVERWRITE: self.overwrite,
self.DRY_RUN: self.dryrun,
self.INFO: self._clean_up_record(info),
self.BUCKET: self.bucket_name,
self.PREFIX: self.prefix,
self.VERIFY_MD5: self.verify_md5
})
if self.files_processed >= self.count > 0:
break
return files
# Use this method in solo mode
def copy_all(self):
"""
Read file information from pre-manifest and copy them all to destination bucket
:return:
"""
if self.mode != SOLO_MODE:
self.log.critical(f'Function only works in {SOLO_MODE} mode!')
return False
self.copier = Copier(self.bucket_name, self.prefix, self.adapter)
file_queue = deque(self._read_pre_manifest())
indexd_manifest = self.get_indexd_manifest_name(self.pre_manifest)
neo4j_manifest = self.get_neo4j_manifest_name(self.pre_manifest)
with open(indexd_manifest, 'w', newline='\n') as indexd_f:
indexd_writer = csv.DictWriter(indexd_f,
delimiter='\t',
fieldnames=self.MANIFEST_FIELDS)
indexd_writer.writeheader()
with open(neo4j_manifest, 'w', newline='\n') as neo4j_f:
fieldnames = self.DATA_FIELDS
for field in self.adapter.filter_fields(self.field_names):
if field not in fieldnames:
fieldnames.append(field)
neo4j_writer = csv.DictWriter(neo4j_f,
delimiter='\t',
fieldnames=fieldnames)
neo4j_writer.writeheader()
while file_queue:
job = file_queue.popleft()
job[self.TTL] -= 1
file_info = job[self.INFO]
try:
result = self.copier.copy_file(file_info,
self.overwrite,
self.dryrun,
self.verify_md5)
if result[Copier.STATUS]:
indexd_record = {}
self.populate_indexd_record(indexd_record, result)
indexd_writer.writerow(indexd_record)
neo4j_record = result[Copier.FIELDS]
self.populate_neo4j_record(neo4j_record, result)
neo4j_writer.writerow(neo4j_record)
else:
self._deal_with_failed_file(job, file_queue)
except Exception as e:
self.log.debug(e)
self._deal_with_failed_file(job, file_queue)
if self.files_skipped > 0:
self.log.info(f'Files skipped: {self.files_skipped}')
self.log.info(f'Files processed: {self.files_processed}')
self.log.info(
f'Files not found: {len(self.copier.files_not_found)}')
self.log.info(f'Files copied: {self.copier.files_copied}')
self.log.info(
f'Files exist at destination: {self.copier.files_exist_at_dest}'
)
self.log.info(f'Files failed: {self.files_failed}')
def _deal_with_failed_file(self, job, queue):
if job[self.TTL] > 0:
self.log.error(
f'Line: {job[self.LINE]} - Copying file FAILED! Retry left: {job[self.TTL]}'
)
queue.append(job)
else:
self.log.critical(
f'Copying file failure exceeded maximum retry times, abort!')
self.files_failed += 1
# Use this method in master mode
def process_all(self):
"""
Read file information from pre-manifest and push jobs into job queue
Listen on result queue for loading result
:return:
"""
if self.mode != MASTER_MODE:
self.log.critical(f'Function only works in {MASTER_MODE} mode!')
return False
try:
files = self._read_pre_manifest()
count = 0
for job in files:
if self.dryrun:
self.log.info(
f'Dry run mode, jobs will be sent to queue but files won\'t be copied!'
)
else:
self.log.info(
f'Line {job[self.LINE]}: file info sent to queue: {self.job_queue_name}'
)
self.job_queue.sendMsgToQueue(
job, f'{job[self.LINE]}_{get_time_stamp()}')
count += 1
self.log.info(f'Files sent to queue: {count}')
self.read_result(count)
except Exception as e:
self.log.debug(e)
self.log.critical(
f'Process files FAILED! Check debug log for detailed information.'
)
# read result from result queue - master mode
def read_result(self, num_files):
if self.mode != MASTER_MODE:
self.log.critical(f'Function only works in {MASTER_MODE} mode!')
return False
indexd_manifest = self.get_indexd_manifest_name(self.pre_manifest)
neo4j_manifest = self.get_neo4j_manifest_name(self.pre_manifest)
with open(indexd_manifest, 'w', newline='\n') as indexd_f:
indexd_writer = csv.DictWriter(indexd_f,
delimiter='\t',
fieldnames=self.MANIFEST_FIELDS)
indexd_writer.writeheader()
with open(neo4j_manifest, 'w', newline='\n') as neo4j_f:
fieldnames = self.adapter.filter_fields(self.field_names)
fieldnames += self.DATA_FIELDS
neo4j_writer = csv.DictWriter(neo4j_f,
delimiter='\t',
fieldnames=fieldnames)
neo4j_writer.writeheader()
count = 0
while count < num_files:
self.log.info(
f'Waiting for results on queue: {self.result_queue_name}, {num_files - count} files pending'
)
for msg in self.result_queue.receiveMsgs(
self.VISIBILITY_TIMEOUT):
self.log.info(f'Received a result!')
extender = None
try:
result = json.loads(msg.body)
# Make sure result is in correct format
if (result and Copier.STATUS in result
and Copier.MD5 in result
and Copier.NAME in result
and Copier.KEY in result
and Copier.FIELDS in result):
extender = VisibilityExtender(
msg, self.VISIBILITY_TIMEOUT)
if result[Copier.STATUS]:
indexd_record = {}
self.populate_indexd_record(
indexd_record, result)
indexd_writer.writerow(indexd_record)
neo4j_record = result[Copier.FIELDS]
self.populate_neo4j_record(
neo4j_record, result)
neo4j_writer.writerow(neo4j_record)
else:
self.log.error(
f'Copy file {result[Copier.NAME]} FAILED!'
)
self.files_failed += 1
extender.stop()
extender = None
count += 1
self.log.info(
f'{count} of {num_files} files finished!')
msg.delete()
else:
self.log.error(f'Wrong message type!')
self.log.error(result)
msg.delete()
except Exception as e:
self.log.debug(e)
self.log.critical(
f'Something wrong happened while processing file! Check debug log for details.'
)
finally:
if extender:
extender.stop()
extender = None
self.log.info(f'All {num_files} files finished!')
# Use this method in slave mode
def start_work(self):
if self.mode != SLAVE_MODE:
self.log.critical(f'Function only works in {SLAVE_MODE} mode!')
return False
while True:
try:
self.log.info(
f'Waiting for jobs on queue: {self.job_queue_name}, {self.files_processed} files have been processed so far'
)
for msg in self.job_queue.receiveMsgs(self.VISIBILITY_TIMEOUT):
self.log.info(f'Received a job!')
extender = None
data = None
try:
data = json.loads(msg.body)
self.log.debug(data)
# Make sure job is in correct format
if (self.ADAPTER_CONF in data and self.BUCKET in data
and self.INFO in data and self.TTL in data
and self.OVERWRITE in data
and self.PREFIX in data
and self.DRY_RUN in data
and self.VERIFY_MD5 in data):
extender = VisibilityExtender(
msg, self.VISIBILITY_TIMEOUT)
dryrun = data[self.DRY_RUN]
verify_md5 = data[self.VERIFY_MD5]
adapter_config = data[self.ADAPTER_CONF]
bucket_name = data[self.BUCKET]
prefix = data[self.PREFIX]
if self.adapter_config != adapter_config:
self.adapter_config = adapter_config
self._init_adapter(
adapter_module=adapter_config[
self.ADAPTER_MODULE],
adapter_class=adapter_config[
self.ADAPTER_CLASS],
params=adapter_config[self.ADAPTER_PARAMS])
self.bucket_name = bucket_name
self.prefix = prefix
self.copier = Copier(bucket_name, prefix,
self.adapter)
if bucket_name != self.bucket_name:
self.bucket_name = bucket_name
self.copier.set_bucket(bucket_name)
if prefix != self.prefix:
self.prefix = prefix
self.copier.set_prefix(prefix)
result = self.copier.copy_file(
data[self.INFO], data[self.OVERWRITE], dryrun
or self.dryrun, verify_md5)
if result[Copier.STATUS]:
self.result_queue.sendMsgToQueue(
result,
f'{result[Copier.NAME]}_{get_time_stamp()}'
)
else:
self._deal_with_failed_file_sqs(data)
extender.stop()
extender = None
self.files_processed += 1
self.log.info(f'Copying file finished!')
msg.delete()
else:
self.log.error(f'Wrong message type!')
self.log.error(data)
msg.delete()
except Exception as e:
self.log.debug(e)
self.log.critical(
f'Something wrong happened while processing file! Check debug log for details.'
)
if data:
self._deal_with_failed_file_sqs(data)
finally:
if extender:
extender.stop()
extender = None
except KeyboardInterrupt:
self.log.info('Good bye!')
return
def _deal_with_failed_file_sqs(self, job):
self.log.info(f'Copy file FAILED, {job[self.TTL] - 1} retry left!')
job[self.TTL] -= 1
self.job_queue.sendMsgToQueue(job, f'{job[self.LINE]}_{job[self.TTL]}')
def run(self):
if self.mode == SOLO_MODE:
self.copy_all()
elif self.mode == MASTER_MODE:
self.process_all()
elif self.mode == SLAVE_MODE:
self.start_work()
class BaseManager:
def __init__(self, filesystem, logger=None):
self.filesystem = cache_directory(read_only(filesystem))
self.filesystem.desc('/')
self.caps_list = {}
self.results = []
self.logger = logger
self.copier = Copier()
def close(self):
self.copier.stop()
self.filesystem.close()
def list_dir(self, top):
return [i.name for i in self.filesystem.scandir(top) if i.is_dir]
def download(self, src_pth, src_file, dest_pth, dest_file):
ff = OSFS(dest_pth)
self.copier.copy(self.filesystem, join(src_pth, src_file), ff,
join('/', dest_file))
def find_nexts(self, top='/', deep=0, maxdeep=2):
if deep == 0:
self.results = []
# print(top)
if deep > maxdeep:
return
# if self.logger:
# self.logger.emit(top, INFORMATION)
dirs, nondirs = [], []
for name in self.filesystem.scandir(top):
if name.is_dir:
dirs.append(name)
elif splitext(name.name)[1].lower() in video_formats:
nondirs.append(name)
# print(dirs,nondirs)
for fil in nondirs:
pp = rename(fil.name)
if pp.error:
pp = parse(j.name)
t1 = ''
t2 = 0
try:
if pp.is_video:
if pp.episode:
t1 = transform(pp.title)
fill = t1
if pp.season:
fill += ' - ' + str(pp.season) + \
'x' + str(pp.episode)
else:
fill += ' - ' + str(pp.episode)
fill += pp.ext
else:
continue
t2 = pp.episode
else:
continue
except KeyError:
if self.logger:
self.logger.emit("Error procesando: " + fil.name, WARNING)
continue
bedd = 100
gap = 2
near = ''
for j in self.caps_list.keys():
edd = editDistance(t1, j, True)
if edd <= gap and edd < bedd:
near = j
bedd = edd
if edd == 0:
break
if near != '':
if isinstance(t2, str):
if 'x' in t2:
t2 = t2.split('x')[1]
if int(t2) > self.caps_list[near]:
best = (near, fil.name, top, fill)
self.results.append(best)
if self.logger:
self.logger.emit('Encontrado: ' + str(best),
INFORMATION)
for name in sorted(dirs, key=skey):
path = join(top, name.name)
if not self.filesystem.islink(path):
try:
self.find_nexts(path, deep + 1, maxdeep)
except (PermissionError, fs.errors.DirectoryExpected) as e:
# print(e)
self.logger.emit(str(e), ERROR)
def last(self, base):
with read_only(OSFS(base)) as ff:
proces = []
folds = []
for i in ff.scandir('/'):
if i.is_dir:
folds.append(i)
for i in folds:
path = join('/', i.name)
try:
for j in ff.scandir(path):
if j.is_file and splitext(
j.name)[1].lower() in video_formats:
proces.append((j, i))
except (PermissionError, DirectoryExpected) as e:
self.logger.emit("Acceso denegado a" + join(base, i.name),
ERROR)
folds = {}
for filee, fold in proces:
fold = fold.name
filee = filee.name
try:
pp = parse2(filee)
if pp.error:
pp = parse(filee)
except Exception as e:
self.logger.emit(
"Error procesando: " + join(base, fold, filee),
WARNING)
self.logger.emit(str(e), ERROR)
continue
t1 = transform(pp.title)
if not pp.episode:
continue
t2 = pp.episode
if t1 in folds:
if folds[t1] < int(t2):
folds[t1] = int(t2)
else:
tt = best_ed(t1, folds.keys())
if tt in folds:
if folds[tt] < int(t2):
folds[tt] = int(t2)
folds[tt] = int(t2)
self.caps_list = folds
def find_nexts2(self, top='/', deep=0, maxdeep=2):
if deep == 0:
self.results = []
# print(top)
if deep > maxdeep:
return
# if self.logger:
# self.logger.emit(top, INFORMATION)
dirs, nondirs = [], []
for name in self.filesystem.scandir(top):
if name.is_dir:
dirs.append(name)
elif splitext(name.name)[1].lower() in video_formats:
nondirs.append(name)
# print(dirs,nondirs)
for fil in nondirs:
pp = parse2(fil.name)
if pp.error:
pp = parse(fil.name)
t1 = ''
t2 = 0
try:
if pp.is_video:
if pp.episode:
t1 = transform(pp.title)
fill = t1
if pp.season:
fill += ' - ' + str(pp.season) + \
'x' + str(pp.episode)
else:
fill += ' - ' + str(pp.episode)
fill += pp.ext
else:
continue
t2 = pp.episode
else:
continue
except KeyError:
if self.logger:
self.logger.emit("Error procesando: " + fil.name, WARNING)
continue
bedd = 100
gap = 2
near = ''
for j in self.caps_list.keys():
edd = editDistance(t1, j, True)
if edd <= gap and edd < bedd:
near = j
bedd = edd
if edd == 0:
break
if near != '':
if isinstance(t2, str):
if 'x' in t2:
t2 = t2.split('x')[1]
if not (int(t2) in self.caps_list[near]):
best = (near, fil.name, top, fill)
self.results.append(best)
if self.logger:
self.logger.emit('Encontrado: ' + str(best),
INFORMATION)
for name in sorted(dirs, key=skey):
path = join(top, name.name)
if not self.filesystem.islink(path):
self.find_nexts2(path, deep + 1, maxdeep)
def last2(self, base):
with read_only(OSFS(base)) as ff:
proces = []
folds = []
for i in ff.scandir('/'):
if i.is_dir:
folds.append(i)
for i in folds:
path = join('/', i.name)
try:
for j in ff.scandir(path):
if j.is_file and splitext(
j.name)[1].lower() in video_formats:
proces.append((j, i))
except (PermissionError, DirectoryExpected) as e:
self.logger.emit("Acceso denegado a" + join(base, i.name),
ERROR)
folds = {}
for filee, fold in proces:
fold = fold.name
filee = filee.name
try:
pp = parse2(filee)
if pp.error:
pp = parse(filee)
except Exception as e:
self.logger.emit(
"Error procesando: " + join(base, fold, filee),
WARNING)
self.logger.emit(str(e), ERROR)
continue
t1 = transform(pp.title)
if not pp.episode:
continue
t2 = pp.episode
if t1 in folds:
folds[t1].add(int(t2))
else:
tt = best_ed(t1, folds.keys())
if tt in folds:
folds[t1].add(int(t2))
else:
folds[t1] = set()
folds[t1].add(int(t2))
self.caps_list = folds
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.filesystem.close()
self.copier.stop()
return False
def test_copier_function(self):
dut = Copier()
dut.copier_test() #bs test method
assert os.path.isfile('total_test_bs_file.py')
os.remove('total_test_bs_file.py')
def test_copier_filenotfounderror(self):
dut = Copier()
with pytest.raises(FileNotFoundError):
dut.copy_prod_machine_files_to_verify(
) #this is failing to raise an error, why??
def main():
copier = Copier()
copier.copy()
def main():
my_runner = Runner(Getter(), Writer(), Deleter(), Copier())
my_runner.run_it()