def main(): copier = Copier() keyboard = Keyboard() scanner = Scanner() printer = Printer() database = Database() copier.copy(scanner, database)
def go(): """ Requests from users go here :return: """ if request.method == 'POST': # extract all details from user request json = request.get_json() public = json['public'] links = json['links'] folder_id = json['folderId'] cred = Credentials(session[ACCESS_TOKEN]) service = build('drive', 'v3', credentials=cred) # extract ids from links # match pattern is 25 words or dash characters, might change this later pattern = '[-\w]{25,}' file_ids = extractIds(pattern, links) response_body = dict() response_body['response'] = [] c = Copier(service) response_body['response'] = c.make_copies(file_ids, public=public, destination=folder_id) return jsonify(response_body), 200 else: # fun message = {'greeting': 'Hello boss'} return jsonify(message)
def __init__(self, filesystem, logger=None): self.filesystem = cache_directory(read_only(filesystem)) self.filesystem.desc('/') self.caps_list = {} self.results = [] self.logger = logger self.copier = Copier()
def copy_origin(self, origins, destination, link = False): copier = Copier(link=link) for origin in origins: self._logger.info( "Copying media from '%s' to '%s'" % (origin, destination) ) try: components = origin.split(':') mod, directory = components[:2] except ValueError: raise zc.buildout.UserError( "Error in '%s': media_origin must be in the form " "'custom.module:directory'" % self.name ) try: __, ws = self.rws mod = dotted_import(mod, [d.location for d in ws]) except ImportError: raise zc.buildout.UserError( "Error in '%s': media_origin is '%s' " "but we cannot find module '%s'" % (self.name, origin, mod) ) orig_directory = os.path.join( os.path.dirname(mod.__file__), directory ) if not os.path.isdir(orig_directory): raise zc.buildout.UserError( "Error in '%s': media_origin is '%s' " "but '%s' does not seem to be a directory" % ( self.name, origin, directory ) ) if len(components) > 2: target = os.path.join(destination, components[2]) else: target = destination copier.copy(orig_directory, target) try: copier.execute() except OSError, e: raise zc.buildout.UserError( "Failed to copy %s into '%s': %s" % ( ', '.join([ "'%s'" % o for o in origins ]), destination, e ) )
def run(self, window): if not self.meta_info.finished: messagebox.showinfo(message="Modification is already happening.", title="Error") window.after(50, lambda: self.listen_for_result(window)) return c = Copier(self.meta_info) self.meta_info.finished = False self.new_thread = threading.Thread(target=c.run) self.new_thread.start() window.after(50, lambda: self.listen_for_result(window))
def main(): utils.print_header("MUSIC BACKUP PROGRAM") configs = Configs.from_json("config.json") analyzer = Analyzer(configs) dst_files = analyzer.get_backup_files() src_files = analyzer.get_source_files() analyzer.compare_directories() summary = "" if analyzer.files_to_backup > 0 and configs.backup_enabled: utils.print_header("COPYING TO BACKUP") print("Starting copying process...\n") copier = Copier(configs.source_path, configs.backup_path, src_files, dst_files, analyzer.files_to_backup) backed_up_count = copier.copy() summary += "Backed up a total of {} files!".format(backed_up_count) if analyzer.files_to_backcopy > 0 and configs.backcopy_enabled: utils.print_header("COPYING TO LOCAL") print("Starting copying process...") copier = Copier(configs.backup_path, configs.backcopy_path, dst_files, src_files, analyzer.files_to_backcopy) backcopied_count = copier.copy() summary += "Copied a total of {} files to your local!".format( backcopied_count) if summary and (configs.backcopy_enabled or configs.backup_enabled): utils.print_header("SUMMARY") print(summary) print("\nComplete!") return
def copy(self, S_DIR: str, test_num: str, setting: List[int]): # Tests Tk() meta_info = MetaInformation() meta_info.set_dirs(S_DIR + "_original/" + test_num, S_DIR + "_replacement", S_DIR + "_copy/" + test_num) meta_info.finished = False # Set the meta_info data meta_info.air_pockets.set(setting[0]) meta_info.water_blocks.set(setting[1]) meta_info.repl_blocks.set(setting[2]) meta_info.apocket_size.set(setting[3]) meta_info.wpocket_size.set(setting[4]) meta_info.repl_area.set(setting[5]) # Run the copy process c = Copier(meta_info) c.run() while not meta_info.text_queue.empty(): print(meta_info.text_queue.get(0))
def main(): copier = Copier() keyboard = Keyboard() # scanner = Scanner() copier.copy(keyboard)
def start_work(self): if self.mode != SLAVE_MODE: self.log.critical(f'Function only works in {SLAVE_MODE} mode!') return False while True: try: self.log.info( f'Waiting for jobs on queue: {self.job_queue_name}, {self.files_processed} files have been processed so far' ) for msg in self.job_queue.receiveMsgs(self.VISIBILITY_TIMEOUT): self.log.info(f'Received a job!') extender = None data = None try: data = json.loads(msg.body) self.log.debug(data) # Make sure job is in correct format if (self.ADAPTER_CONF in data and self.BUCKET in data and self.INFO in data and self.TTL in data and self.OVERWRITE in data and self.PREFIX in data and self.DRY_RUN in data and self.VERIFY_MD5 in data): extender = VisibilityExtender( msg, self.VISIBILITY_TIMEOUT) dryrun = data[self.DRY_RUN] verify_md5 = data[self.VERIFY_MD5] adapter_config = data[self.ADAPTER_CONF] bucket_name = data[self.BUCKET] prefix = data[self.PREFIX] if self.adapter_config != adapter_config: self.adapter_config = adapter_config self._init_adapter( adapter_module=adapter_config[ self.ADAPTER_MODULE], adapter_class=adapter_config[ self.ADAPTER_CLASS], params=adapter_config[self.ADAPTER_PARAMS]) self.bucket_name = bucket_name self.prefix = prefix self.copier = Copier(bucket_name, prefix, self.adapter) if bucket_name != self.bucket_name: self.bucket_name = bucket_name self.copier.set_bucket(bucket_name) if prefix != self.prefix: self.prefix = prefix self.copier.set_prefix(prefix) result = self.copier.copy_file( data[self.INFO], data[self.OVERWRITE], dryrun or self.dryrun, verify_md5) if result[Copier.STATUS]: self.result_queue.sendMsgToQueue( result, f'{result[Copier.NAME]}_{get_time_stamp()}' ) else: self._deal_with_failed_file_sqs(data) extender.stop() extender = None self.files_processed += 1 self.log.info(f'Copying file finished!') msg.delete() else: self.log.error(f'Wrong message type!') self.log.error(data) msg.delete() except Exception as e: self.log.debug(e) self.log.critical( f'Something wrong happened while processing file! Check debug log for details.' ) if data: self._deal_with_failed_file_sqs(data) finally: if extender: extender.stop() extender = None except KeyboardInterrupt: self.log.info('Good bye!') return
def copy_all(self): """ Read file information from pre-manifest and copy them all to destination bucket :return: """ if self.mode != SOLO_MODE: self.log.critical(f'Function only works in {SOLO_MODE} mode!') return False self.copier = Copier(self.bucket_name, self.prefix, self.adapter) file_queue = deque(self._read_pre_manifest()) indexd_manifest = self.get_indexd_manifest_name(self.pre_manifest) neo4j_manifest = self.get_neo4j_manifest_name(self.pre_manifest) with open(indexd_manifest, 'w', newline='\n') as indexd_f: indexd_writer = csv.DictWriter(indexd_f, delimiter='\t', fieldnames=self.MANIFEST_FIELDS) indexd_writer.writeheader() with open(neo4j_manifest, 'w', newline='\n') as neo4j_f: fieldnames = self.DATA_FIELDS for field in self.adapter.filter_fields(self.field_names): if field not in fieldnames: fieldnames.append(field) neo4j_writer = csv.DictWriter(neo4j_f, delimiter='\t', fieldnames=fieldnames) neo4j_writer.writeheader() while file_queue: job = file_queue.popleft() job[self.TTL] -= 1 file_info = job[self.INFO] try: result = self.copier.copy_file(file_info, self.overwrite, self.dryrun, self.verify_md5) if result[Copier.STATUS]: indexd_record = {} self.populate_indexd_record(indexd_record, result) indexd_writer.writerow(indexd_record) neo4j_record = result[Copier.FIELDS] self.populate_neo4j_record(neo4j_record, result) neo4j_writer.writerow(neo4j_record) else: self._deal_with_failed_file(job, file_queue) except Exception as e: self.log.debug(e) self._deal_with_failed_file(job, file_queue) if self.files_skipped > 0: self.log.info(f'Files skipped: {self.files_skipped}') self.log.info(f'Files processed: {self.files_processed}') self.log.info( f'Files not found: {len(self.copier.files_not_found)}') self.log.info(f'Files copied: {self.copier.files_copied}') self.log.info( f'Files exist at destination: {self.copier.files_exist_at_dest}' ) self.log.info(f'Files failed: {self.files_failed}')
class FileLoader: GUID = 'GUID' MD5 = 'md5' SIZE = 'size' URL = 'url' MANIFEST_FIELDS = [GUID, MD5, SIZE, Copier.ACL, URL] NODE_TYPE = 'type' FILE_NAME = 'file_name' FILE_SIZE = "file_size" MD5_SUM = 'md5sum' FILE_STAT = 'file_status' FILE_LOC = 'file_location' FILE_FORMAT = 'file_format' DATA_FIELDS = [ NODE_TYPE, FILE_NAME, UUID, FILE_SIZE, MD5_SUM, FILE_STAT, FILE_LOC, FILE_FORMAT, Copier.ACL ] DEFAULT_NODE_TYPE = 'file' DEFAULT_STAT = 'uploaded' INDEXD_GUID_PREFIX = 'dg.4DFC/' INDEXD_MANIFEST_EXT = '.tsv' VISIBILITY_TIMEOUT = 30 ADAPTER_MODULE = 'adapter_module' ADAPTER_CLASS = 'adapter_class' ADAPTER_PARAMS = 'adapter_params' # keys in job dict ADAPTER_CONF = 'adapter_config' TTL = 'ttl' INFO = 'file_info' LINE = 'line_num' OVERWRITE = 'overwrite' DRY_RUN = 'dry_run' BUCKET = 'bucket' PREFIX = 'prefix' VERIFY_MD5 = 'verify_md5' def __init__(self, mode, adapter_module=None, adapter_class=None, adapter_params=None, domain=None, bucket=None, prefix=None, pre_manifest=None, first=1, count=-1, job_queue=None, result_queue=None, retry=3, overwrite=False, dryrun=False, verify_md5=False): """" :param bucket: string type :param pre_manifest: string type, holds path to pre-manifest :param first: first file of files to process, file 1 is in line 2 of pre-manifest :param count: number of files to process :param adapter: any object that has following methods/properties defined in adapter_attrs """ if mode not in Config.valid_modes: raise ValueError(f'Invalid loading mode: {mode}') self.mode = mode if mode != SOLO_MODE: if not job_queue: raise ValueError( f'Job queue name is required in {self.mode} mode!') self.job_queue_name = job_queue self.job_queue = Queue(job_queue) if not result_queue: raise ValueError( f'Result queue name is required in {self.mode} mode!') self.result_queue_name = result_queue self.result_queue = Queue(result_queue) if self.mode != SLAVE_MODE: if not bucket: raise ValueError('Empty destination bucket name') self.bucket_name = bucket if prefix and isinstance(prefix, str): self.prefix = removeTrailingSlash(prefix) else: raise ValueError(f'Invalid prefix: "{prefix}"') if not pre_manifest or not os.path.isfile(pre_manifest): raise ValueError( f'Pre-manifest: "{pre_manifest}" dosen\'t exist') self.pre_manifest = pre_manifest if not domain: raise ValueError(f'Empty domain!') self.domain = domain self.adapter_config = { self.ADAPTER_PARAMS: adapter_params, self.ADAPTER_CLASS: adapter_class, self.ADAPTER_MODULE: adapter_module } self._init_adapter(adapter_module, adapter_class, adapter_params) else: self.adapter = None self.adapter_config = {} self.copier = None if not first > 0 or count == 0: raise ValueError(f'Invalid first ({first}) or count ({count})') self.skip = first - 1 self.count = count if not isinstance(retry, int) and retry > 0: raise ValueError(f'Invalid retry value: {retry}') self.retry = retry if not isinstance(overwrite, bool): raise TypeError(f'Invalid overwrite value: {overwrite}') self.overwrite = overwrite if not isinstance(dryrun, bool): raise TypeError(f'Invalid dryrun value: {dryrun}') self.dryrun = dryrun self.verify_md5 = verify_md5 self.log = get_logger('FileLoader') # Statistics self.files_processed = 0 self.files_skipped = 0 self.files_failed = 0 def _init_adapter(self, adapter_module, adapter_class, params): """ Initialize different adapters base on given adapter_name :param adapter_name: :return: """ self.adapter = load_plugin(adapter_module, adapter_class, params) if not hasattr(self.adapter, 'filter_fields'): raise TypeError( f'Adapter "{adapter_class}" does not have a "filter_fields" method' ) def get_indexd_manifest_name(self, file_name): folder = os.path.dirname(file_name) base_name = os.path.basename(file_name) name, _ = os.path.splitext(base_name) new_name = '{}_indexd{}'.format(name, self.INDEXD_MANIFEST_EXT) return os.path.join(folder, new_name) @staticmethod def get_s3_location(bucket, key): return "s3://{}/{}".format(bucket, key) @staticmethod def get_neo4j_manifest_name(file_name): folder = os.path.dirname(file_name) base_name = os.path.basename(file_name) name, ext = os.path.splitext(base_name) new_name = '{}_neo4j{}'.format(name, ext) return os.path.join(folder, new_name) def populate_indexd_record(self, record, result): record[self.SIZE] = result[Copier.SIZE] record[self.MD5] = result[Copier.MD5] record[Copier.ACL] = result[Copier.ACL] record[self.URL] = self.get_s3_location(self.bucket_name, result[Copier.KEY]) record[self.GUID] = '{}{}'.format( self.INDEXD_GUID_PREFIX, get_uuid(self.domain, "file", record[self.URL])) return record def populate_neo4j_record(self, record, result): if self.NODE_TYPE not in record: record[self.NODE_TYPE] = self.DEFAULT_NODE_TYPE record[self.FILE_NAME] = result[Copier.NAME] record[self.FILE_SIZE] = result[Copier.SIZE] record[self.FILE_LOC] = self.get_s3_location(self.bucket_name, result[Copier.KEY]) file_name = result[Copier.NAME] record[self.MD5_SUM] = result[Copier.MD5] record[self.FILE_FORMAT] = ( os.path.splitext(file_name)[1]).split('.')[1].lower() record[UUID] = get_uuid(self.domain, "file", record[self.FILE_LOC]) record[self.FILE_STAT] = self.DEFAULT_STAT record[Copier.ACL] = result[Copier.ACL] return record @staticmethod def _clean_up_field_names(headers): ''' Removes leading and trailing spaces from header names :param headers: :return: ''' return [header.strip() for header in headers] @staticmethod def _clean_up_record(record): ''' Removes leading and trailing spaces from keys in org_record :param record: :return: ''' return {key.strip(): value for key, value in record.items()} def _read_pre_manifest(self): files = [] with open(self.pre_manifest) as pre_m: reader = csv.DictReader(pre_m, delimiter='\t') self.field_names = self._clean_up_field_names(reader.fieldnames) for _ in range(self.skip): next(reader) self.files_skipped += 1 line_num = self.files_skipped + 1 for info in reader: self.files_processed += 1 line_num += 1 files.append({ self.ADAPTER_CONF: self.adapter_config, self.LINE: line_num, self.TTL: self.retry, self.OVERWRITE: self.overwrite, self.DRY_RUN: self.dryrun, self.INFO: self._clean_up_record(info), self.BUCKET: self.bucket_name, self.PREFIX: self.prefix, self.VERIFY_MD5: self.verify_md5 }) if self.files_processed >= self.count > 0: break return files # Use this method in solo mode def copy_all(self): """ Read file information from pre-manifest and copy them all to destination bucket :return: """ if self.mode != SOLO_MODE: self.log.critical(f'Function only works in {SOLO_MODE} mode!') return False self.copier = Copier(self.bucket_name, self.prefix, self.adapter) file_queue = deque(self._read_pre_manifest()) indexd_manifest = self.get_indexd_manifest_name(self.pre_manifest) neo4j_manifest = self.get_neo4j_manifest_name(self.pre_manifest) with open(indexd_manifest, 'w', newline='\n') as indexd_f: indexd_writer = csv.DictWriter(indexd_f, delimiter='\t', fieldnames=self.MANIFEST_FIELDS) indexd_writer.writeheader() with open(neo4j_manifest, 'w', newline='\n') as neo4j_f: fieldnames = self.DATA_FIELDS for field in self.adapter.filter_fields(self.field_names): if field not in fieldnames: fieldnames.append(field) neo4j_writer = csv.DictWriter(neo4j_f, delimiter='\t', fieldnames=fieldnames) neo4j_writer.writeheader() while file_queue: job = file_queue.popleft() job[self.TTL] -= 1 file_info = job[self.INFO] try: result = self.copier.copy_file(file_info, self.overwrite, self.dryrun, self.verify_md5) if result[Copier.STATUS]: indexd_record = {} self.populate_indexd_record(indexd_record, result) indexd_writer.writerow(indexd_record) neo4j_record = result[Copier.FIELDS] self.populate_neo4j_record(neo4j_record, result) neo4j_writer.writerow(neo4j_record) else: self._deal_with_failed_file(job, file_queue) except Exception as e: self.log.debug(e) self._deal_with_failed_file(job, file_queue) if self.files_skipped > 0: self.log.info(f'Files skipped: {self.files_skipped}') self.log.info(f'Files processed: {self.files_processed}') self.log.info( f'Files not found: {len(self.copier.files_not_found)}') self.log.info(f'Files copied: {self.copier.files_copied}') self.log.info( f'Files exist at destination: {self.copier.files_exist_at_dest}' ) self.log.info(f'Files failed: {self.files_failed}') def _deal_with_failed_file(self, job, queue): if job[self.TTL] > 0: self.log.error( f'Line: {job[self.LINE]} - Copying file FAILED! Retry left: {job[self.TTL]}' ) queue.append(job) else: self.log.critical( f'Copying file failure exceeded maximum retry times, abort!') self.files_failed += 1 # Use this method in master mode def process_all(self): """ Read file information from pre-manifest and push jobs into job queue Listen on result queue for loading result :return: """ if self.mode != MASTER_MODE: self.log.critical(f'Function only works in {MASTER_MODE} mode!') return False try: files = self._read_pre_manifest() count = 0 for job in files: if self.dryrun: self.log.info( f'Dry run mode, jobs will be sent to queue but files won\'t be copied!' ) else: self.log.info( f'Line {job[self.LINE]}: file info sent to queue: {self.job_queue_name}' ) self.job_queue.sendMsgToQueue( job, f'{job[self.LINE]}_{get_time_stamp()}') count += 1 self.log.info(f'Files sent to queue: {count}') self.read_result(count) except Exception as e: self.log.debug(e) self.log.critical( f'Process files FAILED! Check debug log for detailed information.' ) # read result from result queue - master mode def read_result(self, num_files): if self.mode != MASTER_MODE: self.log.critical(f'Function only works in {MASTER_MODE} mode!') return False indexd_manifest = self.get_indexd_manifest_name(self.pre_manifest) neo4j_manifest = self.get_neo4j_manifest_name(self.pre_manifest) with open(indexd_manifest, 'w', newline='\n') as indexd_f: indexd_writer = csv.DictWriter(indexd_f, delimiter='\t', fieldnames=self.MANIFEST_FIELDS) indexd_writer.writeheader() with open(neo4j_manifest, 'w', newline='\n') as neo4j_f: fieldnames = self.adapter.filter_fields(self.field_names) fieldnames += self.DATA_FIELDS neo4j_writer = csv.DictWriter(neo4j_f, delimiter='\t', fieldnames=fieldnames) neo4j_writer.writeheader() count = 0 while count < num_files: self.log.info( f'Waiting for results on queue: {self.result_queue_name}, {num_files - count} files pending' ) for msg in self.result_queue.receiveMsgs( self.VISIBILITY_TIMEOUT): self.log.info(f'Received a result!') extender = None try: result = json.loads(msg.body) # Make sure result is in correct format if (result and Copier.STATUS in result and Copier.MD5 in result and Copier.NAME in result and Copier.KEY in result and Copier.FIELDS in result): extender = VisibilityExtender( msg, self.VISIBILITY_TIMEOUT) if result[Copier.STATUS]: indexd_record = {} self.populate_indexd_record( indexd_record, result) indexd_writer.writerow(indexd_record) neo4j_record = result[Copier.FIELDS] self.populate_neo4j_record( neo4j_record, result) neo4j_writer.writerow(neo4j_record) else: self.log.error( f'Copy file {result[Copier.NAME]} FAILED!' ) self.files_failed += 1 extender.stop() extender = None count += 1 self.log.info( f'{count} of {num_files} files finished!') msg.delete() else: self.log.error(f'Wrong message type!') self.log.error(result) msg.delete() except Exception as e: self.log.debug(e) self.log.critical( f'Something wrong happened while processing file! Check debug log for details.' ) finally: if extender: extender.stop() extender = None self.log.info(f'All {num_files} files finished!') # Use this method in slave mode def start_work(self): if self.mode != SLAVE_MODE: self.log.critical(f'Function only works in {SLAVE_MODE} mode!') return False while True: try: self.log.info( f'Waiting for jobs on queue: {self.job_queue_name}, {self.files_processed} files have been processed so far' ) for msg in self.job_queue.receiveMsgs(self.VISIBILITY_TIMEOUT): self.log.info(f'Received a job!') extender = None data = None try: data = json.loads(msg.body) self.log.debug(data) # Make sure job is in correct format if (self.ADAPTER_CONF in data and self.BUCKET in data and self.INFO in data and self.TTL in data and self.OVERWRITE in data and self.PREFIX in data and self.DRY_RUN in data and self.VERIFY_MD5 in data): extender = VisibilityExtender( msg, self.VISIBILITY_TIMEOUT) dryrun = data[self.DRY_RUN] verify_md5 = data[self.VERIFY_MD5] adapter_config = data[self.ADAPTER_CONF] bucket_name = data[self.BUCKET] prefix = data[self.PREFIX] if self.adapter_config != adapter_config: self.adapter_config = adapter_config self._init_adapter( adapter_module=adapter_config[ self.ADAPTER_MODULE], adapter_class=adapter_config[ self.ADAPTER_CLASS], params=adapter_config[self.ADAPTER_PARAMS]) self.bucket_name = bucket_name self.prefix = prefix self.copier = Copier(bucket_name, prefix, self.adapter) if bucket_name != self.bucket_name: self.bucket_name = bucket_name self.copier.set_bucket(bucket_name) if prefix != self.prefix: self.prefix = prefix self.copier.set_prefix(prefix) result = self.copier.copy_file( data[self.INFO], data[self.OVERWRITE], dryrun or self.dryrun, verify_md5) if result[Copier.STATUS]: self.result_queue.sendMsgToQueue( result, f'{result[Copier.NAME]}_{get_time_stamp()}' ) else: self._deal_with_failed_file_sqs(data) extender.stop() extender = None self.files_processed += 1 self.log.info(f'Copying file finished!') msg.delete() else: self.log.error(f'Wrong message type!') self.log.error(data) msg.delete() except Exception as e: self.log.debug(e) self.log.critical( f'Something wrong happened while processing file! Check debug log for details.' ) if data: self._deal_with_failed_file_sqs(data) finally: if extender: extender.stop() extender = None except KeyboardInterrupt: self.log.info('Good bye!') return def _deal_with_failed_file_sqs(self, job): self.log.info(f'Copy file FAILED, {job[self.TTL] - 1} retry left!') job[self.TTL] -= 1 self.job_queue.sendMsgToQueue(job, f'{job[self.LINE]}_{job[self.TTL]}') def run(self): if self.mode == SOLO_MODE: self.copy_all() elif self.mode == MASTER_MODE: self.process_all() elif self.mode == SLAVE_MODE: self.start_work()
class BaseManager: def __init__(self, filesystem, logger=None): self.filesystem = cache_directory(read_only(filesystem)) self.filesystem.desc('/') self.caps_list = {} self.results = [] self.logger = logger self.copier = Copier() def close(self): self.copier.stop() self.filesystem.close() def list_dir(self, top): return [i.name for i in self.filesystem.scandir(top) if i.is_dir] def download(self, src_pth, src_file, dest_pth, dest_file): ff = OSFS(dest_pth) self.copier.copy(self.filesystem, join(src_pth, src_file), ff, join('/', dest_file)) def find_nexts(self, top='/', deep=0, maxdeep=2): if deep == 0: self.results = [] # print(top) if deep > maxdeep: return # if self.logger: # self.logger.emit(top, INFORMATION) dirs, nondirs = [], [] for name in self.filesystem.scandir(top): if name.is_dir: dirs.append(name) elif splitext(name.name)[1].lower() in video_formats: nondirs.append(name) # print(dirs,nondirs) for fil in nondirs: pp = rename(fil.name) if pp.error: pp = parse(j.name) t1 = '' t2 = 0 try: if pp.is_video: if pp.episode: t1 = transform(pp.title) fill = t1 if pp.season: fill += ' - ' + str(pp.season) + \ 'x' + str(pp.episode) else: fill += ' - ' + str(pp.episode) fill += pp.ext else: continue t2 = pp.episode else: continue except KeyError: if self.logger: self.logger.emit("Error procesando: " + fil.name, WARNING) continue bedd = 100 gap = 2 near = '' for j in self.caps_list.keys(): edd = editDistance(t1, j, True) if edd <= gap and edd < bedd: near = j bedd = edd if edd == 0: break if near != '': if isinstance(t2, str): if 'x' in t2: t2 = t2.split('x')[1] if int(t2) > self.caps_list[near]: best = (near, fil.name, top, fill) self.results.append(best) if self.logger: self.logger.emit('Encontrado: ' + str(best), INFORMATION) for name in sorted(dirs, key=skey): path = join(top, name.name) if not self.filesystem.islink(path): try: self.find_nexts(path, deep + 1, maxdeep) except (PermissionError, fs.errors.DirectoryExpected) as e: # print(e) self.logger.emit(str(e), ERROR) def last(self, base): with read_only(OSFS(base)) as ff: proces = [] folds = [] for i in ff.scandir('/'): if i.is_dir: folds.append(i) for i in folds: path = join('/', i.name) try: for j in ff.scandir(path): if j.is_file and splitext( j.name)[1].lower() in video_formats: proces.append((j, i)) except (PermissionError, DirectoryExpected) as e: self.logger.emit("Acceso denegado a" + join(base, i.name), ERROR) folds = {} for filee, fold in proces: fold = fold.name filee = filee.name try: pp = parse2(filee) if pp.error: pp = parse(filee) except Exception as e: self.logger.emit( "Error procesando: " + join(base, fold, filee), WARNING) self.logger.emit(str(e), ERROR) continue t1 = transform(pp.title) if not pp.episode: continue t2 = pp.episode if t1 in folds: if folds[t1] < int(t2): folds[t1] = int(t2) else: tt = best_ed(t1, folds.keys()) if tt in folds: if folds[tt] < int(t2): folds[tt] = int(t2) folds[tt] = int(t2) self.caps_list = folds def find_nexts2(self, top='/', deep=0, maxdeep=2): if deep == 0: self.results = [] # print(top) if deep > maxdeep: return # if self.logger: # self.logger.emit(top, INFORMATION) dirs, nondirs = [], [] for name in self.filesystem.scandir(top): if name.is_dir: dirs.append(name) elif splitext(name.name)[1].lower() in video_formats: nondirs.append(name) # print(dirs,nondirs) for fil in nondirs: pp = parse2(fil.name) if pp.error: pp = parse(fil.name) t1 = '' t2 = 0 try: if pp.is_video: if pp.episode: t1 = transform(pp.title) fill = t1 if pp.season: fill += ' - ' + str(pp.season) + \ 'x' + str(pp.episode) else: fill += ' - ' + str(pp.episode) fill += pp.ext else: continue t2 = pp.episode else: continue except KeyError: if self.logger: self.logger.emit("Error procesando: " + fil.name, WARNING) continue bedd = 100 gap = 2 near = '' for j in self.caps_list.keys(): edd = editDistance(t1, j, True) if edd <= gap and edd < bedd: near = j bedd = edd if edd == 0: break if near != '': if isinstance(t2, str): if 'x' in t2: t2 = t2.split('x')[1] if not (int(t2) in self.caps_list[near]): best = (near, fil.name, top, fill) self.results.append(best) if self.logger: self.logger.emit('Encontrado: ' + str(best), INFORMATION) for name in sorted(dirs, key=skey): path = join(top, name.name) if not self.filesystem.islink(path): self.find_nexts2(path, deep + 1, maxdeep) def last2(self, base): with read_only(OSFS(base)) as ff: proces = [] folds = [] for i in ff.scandir('/'): if i.is_dir: folds.append(i) for i in folds: path = join('/', i.name) try: for j in ff.scandir(path): if j.is_file and splitext( j.name)[1].lower() in video_formats: proces.append((j, i)) except (PermissionError, DirectoryExpected) as e: self.logger.emit("Acceso denegado a" + join(base, i.name), ERROR) folds = {} for filee, fold in proces: fold = fold.name filee = filee.name try: pp = parse2(filee) if pp.error: pp = parse(filee) except Exception as e: self.logger.emit( "Error procesando: " + join(base, fold, filee), WARNING) self.logger.emit(str(e), ERROR) continue t1 = transform(pp.title) if not pp.episode: continue t2 = pp.episode if t1 in folds: folds[t1].add(int(t2)) else: tt = best_ed(t1, folds.keys()) if tt in folds: folds[t1].add(int(t2)) else: folds[t1] = set() folds[t1].add(int(t2)) self.caps_list = folds def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.filesystem.close() self.copier.stop() return False
def test_copier_function(self): dut = Copier() dut.copier_test() #bs test method assert os.path.isfile('total_test_bs_file.py') os.remove('total_test_bs_file.py')
def test_copier_filenotfounderror(self): dut = Copier() with pytest.raises(FileNotFoundError): dut.copy_prod_machine_files_to_verify( ) #this is failing to raise an error, why??
def main(): copier = Copier() copier.copy()
def main(): my_runner = Runner(Getter(), Writer(), Deleter(), Copier()) my_runner.run_it()