def recursive_include(self, dir, pattern): """ Include all files anywhere in 'dir/' that match the pattern. """ full_pattern = os.path.join(dir, '**', pattern) found = [f for f in glob(full_pattern, recursive=True) if not os.path.isdir(f)] self.extend(found) return bool(found)
def graft(self, dir): """Include all files from 'dir/'.""" found = [ item for match_dir in glob(dir) for item in distutils.filelist.findall(match_dir) ] self.extend(found) return bool(found)
def create_dir_with_structure_files(queryset): os.mkdir(os.path.join(settings.MEDIA_ROOT, 'structures')) for name in queryset.values_list('pdb_id', flat=True): os.mkdir(os.path.join(settings.MEDIA_ROOT, 'structures', name)) os.chdir(os.path.join(settings.MEDIA_ROOT, 'structures', name)) files = glob.glob(os.path.join(settings.MEDIA_ROOT, '**', f'{name}*.*'), recursive=True) copy_files_to_current_dir(files)
def recursive_include(self, dir, pattern): """ Include all files anywhere in 'dir/' that match the pattern. """ full_pattern = os.path.join(dir, '**', pattern) found = [ f for f in glob(full_pattern, recursive=True) if not os.path.isdir(f) ] self.extend(found) return bool(found)
def build(self): Window.size = (480, 800) Window.fullscreen = False layout = BoxLayout(orientation='vertical') carousel = Carousel(direction='right') layout.add_widget(carousel) b = BackButton(text='Back ', size_hint=(None, None), size=(100, 50)) layout.add_widget(b) # get any files into images directory curdir = dirname(__file__) for filename in glob(join(curdir, 'DCIM', '*')): image = AsyncImage(source=filename, allow_stretch=True) carousel.add_widget(image) return layout
def refreshSavedLayouts(self): self.layout_list_menu.clear() layout_paths = glob.glob( os.path.join(self.layout_directory, "*.settings")) if not layout_paths: self.layout_list_menu.addAction("No available layout").setDisabled( True) else: for p in layout_paths: settings = QtCore.QSettings(p, QtCore.QSettings.IniFormat) action = QtWidgets.QAction(settings.value("layoutName"), self) action.triggered.connect( functools.partial(self.loadLayout, settings)) self.layout_list_menu.addAction(action)
def collect_and_sort_migrations(directory: str) -> List[Migration]: if not is_directory(directory): msg = "%s is not a directory." % directory raise Exception(msg) wildcard = os.path.join(directory, '*', 'migrate.py') migration_files = glob.glob(wildcard) migrations = [] for path in migration_files: elements = path.split('/') dir = elements[-2] parts = dir.split('_') migrations.append( Migration(path=path, type=parts[1], stamp=int(parts[0]))) migrations.sort(key=lambda m: m['stamp']) return migrations
def sqlitefuncts_db_not_initialized(): global CONNECT_PARAMS table_names = { "data_table": DATA_TABLE, "topics_table": TOPICS_TABLE, "meta_table": META_TABLE, "agg_topics_table": AGG_TOPICS_TABLE, "agg_meta_table": AGG_META_TABLE, } client = SqlLiteFuncts(CONNECT_PARAMS, table_names) yield client # Teardown if os.path.isdir("./data"): files = glob.glob("./data/*", recursive=True) for f in files: os.remove(f) os.rmdir("./data/")
def load(self, head=None, strip=True): file_list = glob.glob(DBPEDIA_DATA_DIR + '/*.gz') for file_path in file_list: with gzip.open(file_path, 'rt', encoding='cp1250', errors='ignore') as f: for line in f: fact = line.rstrip('\n')[:-2].split(' ') e1, r = fact[:2] e2 = ' '.join(fact[2:]) if not e1 or not e2: continue triple = (e1, r, e2) self.add_triple(triple) if self.number_of_triples() == head: return
def run_smart_parser(filepath, args): """start SmartParser for one file""" start_time = datetime.now() sourcefile = filepath[:filepath.rfind('.')] json_list = glob.glob("%s*.json" % glob.escape(sourcefile)) if json_list: if not args.usecache: # logger.info("Delete existed JSON file(s).") for jf in json_list: os.remove(jf) else: # logger.info("Skipping existed JSON file %s.json" % sourcefile) return 1, "" smart_parser_options = "-adapter prod -converted-storage-url http://disclosures.ru:8091" log = filepath + ".log" if os.path.exists(log): os.remove(log) cmd = "{} {} \"{}\"".format(SMART_PARSER, smart_parser_options, filepath.replace("`", "\\`")) return (datetime.now() - start_time).total_seconds(), os.popen(cmd).read()
def include(self, pattern): """Include files that match 'pattern'.""" found = [f for f in glob(pattern) if not os.path.isdir(f)] self.extend(found) return bool(found)
from setuptools import setup, find_packages, glob package_name = 'imu' setup( name=package_name, version='0.0.0', packages=find_packages(), data_files=[ ('share/ament_index/resource_index/packages', ['resource/' + package_name]), ('share/' + package_name, ['package.xml']), ('share/' + package_name + '/param', glob.glob('param/*')), ], install_requires=['setuptools'], zip_safe=True, maintainer='grisw', maintainer_email='*****@*****.**', description='TODO: Package description', license='TODO: License declaration', tests_require=['pytest'], entry_points={ 'console_scripts': [ 'node = imu.node:main', ], }, )
def __init__(self, useGPU=False, useDepth=False): self.useGPU = useGPU self.useDepth = useDepth self.eps_start = 0.9 self.eps_end = 0.05 self.eps_decay = 30000 self.gamma = 0.8 self.learning_rate = 0.001 self.batch_size = 512 self.max_episodes = 10000 self.save_interval = 10 self.episode = -1 self.steps_done = 0 if self.useGPU: self.device = torch.device( 'cuda:0' if torch.cuda.is_available() else 'cpu') else: self.device = torch.device('cpu') self.dqn = DQN() self.env = DroneEnv(useGPU, useDepth) self.memory = deque(maxlen=10000) self.optimizer = optim.Adam(self.dqn.parameters(), self.learning_rate) print('Using device:', self.device) if self.device.type == 'cuda': print(torch.cuda.get_device_name(0)) # LOGGING cwd = os.getcwd() self.save_dir = os.path.join(cwd, "saved models") if not os.path.exists(self.save_dir): os.mkdir("saved models") if self.useGPU: self.dqn = self.dqn.to(self.device) # to use GPU # model backup files = glob.glob(self.save_dir + '\\*.pt') if len(files) > 0: files.sort(key=os.path.getmtime) file = files[-1] checkpoint = torch.load(file) self.dqn.load_state_dict(checkpoint['state_dict']) self.optimizer.load_state_dict(checkpoint['optimizer']) self.episode = checkpoint['episode'] self.steps_done = checkpoint['steps_done'] print("Saved parameters loaded" "\nModel: ", file, "\nSteps done: ", self.steps_done, "\nEpisode: ", self.episode) else: if os.path.exists("log.txt"): open('log.txt', 'w').close() if os.path.exists("last_episode.txt"): open('last_episode.txt', 'w').close() if os.path.exists("last_episode.txt"): open('saved_model_params.txt', 'w').close() obs = self.env.reset() tensor = self.transformToTensor(obs) writer.add_graph(self.dqn, tensor)
def decrypt_stored_files(newer_than, search_range, append_to_thread=False, remain_decrypted_file=True): unzipper = MailUnzipper() label_name2id_table = client.get_label_name2id_table() my_address = client.get_my_address() query_builder = GmailSearchQueryBuilder(newer_than, exclude_from=my_address) # processing processing_zip_mails = client.search_mails( query_builder.build_processing_mails_query(), { 'format': 'metadata', 'metadataHeaders': ['From', 'Subject', 'subject'] }) logger.info("hit {} processing message".format(len(processing_zip_mails))) # processingメールがあれば、同一人物からのメールを検索し、パスワード候補を抽出、復号化を試みる。対象ファイルはまずtmp以下を探し、なければダウンロードしてくる # 復号に成功すればdoneフラグを立てて解凍済みファイルを自分宛てに再送 # 失敗すればdoingのまま次の試行を待つ # TODO 関係ないメールが間に入った場合、何度も解析することになってしまう。フラグを立てるべき。ただし複数ファイルに対して暗号化されることもあるのでn:nのラベル管理が必要 # WARNING 時間軸は考慮してない。いったんnewer_than以降のメールはすべて解析する(パスワードを先に送るケースがありえる?) address_extractor = re.compile(r'.*<(.+)@(.+)>.*') for mail in processing_zip_mails: # c.f. http://srgia.com/docs/rfc2822j.html # Fromの他にSenderやReply-toも考えられるが、オプション扱いなのでまずはFromで運用してみる from_kvs = list( filter(lambda x: x['name'] == 'From', mail['payload']['headers'])) if len(from_kvs) == 0: logger.error("No From header message:", mail) continue from_value = from_kvs[0]['value'] match = address_extractor.match(from_value) if match is None: logger.error("Unknown address format:", from_value) continue if search_range == 'himself': from_address = match.group(1) + "@" + match.group(2) else: # domain from_address = "@" + match.group(2) candidate_mails = client.search_mails( query_builder.build_password_candidate_mails_query(from_address)) # TODO 対象ファイルよりあとに送られたことを前提にできるようにする? password_candidates = [] for message in candidate_mails: content = client.extract_body_text(message) password_candidates += unzipper.search_password_candidates(content) password_candidates = list(set(password_candidates)) # make unique if len(password_candidates) == 0: # password mail is not comming yet. continue message_id = mail['id'] # attachment_idは毎回変わってしまう・・ # message_with_detail = client.get_message(message_id) # zip_attachment_ids = client.extract_zip_attachment_ids(message_with_detail) # del mail encrypted_file_paths = glob.glob( os.path.join(TMP_CACHE_DIR, message_id, "*.zip")) # TODO 別サーバーでやってると保存されていない。そもそもlabelをサーバー側で管理するのを辞めるべき # 基本的には1つ想定 for enc_fpath in encrypted_file_paths: if enc_fpath.endswith(NOPASS_SUFFIX + ".zip"): continue with zipfile.ZipFile(enc_fpath) as zf: logger.debug("try: {}".format(password_candidates)) matched_password = unzipper.try_passwords( zf, password_candidates) if matched_password is None: continue logger.info("find correct password {} for {}".format( matched_password, enc_fpath)) dec_dir_path = enc_fpath.replace(".zip", NOPASS_SUFFIX) unzipper.extract_all(zf, dec_dir_path, password=matched_password.encode('ascii')) shutil.make_archive(dec_dir_path, 'zip', dec_dir_path) if append_to_thread: rezipped_path = dec_dir_path + ".zip" with open(rezipped_path, 'rb') as f: decrypted_zip_binary = f.read() received_message_subject = client.extract_message_subject(mail) fname = os.path.basename(rezipped_path) reply = create_message_with_zip( my_address, my_address, 'Re: ' + received_message_subject, 'decrypted zip message', decrypted_zip_binary, fname) reply['threadId'] = mail['threadId'] sent = client.send_message(reply) os.remove(rezipped_path) # TODO 複数添付のときに1つだけ解凍に成功した場合が難しい・・。どうするか考える client.add_label(mail['id'], label_name2id_table[DONE_LABEL]) client.remove_label(mail['id'], label_name2id_table[PROCESSING_LABEL]) # remove cached files os.remove(enc_fpath) if not remain_decrypted_file: shutil.rmtree(dec_dir_path)
def fiFindByWildcard(wildcard): return glob.glob(os.path.expanduser(wildcard), recursive=True)
def scan(self): @contextmanager def open_file(_file): if sys.version_info[0] < 3: with open(_file) as _doc: _source = _doc.read() yield (_source, _doc) else: with open(_file, encoding="utf-8") as _doc: _source = _doc.read() yield (_source, _doc) def parse_file(_file): with open_file(_file) as __source: suite_imported_as_alias = re.findall( CliRunner.__REGEX_ALIAS_IMPORT, __source[0]) if suite_imported_as_alias: suite_alias = suite_imported_as_alias[-1].split( "Suite")[-1].split("as")[-1].split(",")[0].strip() CliRunner.start_in_a_thread( target=self.__find_and_register_suite, args=(suite_alias, __source[0], __source[1].name)) return True suite_imported = re.findall(CliRunner.__REGEX_NO_ALIAS_IMPORT, __source[0]) if suite_imported: CliRunner.start_in_a_thread( target=self.__find_and_register_suite, args=("Suite", __source[0], __source[1].name)) return True try: print("\n[{status}] Scanning: {location} ...".format( location=CliUtils.format_color_string(value=",".join( self.sources), color="green"), status=CliUtils.format_color_string(value="INFO", color="blue"))) start = time.time() for source in self.sources: if source.endswith(".py"): parse_file(source) else: for dirName, subdirList, fileList in os.walk(source, topdown=True): if self.__skip(source, dirName): continue for file_path in glob( os.path.join(os.path.dirname(dirName + "\\"), "*.py")): if parse_file(file_path) is True: continue for thread in CliRunner.__SCANNER_THREADS: thread.join() print( "[{status}] Scan finished in: {time} seconds. Found: {suites} suite(s)." .format(status=CliUtils.format_color_string(value="INFO", color="blue"), time="{0:.2f}".format(time.time() - start), suites=CliUtils.format_bold_string(len(self.suites)))) except KeyboardInterrupt: print("(Ctrl+C) Exiting!") exit(12) except BadCliParameters as err: print("[{status}] {error}.".format( status=CliUtils.format_color_string(value="ERROR", color="red"), error=err)) exit(120) except: print("[{status}] Unexpected error during scan for test suites.". format(status=CliUtils.format_color_string(value="ERROR", color="red"))) CliUtils.print_color_traceback() exit(120)
def __init__(self, directory_path): super().__init__() self.bandList = [] for path in glob.glob(directory_path + "/*.tif"): self.bandList.append(gdal.Open(path))
def __init__(self, useDepth=False): self.useDepth = useDepth self.eps_start = 0.9 self.eps_end = 0.05 self.eps_decay = 30000 self.gamma = 0.8 self.learning_rate = 0.001 self.batch_size = 512 self.memory = Memory(10000) self.max_episodes = 10000 self.save_interval = 2 self.test_interval = 10 self.network_update_interval = 10 self.episode = -1 self.steps_done = 0 self.max_steps = 34 self.policy = DQN() self.target = DQN() self.test_network = DQN() self.target.eval() self.test_network.eval() self.updateNetworks() self.env = DroneEnv(useDepth) self.optimizer = optim.Adam(self.policy.parameters(), self.learning_rate) if torch.cuda.is_available(): print('Using device:', device) print(torch.cuda.get_device_name(0)) else: print("Using CPU") # LOGGING cwd = os.getcwd() self.save_dir = os.path.join(cwd, "saved models") if not os.path.exists(self.save_dir): os.mkdir("saved models") if not os.path.exists(os.path.join(cwd, "videos")): os.mkdir("videos") if torch.cuda.is_available(): self.policy = self.policy.to(device) # to use GPU self.target = self.target.to(device) # to use GPU self.test_network = self.test_network.to(device) # to use GPU # model backup files = glob.glob(self.save_dir + '\\*.pt') if len(files) > 0: files.sort(key=os.path.getmtime) file = files[-1] checkpoint = torch.load(file) self.policy.load_state_dict(checkpoint['state_dict']) self.episode = checkpoint['episode'] self.steps_done = checkpoint['steps_done'] self.updateNetworks() print("Saved parameters loaded" "\nModel: ", file, "\nSteps done: ", self.steps_done, "\nEpisode: ", self.episode) else: if os.path.exists("log.txt"): open('log.txt', 'w').close() if os.path.exists("last_episode.txt"): open('last_episode.txt', 'w').close() if os.path.exists("last_episode.txt"): open('saved_model_params.txt', 'w').close() self.optimizer = optim.Adam(self.policy.parameters(), self.learning_rate) obs, _ = self.env.reset() tensor = self.transformToTensor(obs) writer.add_graph(self.policy, tensor)
def post_results(sourcefile, job, time_delta=None, skip_upload=False): df_id, archive_file = job.get('document_file', None), job.get('archive_file', None) filename = sourcefile[:sourcefile.rfind('.')] json_list = glob.glob("%s.json" % glob.escape(sourcefile)) if len(json_list) == 1: # Properly constructed final JSON found data = json.load(open(json_list[0], encoding='utf8')) else: json_list = glob.glob("%s*.json" % glob.escape(sourcefile)) if not json_list: # Build empty JSON to post report in API and skip parsing attemp in a future # print("Build empty JSON for %s" % sourcefile) data = {'document': {'documentfile_id': df_id}, 'persons': []} if archive_file: data['document']['archive_file'] = archive_file else: # Join separated JSON files (of XLSX lists) data = {'persons': [], 'document': {}} for json_file in json_list: file_data = json.load(open(json_file, encoding='utf8')) file_year = file_data.get('document', {}).get('year') expected_year = job.get('income_year', None) if file_year is not None and expected_year is not None and file_year != expected_year: logger.warning( "%s: Skip wrong declaration year %i (expected %i)" % (json_file, file_year, expected_year)) continue data['persons'] += file_data['persons'] # if data['document']: # if data['document'].get('sheet_title') != file_data['document'].get('sheet_title'): # logger.warning( # "Document sheet title changed in one XLSX!") data['document'] = file_data['document'] if 'sheet_number' in data['document']: del data['document']['sheet_number'] try: data['document']['file_size'] = os.path.getsize(sourcefile) except: data['document']['file_size'] = 0 try: data['document']['parser_log'] = open( sourcefile + ".log", 'rb').read().decode('utf-8', errors='ignore') except FileNotFoundError: data['document']['parser_log'] = "FileNotFoundError: " + \ sourcefile + ".log" data['document']['documentfile_id'] = df_id if archive_file: data['document']['archive_file'] = archive_file # if time_delta == PARSER_TIMEOUT: # data['document']['parser_log'] += "\nTimeout %i exceeded for smart_parser.exe" % PARSER_TIMEOUT if time_delta: data['document']['parser_time'] = time_delta body = json.dumps(data, ensure_ascii=False, indent=4).encode('utf-8', errors='ignore') with open(filename + ".json", "wb") as fp: fp.write(body) parsed = len(data['persons']) > 0 result = job.copy() result['parser_log'] = data['document']['parser_log'] result['sourcefile'] = sourcefile if not parsed: result['new_status'] = 'error' else: result['new_status'] = 'ok' if df_id: if job['status'] == 'ok' and not parsed: result['new_status'] = 'degrade' else: # logger.info("POSTing results (id=%i): %i persons, %i files, file_size %i" % ( # df_id, len(data['persons']), len(json_list), data['document']['file_size'])) response = None if not skip_upload: while response is None: try: response = client.post(declarator_domain + '/api/jsonfile/validate/', data=body) except requests.exceptions.ConnectionError: logger.error( "requests.exceptions.ConnectionError, retrying...") if response.status_code != requests.codes.ok: logger.error(response) logger.error(response.text) return result
#Initiate webcam webCam = cv2.VideoCapture(0) # criteria for termination criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001) # prepering obk=ject point like (0, 0, 0), (1,0,0) and etc objp = np.zeros((6 * 7, 3), np.float32) objp[:, :2] = np.mgrid[0:7, 0:6].T.reshape(-1, 2) # Arrays to store image and object points imgP = [] objP = [] # Load calibration images images = glob.glob('CalibImages/*.jpg') # loop to read each calibration image in the folder for fname in images: img = cv2.imread(fname) gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) # Find the corners of the chess board ret, corners = cv2.findChessboardCorners(gray, (7, 6), None) # if the corners are found extract object and image points if ret == True: objP.append(objp) corners1 = cv2.cornerSubPix(gray, corners, (11, 11), (-1, -1),
def test_glob(monkeypatch, tmpdir, tree, pattern, matches): monkeypatch.chdir(tmpdir) build_files({name: '' for name in tree.split()}) assert list(sorted(glob(pattern))) == list(sorted(matches))
def get_all_files(dirname): return dirname, glob.glob(dirname + '/**/*.*', True)
def test_glob(monkeypatch, tmpdir, tree, pattern, matches): monkeypatch.chdir(tmpdir) path.build({name: '' for name in tree.split()}) assert list(sorted(glob(pattern))) == list(sorted(matches))
from setuptools import setup, glob package_name = 'robot' setup( name=package_name, version='0.0.0', packages=[package_name], data_files=[ ('share/ament_index/resource_index/packages', ['resource/' + package_name]), ('share/' + package_name, ['package.xml']), ('share/' + package_name + '/launch', glob.glob('launch/*')), ], install_requires=['setuptools'], zip_safe=True, maintainer='grisw', maintainer_email='*****@*****.**', description='TODO: Package description', license='TODO: License declaration', tests_require=['pytest'], entry_points={ 'console_scripts': [], }, )