def recursive_include(self, dir, pattern):
"""
Include all files anywhere in 'dir/' that match the pattern.
"""
full_pattern = os.path.join(dir, '**', pattern)
found = [f for f in glob(full_pattern, recursive=True)
if not os.path.isdir(f)]
self.extend(found)
return bool(found)
def graft(self, dir):
"""Include all files from 'dir/'."""
found = [
item
for match_dir in glob(dir)
for item in distutils.filelist.findall(match_dir)
]
self.extend(found)
return bool(found)
def create_dir_with_structure_files(queryset):
os.mkdir(os.path.join(settings.MEDIA_ROOT, 'structures'))
for name in queryset.values_list('pdb_id', flat=True):
os.mkdir(os.path.join(settings.MEDIA_ROOT, 'structures', name))
os.chdir(os.path.join(settings.MEDIA_ROOT, 'structures', name))
files = glob.glob(os.path.join(settings.MEDIA_ROOT, '**',
f'{name}*.*'),
recursive=True)
copy_files_to_current_dir(files)
def recursive_include(self, dir, pattern):
"""
Include all files anywhere in 'dir/' that match the pattern.
"""
full_pattern = os.path.join(dir, '**', pattern)
found = [
f for f in glob(full_pattern, recursive=True)
if not os.path.isdir(f)
]
self.extend(found)
return bool(found)
def build(self):
Window.size = (480, 800)
Window.fullscreen = False
layout = BoxLayout(orientation='vertical')
carousel = Carousel(direction='right')
layout.add_widget(carousel)
b = BackButton(text='Back ', size_hint=(None, None), size=(100, 50))
layout.add_widget(b)
# get any files into images directory
curdir = dirname(__file__)
for filename in glob(join(curdir, 'DCIM', '*')):
image = AsyncImage(source=filename, allow_stretch=True)
carousel.add_widget(image)
return layout
def refreshSavedLayouts(self):
self.layout_list_menu.clear()
layout_paths = glob.glob(
os.path.join(self.layout_directory, "*.settings"))
if not layout_paths:
self.layout_list_menu.addAction("No available layout").setDisabled(
True)
else:
for p in layout_paths:
settings = QtCore.QSettings(p, QtCore.QSettings.IniFormat)
action = QtWidgets.QAction(settings.value("layoutName"), self)
action.triggered.connect(
functools.partial(self.loadLayout, settings))
self.layout_list_menu.addAction(action)
def collect_and_sort_migrations(directory: str) -> List[Migration]:
if not is_directory(directory):
msg = "%s is not a directory." % directory
raise Exception(msg)
wildcard = os.path.join(directory, '*', 'migrate.py')
migration_files = glob.glob(wildcard)
migrations = []
for path in migration_files:
elements = path.split('/')
dir = elements[-2]
parts = dir.split('_')
migrations.append(
Migration(path=path, type=parts[1], stamp=int(parts[0])))
migrations.sort(key=lambda m: m['stamp'])
return migrations
def sqlitefuncts_db_not_initialized():
global CONNECT_PARAMS
table_names = {
"data_table": DATA_TABLE,
"topics_table": TOPICS_TABLE,
"meta_table": META_TABLE,
"agg_topics_table": AGG_TOPICS_TABLE,
"agg_meta_table": AGG_META_TABLE,
}
client = SqlLiteFuncts(CONNECT_PARAMS, table_names)
yield client
# Teardown
if os.path.isdir("./data"):
files = glob.glob("./data/*", recursive=True)
for f in files:
os.remove(f)
os.rmdir("./data/")
def load(self, head=None, strip=True):
file_list = glob.glob(DBPEDIA_DATA_DIR + '/*.gz')
for file_path in file_list:
with gzip.open(file_path, 'rt', encoding='cp1250', errors='ignore') as f:
for line in f:
fact = line.rstrip('\n')[:-2].split(' ')
e1, r = fact[:2]
e2 = ' '.join(fact[2:])
if not e1 or not e2:
continue
triple = (e1, r, e2)
self.add_triple(triple)
if self.number_of_triples() == head:
return
def run_smart_parser(filepath, args):
"""start SmartParser for one file"""
start_time = datetime.now()
sourcefile = filepath[:filepath.rfind('.')]
json_list = glob.glob("%s*.json" % glob.escape(sourcefile))
if json_list:
if not args.usecache:
# logger.info("Delete existed JSON file(s).")
for jf in json_list:
os.remove(jf)
else:
# logger.info("Skipping existed JSON file %s.json" % sourcefile)
return 1, ""
smart_parser_options = "-adapter prod -converted-storage-url http://disclosures.ru:8091"
log = filepath + ".log"
if os.path.exists(log):
os.remove(log)
cmd = "{} {} \"{}\"".format(SMART_PARSER, smart_parser_options,
filepath.replace("`", "\\`"))
return (datetime.now() - start_time).total_seconds(), os.popen(cmd).read()
def include(self, pattern):
"""Include files that match 'pattern'."""
found = [f for f in glob(pattern) if not os.path.isdir(f)]
self.extend(found)
return bool(found)
from setuptools import setup, find_packages, glob
package_name = 'imu'
setup(
name=package_name,
version='0.0.0',
packages=find_packages(),
data_files=[
('share/ament_index/resource_index/packages',
['resource/' + package_name]),
('share/' + package_name, ['package.xml']),
('share/' + package_name + '/param', glob.glob('param/*')),
],
install_requires=['setuptools'],
zip_safe=True,
maintainer='grisw',
maintainer_email='*****@*****.**',
description='TODO: Package description',
license='TODO: License declaration',
tests_require=['pytest'],
entry_points={
'console_scripts': [
'node = imu.node:main',
],
},
)
def __init__(self, useGPU=False, useDepth=False):
self.useGPU = useGPU
self.useDepth = useDepth
self.eps_start = 0.9
self.eps_end = 0.05
self.eps_decay = 30000
self.gamma = 0.8
self.learning_rate = 0.001
self.batch_size = 512
self.max_episodes = 10000
self.save_interval = 10
self.episode = -1
self.steps_done = 0
if self.useGPU:
self.device = torch.device(
'cuda:0' if torch.cuda.is_available() else 'cpu')
else:
self.device = torch.device('cpu')
self.dqn = DQN()
self.env = DroneEnv(useGPU, useDepth)
self.memory = deque(maxlen=10000)
self.optimizer = optim.Adam(self.dqn.parameters(), self.learning_rate)
print('Using device:', self.device)
if self.device.type == 'cuda':
print(torch.cuda.get_device_name(0))
# LOGGING
cwd = os.getcwd()
self.save_dir = os.path.join(cwd, "saved models")
if not os.path.exists(self.save_dir):
os.mkdir("saved models")
if self.useGPU:
self.dqn = self.dqn.to(self.device) # to use GPU
# model backup
files = glob.glob(self.save_dir + '\\*.pt')
if len(files) > 0:
files.sort(key=os.path.getmtime)
file = files[-1]
checkpoint = torch.load(file)
self.dqn.load_state_dict(checkpoint['state_dict'])
self.optimizer.load_state_dict(checkpoint['optimizer'])
self.episode = checkpoint['episode']
self.steps_done = checkpoint['steps_done']
print("Saved parameters loaded"
"\nModel: ", file, "\nSteps done: ", self.steps_done,
"\nEpisode: ", self.episode)
else:
if os.path.exists("log.txt"):
open('log.txt', 'w').close()
if os.path.exists("last_episode.txt"):
open('last_episode.txt', 'w').close()
if os.path.exists("last_episode.txt"):
open('saved_model_params.txt', 'w').close()
obs = self.env.reset()
tensor = self.transformToTensor(obs)
writer.add_graph(self.dqn, tensor)
def decrypt_stored_files(newer_than,
search_range,
append_to_thread=False,
remain_decrypted_file=True):
unzipper = MailUnzipper()
label_name2id_table = client.get_label_name2id_table()
my_address = client.get_my_address()
query_builder = GmailSearchQueryBuilder(newer_than,
exclude_from=my_address)
# processing
processing_zip_mails = client.search_mails(
query_builder.build_processing_mails_query(), {
'format': 'metadata',
'metadataHeaders': ['From', 'Subject', 'subject']
})
logger.info("hit {} processing message".format(len(processing_zip_mails)))
# processingメールがあれば、同一人物からのメールを検索し、パスワード候補を抽出、復号化を試みる。対象ファイルはまずtmp以下を探し、なければダウンロードしてくる
# 復号に成功すればdoneフラグを立てて解凍済みファイルを自分宛てに再送
# 失敗すればdoingのまま次の試行を待つ
# TODO 関係ないメールが間に入った場合、何度も解析することになってしまう。フラグを立てるべき。ただし複数ファイルに対して暗号化されることもあるのでn:nのラベル管理が必要
# WARNING 時間軸は考慮してない。いったんnewer_than以降のメールはすべて解析する（パスワードを先に送るケースがありえる？）
address_extractor = re.compile(r'.*<(.+)@(.+)>.*')
for mail in processing_zip_mails:
# c.f. http://srgia.com/docs/rfc2822j.html
# Fromの他にSenderやReply-toも考えられるが、オプション扱いなのでまずはFromで運用してみる
from_kvs = list(
filter(lambda x: x['name'] == 'From', mail['payload']['headers']))
if len(from_kvs) == 0:
logger.error("No From header message:", mail)
continue
from_value = from_kvs[0]['value']
match = address_extractor.match(from_value)
if match is None:
logger.error("Unknown address format:", from_value)
continue
if search_range == 'himself':
from_address = match.group(1) + "@" + match.group(2)
else: # domain
from_address = "@" + match.group(2)
candidate_mails = client.search_mails(
query_builder.build_password_candidate_mails_query(from_address))
# TODO 対象ファイルよりあとに送られたことを前提にできるようにする？
password_candidates = []
for message in candidate_mails:
content = client.extract_body_text(message)
password_candidates += unzipper.search_password_candidates(content)
password_candidates = list(set(password_candidates)) # make unique
if len(password_candidates) == 0:
# password mail is not comming yet.
continue
message_id = mail['id']
# attachment_idは毎回変わってしまう・・
# message_with_detail = client.get_message(message_id)
# zip_attachment_ids = client.extract_zip_attachment_ids(message_with_detail)
# del mail
encrypted_file_paths = glob.glob(
os.path.join(TMP_CACHE_DIR, message_id, "*.zip"))
# TODO 別サーバーでやってると保存されていない。そもそもlabelをサーバー側で管理するのを辞めるべき
# 基本的には1つ想定
for enc_fpath in encrypted_file_paths:
if enc_fpath.endswith(NOPASS_SUFFIX + ".zip"): continue
with zipfile.ZipFile(enc_fpath) as zf:
logger.debug("try: {}".format(password_candidates))
matched_password = unzipper.try_passwords(
zf, password_candidates)
if matched_password is None:
continue
logger.info("find correct password {} for {}".format(
matched_password, enc_fpath))
dec_dir_path = enc_fpath.replace(".zip", NOPASS_SUFFIX)
unzipper.extract_all(zf,
dec_dir_path,
password=matched_password.encode('ascii'))
shutil.make_archive(dec_dir_path, 'zip', dec_dir_path)
if append_to_thread:
rezipped_path = dec_dir_path + ".zip"
with open(rezipped_path, 'rb') as f:
decrypted_zip_binary = f.read()
received_message_subject = client.extract_message_subject(mail)
fname = os.path.basename(rezipped_path)
reply = create_message_with_zip(
my_address, my_address, 'Re: ' + received_message_subject,
'decrypted zip message', decrypted_zip_binary, fname)
reply['threadId'] = mail['threadId']
sent = client.send_message(reply)
os.remove(rezipped_path)
# TODO 複数添付のときに1つだけ解凍に成功した場合が難しい・・。どうするか考える
client.add_label(mail['id'], label_name2id_table[DONE_LABEL])
client.remove_label(mail['id'],
label_name2id_table[PROCESSING_LABEL])
# remove cached files
os.remove(enc_fpath)
if not remain_decrypted_file:
shutil.rmtree(dec_dir_path)
def fiFindByWildcard(wildcard):
return glob.glob(os.path.expanduser(wildcard), recursive=True)
def scan(self):
@contextmanager
def open_file(_file):
if sys.version_info[0] < 3:
with open(_file) as _doc:
_source = _doc.read()
yield (_source, _doc)
else:
with open(_file, encoding="utf-8") as _doc:
_source = _doc.read()
yield (_source, _doc)
def parse_file(_file):
with open_file(_file) as __source:
suite_imported_as_alias = re.findall(
CliRunner.__REGEX_ALIAS_IMPORT, __source[0])
if suite_imported_as_alias:
suite_alias = suite_imported_as_alias[-1].split(
"Suite")[-1].split("as")[-1].split(",")[0].strip()
CliRunner.start_in_a_thread(
target=self.__find_and_register_suite,
args=(suite_alias, __source[0], __source[1].name))
return True
suite_imported = re.findall(CliRunner.__REGEX_NO_ALIAS_IMPORT,
__source[0])
if suite_imported:
CliRunner.start_in_a_thread(
target=self.__find_and_register_suite,
args=("Suite", __source[0], __source[1].name))
return True
try:
print("\n[{status}] Scanning: {location} ...".format(
location=CliUtils.format_color_string(value=",".join(
self.sources),
color="green"),
status=CliUtils.format_color_string(value="INFO",
color="blue")))
start = time.time()
for source in self.sources:
if source.endswith(".py"):
parse_file(source)
else:
for dirName, subdirList, fileList in os.walk(source,
topdown=True):
if self.__skip(source, dirName):
continue
for file_path in glob(
os.path.join(os.path.dirname(dirName + "\\"),
"*.py")):
if parse_file(file_path) is True:
continue
for thread in CliRunner.__SCANNER_THREADS:
thread.join()
print(
"[{status}] Scan finished in: {time} seconds. Found: {suites} suite(s)."
.format(status=CliUtils.format_color_string(value="INFO",
color="blue"),
time="{0:.2f}".format(time.time() - start),
suites=CliUtils.format_bold_string(len(self.suites))))
except KeyboardInterrupt:
print("(Ctrl+C) Exiting!")
exit(12)
except BadCliParameters as err:
print("[{status}] {error}.".format(
status=CliUtils.format_color_string(value="ERROR",
color="red"),
error=err))
exit(120)
except:
print("[{status}] Unexpected error during scan for test suites.".
format(status=CliUtils.format_color_string(value="ERROR",
color="red")))
CliUtils.print_color_traceback()
exit(120)
def __init__(self, directory_path):
super().__init__()
self.bandList = []
for path in glob.glob(directory_path + "/*.tif"):
self.bandList.append(gdal.Open(path))
def __init__(self, useDepth=False):
self.useDepth = useDepth
self.eps_start = 0.9
self.eps_end = 0.05
self.eps_decay = 30000
self.gamma = 0.8
self.learning_rate = 0.001
self.batch_size = 512
self.memory = Memory(10000)
self.max_episodes = 10000
self.save_interval = 2
self.test_interval = 10
self.network_update_interval = 10
self.episode = -1
self.steps_done = 0
self.max_steps = 34
self.policy = DQN()
self.target = DQN()
self.test_network = DQN()
self.target.eval()
self.test_network.eval()
self.updateNetworks()
self.env = DroneEnv(useDepth)
self.optimizer = optim.Adam(self.policy.parameters(), self.learning_rate)
if torch.cuda.is_available():
print('Using device:', device)
print(torch.cuda.get_device_name(0))
else:
print("Using CPU")
# LOGGING
cwd = os.getcwd()
self.save_dir = os.path.join(cwd, "saved models")
if not os.path.exists(self.save_dir):
os.mkdir("saved models")
if not os.path.exists(os.path.join(cwd, "videos")):
os.mkdir("videos")
if torch.cuda.is_available():
self.policy = self.policy.to(device) # to use GPU
self.target = self.target.to(device) # to use GPU
self.test_network = self.test_network.to(device) # to use GPU
# model backup
files = glob.glob(self.save_dir + '\\*.pt')
if len(files) > 0:
files.sort(key=os.path.getmtime)
file = files[-1]
checkpoint = torch.load(file)
self.policy.load_state_dict(checkpoint['state_dict'])
self.episode = checkpoint['episode']
self.steps_done = checkpoint['steps_done']
self.updateNetworks()
print("Saved parameters loaded"
"\nModel: ", file,
"\nSteps done: ", self.steps_done,
"\nEpisode: ", self.episode)
else:
if os.path.exists("log.txt"):
open('log.txt', 'w').close()
if os.path.exists("last_episode.txt"):
open('last_episode.txt', 'w').close()
if os.path.exists("last_episode.txt"):
open('saved_model_params.txt', 'w').close()
self.optimizer = optim.Adam(self.policy.parameters(), self.learning_rate)
obs, _ = self.env.reset()
tensor = self.transformToTensor(obs)
writer.add_graph(self.policy, tensor)
def post_results(sourcefile, job, time_delta=None, skip_upload=False):
df_id, archive_file = job.get('document_file',
None), job.get('archive_file', None)
filename = sourcefile[:sourcefile.rfind('.')]
json_list = glob.glob("%s.json" % glob.escape(sourcefile))
if len(json_list) == 1:
# Properly constructed final JSON found
data = json.load(open(json_list[0], encoding='utf8'))
else:
json_list = glob.glob("%s*.json" % glob.escape(sourcefile))
if not json_list:
# Build empty JSON to post report in API and skip parsing attemp in a future
# print("Build empty JSON for %s" % sourcefile)
data = {'document': {'documentfile_id': df_id}, 'persons': []}
if archive_file:
data['document']['archive_file'] = archive_file
else:
# Join separated JSON files (of XLSX lists)
data = {'persons': [], 'document': {}}
for json_file in json_list:
file_data = json.load(open(json_file, encoding='utf8'))
file_year = file_data.get('document', {}).get('year')
expected_year = job.get('income_year', None)
if file_year is not None and expected_year is not None and file_year != expected_year:
logger.warning(
"%s: Skip wrong declaration year %i (expected %i)" %
(json_file, file_year, expected_year))
continue
data['persons'] += file_data['persons']
# if data['document']:
# if data['document'].get('sheet_title') != file_data['document'].get('sheet_title'):
# logger.warning(
# "Document sheet title changed in one XLSX!")
data['document'] = file_data['document']
if 'sheet_number' in data['document']:
del data['document']['sheet_number']
try:
data['document']['file_size'] = os.path.getsize(sourcefile)
except:
data['document']['file_size'] = 0
try:
data['document']['parser_log'] = open(
sourcefile + ".log", 'rb').read().decode('utf-8', errors='ignore')
except FileNotFoundError:
data['document']['parser_log'] = "FileNotFoundError: " + \
sourcefile + ".log"
data['document']['documentfile_id'] = df_id
if archive_file:
data['document']['archive_file'] = archive_file
# if time_delta == PARSER_TIMEOUT:
# data['document']['parser_log'] += "\nTimeout %i exceeded for smart_parser.exe" % PARSER_TIMEOUT
if time_delta:
data['document']['parser_time'] = time_delta
body = json.dumps(data, ensure_ascii=False,
indent=4).encode('utf-8', errors='ignore')
with open(filename + ".json", "wb") as fp:
fp.write(body)
parsed = len(data['persons']) > 0
result = job.copy()
result['parser_log'] = data['document']['parser_log']
result['sourcefile'] = sourcefile
if not parsed:
result['new_status'] = 'error'
else:
result['new_status'] = 'ok'
if df_id:
if job['status'] == 'ok' and not parsed:
result['new_status'] = 'degrade'
else:
# logger.info("POSTing results (id=%i): %i persons, %i files, file_size %i" % (
# df_id, len(data['persons']), len(json_list), data['document']['file_size']))
response = None
if not skip_upload:
while response is None:
try:
response = client.post(declarator_domain +
'/api/jsonfile/validate/',
data=body)
except requests.exceptions.ConnectionError:
logger.error(
"requests.exceptions.ConnectionError, retrying...")
if response.status_code != requests.codes.ok:
logger.error(response)
logger.error(response.text)
return result
#Initiate webcam
webCam = cv2.VideoCapture(0)
# criteria for termination
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
# prepering obk=ject point like (0, 0, 0), (1,0,0) and etc
objp = np.zeros((6 * 7, 3), np.float32)
objp[:, :2] = np.mgrid[0:7, 0:6].T.reshape(-1, 2)
# Arrays to store image and object points
imgP = []
objP = []
# Load calibration images
images = glob.glob('CalibImages/*.jpg')
# loop to read each calibration image in the folder
for fname in images:
img = cv2.imread(fname)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# Find the corners of the chess board
ret, corners = cv2.findChessboardCorners(gray, (7, 6), None)
# if the corners are found extract object and image points
if ret == True:
objP.append(objp)
corners1 = cv2.cornerSubPix(gray, corners, (11, 11), (-1, -1),
def test_glob(monkeypatch, tmpdir, tree, pattern, matches):
monkeypatch.chdir(tmpdir)
build_files({name: '' for name in tree.split()})
assert list(sorted(glob(pattern))) == list(sorted(matches))
def get_all_files(dirname):
return dirname, glob.glob(dirname + '/**/*.*', True)
def test_glob(monkeypatch, tmpdir, tree, pattern, matches):
monkeypatch.chdir(tmpdir)
path.build({name: '' for name in tree.split()})
assert list(sorted(glob(pattern))) == list(sorted(matches))
def include(self, pattern):
"""Include files that match 'pattern'."""
found = [f for f in glob(pattern) if not os.path.isdir(f)]
self.extend(found)
return bool(found)
from setuptools import setup, glob
package_name = 'robot'
setup(
name=package_name,
version='0.0.0',
packages=[package_name],
data_files=[
('share/ament_index/resource_index/packages',
['resource/' + package_name]),
('share/' + package_name, ['package.xml']),
('share/' + package_name + '/launch', glob.glob('launch/*')),
],
install_requires=['setuptools'],
zip_safe=True,
maintainer='grisw',
maintainer_email='*****@*****.**',
description='TODO: Package description',
license='TODO: License declaration',
tests_require=['pytest'],
entry_points={
'console_scripts': [],
},
)
