def store_preprocessed_metadata(
self,
node_types_number: Optional[int],
nodes_number: int,
edge_types_number: Optional[int],
edges_number: int,
):
"""Store the provided metadata.
Parameters
--------------------------------
node_types_number: Optional[int],
The number of unique node types existing in this graph.
nodes_number: int,
The number of unique nodes existing in this graph.
edge_types_number: Optional[int],
The number of unique edge types existing in this graph.
edges_number: int,
The number of edges existing in this graph.
"""
compress_json.dump(
dict(
node_types_number=node_types_number,
nodes_number=nodes_number,
edge_types_number=edge_types_number,
edges_number=edges_number
),
self.get_preprocessed_graph_metadata_path()
)
def save_data(self, compress=False):
if compress:
compress_json.dump(self.chile, "./data/chile.json.gz")
return
with open("./data/chile.json", "w") as outfile:
if minify:
json.dump(self.chile, outfile)
else:
json.dump(self.chile, outfile, indent=4)
def compress_all_db():
"""
Iterate through all available databases
and compress them one by one
"""
#взять все файлы, которые есть в папке с дб,
#и всем сделать compress_data, по их порядку создания
_db_files = sorted(glob.iglob(
'{}\\jsons\\DB_*.json'.format(db_folder_path)),
key=os.path.getctime)
#iterate through all dbs
for _n in range(len(_db_files)):
print('compressing {} db out of {}'.format(_n + 1, len(_db_files)))
#берем файл, достаем из него инфу и компрессим ее в новый файл
with open(_db_files[_n], encoding="utf8") as json_file:
compress_json.dump(json.load(json_file),
"{}.gz".format(_db_files[_n]))
def test_compress_json():
D = random_string_dict(10, 10)
key = sha256(D)
extensions = compress_json.compress_json._DEFAULT_EXTENSION_MAP.keys()
for ext in extensions:
path = f"random_dirs/test.json.{ext}"
compress_json.dump(D, path)
assert key == sha256(compress_json.load(path))
shutil.rmtree("random_dirs")
for ext in extensions:
path = f"random_dirs/test.json.{ext}"
compress_json.local_dump(D, path)
assert key == sha256(compress_json.local_load(path))
shutil.rmtree("tests/random_dirs")
def on_exit():
if establishConnection():
if __version__ != (version := win.session.get_version()):
print(
f"You can only update the private database if you're using the latest version! (v{version})"
)
else:
print('Updating private database...')
subprocess.call(
[github_db, '-p', win.database.folder, '-g'],
shell=True,
stdout=sys.stdout)
content = retreive_temp_data(win.database.folder)
content.update(win.database.all())
win.database.store(content)
content = win.database.organise()
compress_json.dump(content, win.database.filename)
subprocess.call([github_db, '-e', win.database.filename],
shell=True)
print('Successfully updated!')
def create_JSON(case):
""""Create JSON with all Data"""
# Read CSV
key_file = "Master_Key/master_key.csv"
key_df = pd.read_csv(key_file)
key_df = key_df.sort_values(by=["File"])
# Remove Unacast
key_df = key_df[
key_df["File"] !=
"../../../covid19demographics-fall/unacast/data/2020-04-07 22:39:47.057978.csv"]
new_files = vlookup(key_df, case)
out = {"USA": {}, "Intl.": {}}
intl_keys = ["Iceland", "SouthKorea", "SouthKoreaCensus"]
# Add all rows to state key
for state in new_files:
international = False
if state in intl_keys:
out["Intl."][state] = []
international = True
else:
out["USA"][state] = []
for dic in new_files[state]:
for key in dic:
rows = dic[key].to_dict(orient="records")
if international:
out["Intl."][state].extend(rows)
else:
out["USA"][state].extend(rows)
now = str(datetime.now())
# Export JSON - works when running on Andrew's PC
compress_json.dump(out, "../../../covid19demographics/data/data.json.gz")
def demo_pipeline(
image_path: str,
output_path: str = None,
cache: bool = False,
errors_path: str = None,
):
"""Executes demosaicking pipeline on given image.
Parameters
---------------------------
image_path: str,
Path from where to load the given image.
output_path: str = None,
Path where to save the demosaicked image.
Use None to not save the image.
cache: bool = False,
Whenever to skip processing an image if it was already processed.
errors_path: str = None,
Path where to store the error info.
Use None to not log the errors and raise them.
"""
try:
# Check if we have already this image caches
if cache and output_path is not None and os.path.exists(output_path):
# If this is the case we skip this image.
return None
# Loading the image.
original = load_image(image_path)
# If the user hit a keyboard interrupt we just stop.
except KeyboardInterrupt as e:
raise e
# Otherwise we optionally write to disk to encountered exception.
except Exception as e:
if errors_path is None:
raise e
os.makedirs(errors_path, exist_ok=True)
compress_json.dump(
{
"image-path": image_path,
"text": str(e),
"class": str(type(e)),
"traceback": " ".join(traceback.format_exc().splitlines())
},
"{}/{}.json".format(errors_path,
os.path.basename(image_path).split(".")[0]),
json_kwargs=dict(indent=4))
return None
if output_path is not None:
directory_name = os.path.dirname(output_path)
os.makedirs(directory_name, exist_ok=True)
image_demo = demosaicking(original, 'menon')
if output_path is not None:
# Saving image to given path
cv2.imwrite( # pylint: disable=no-member
output_path,
# Resize given image
image_demo)
return image_demo
destinations_column="object",
directed=True,
edge_types_column="edge_label",
node_path="../embiggen/pos_train_nodes.tsv",
nodes_column="id",
node_types_column="category")
completed_graph = time() - start
start_walk = time()
graph.walk(iterations=10,
length=80,
min_length=0,
return_weight=1,
explore_weight=1,
change_node_type_weight=1,
change_edge_type_weight=1)
delta = time() - start
total_walk_time = time() - start_walk
response = {
"required_time": delta,
"human_time": naturaldelta(delta),
"building_graph_required_time": completed_graph,
"building_graph_required_human_time": naturaldelta(completed_graph),
"random_walk_time": total_walk_time,
"random_walk_human_time": naturaldelta(total_walk_time)
}
print(json.dumps(response, indent=4))
compress_json.dump(response, "time_required.json", json_kwargs={"indent": 4})
input_dir: str = "json"
output_dir: str = "mini-json"
log_file: str = "failed-minification.txt"
files: List[str] = os.listdir(input_dir)
if not os.path.exists(output_dir):
os.mkdir(output_dir)
for file in files:
print(f"Minifying and Compressing {file}")
try:
json_file_path: str = os.path.join(input_dir, file)
mini_file_path: str = os.path.join(output_dir, f"{file}.lzma")
input_file: TextIO = open(file=json_file_path, mode="r")
input_contents_list: List[str] = input_file.readlines()
input_contents: str = "\n".join(input_contents_list)
input_dict: dict = json.loads(input_contents)
compress_json.dump(input_dict, mini_file_path)
except Exception as e: # JSONDecodeError
print(f"Failed To Minify {file}!!! Reason: {e}!!! Logging To {log_file}")
with open(file=log_file, mode="a+") as f:
f.writelines(f"{file}; {e}\n")
f.close()
class Session(QObject):
logger = pyqtSignal(str, dict)
show = pyqtSignal()
close = pyqtSignal()
def __init__(self, base, parent=None):
super().__init__(parent)
self.running = False
self.shownStats = False
self.ui = base
self.database = base.database
self.cache = base.cache
def actually_start(self):
asyncio.run(self.start())
async def start(self):
"""
Start the Session.
"""
self.ui.ui.table.clearContents()
for i in range(6):
self.ui.ui.table.setItem(0, i, QTableWidgetItem())
self.ui.ui.table.setItem(1, i, QTableWidgetItem())
self.running = True
self.ui.ui.startButton.setEnabled(False)
self.ui.ui.stopButton.setEnabled(True)
self.email = self.ui.ui.emailTassomai.text()
self.password = self.ui.ui.passwordTassomai.text()
self.maxQuizes = self.ui.ui.maxQuizes.value()
self.dailyGoal = self.ui.ui.dailyGoal.isChecked()
self.bonusGoal = self.ui.ui.bonusGoal.isChecked()
self.row = self.ui.row
self.correct = 0
self.incorrect = 0
self.quizes = 0
self.until = 0
self.until_incorrect = self.ui.ui.randomnessAmount.value()
if '@' not in self.email:
self.logger.emit('TYPES=[(#c8001a, BOLD), Invalid email.]', {})
print("Invalid email.")
return
if len(self.password) < 1:
self.logger.emit('TYPES=[(#c8001a, BOLD), Must include password.]',
{})
print("Must include password.")
return
self.timer = time.perf_counter()
self.cache.store({'email': self.email, 'password': self.password})
def connect():
self.logger.emit(
'TYPES=[(#0066cc, BOLD), Establishing a connection...]', {})
print("Establishing a connection...")
r = establishConnection()
if r:
self.logger.emit(
'TYPES=[(#0c5d09, BOLD), Successfully established connection.]',
{'newlinesafter': 2})
print("Successfully established connection.\n")
else:
self.logger.emit(
'TYPES=[(#c8001a, BOLD), Unable to establish connection.]',
{})
print("Unable to establish connection.")
time.sleep(0.5)
connect()
connect()
if __version__ != (version := self.get_version()):
self.logger.emit(
'TYPES=[(#c8001a, BOLD), Your Tassomai Automation is outdated! '
''
f'Please update to the newest version v{version} for better performance.]',
{})
print(
f"Your Tassomai Automation is outdated! Please update to the newest version v{version} for better performance."
)
return
subprocess.call([github_db, '-p', self.database.folder, '-g'],
shell=True,
stdout=sys.stdout)
content = retreive_temp_data(self.database.folder)
self.database.store(content)
content = self.database.organise()
compress_json.dump(content, self.database.filename)
self.logger.emit(
'TYPES=[(BOLD, #000000), Successfully updated local database by fetching the Private Answers Database!]',
{'newlinesafter': 2})
print(
"Successfully updated local database by fetching the Private Answers Database!\n"
)
if not self.running:
return
self.tassomai = Tassomai(self.database.all())
await self.tassomai.login(self.email, self.password) # logging in
if not self.running:
return
self.logger.emit(
'TYPES=[(#0c5d09, BOLD), Successfully logged into Tassomai.]', {})
print("Successfully logged into Tassomai.")
await asyncio.sleep(2.00)
loop = 1
try:
for quiz in range(self.maxQuizes):
if not self.running:
subprocess.call(
[github_db, '-p', self.database.folder, '-g'],
shell=True,
stdout=sys.stdout)
content = retreive_temp_data(self.database.folder)
content.update(self.tassomai.database)
database = {
key: value
for key, value in sorted(content.items(),
key=lambda item: item[0])
}
self.database.store(database)
return
if self.tassomai.is_complete and self.dailyGoal:
break
if self.tassomai.is_bonus_complete and self.bonusGoal:
break
self.logger.emit(
f'COLOR=(#0c5d09, Starting up quiz {quiz+1}.)',
{'newlinesbefore': 1})
print(f"\nQuiz {quiz+1}")
time.sleep(0.50)
self.quiz_data = await self.tassomai.extract_quiz_data()
title = self.tassomai.title
self.logger.emit(title, {
'color': '#7214ff',
'bold': True,
'newlinesafter': 2
})
print(f"{title}\n")
quiz_timer = time.perf_counter()
for index, question in enumerate(self.quiz_data['questions']):
force_incorrect = False
if self.ui.ui.randomness.isChecked():
if self.until == self.until_incorrect:
self.until = 0
force_incorrect = True
question['text'] = question['text'].replace(" ", " ")
for ie, ans in enumerate(question['answers']):
question['answers'][ie]['text'] = ans['text'].replace(
" ", " ")
question_data, database = await self.tassomai.answer_question(
Variables(question, force_incorrect))
item = self.ui.ui.table.item(self.row, 0)
if item is None:
for i in range(6):
self.ui.ui.table.setItem(self.row, i,
QTableWidgetItem())
item = self.ui.ui.table.item(self.row, 0)
item2 = self.ui.ui.table.item(self.row, 1)
item3 = self.ui.ui.table.item(self.row, 2)
item4 = self.ui.ui.table.item(self.row, 3)
item5 = self.ui.ui.table.item(self.row, 4)
item6 = self.ui.ui.table.item(self.row, 5)
item.setText(str(loop))
item2.setText(str(index + 1))
item3.setText(question_data['question'])
item4.setText(str(question_data['correct']))
item5.setText(str(question_data['time']) + 's')
item6.setText(' **OR** '.join(question_data['answer']))
self.logger.emit(
f'Question {index+1}: {question_data["question"]}', {
'color': '#0c5d09',
'bold': True
})
self.logger.emit(
f'TYPES=[(BOLD, #0066cc), Correct:] TYPES=[(BOLD, '
f'{"#0c5d09" if question_data["correct"] else "#c8001a"}), '
f'{question_data["correct"]}]', {})
self.row += 1
self.ui.row = self.row
if question_data['correct']:
self.correct += 1
else:
self.incorrect += 1
if self.ui.ui.delay.isChecked():
if self.ui.ui.whenDelay.currentText() == "question":
if self.ui.ui.amountOfDelay.value(
) == 0 and self.ui.ui.amountOfDelay2.value() == 0:
rand = round(random.uniform(1, 4), 2)
elif self.ui.ui.amountOfDelay.value(
) == self.ui.ui.amountOfDelay2.value():
rand = self.ui.ui.amountOfDelay.value()
elif self.ui.ui.amountOfDelay.value(
) != self.ui.ui.amountOfDelay2.value():
rand = round(
random.uniform(
self.ui.ui.amountOfDelay.value(),
self.ui.ui.amountOfDelay2.value()), 2)
item5.setText(
f"{float(question_data['time']) + rand}s")
await asyncio.sleep(rand)
self.until += 1
end_time = time.perf_counter() - quiz_timer
print(f"Completed quiz {loop} in {end_time:0.2f}s")
self.logger.emit(f"Completed quiz {loop} in {end_time:0.2f}s",
{
'color': '#7214ff',
'bold': True
})
print(len(self.database.all()), "total questions answered!\n")
loop += 1
if not self.running:
subprocess.call(
[github_db, '-p', self.database.folder, '-g'],
shell=True,
stdout=sys.stdout)
content = retreive_temp_data(self.database.folder)
content.update(self.tassomai.database)
database = {
key: value
for key, value in sorted(content.items(),
key=lambda item: item[0])
}
self.database.store(database)
content = self.database.organise()
compress_json.dump(content, self.database.filename)
return
self.quizes += 1
if self.ui.ui.delay.isChecked():
if self.ui.ui.whenDelay.currentText() == "quiz":
if self.ui.ui.amountOfDelay.value(
) == 0 and self.ui.ui.amountOfDelay2.value() == 0:
rand = round(random.uniform(1, 4), 2)
elif self.ui.ui.amountOfDelay.value(
) == self.ui.ui.amountOfDelay2.value():
rand = self.ui.ui.amountOfDelay.value()
elif self.ui.ui.amountOfDelay.value(
) != self.ui.ui.amountOfDelay2.value():
rand = round(
random.uniform(
self.ui.ui.amountOfDelay.value(),
self.ui.ui.amountOfDelay2.value()), 2)
await asyncio.sleep(rand)
except:
logging.error(traceback.format_exc())
subprocess.call([github_db, '-p', self.database.folder, '-g'],
shell=True,
stdout=sys.stdout)
content = retreive_temp_data(self.database.folder)
content.update(self.tassomai.database)
database = {
key: value
for key, value in sorted(content.items(), key=lambda item: item[0])
}
self.database.store(database)
content = self.database.organise()
compress_json.dump(content, self.database.filename)
self.shownStats = True
self.show_stats()
self.ui.terminate_session()
def build_all(self):
"""Build graph retrieval methods."""
target_directory_path = os.path.join(
"../bindings/python/ensmallen/datasets",
self.repository_package_name,
)
file_path = "{}.py".format(target_directory_path)
imports = []
for graph_data_path in tqdm(
glob(os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"graph_repositories",
self.get_formatted_repository_name(),
"*.json.gz"
)),
desc="Building graph retrieval methods for {}".format(self.name),
leave=False
):
graph_data = compress_json.load(graph_data_path)
first_graph_version_data = list(graph_data.values())[0]
graph_name = first_graph_version_data["graph_name"]
packages_to_import = self.get_imports(
graph_name, list(graph_data.keys())[-1])
if packages_to_import:
imports.append(packages_to_import)
imports = list(set(imports))
first_references = list(compress_json.load(glob(os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"graph_repositories",
self.get_formatted_repository_name(),
"*.json.gz"
))[0]).values())[0]["references"]
has_unique_references = all(
list(compress_json.load(path).values())[
0]["references"] == first_references
for path in glob(os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"graph_repositories",
self.get_formatted_repository_name(),
"*.json.gz"
))
) and first_references
file = open(file_path, "w")
file.write("\n".join([
"\"\"\"Module providing graphs available from {repository_name}.{references}\"\"\"".format(
repository_name=self.get_formatted_repository_name(),
references="\n\n{}\n".format(self.format_references(
first_references)) if has_unique_references else ""
),
"from ensmallen import Graph # pylint: disable=import-error",
self.get_graph_retrieval_import(),
*imports,
"",
""
]))
graph_repository_metadata = {}
for graph_data_path in tqdm(
glob(os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"graph_repositories",
self.get_formatted_repository_name(),
"*.json.gz"
)),
desc="Building graph retrieval methods for {}".format(self.name),
leave=False,
dynamic_ncols=True
):
graph_data = compress_json.load(graph_data_path)
first_graph_version_data = list(graph_data.values())[0]
graph_name = first_graph_version_data["graph_name"]
graph_method_name = first_graph_version_data["graph_method_name"]
graph_retrieval_file = self.format_graph_retrieval_file(
graph_name=graph_name,
graph_method_name=graph_method_name,
references=first_graph_version_data["references"],
versions=list(graph_data.keys()),
has_unique_references=has_unique_references
)
for value in graph_data.values():
value.pop("references")
graph_repository_metadata[graph_method_name] = graph_data
file.write(graph_retrieval_file)
file.close()
compress_json.dump(
graph_repository_metadata,
"{}.json.gz".format(target_directory_path)
)
def api_db_load(vend, buy, db_compare=False, compressed=True):
"""
Make api call, save new db, compare new db with old one and save demand
Parameters
----------
vend : list
list that will store vend data.
buy : list
list that will store vend data.
db_compare : TYPE, optional
if we want to compare all available dbs. The default is False.
Returns
-------
None.
"""
#находим последнюю сжатую базу
if compressed:
db_conv_time = last_db_time_get()
if not db_compare:
if convert_time(db_conv_time, 'm') > 9: #9
print("Old database...")
#собираем ссылку
link = 'https://api.originsro.org/api/v1/market/list?api_key='
api_key = ''
#делаем вызов и сохраняем получаемое в переменной
response = rq.get('{}{}'.format(link, api_key))
#создаем путь/имя для нашей новой базы
latest_file = '{}\\jsons\\DB_{}.json.gz'.format(
db_folder_path, now.strftime("%d-%m-%Y_%H-%M-%S"))
#сохраняем нашу новую базу
compress_json.dump(response.text, latest_file)
print('compressed db saved')
#и сразу же загружаем
#actual_db = compress_json.local_load(latest_file)
#находим 2 по старости дб (то есть ту, которая была перед нашей только что выкачанной)
previous_db = sorted(glob.iglob(
'{}\\jsons\\DB_*.json.gz'.format(db_folder_path)),
key=os.path.getctime)[-2]
#распаковываем его
#previous_db_uncompressed = compress_json.local_load(previous_db)
#создаем временные листы с данными
#из прошлой бд
pre_vendor_shops = []
pre_buy_shops = []
get_vend_data(previous_db, pre_vendor_shops, pre_buy_shops)
#из новой бд
print(latest_file)
get_vend_data(latest_file, vend, buy)
#fill demand file
compare_data(pre_vendor_shops, vend)
compare_data(pre_buy_shops, buy)
print('demand refreshed')
else:
print('Database ok...')
latest_file = max(glob.glob(
'{}\\jsons\\DB_*.json.gz'.format(db_folder_path)),
key=os.path.getctime)
get_vend_data(latest_file, vend, buy)
else:
compare_db()
else:
#находим самую свежую базу из имеющихся
latest_file = max(glob.glob(
'{}\\jsons\\DB_*.json'.format(db_folder_path)),
key=os.path.getctime)
#Загрузка БД и ее обновление
with open(latest_file, encoding="utf8") as json_file:
#делаем удобное обозначение той же базы
api_db = json.load(json_file)['generation_timestamp']
db_time = datetime.datetime.strptime(api_db,
"%Y-%m-%dT%H:%M:%S.%fZ")
if not db_compare:
#если базе больше 10 минут, надо обновить
if convert_time(db_time, 'm') > 10: #9
print("Old database...")
link = 'https://api.originsro.org/api/v1/market/list?api_key='
api_key = 'r3bkd0q8umxhuj75ahtvwpgkd3yzi3rm'
response = rq.get('{}{}'.format(link, api_key))
latest_file = '{}\\jsons\\DB_{}.json'.format(
db_folder_path, now.strftime("%d-%m-%Y_%H-%M-%S"))
#создаем новую бд с названием из latest_file
with open(latest_file, "w+", encoding="utf8") as new_file:
new_file.write(response.text)
#создаем новую, сжатую, бд
#compress_json.dump(response.text, latest_file)
#находим второй по новости файл, то есть прошлую бд
previous_db = sorted(glob.iglob(
'{}\\jsons\\DB_*.json'.format(db_folder_path)),
key=os.path.getctime)[-2]
#создаем временные листы с данными
#из прошлой бд
pre_vendor_shops = []
pre_buy_shops = []
get_vend_data(previous_db, pre_vendor_shops, pre_buy_shops)
#из новой бд
get_vend_data(latest_file, vend, buy)
#fill demand file
compare_data(pre_vendor_shops, vend)
compare_data(pre_buy_shops, buy)
else:
print('Database ok...')
get_vend_data(latest_file, vend, buy)
else:
compare_db()
def _write_compressed_json(self, filename, content):
compress_json.dump(
content,
os.path.join(self._destination_folder, (filename + ".json.gz")))
def reference_path_example(mode):
"""
Saves a video of a shortest path follower agent navigating from a start
position to a goal. Agent follows the ground truth reference path by
navigating to intermediate viewpoints en route to goal.
Args:
mode: 'geodesic_path' or 'greedy'
"""
show_waypoint_indicators = False
config = habitat.get_config(
config_paths="configs/test/habitat_r2r_vln_test.yaml")
config.defrost()
config.TASK.MEASUREMENTS.append("TOP_DOWN_MAP")
config.TASK.SENSORS.append("HEADING_SENSOR")
config.freeze()
split = 'train'
vln_data_path = '/home/mirshad7/habitat-lab/data/datasets/vln/mp3d/r2r/v1/' + split + '/' + split + '.json.gz'
with gzip.open(vln_data_path, "rt") as f:
deserialized = json.loads(f.read())
val_ids = {}
for i in range(len(deserialized['episodes'])):
val_ids[deserialized['episodes'][i]['episode_id']] = i
new_data_dict = {}
new_data_dict['episodes'] = {}
new_data_dict['instruction_vocab'] = deserialized['instruction_vocab']
new_data_list = []
save_fig = False
steps_dict = {}
with SimpleRLEnv(config=config) as env:
print("Environment creation successful")
# obj_attr_mgr = env.habitat_env._sim.get_object_template_manager()
# remove_all_objects(env.habitat_env._sim)
sim_time = 30 # @param {type:"integer"}
continuous_nav = True # @param {type:"boolean"}
if continuous_nav:
control_frequency = 10 # @param {type:"slider", min:1, max:30, step:1}
frame_skip = 6 # @param {type:"slider", min:1, max:30, step:1}
fps = control_frequency * frame_skip
print("fps = " + str(fps))
control_sequence = []
for action in range(int(sim_time * control_frequency)):
if continuous_nav:
# allow forward velocity and y rotation to vary
control_sequence.append({
"forward_velocity":
random.random() * 2.0, # [0,2)
"rotation_velocity":
(random.random() - 0.5) * 2.0, # [-1,1)
})
else:
control_sequence.append(random.choice(action_names))
# create and configure a new VelocityControl structure
vel_control = habitat_sim.physics.VelocityControl()
vel_control.controlling_lin_vel = True
vel_control.lin_vel_is_local = True
vel_control.controlling_ang_vel = True
vel_control.ang_vel_is_local = True
vis_ids = []
collided_trajectories = []
trajectory_without_collision = True
for episode in range(len(deserialized['episodes'])):
counter = 0
env.reset()
episode_id = env.habitat_env.current_episode.episode_id
print(
f"Agent stepping around inside environment. Episode id: {episode_id}"
)
dirname = os.path.join(IMAGE_DIR, "vln_reference_path_example",
mode, "%02d" % episode)
if os.path.exists(dirname):
shutil.rmtree(dirname)
os.makedirs(dirname)
images = []
steps = 0
reference_path = env.habitat_env.current_episode.reference_path + [
env.habitat_env.current_episode.goals[0].position
]
# manually control the object's kinematic state via velocity integration
time_step = 1.0 / (30)
x = []
y = []
yaw = []
vel = []
omega = []
continuous_path_follower = ContinuousPathFollower(
env.habitat_env._sim, reference_path, waypoint_threshold=0.4)
max_time = 30.0
done = False
EPS = 1e-4
prev_pos = np.linalg.norm(
env.habitat_env._sim.get_agent_state().position)
if show_waypoint_indicators:
for id in vis_ids:
sim.remove_object(id)
vis_ids = setup_path_visualization(env.habitat_env._sim,
continuous_path_follower)
while continuous_path_follower.progress < 1.0:
# print("done",done)
if done:
break
if counter == 150:
counter = 0
collided_trajectories.append(
env.habitat_env.current_episode.episode_id)
trajectory_without_collision = False
break
continuous_path_follower.update_waypoint()
if show_waypoint_indicators:
sim.set_translation(continuous_path_follower.waypoint,
vis_ids[0])
agent_state = env.habitat_env._sim.get_agent_state()
pos = np.linalg.norm(
env.habitat_env._sim.get_agent_state().position)
if abs(pos - prev_pos) < EPS:
counter += 1
previous_rigid_state = habitat_sim.RigidState(
quat_to_magnum(agent_state.rotation), agent_state.position)
v, w = track_waypoint(
continuous_path_follower.waypoint,
previous_rigid_state,
vel_control,
dt=time_step,
)
observations, reward, done, info = env.step(vel_control)
# print(observations)
# save_map(observations, info, images)
prev_pos = pos
x.append(env.habitat_env._sim.get_agent_state().position[0])
y.append(-env.habitat_env._sim.get_agent_state().position[2])
yaw.append(
quaternion.as_euler_angles(
env.habitat_env._sim.get_agent_state().rotation)[1])
vel.append(v)
omega.append(w)
steps += 1
if save_fig: # pragma: no cover
plt.close()
plt.subplots(1)
plt.plot(x, y, "xb", label="input")
plt.grid(True)
plt.axis("equal")
plt.xlabel("x[m]")
plt.ylabel("y[m]")
plt.legend()
pose_title = dirname + 'pose.png'
plt.savefig(pose_title)
plt.subplots(1)
plt.plot(yaw, "-r", label="yaw")
plt.grid(True)
plt.legend()
plt.xlabel("line length[m]")
plt.ylabel("yaw angle[deg]")
yaw_title = dirname + 'yaw.png'
plt.savefig(yaw_title)
plt.subplots(1)
plt.plot(vel, "-r", label="vel")
plt.grid(True)
plt.legend()
plt.xlabel("line length[m]")
plt.ylabel("omega_reference [rad/s^2]")
vel_title = dirname + 'vel.png'
plt.savefig(vel_title)
plt.subplots(1)
plt.plot(omega, "-r", label="v_reference")
plt.grid(True)
plt.legend()
plt.xlabel("line length[m]")
plt.ylabel("v_reference [m/s]")
omega_title = dirname + 'omega.png'
plt.savefig(omega_title)
x = []
y = []
yaw = []
vel = []
omega = []
if trajectory_without_collision:
ids = val_ids[episode_id]
single_data_dict = deserialized['episodes'][ids]
new_data_list.append(single_data_dict)
trajectory_without_collision = True
print(f"Navigated to goal in {steps} steps.")
steps_dict[episode] = steps
# images_to_video(images, dirname, str(episode_id), fps = int (1.0/time_step))
images = []
steps_path = '/home/mirshad7/habitat-lab/train_steps.json.gz'
# new_data_dict['episodes'] = new_data_list
# path = '/home/mirshad7/habitat-lab/data/datasets/vln/mp3d/r2r/robo_vln/train/train.json.gz'
# compress_json.dump(new_data_dict, path)
compress_json.dump(steps_dict, steps_path)
print("collided trajectories:", collided_trajectories)
@app.route('/', methods=['GET', 'POST'])
def home():
global result
global count
global search_word
if request.method == 'POST':
search_word = request.form["search bar"]
result = search(search_word)
count = 1
return redirect(url_for("result_found"))
else:
return render_template("search.html")
if __name__ == '__main__':
if not (os.path.isfile('comp_data.json.gz')
and os.access('comp_data.json.gz', os.R_OK)):
if not (os.path.isfile('data.txt') and os.access('data.txt', os.R_OK)):
with open("data.txt", "w", encoding='utf8') as jsonfile:
open_files_and_collect_data()
collect_yap_data()
json.dump(data, jsonfile, ensure_ascii=False)
with open('data.txt', encoding='utf8') as data_json:
data_load = json.load(data_json)
compress_json.dump(data_load, "comp_data.json.gz")
data = compress_json.load("comp_data.json.gz")
app.run()
def image_pipeline(
image_path: str,
output_path: str = None,
blur_bbox_padding: int = 50,
width: int = 480,
initial_width: int = 1024,
thumbnail_width: int = 256,
hardness: float = 0.7,
save_steps: bool = False,
cache: bool = False,
errors_path: str = None,
):
"""Executes complete pipeline on given image.
Parameters
---------------------------
image_path: str,
Path from where to load the given image.
output_path: str = None,
Path where to save the processed image.
Use None to not save the image.
blur_bbox_padding: int = 50,
The padding to use around the blur bbox cut.
width: int = 480,
The size to resize the image.
initial_width: int = 1024,
The initial resize width.
thumbnail_width: int = 256,
Width to use for the thumbnails during processing.
hardness: float = 0.7,
Hardness to use for the body cut.
save_steps: bool = False,
Wethever to save the partial steps instead othe processed image.
This option is useful to debug which parameters are to blaim for
unexpected pipeline behaviour.
By default, this is False.
cache: bool = False,
Wethever to skip processing an image if it was already processed.
errors_path: str = None,
Path where to store the error info.
Use None to not log the errors and raise them.
"""
try:
# Check if we have already this image caches
if cache and output_path is not None and os.path.exists(output_path):
# If this is the case we skip this image.
return None
# Loading the image.
original = load_image(image_path)
# Executes perspective correction
image_perspective = perspective_correction(original)
# Computing thumbnail
image_perspective = get_thumbnail(image_perspective,
width=initial_width)
# Executes blur bbox cut
image_bbox = blur_bbox(image_perspective, padding=blur_bbox_padding)
# Determines optimal counter rotation
image_rotated, angle, x = counter_rotate(image_bbox,
width=thumbnail_width)
# Cuts the body lower part
image_body_cut, _ = get_body_cut(image_bbox,
image_rotated,
angle,
simmetry_axis=x,
width=thumbnail_width,
hardness=hardness)
# Executes secondary blur bbox cut
image_body_cut = blur_bbox(image_body_cut, padding=blur_bbox_padding)
# Final result
final_result = cv2.cvtColor( # pylint: disable=no-member
ace(
cv2.cvtColor( # pylint: disable=no-member
image_body_cut,
cv2.COLOR_GRAY2RGB # pylint: disable=no-member
),
slope=10),
cv2.COLOR_RGB2GRAY # pylint: disable=no-member
)
# If the user hit a keyboard interrupt we just stop.
except KeyboardInterrupt as e:
raise e
# Otherwise we optionally write to disk to encountered exception.
except Exception as e:
if errors_path is None:
raise e
os.makedirs(errors_path, exist_ok=True)
compress_json.dump(
{
"image-path": image_path,
"text": str(e),
"class": str(type(e)),
"traceback": " ".join(traceback.format_exc().splitlines())
},
"{}/{}.json".format(errors_path,
os.path.basename(image_path).split(".")[0]),
json_kwargs=dict(indent=4))
return None
if output_path is not None:
directory_name = os.path.dirname(output_path)
os.makedirs(directory_name, exist_ok=True)
thumb = get_thumbnail(image_body_cut, width=width)
if not save_steps:
if output_path is not None:
# Saving image to given path
cv2.imwrite( # pylint: disable=no-member
output_path,
# Resize given image
thumb)
return thumb
fig, axes = plt.subplots(nrows=2, ncols=3, figsize=(15, 10))
axes = axes.ravel()
axes[0].imshow(original, cmap="gray")
axes[0].set_title("Original image")
axes[1].imshow(image_perspective, cmap="gray")
axes[1].set_title("Perspective correction")
axes[2].imshow(image_bbox, cmap="gray")
axes[2].set_title("Blur BBox image")
axes[3].imshow(image_rotated, cmap="gray")
axes[3].set_title("Rotated image")
axes[4].imshow(image_body_cut, cmap="gray")
axes[4].set_title("Body cut image")
axes[5].imshow(final_result, cmap="gray")
axes[5].set_title("ACE")
[ax.set_axis_off() for ax in axes.ravel()]
fig.tight_layout()
fig.savefig(output_path)
plt.close(fig)
from typing import List, TextIO
input_dir: str = "mini-json"
output_dir: str = "decompressed-json"
log_file: str = "failed-decompression.txt"
files: List[str] = os.listdir(input_dir)
if not os.path.exists(output_dir):
os.mkdir(output_dir)
for file in files:
print(f"Decompressing {file}")
try:
mini_json_file_path: str = os.path.join(input_dir, file)
decompressed_file_path: str = os.path.join(output_dir, f"{file[:-5]}")
input_dict: dict = compress_json.load(mini_json_file_path)
compress_json.dump(input_dict, decompressed_file_path)
except Exception as e: # JSONDecodeError
print(
f"Failed To Decompress {file}!!! Reason: {e}!!! Logging To {log_file}"
)
with open(file=log_file, mode="a+") as f:
f.writelines(f"{file}; {e}\n")
f.close()
def sanitize_graph(graph_data: str, root: str):
"""Convert all the graphs to a standard format.
Parameters
----------
graph_data: List[Dict],
Informations of the graph to sanitize
root: str,
The working folder. All the files will be read and written from here.
"""
kwargs = graph_data["loading_settings"]
kwargs["edge_path"] = os.path.join(root, graph_data["folder_name"],
graph_data["edge_file"])
kwargs.setdefault("directed", False)
directed_dst_path = os.path.join(root, graph_data["folder_name"],
"directed_sanitized.tsv")
undirected_dst_path = os.path.join(root, graph_data["folder_name"],
"undirected_sanitized.tsv")
report_path = os.path.join(root, graph_data["folder_name"], "report.json")
textual_report_path = os.path.join(root, graph_data["folder_name"],
"report.txt")
if all(
os.path.exists(p) for p in (directed_dst_path, undirected_dst_path,
report_path, textual_report_path)):
return
logger.info("Loading the file %s" % kwargs["edge_path"])
graph: EnsmallenGraph = EnsmallenGraph.from_unsorted_csv(
**kwargs, name=graph_data["graph"])
logger.info("Enabling fast version")
graph.enable_fast_walk()
logger.info("Computing metadata")
if not os.path.exists(report_path):
logger.info("Computing JSON report")
report = graph.report()
compress_json.dump(report, report_path)
if not os.path.exists(textual_report_path):
logger.info("Computing textual report")
textual_report = str(graph)
with open(textual_report_path, "w") as f:
f.write(textual_report)
if not os.path.exists(undirected_dst_path):
logger.info("Writing the file {}".format(undirected_dst_path))
graph.dump_edges(
path=undirected_dst_path,
header=False,
sources_column_number=0,
destinations_column_number=1,
weights_column_number=2,
numeric_node_ids=True,
# We dump with directed=True for the undirected file to have in the file the bidirectional edges.
directed=True)
if not os.path.exists(directed_dst_path):
logger.info("Writing the file {}".format(directed_dst_path))
graph.dump_edges(
path=directed_dst_path,
header=False,
sources_column_number=0,
destinations_column_number=1,
weights_column_number=2,
numeric_node_ids=True,
# We dump with directed=False for the directed file to have no doubled bidirectional edge in the write out.
directed=False)
