def get_sp3(sp3file, directory=os.getcwd()):
"""
This function downloads IGS orbit file from NASA CDDIS ftp server.
"""
fileName = sp3file + ".Z"
if os.path.exists(fileName) == True:
if os.path.exists(fileName[:-2]) == True:
print(fileName[-2] + " exists in working directory")
return
else:
print(fileName + " exists in working directory | Extracting...")
Archive(fileName + ".Z").extractall(os.getcwd())
return
internet = check_internet()
if internet == False:
raise Warning('No internet connection! | Cannot download orbit file')
ftpserver = 'ftp://cddis.gsfc.nasa.gov'
sp3FileDir = 'gnss/products'
if sp3file.startswith("wum"):
sp3FileDir += '/mgex'
file_topath = os.path.join(directory, fileName)
fileDir = [ftpserver, sp3FileDir, fileName[3:-7], fileName]
ftp = '/'.join(fileDir) # FTP link of file
try:
print('Downloading:', fileName, end = '')
url.urlretrieve(ftp, file_topath)
print(' | Download completed for', fileName)
Archive(fileName).extractall(os.getcwd())
except:
print(" | Requested file", fileName, "cannot be not found!")
def ref_files():
my_os = os.name
if (my_os is 'posix'):
# cmd_text = '/usr/bin/ogr2ogr'
slash = '/'
else:
# cmd_text = 'C:\\OSGeo\\bin\\ogr2ogr.exe'
slash = '\\'
if not os.path.isfile(
'shapefiles{slash}AUS_2016_AUST.shp'.format(slash=slash)):
print('Downloading ABS Australia file in Shape file format')
url = 'http://www.abs.gov.au/AUSSTATS/subscriber.nsf/log?openagent&1270055001_aus_2016_aust_shape.zip&1270.0.55.001&Data%20Cubes&5503B37F8055BFFECA2581640014462C&0&July%202016&24.07.2017&Latest'
urllib.request.urlretrieve(
url, 'shapefiles{slash}1270055001_aus_2016_aust_shape.zip'.format(
slash=slash))
print('Unzipping ABS Australia file in Shape file format')
Archive('shapefiles{slash}1270055001_aus_2016_aust_shape.zip'.format(
slash=slash)).extractall('shapefiles'.format(slash=slash))
else:
print('ABS Australia file in Shape file format exists')
if not os.path.isfile(
'tabfiles{slash}AUS_2016_AUST.tab'.format(slash=slash)):
print('Downloading ABS Australia file in Tab file format')
url = 'http://www.abs.gov.au/AUSSTATS/subscriber.nsf/log?openagent&1270055001_aus_2016_aust_tab.zip&1270.0.55.001&Data%20Cubes&F18065BF058615F9CA2581640014491B&0&July%202016&24.07.2017&Latest'
urllib.request.urlretrieve(
url, 'tabfiles{slash}1270055001_aus_2016_aust_tab.zip'.format(
slash=slash))
print('Unzipping ABS Australia file in Tab file format')
Archive('tabfiles{slash}1270055001_aus_2016_aust_tab.zip'.format(
slash=slash)).extractall('tabfiles'.format(slash=slash))
else:
print('ABS Australia file in Tab file format exists')
def test2(self):
d = tempfile.mkdtemp()
x_txt = os.path.join(d, 'x.txt')
open(x_txt, 'w').write('123')
x_zip = os.path.join(d, 'x.zip')
Proc(['zip', '--no-dir-entries', x_zip, 'x.txt'], cwd=d).call()
self.assertRaises(ValueError,
lambda: Archive(x_zip).extractall('blabla'))
d = tempfile.mkdtemp()
Archive(x_zip, backend='patool').extractall(d)
ok_file(d, 'x.txt')
d = tempfile.mkdtemp()
Archive(x_zip).extractall(d)
ok_file(d, 'x.txt')
d = tempfile.mkdtemp()
Archive(x_zip, backend='auto').extractall(d)
ok_file(d, 'x.txt')
if sys.version_info >= (2, 6):
d = tempfile.mkdtemp()
Archive(x_zip, backend='zipfile').extractall(d)
ok_file(d, 'x.txt')
d = tempfile.mkdtemp()
cli.extractall(x_zip, d)
ok_file(d, 'x.txt')
def unpackArchives(path):
print("Downloading Flutter....")
flutterArchive = wget.download('https://storage.googleapis.com/flutter_infra/releases/stable/linux/flutter_linux_v1.7.8+hotfix.4-stable.tar.xz')
print("\n\nDownloading AndroidSDK....")
androidSDKArchive = wget.download('https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip')
# Unpacks archives in SDK folder
# Creates SDK folder if non exists
try:
os.mkdir(path)
except FileExistsError:
print("\n\nSDK folder already made!")
finally:
try:
os.mkdir(path+'/androidSDK')
except FileExistsError:
print("androidSDK folder already made!")
# Unpacks flutter and androidsdk in SDK folder
Archive(flutterArchive).extractall(path)
Archive(androidSDKArchive).extractall(path+'/androidSDK')
# Deletes archive files from current directory
os.remove(flutterArchive)
os.remove(androidSDKArchive)
def download_from_dump(self, home, articles, key):
if not os.path.isdir(home + '/knolml_dataset/phase_details'):
download('knolml_dataset',
verbose=True,
glob_pattern='phase_details.7z',
destdir=home)
Archive('~/knolml_dataset/phase_details.7z').extractall(
'~/knolml_dataset')
if not os.path.isdir(home + '/knolml_dataset/bz2t'):
download('knolml_dataset',
verbose=True,
glob_pattern='bz2t.7z',
destdir=home)
Archive('~/knolml_dataset/bz2t.7z').extractall(home +
'/knolml_dataset')
fileList = glob.glob(home + '/knolml_dataset/phase_details/*.txt')
for files in fileList:
if 'phase' in files:
with open(files, 'r') as myFile:
for line in myFile:
l = line.split('#$*$#')
if l[0] in articles:
print("Found hit for article " + l[0])
# file, art, index, home, key
self.extract_from_bzip(file=l[1],
art=l[0],
index=int(l[2]),
home=home,
key=key)
def test2():
x_zip = create_zip()
assert_raises(ValueError, lambda: Archive(x_zip).extractall('blabla'))
d = tmpdir()
Archive(x_zip, backend='patool').extractall(d)
ok_file(d, 'x.txt')
d = tmpdir()
Archive(x_zip).extractall(d)
ok_file(d, 'x.txt')
d = tmpdir()
Archive(x_zip, backend='auto').extractall(d)
ok_file(d, 'x.txt')
if sys.version_info >= (2, 6):
d = tmpdir()
Archive(x_zip, backend='zipfile').extractall(d)
ok_file(d, 'x.txt')
d = tmpdir()
cli.extractall(x_zip, d)
ok_file(d, 'x.txt')
def test2():
for f in formats:
print(f)
x_zip = create_arc(f)
with pytest.raises(ValueError):
Archive(x_zip).extractall("blabla")
d = tmpdir()
Archive(x_zip, backend="patool").extractall(d)
ok_file(d, "x.txt")
d = tmpdir()
Archive(x_zip).extractall(d)
ok_file(d, "x.txt")
d = tmpdir()
Archive(x_zip, backend="auto").extractall(d)
ok_file(d, "x.txt")
if f == "zip":
d = tmpdir()
Archive(x_zip, backend="zipfile").extractall(d)
ok_file(d, "x.txt")
d = tmpdir()
cli.extractall(x_zip, d)
ok_file(d, "x.txt")
def isexist(fileName):
if os.path.exists(fileName) == False:
if (os.path.exists(fileName + ".Z") == False) and (fileName.endswith(".Z")==False):
extension = fileName.split(".")[1]
if extension[-1].lower() == "o":
if is_IGS(fileName[:4]) == True:
print(fileName + ".Z does not exist in working directory | Downloading...")
fileEpoch = doy2date(fileName)
download.get_rinex([fileName[:4]], fileEpoch, Datetime = True)
Archive(fileName + ".Z").extractall(_CWD)
else:
raise Warning(fileName,"does not exist in directory and cannot be found in IGS Station list!")
elif extension.lower() == "rnx":
if is_IGS(fileName[:4]) == True:
print(fileName + " does not exist in working directory | Downloading...")
fileName = fileName.split(".")[0] + ".crx"
fileEpoch = doy2date(fileName)
download.get_rinex3([fileName[:4]], fileEpoch, Datetime = True)
Archive(fileName + ".gz").extractall(_CWD)
crx2rnx(fileName)
else:
raise Warning(fileName,"does not exist in directory and cannot be found in IGS Station list!")
elif extension.lower() == "crx":
if is_IGS(fileName[:4]) == True:
print(fileName + ".gz does not exist in working directory | Downloading...")
fileEpoch = doy2date(fileName)
download.get_rinex3([fileName[:4]], fileEpoch, Datetime = True)
Archive(fileName + ".gz").extractall(_CWD)
crx2rnx(fileName)
else:
raise Warning(fileName,"does not exist in directory and cannot be found in IGS Station list!")
elif extension[-1].lower() in {"n","p","g"}:
if is_IGS(fileName[:4]) == True:
print(fileName + ".Z does not exist in working directory | Downloading...")
fileEpoch = doy2date(fileName)
download.get_navigation([fileName[:4]], fileEpoch, Datetime = True)
Archive(fileName + ".Z").extractall(_CWD)
elif extension.lower() in {"clk","clk_05s"}:
download.get_clock(fileName)
Archive(fileName + ".Z").extractall(_CWD)
elif extension.lower() == "sp3":
download.get_sp3(fileName)
Archive(fileName + ".Z").extractall(_CWD)
elif extension[-1].lower() == "i":
download.get_ionosphere(fileName)
Archive(fileName + ".Z").extractall(_CWD)
else:
raise Warning("Unknown file extension:",extension.lower())
elif (os.path.exists(fileName + ".Z") == True) and (fileName.endswith(".Z")==True):
print(fileName + " exists in working directory | Extracting...")
Archive(fileName).extractall(_CWD)
else:
print(fileName + ".Z exists in working directory | Extracting...")
Archive(fileName + ".Z").extractall(_CWD)
elif (os.path.exists(fileName) == True) and (fileName.endswith(".Z")==True):
print(fileName + " exists in working directory | Extracting...")
Archive(fileName).extractall(_CWD)
else:
print(fileName + " exist in working directory | Reading...")
def get_rinex(stationList, date_start, date_finish=None, period='day', Datetime=False, directory=os.getcwd()):
"""
This function downloads IGS rinex observation file from NASA CDDIS ftp server.
Usage:
get_rinex(['mate'],'02-01-2017')
get_rinex(['mate', 'onsa'],'01-01-2017')
get_rinex(['mate'], date_start = '01-01-2017', date_finish = '05-01-2017', period = 'day')
get_rinex(['mate'], date_start = '01-01-2017', date_finish = '01-06-2017', period = 'month')
get_rinex(['mate', 'onsa'], date_start = '01-01-2017', date_finish = '03-01-2017', period = 'month')
get_rinex(['mate'], date_start = '01-01-2017', date_finish = '01-01-2018', period = 'year')
"""
internet = check_internet()
if internet == False:
raise Warning('No internet connection! | Cannot download RINEX file...')
if Datetime == False:
date_start = datetime.date(year = int(date_start[-4:]), month = int(date_start[-7:-5]), day = int(date_start[-10:-8]))
if date_finish != None:
date_finish = datetime.date(year = int(date_finish[-4:]), month = int(date_finish[-7:-5]), day = int(date_finish[-10:-8]))
timedelta = {'day' : relativedelta(days = 1),
'month' : relativedelta(months = 1),
'year' : relativedelta(years = 1)}[period]
dateList = [date_start] # dates of observation files
if date_finish != None:
while dateList[-1] != date_finish:
date = dateList[-1] + timedelta
dateList.append(date)
ftpserver = 'ftp://cddis.gsfc.nasa.gov'
obsFileDir = 'gnss/data/daily' # observation file directory in ftp server
for stationName in stationList:
for date in dateList:
doy = datetime2doy(date, string = True)
fileName = obsFileName(stationName, date, zipped = True)
# check if the file already exist in the directory
if os.path.exists(fileName) == True:
if os.path.exists(fileName[:-2]) == True:
print(fileName[:-2] + " exists in working directory")
continue
else:
print(fileName + " exists in working directory | Extracting...")
Archive(fileName).extractall(os.getcwd())
continue
file_topath = os.path.join(directory, fileName)
fileDir = [ftpserver, obsFileDir, str(date.year), doy, str(date.year)[-2:] + 'o', fileName] # file directory
ftp = '/'.join(fileDir)
# Download the file
try:
print('Downloading:', fileName, end= '')
url.urlretrieve(ftp, file_topath)
print(" | Download completed for", fileName, " | Extracting...")
Archive(fileName).extractall(os.getcwd())
except:
raise Warning("Requested file", fileName, "cannot be not found!")
def test_subdir():
x_zip = create_zip()
d = tmpdir() / 'subdir'
assert_raises(ValueError,
lambda: Archive(x_zip).extractall(d, auto_create_dir=False))
d = tmpdir() / 'subdir'
Archive(x_zip, backend='auto').extractall(d, auto_create_dir=True)
ok_file(d, 'x.txt')
def get_files(html):
soup = BeautifulSoup(html, 'lxml')
first = soup.find('a', href = re.compile(r'vpp_\w+\_u.xlsx')).get('href')
wget.download('http://www.ukrstat.gov.ua/' + str(first)[9:], out='C:/ProgramData/MySQL/MySQL Server 8.0/Data/vpp_month/')
Archive('C:/ProgramData/MySQL/MySQL Server 8.0/Data/vpp_month/').extractall('C:/ProgramData/MySQL/MySQL Server 8.0/Data/vpp_month/')
second = soup.find('a', href = re.compile(r'vppr_\w+\_u.xlsx')).get('href')
wget.download('http://www.ukrstat.gov.ua/' + str(second)[9:], out='C:/ProgramData/MySQL/MySQL Server 8.0/Data/vpp_month/')
Archive('C:/ProgramData/MySQL/MySQL Server 8.0/Data/vpp_month/').extractall('C:/ProgramData/MySQL/MySQL Server 8.0/Data/vpp_month/')
return first,second
def test_subdir():
for f in formats:
x_zip = create_arc(f)
d = tmpdir() / "subdir"
with pytest.raises(ValueError):
Archive(x_zip).extractall(d, auto_create_dir=False)
d = tmpdir() / "subdir"
Archive(x_zip, backend="auto").extractall(d, auto_create_dir=True)
ok_file(d, "x.txt")
def extraction(file):
Archive(file).extractall('')
list_of_files = glob.glob('*')
latest_file = max(list_of_files, key=os.path.getctime)
path = ''
fin_path = os.path.join(path, latest_file)
return fin_path
def _import(src_file, task_data):
with TemporaryDirectory() as tmp_dir:
Archive(src_file.name).extractall(tmp_dir)
dataset = dm_env.make_importer('tf_detection_api')(
tmp_dir).make_dataset()
import_dm_annotations(dataset, task_data)
def _import(src_file, instance_data, load_data_callback=None):
with TemporaryDirectory() as tmp_dir:
Archive(src_file.name).extractall(tmp_dir)
# put label map from the task if not present
labelmap_file = osp.join(tmp_dir, 'labelmap.txt')
if not osp.isfile(labelmap_file):
labels_meta = instance_data.meta['project']['labels'] \
if isinstance(instance_data, ProjectData) else instance_data.meta['task']['labels']
labels = (label['name'] + ':::' for _, label in labels_meta)
with open(labelmap_file, 'w') as f:
f.write('\n'.join(labels))
# support flat archive layout
anno_dir = osp.join(tmp_dir, 'Annotations')
if not osp.isdir(anno_dir):
anno_files = glob(osp.join(tmp_dir, '**', '*.xml'), recursive=True)
subsets_dir = osp.join(tmp_dir, 'ImageSets', 'Main')
os.makedirs(subsets_dir, exist_ok=True)
with open(osp.join(subsets_dir, 'train.txt'), 'w') as subset_file:
for f in anno_files:
subset_file.write(osp.splitext(osp.basename(f))[0] + '\n')
os.makedirs(anno_dir, exist_ok=True)
for f in anno_files:
shutil.move(f, anno_dir)
dataset = Dataset.import_from(tmp_dir, 'voc', env=dm_env)
dataset.transform('masks_to_polygons')
if load_data_callback is not None:
load_data_callback(dataset, instance_data)
import_dm_annotations(dataset, instance_data)
def file_deploy(resource_data, slash='/'):
"""
Deploy downloaded files
Input:
resource_data: dictionary string containing file configuration data
slash: os dependant directory delimiter:
forward slash '/' - 'posix'
backwards slash '\\' - 'nt'
Output:
files dwonloaded if necessary and deployed to file system with
reference to os dependant slash
"""
if not os.path.isfile(path_slash(resource_data['file_path'], '/', slash)):
print('Downloading {} file in {} file format'\
.format(resource_data['format'], resource_data['description']))
try:
urllib.request.urlretrieve(resource_data['down_url'],
resource_data['zip_path'].\
replace('/', slash))
except urllib.error.HTTPError:
print("URL not found")
if resource_data['zip_path'].endswith('zip') is True:
print('Unzipping {} file in {} file format'\
.format(resource_data['description'], resource_data['format']))
print('extracting files')
Archive(path_slash(resource_data['zip_path'],'/', slash)).\
extractall(path_slash(resource_data['zip_dir'],'/', slash))
else:
print('{} file in {} file format exists'\
.format(resource_data['description'], resource_data['format']))
def unarchive(list_files, dir_files, old_file):
for infile in list_files:
if infile[-3:] == "rar" or infile[-3:] == "zip":
archive = Archive(dir_files + infile)
archive.extractall(dir_files)
print("move archive: {}".format(infile))
shutil.move(dir_files + infile, old_file + infile)
def unpack():
path = pathlib.Path('C:\\Users\\Letícia\\Documents\\Desafios Python do Carlo')
extensoes = (".zip", ".tar", ".rar", ".7z")
for i in extensoes:
for file in path.glob('**/*' + i):
Archive(file).extractall(path)
def _import(src_file, instance_data):
with TemporaryDirectory() as tmp_dir:
Archive(src_file.name).extractall(tmp_dir)
dataset = Dataset.import_from(tmp_dir, 'label_me', env=dm_env)
dataset.transform('masks_to_polygons')
import_dm_annotations(dataset, instance_data)
def walk(p: Path):
for f in p.iterdir():
f: Path
if f.is_file():
if '.py' in f.name or '.out' in f.name:
continue
if f.name.lower().endswith('.iso'):
continue
try:
Archive(f.as_posix()).extractall(p.as_posix())
except PatoolError as e:
if '.' in f.name:
logging.warning(f"Can't unpack {f.name}")
# shutil.unpack_archive(f.as_posix(), p.as_posix())
# extract(f)
if '.' not in f.name:
data = f.read_bytes()
if len(data) == 33:
data = b64decode(data)
logging.info(f"{f.as_posix().count('/'):0>2}", data)
try:
print(f"FOUND {data.decode()}")
except UnicodeDecodeError:
pass
if b'ctf' in data:
print(f"FOUND {data}")
dump.append(data)
for f in p.iterdir():
if f.is_dir():
if 'iso' in f.name.lower():
logging.warning(f"Skipping {f.name}")
walk(f)
def _get_gdalDRV_filepath(resource, resource_tmp_folder, file_name):
'''Tries to find the vector file which is going to be read by GDAL.
Returns the gdal driver name, the vector file path and if the vector
file is a Shapefile , the existence if the .prj file.
param resource_tmp_folder: The path of the created folder containing
the resource as it was downloaded
'''
global archive_mime_types
actual_resource_parent_folder = resource_tmp_folder
downloaded_res_file = os.path.join(resource_tmp_folder, file_name)
mimetype = magic.from_file(downloaded_res_file, mime=True)
# If the mimetype of the resource is an Archive format, extract the files
# and set the extaction folder as the folder to search for the vector file
if mimetype in archive_mime_types:
base = os.path.basename(downloaded_res_file)
file_name_wo_ext = os.path.splitext(base)[0]
extraction_folder = os.path.join(resource_tmp_folder, file_name_wo_ext)
os.makedirs(extraction_folder)
Archive(downloaded_res_file).extractall(extraction_folder)
file_extracted_folder = _get_file_folder(extraction_folder)
actual_resource_parent_folder = file_extracted_folder
gdal_driver, vector_file_name, prj_exists = _get_file_path(
actual_resource_parent_folder, resource)
vector_file_path = os.path.join(actual_resource_parent_folder,
vector_file_name)
return gdal_driver, vector_file_path, prj_exists
def extractArchive(modPath: str) -> str:
extractedDir = normalizePath(data.config.extracted)
modPath = normalizePath(modPath)
if (path.exists(extractedDir)):
removeDirectory(extractedDir)
while path.isdir(extractedDir):
pass
mkdir(extractedDir)
if platform == "win32" or platform == "cygwin":
si = subprocess.STARTUPINFO()
CREATE_NO_WINDOW = 0x08000000
si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
exe = getProgramRootFolder() + "/tools/7zip/7z.exe"
result = subprocess.run([exe, "x", modPath, "-o" + extractedDir, "-y"],
creationflags=CREATE_NO_WINDOW,
startupinfo=si,
stdin=subprocess.DEVNULL,
capture_output=True)
if result.returncode != 0:
raise IOError(
result.stderr.decode('utf-8') if result.
stderr else 'Could not extract archive')
else:
from pyunpack import Archive
Archive(modPath).extractall(extractedDir)
return extractedDir
def _import(src_file, instance_data):
with TemporaryDirectory() as tmp_dir:
Archive(src_file.name).extractall(tmp_dir)
dataset = Dataset.import_from(tmp_dir, 'datumaro', env=dm_env)
import_dm_annotations(dataset, instance_data)
def downloadQuota(count, url):
quota = count
while quota > 0:
html = requests.get(url + page.format(randint(1, 400))).text
soup = BeautifulSoup(html, "html.parser")
try:
for mod in soup.select("records > record"):
modPage = mod.select("a.Name")[0].get("href")
modPage = requests.get(modPage).text
soup = BeautifulSoup(modPage, "html.parser")
modFileName = soup.select(
"div.FileInfo > span > code")[0].string
modPage = soup.select(
".SmallManualDownloadIcon")[0].parent.get("href")
modPage = requests.get(modPage).text
soup = BeautifulSoup(modPage, "html.parser")
download = soup.select(
".SmallManualDownloadIcon")[0].parent.get("href")
print("downloading", modFileName, "with url", download)
filename = "./custom/TF2AutoMod/" + modFileName
# NOTE the stream=True parameter below
with requests.get(download, stream=True) as r:
r.raise_for_status()
with open(filename, 'wb') as f:
for chunk in tqdm(r.iter_content(chunk_size=1024),
unit="KB",
desc=filename):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
if modFileName.endswith(".zip") or modFileName.endswith(
".rar"):
Archive(filename).extractall('../')
quota -= 1
except Exception as e:
print(e)
def _import(src_file, instance_data, load_data_callback=None):
with TemporaryDirectory() as tmp_dir:
Archive(src_file.name).extractall(tmp_dir)
image_info = {}
frames = [YoloExtractor.name_from_path(osp.relpath(p, tmp_dir))
for p in glob(osp.join(tmp_dir, '**', '*.txt'), recursive=True)]
root_hint = find_dataset_root(
[DatasetItem(id=frame) for frame in frames], instance_data)
for frame in frames:
frame_info = None
try:
frame_id = match_dm_item(DatasetItem(id=frame), instance_data,
root_hint=root_hint)
frame_info = instance_data.frame_info[frame_id]
except Exception: # nosec
pass
if frame_info is not None:
image_info[frame] = (frame_info['height'], frame_info['width'])
dataset = Dataset.import_from(tmp_dir, 'yolo',
env=dm_env, image_info=image_info)
if load_data_callback is not None:
load_data_callback(dataset, instance_data)
import_dm_annotations(dataset, instance_data)
def extractSourceZipFiles(self, source_meta_path, zip_files_path_prefix,
data_destination_directory):
args = sys.argv
file_extension = ".7z"
source_folder_path = ""
domains = []
zip_file_paths = []
#parse source meta file
context = xml.iterparse(source_meta_path, events=("start", "end"))
context = iter(context)
event, root = context.next()
for event, elem in context:
if event == "end":
if elem.attrib.has_key(
'Url') and "meta." not in elem.attrib['Url']:
domains.append(elem.attrib['TinyName'])
zip_file_paths.append(
urlparse(elem.attrib['Url']).hostname)
# subsample if source is not ssd
if not len(args) > 1 or not args[1] == 'ssd':
# too big for my machine, only trying random 10 sources
zip_file_paths = random.sample(zip_file_paths, 10)
# iterate through zip file and extract to destination
for zip_file in zip_file_paths:
print("Started extracting file: " + zip_file)
path_to_zip_file = zip_files_path_prefix + zip_file + file_extension
directory_to_extract_to = data_destination_directory + zip_file
os.mkdir(directory_to_extract_to)
Archive(path_to_zip_file).extractall(directory_to_extract_to)
print("Processed extracting file: " + zip_file)
def load(file_object, annotations):
from pyunpack import Archive
import os.path as osp
from tempfile import TemporaryDirectory
from glob import glob
from datumaro.plugins.yolo_format.importer import YoloImporter
from cvat.apps.dataset_manager.bindings import import_dm_annotations
archive_file = file_object if isinstance(file_object, str) else getattr(
file_object, "name")
with TemporaryDirectory() as tmp_dir:
Archive(archive_file).extractall(tmp_dir)
image_info = {}
anno_files = glob(osp.join(tmp_dir, '**', '*.txt'), recursive=True)
for filename in anno_files:
filename = osp.basename(filename)
frame_info = None
try:
frame_info = annotations.frame_info[int(
osp.splitext(filename)[0])]
except Exception:
pass
try:
frame_info = annotations.match_frame(filename)
frame_info = annotations.frame_info[frame_info]
except Exception:
pass
if frame_info is not None:
image_info[osp.splitext(filename)[0]] = \
(frame_info['height'], frame_info['width'])
dm_project = YoloImporter()(tmp_dir, image_info=image_info)
dm_dataset = dm_project.make_dataset()
import_dm_annotations(dm_dataset, annotations)
async def download_images(url_dir):
data_path = path / 'dataset_test.rar'
await download_file(url_dir, data_path) # download data from Dropbox
os.makedirs(path / 'reto_deep_learning' / 'test', exist_ok=True)
Archive(data_path).extractall("./app/reto_deep_learning/test")
#rar = rarfile.RarFile(data_path)
#rar.extractall()
local_data_dir = './app/reto_deep_learning'
'''
# r=root, d=directory, f=files
print(path/'reto_deep_learning')
for r, d, f in os.walk(path/'reto_deep_learning'):
for directory in d:
print(len(d))
print('\n')
#print(r)
if directory == 'test':
local_data_dir = os.path.join(r)
else:
print('No existe folder de test. Renombrar folder a test')
os.makedirs(os.path.join(local_data_dir,'test_img/','class0'), exist_ok=True)
for r, d, files in os.walk(path/'reto_deep_learning/test_img'):
for f in files:
os.replace(f'{local_data_dir}/test_img/{f}',f'{local_data_dir}/test_img/class0/{f}')
print(f'Moving ... {local_data_dir}/test_img/{f}')
'''
return local_data_dir
def unRar(self, dirPath, destPath):
# Function to unrar .rar file.
for folder in os.listdir(dirPath):
if folder[-3:] == 'rar':
Archive(dirPath + '/' + folder).extractall(destPath)
print('Done unRAR %s' % folder)
shutil.rmtree(dirPath + '/' + folder)
def dowloadfile(adresse):
adresse = adresse.replace(chr(10), "") #remove new line
chunk_size = 1000024 #1MB
url = adresse
r = requests.get(url, stream=True, allow_redirects=True
) #Permet de télécharger le fichier quand t est utilisé
total_size = int(r.headers['content-length']
) #Calcul le nombre d'octets sans télécharger le fichier
print(total_size)
try: #Avec redirection
fname = re.findall(
"filename=(.+)",
r.headers["Content-Disposition"])[0] #Find name off file
filename = fname.replace(chr(34), "") #enlève les "" de mort
except: #Sans redirection
filename = url.split("/")[-1]
print(filename)
with open(filename, 'wb') as f:
for data in tqdm(iterable=r.iter_content(chunk_size=chunk_size),
total=int((total_size / chunk_size) * 0.954),
unit='MB'): #Download file with progress bar
print("")
f.write(data) #Write in file
print("Download complete!")
print(filename)
foldername = filename.replace(".zip",
"") #Il faudra ajouter d'autres extensions
if os.path.exists(foldername) == True: #Check if folder already exist
print("Already exsit deleting...")
shutil.rmtree(foldername)
os.mkdir(foldername)
Archive(filename).extractall(
os.getcwd() + "/" + foldername) #Décompresse ou le script ce situe
print(foldername + " Downloaded")
