def download(self, post):
url, timestamp = post[0], post[1]
with requests.Session() as s:
r = s.get(url, headers=self.headers)
if r.ok:
soup = BeautifulSoup(r.content, 'lxml')
try:
source = soup.find('source')['src']
except TypeError:
source = soup.find(
'img',
{'class': 'responsive-image margin-bottom-5'})['src']
r = s.get(source, headers=self.headers)
if r.ok:
filename = os.path.join(self.model, source.split('-')[-1])
with open(filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
f.write(chunk)
if self.system == 'Windows':
setctime(filename, timestamp)
os.utime(filename, (timestamp, timestamp))
else:
self.logger.error(f'Unable to download: {source}')
else:
self.logger.error(f'Unable to download: {url}')
self.current_total += 1
def test_file_already_opened_read(tmp_path):
filepath = tmp_path / "test_file_already_opened_read.txt"
timestamp = 123456789
filepath.touch()
with open(str(filepath), "r"):
setctime(filepath, timestamp)
assert getctime(filepath) == timestamp
def download_media(media, session, directory, username):
while True:
link = media["link"]
r = session.head(link, allow_redirects=True)
if r.status_code != 200:
return
link = r.url
file_name = link.rsplit('/', 1)[-1].rsplit("?", 1)[0]
result = file_name.split("_", 1)
if len(result) > 1:
file_name = result[1]
else:
file_name = result[0]
file_name, ext = os.path.splitext(file_name)
ext = ext.replace(".", "")
date_object = datetime.strptime(media["postedAt"], "%d-%m-%Y %H:%M:%S")
directory = reformat(directory, file_name, media["text"], ext,
date_object, username)
timestamp = date_object.timestamp()
if not overwrite_files:
if os.path.isfile(directory):
return
if not os.path.exists(os.path.dirname(directory)):
os.makedirs(os.path.dirname(directory))
r = session.get(link, allow_redirects=True, stream=True)
with open(directory, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
os_name = platform.system()
if os_name != "macOS":
setctime(directory, timestamp)
print(link)
return True
def apply_datetime_attr(path, created_at: str, updated_at: str):
c_time = datetime.strptime(created_at, r'%Y-%m-%dT%H:%M:%S%z').timestamp()
m_time = datetime.strptime(updated_at, r'%Y-%m-%dT%H:%M:%S%z').timestamp()
a_time = time.time()
if is_windows():
setctime(path, c_time)
os.utime(path, (a_time, m_time))
def change_time(origin: Path, target: Path):
stat = origin.stat()
atime = stat.st_atime
mtime = stat.st_mtime
ctime = stat.st_ctime
setctime(target, ctime)
os.utime(target, (atime, mtime))
def test_atime_not_modified(tmp_path):
filepath = tmp_path / "test_atime_not_modified.txt"
filepath.touch()
before = os.path.getatime(str(filepath))
time.sleep(0.1)
setctime(filepath, 123456789)
assert os.path.getatime(str(filepath)) == before
def set_ctime_windows(filepath, timestamp):
if not win32_setctime.SUPPORTED:
return
try:
win32_setctime.setctime(filepath, timestamp)
except (OSError, ValueError):
pass
def test_forward_slash(tmp_path):
folder = tmp_path / "foo" / "bar" / "baz"
filepath = folder / "test_forward_slash.txt"
timestamp = 123456789
folder.mkdir(exist_ok=True, parents=True)
filepath.touch()
setctime(str(filepath).replace(r"\\", "/"), timestamp)
assert getctime(filepath) == timestamp
def create_new_latest_log(path):
# Create new latest.log
f = open(path + 'latest.log', 'x')
# Update creation time (stupid file system tunneling crap)
if (platform.system() == 'Windows'):
setctime(path + 'latest.log', time.time())
f.close()
def modifyFileTime(filename,
createTime,
modifyTime,
accessTime,
format="%Y/%m/%d %H:%M:%S"):
setctime(filename, getTimeStamp(createTime, format))
os.utime(
filename,
(getTimeStamp(modifyTime, format), getTimeStamp(accessTime, format)))
def set_time(file_path, year, month, day, hour, minute, second, microsecond):
date_time_tuple = (year, month, day, hour, minute, second, microsecond)
ctime = datetime.datetime(*date_time_tuple).timestamp()
mtime = datetime.datetime(*date_time_tuple).timestamp()
atime = datetime.datetime(*date_time_tuple).timestamp()
win32_setctime.setctime(file_path, ctime)
os.utime(file_path, (atime, mtime))
def test_fix_file_tunneling(tmp_path):
filepath = tmp_path / "test_fix_file_tunneling.txt"
timestamp = 123456789
filepath.touch()
before = getctime(filepath)
time.sleep(0.1)
os.remove(str(filepath))
filepath.touch()
assert getctime(filepath) == before
setctime(filepath, timestamp)
assert getctime(filepath) == timestamp
def set_creation_time(filepath):
# format filepath
new_filepath = '"{}"'.format(filepath.replace('\"', '\\"'))
if user_os == 'Windows':
# Set on Windows
if os.path.isfile(filepath):
setctime(filepath, get_mod_time())
elif user_os == 'Darwin':
# Set on Mac
os.system('SetFile -d "{}" {}'.format(get_date_str(), new_filepath))
def format_image(filepath, timestamp):
while True:
try:
if os_name == "Windows":
from win32_setctime import setctime
setctime(filepath, timestamp)
print(filepath)
os.utime(filepath, (timestamp, timestamp))
except Exception as e:
continue
break
def condo_win(src_file, dist_file):
src_meta_data = os.stat(
src_file
) # получаем метаданные, 7, 8, 9 - access time, modification time, creation time
shutil.copy2(src_file,
dist_file) # глубокое копирование файла с метаданными
setctime(dist_file, src_meta_data[9]) # меняю creation time
os.utime(
dist_file,
(src_meta_data[7],
src_meta_data[8])) # меняю access time, modification time в дисте
def format_image(filepath, timestamp):
while True:
try:
if os_name == "Windows":
from win32_setctime import setctime
setctime(filepath, timestamp)
print(f"Updated Creation Time {filepath}")
os.utime(filepath, (timestamp, timestamp))
print(f"Updated Modification Time {filepath}")
except Exception as e:
continue
break
def format_image(filepath, timestamp):
if json_global_settings["helpers"]["reformat_media"]:
while True:
try:
if os_name == "Windows":
from win32_setctime import setctime
setctime(filepath, timestamp)
# print(f"Updated Creation Time {filepath}")
os.utime(filepath, (timestamp, timestamp))
# print(f"Updated Modification Time {filepath}")
except Exception as e:
continue
break
def restore_file_times(self, pathlib_obj):
"""Restore original creation and modification times to file."""
os.utime(self.pathlib_obj,
(self.last_modification_time, self.last_modification_time))
if sys.platform == 'win32':
win32_setctime.setctime(self.pathlib_obj, self.creation_time)
elif sys.platform == 'darwin':
date = self.human_readable_date(self.creation_time)
path = str(pathlib_obj).replace(' ', r'\ ')
os.system(f'SetFile -d "{date}" {path} >/dev/null')
print(
f'{G}Restored creation and modification times: {self.human_readable_date(self.creation_time)}, '
f'{self.human_readable_date(self.last_modification_time)}')
def main():
filepath_origin = "AI_2021-02-24-10-18-13-646.png"
stat_origin = os.stat(filepath_origin)
print(stat_origin)
print_timestring(filepath_origin)
filepath = "AI_2021-02-24-10-18-13-646-quality-95.jpg"
stat = os.stat(filepath)
print(stat)
print_timestring(filepath)
# Set atime and mtime
os.utime(filepath, (stat_origin.st_atime, stat_origin.st_mtime))
# Set ctime
setctime(filepath, stat_origin.st_ctime)
print_timestring(filepath)
def copy_stat(src, dest):
stat_origin = os.stat(src)
# Set atime and mtime
os.utime(dest, (stat_origin.st_atime, stat_origin.st_mtime))
# Set ctime
import platform
sys = platform.system()
if sys == "Windows":
from win32_setctime import setctime
setctime(dest, stat_origin.st_ctime)
elif sys == "Darwin":
# Still cannot set ctime on Mac platform
# See: https://stackoverflow.com/questions/56008797/how-to-change-the-creation-date-of-file-using-python-on-a-mac
pass
def download(self, tupl):
url, filename, timestamp = tupl
with requests.Session() as s:
r = s.get(url, headers=HEADERS)
if r.ok:
path = self.check_path_exists(self.album_dir, filename)
with open(path, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
f.write(chunk)
if platform.system() == 'Windows':
setctime(path, timestamp)
os.utime(path, (timestamp, timestamp))
else:
print(
f"STATUS CODE: {r.status_code} -- Unable to download '{url}'")
def copy_file(self, srcfile=None, destfile=None, ctime=None):
path, fn = os.path.split(destfile)
if not os.path.exists(path):
os.makedirs(path)
try:
#shutil.copyfile(srcfile, destfile)
shutil.copy2(srcfile, destfile)
if ctime is not None and os.name == 'nt':
setctime(destfile, ctime)
logger.debug(f"Successfully copy {srcfile} to {destfile}.")
return True
except:
logger.error(f"{traceback.format_exc()}")
logger.error(f"Failed to copy {srcfile} to {destfile}!!!")
self.delete_if_exist(thisfile=destfile) # in case partially copied
return False
def set_creation_date_from_str(file: Path, str_datetime):
try:
# Turns out exif can have different formats - YYYY:MM:DD, YYYY/..., YYYY-... etc
# God wish that americans won't have something like MM-DD-YYYY
# The replace ': ' to ':0' fixes issues when it reads the string as 2006:11:09 10:54: 1.
# It replaces the extra whitespace with a 0 for proper parsing
str_datetime = str_datetime.replace('-', ':').replace('/', ':').replace('.', ':') \
.replace('\\', ':').replace(': ', ':0')[:19]
timestamp = timestamp_from_datetime(
_datetime.strptime(str_datetime, '%Y:%m:%d %H:%M:%S'))
_os.utime(file, (timestamp, timestamp))
if _os.name == 'nt':
_windoza_setctime.setctime(str(file), timestamp)
except Exception as e:
raise ValueError(
f"Error setting creation date from string: {str_datetime}")
def downloadMemories(path):
clear()
with open(path, 'r') as f:
content = json.load(f)
media = content['Saved Media']
print(f'[OK] Found {len(media)} files')
if not os.path.exists('memories'):
try:
os.mkdir('memories')
print('[OK] Directory created\n')
except Exception as e:
input(f'[ERROR] Could not create directory: {e}')
exit()
for data in tqdm(media, desc="[OK] Downloading: ", unit="file", ncols=70, bar_format="{desc}{n_fmt}/{total_fmt} {bar}{percentage:3.0f}%"):
date = data['Date']
url = data['Download Link']
filetype = data['Media Type']
day = date.split(" ")[0]
time = date.split(" ")[1].replace(':', '-')
filename = f'memories/{day}_{time}.mp4' if filetype == 'VIDEO' else f'memories/{day}_{time}.jpg'
if not os.path.exists(filename):
req = requests.post(url, allow_redirects=True)
response = req.text
if response == '':
print(f'\n\n[ERROR] Could not download memory: {filename[9:]}')
print('[!] If this error persists request new data\n')
continue
file = requests.get(response)
timestamp = datetime.datetime.timestamp(datetime.datetime.strptime(day + '-' + time, "%Y-%m-%d-%H-%M-%S"))
with open(filename, 'wb') as f:
f.write(file.content)
os.utime(filename, (timestamp, timestamp))
if os.name=='nt': ## only for windows overrite creation time
setctime(filename, timestamp)
print('\n\n---------------- ')
input('[OK] Finished ')
exit()
def file_read_v2(self, handle, sz, local_file, infos):
MAXIMUM_READ_SIZE = 1 << 16
full_size = sz
data = ""
if PY3:
data = b""
while sz > 0:
with open(local_file, "ab") as local_file_handle:
if sz > MAXIMUM_READ_SIZE:
toRead = MAXIMUM_READ_SIZE
else:
toRead = sz
try:
self.dispatch_packet(AFC_OP_READ,
struct.pack("<QQ", handle, toRead))
s, d = self.receive_data()
if len(d) == 0:
print(local_file + " appears to be an empty file")
local_file_handle.write(b"")
else:
local_file_handle.write(d)
except:
import traceback
traceback.print_exc()
self.lockdown = LockdownClient()
self.service = self.lockdown.startService("com.apple.afc2")
return self.file_read(handle, sz)
if s != AFC_E_SUCCESS:
break
sz -= toRead
#data += d
# Change modify times
modify_time = datetime.fromtimestamp(
int(infos['st_mtime']) // 1000000000)
modTime = time.mktime(modify_time.timetuple())
os.utime(local_file, (modTime, modTime))
birth_time = int(infos['st_birthtime']) // 1000000000
# Change creation time on windows
if os.name == 'nt':
setctime(local_file, birth_time)
return
def downloadMemories(path):
clear()
with open(path, 'r') as f:
content = json.load(f)
media = content['Saved Media']
print(f'[OK] Found {len(media)} files')
if not os.path.exists('memories'):
try:
os.mkdir('memories')
print('[OK] Directory created\n')
except Exception as e:
input(f'[ERROR] Could not create directory: {e}')
exit()
index = 1
for data in media:
date = data['Date']
url = data['Download Link']
filetype = data['Media Type']
day = date.split(" ")[0]
time = date.split(" ")[1].replace(':', '-')
filename = f'memories/{day}_{time}.mp4' if filetype == 'VIDEO' else f'memories/{day}_{time}.jpg'
if not os.path.exists(filename):
print(f'[OK] Downloading [{index}/{len(media)}]\r', end="")
req = requests.post(url, allow_redirects=True)
response = req.text
file = requests.get(response)
timestamp = datetime.datetime.timestamp(
datetime.datetime.strptime(day + '-' + time,
"%Y-%m-%d-%H-%M-%S"))
with open(filename, 'wb') as f:
f.write(file.content)
os.utime(filename, (timestamp, timestamp))
if os.name == 'nt': ## only for windows overrite creation time
setctime(filename, timestamp)
index += 1
print('\n\n---------------- ')
input('[OK] Finished ')
exit()
def excel_writer(filename, path, data, crtime, modifiedTime):
temppath = path
temppath = temppath.split('\\')
if len(temppath) > 1:
try:
os.makedirs(path.rsplit('\\', 1)[0])
except:
pass
fob = open(path, 'w+')
fob.write(data)
fob.close()
setctime(path, float(crtime))
os.utime(path, (modifiedTime, modifiedTime))
#print('Data Successfully Written')
logging.info('\tData Successfully Written for: ' + str(path))
def set_creation_date_from_str(file: Path, str_datetime):
try:
# Turns out exif can have different formats - YYYY:MM:DD, YYYY/..., YYYY-... etc
# God wish that americans won't have something like MM-DD-YYYY
# The replace ': ' to ':0' fixes issues when it reads the string as 2006:11:09 10:54: 1.
# It replaces the extra whitespace with a 0 for proper parsing
str_datetime = (str_datetime.replace("-", ":").replace(
"/", ":").replace(".", ":").replace("\\",
":").replace(": ",
":0")[:19])
timestamp = _datetime.strptime(str_datetime,
"%Y:%m:%d %H:%M:%S").timestamp()
_os.utime(file, (timestamp, timestamp))
if _os.name == "nt":
_windoza_setctime.setctime(str(file), timestamp)
except Exception as e:
print("Error setting creation date from string:")
print(e)
raise ValueError(
f"Error setting creation date from string: {str_datetime}")
def updateImageTakenDate(file):
year = 2022
month = 1
day = 16
hour = 10
minute = 11
second = 12
microsecond = 125
date = datetime.datetime(year, month, day, hour, minute, second,
microsecond)
exif_dict = piexif.load(file)
exif_date = date.strftime("%Y:%m:%d %H:%M:%S")
exif_dict['0th'][piexif.ImageIFD.DateTime] = exif_date
exif_dict['Exif'][piexif.ExifIFD.DateTimeOriginal] = exif_date
exif_dict['Exif'][piexif.ExifIFD.DateTimeDigitized] = exif_date
exif_bytes = piexif.dump(exif_dict)
piexif.insert(exif_bytes, file)
timestamp = date.timestamp()
setctime(file, timestamp)
async def format_image(filepath: str, timestamp: float):
if json_global_settings["helpers"]["reformat_media"]:
while True:
try:
if os_name == "Windows":
from win32_setctime import setctime
setctime(filepath, timestamp)
# Set jpeg Data
if os.path.splitext(filepath)[1] in ['.jpg', '.tiff']:
# print("xxx", filepath)
exif_dict = piexif.load(filepath)
datetime_str = datetime.fromtimestamp(timestamp).strftime("%Y:%m:%d %H:%M:%S")
exif_dict['Exif'][piexif.ExifIFD.DateTimeOriginal] = datetime_str
exif_dict['Exif'][piexif.ExifIFD.DateTimeDigitized] = datetime_str
piexif.remove(filepath)
exif_bytes = piexif.dump(exif_dict)
piexif.insert(exif_bytes, filepath)
os.utime(filepath, (timestamp, timestamp))
except Exception as e:
print(e, filepath, timestamp)
sleep(1)
continue
break
