def read_results(self, calc, label):
'''
Get the results
'''
results = super().read_results(calc, label)
print('THREAD ID {} START READING THE EPSILON'.format(threading.get_native_id()))
# Add the additional information related to the epsilon
prefix = label
epsilon_data = None
eps_real = np.loadtxt(os.path.join(self.local_workdir, 'epsr_{}.dat'.format(prefix)))
eps_imag = np.loadtxt(os.path.join(self.local_workdir, 'epsi_{}.dat'.format(prefix)))
nw = eps_real.shape[0]
epsilon_data = np.zeros((nw, 3), dtype = np.double)
epsilon_data[:,0] = eps_real[:,0]
epsilon_data[:, 1] = np.mean(eps_real[:, 1:], axis = 1)
epsilon_data[:, 2] = np.mean(eps_imag[:, 2:], axis = 1)
results['epsilon'] = epsilon_data
print('THREAD ID {} READED ALSO THE EPSILON!'.format(threading.get_native_id()))
return results
def disconnect(self) -> None:
"""Call this function to terminate the connections with TWS."""
# We would like to call EClient.disconnect, but it does not wait for
# the reader thread to come home which leads to problems if a connect
# is done immediately after the disconnect. The still running reader
# thread snatches the early handshaking messages and leaves the
# connect hanging. The following code is from client.py and is
# modified here to add the thread join to ensure the reader comes
# home before the disconnect returns.
# Note also the use of the disconnect lock to serialize the two known
# cases of disconnect being called from different threads (one from
# mainline through disconnect_from_ib in AlgoApp, and one from the
# EClient run method in the run thread.
call_seq = get_formatted_call_sequence()
logger.debug("%s entered disconnect", call_seq)
with self.disconnect_lock:
logger.debug("%s setting conn state", call_seq)
self.setConnState(EClient.DISCONNECTED)
if self.conn is not None:
logger.info("%s disconnecting", call_seq)
self.conn.disconnect()
self.wrapper.connectionClosed()
reader_id = id(self.reader)
my_id = get_ident()
my_native_id = get_native_id()
logger.debug(
'about to join reader id %d for self id %d to'
' wait for it to come home on thread %d %d', reader_id,
id(self), my_id, my_native_id)
self.reader.join()
logger.debug(
'reader id %d came home for id(self) %d '
'thread id %d %d', reader_id, id(self), my_id,
my_native_id)
self.reset()
def worker_init_fn(self):
"""Initialize processors used for processing the feature on the correct device"""
self = deepcopy(self)
rank = threading.get_native_id()
device = f"cuda:{rank % torch.cuda.device_count()}" if torch.cuda.is_available() else "cpu"
for i, processor in enumerate(self.processors):
processor.initialize(device)
if isinstance(processor.model, torch.nn.Module):
# TODO figure out a way to move this out of worker_init, this is executed on all workers --> inefficient
# must be called AFTER processor.initialize() though
try:
# try to convert Processor's model to TorchScript which avoids python's GIL for heavy computation
self.processors[i].model = torch.jit.script(processor.model) # TODO get example input, use trace()
except Exception as e:
print(e)
print(
f"WARNING: converting {self.processors[i].__class__.__name__} to TorchScript failed, feature calculation might be slower..."
)
# HACK to get around ThreadPool pickling of member functions. This results in the main thread's Feature object
# being "self" rather than the worker's Feature object. Therefore we load the worker's process_batch function
# into thread-local scope and execute it through the above helper function (worker_process_batch)
LOCAL.process_batch = self.process_batch
def log_webhook_execution_result(success, hook_step=None, json_data=None):
worker_name = os.environ["CELERY_WORKER_NAME"]
if worker_name is None:
LOGGER.warning(
"No name set for current worker, falling back to using OS-assigned thread ID"
)
worker_name = threading.get_native_id()
# Store stats for current worker
current_stats = (
CredentialHookStats.objects.filter(worker_id=worker_name).first()
or CredentialHookStats(worker_id=worker_name))
if success is False or hook_step is HookStep.FIRST_ATTEMPT:
current_stats.attempt_count = current_stats.attempt_count + 1
current_stats.total_count = current_stats.total_count + 1
elif success is False and hook_step is HookStep.RETRY:
current_stats.retry_count = current_stats.retry_count + 1
current_stats.total_count = current_stats.total_count + 1
elif success is False and hook_step is HookStep.RETRY_FAIL:
current_stats.retry_fail_count = current_stats.retry_fail_count + 1
elif success is False and hook_step is None:
current_stats.fail_count = current_stats.fail_count + 1
elif success is True:
current_stats.success_count = current_stats.success_count + 1
else:
LOGGER.warning(
f"Unexpected argument combination: success={success}, hook_step={hook_step}"
)
current_stats.save()
def handle_os_interrupt(signum: int, frame: types.FrameType) -> None:
# If we use a POSIX system, then raising an exception in here works perfectly and shuts the loop down
# with an exception, which is good.
# Windows, however, is special on this front. On Windows, the exception is caught by whatever was
# currently running on the event loop at the time, which is annoying for us, as this could be fired into
# the task for an event dispatch, for example, which is a guarded call that is never waited for by design.
# We can't always safely intercept this either, as Windows does not allow us to use asyncio loop
# signal listeners (since Windows doesn't have kernel-level signals, only emulated system calls
# for a remote few standard C signal types). Thus, the best solution here is to set the close bit
# instead, which will let the bot start to clean itself up as if the user closed it manually via a call
# to `bot.close()`.
nonlocal interrupt
signame = signal.strsignal(signum)
assert signame is not None # Will always be True
interrupt = errors.HikariInterrupt(signum, signame)
# The loop may or may not be running, depending on the state of the application when this occurs.
# Signals on POSIX only occur on the main thread usually, too, so we need to ensure this is
# threadsafe if we want the user's application to still shut down if on a separate thread.
# We log native thread IDs purely for debugging purposes.
if _LOGGER.getEffectiveLevel() <= ux.TRACE:
_LOGGER.log(
ux.TRACE,
"interrupt %s occurred on thread %s, bot on thread %s will be notified to shut down shortly\n"
"Stacktrace for developer sanity:\n%s",
signum,
threading.get_native_id(),
loop_thread_id,
"".join(traceback.format_stack(frame)),
)
asyncio.run_coroutine_threadsafe(
self._set_close_flag(signame, signum), loop)
def run(self):
timeoutMS = 1000
print("ebus reader thread id: ", threading.get_native_id())
while not self.quit_thread_flag:
if self.connected and self.streamOpened:
(img_buffer, img_info) = ebus.getImage(timeoutMS)
img_np = np.frombuffer(img_buffer, np.uint16)
info = ebus.expand_img_info_tuple(img_info)
img_np = img_np.reshape(info['Height'], info['Width'])
frame_timestamp = img_np[0, 0]
# img_copy = np.delete(img_np, 1, axis=1) # remove column that contains the frame timestamps, also creates a copy
img_copy = img_np.copy(
) # make a copy so that we can release the ebus buffer for the driver to re-use, while still doing operations on it
ebus.releaseImage()
# img_copy = img_copy[0:img_copy.shape[0], 1:img_copy.shape[1]].copy()
img_copy[:,
0] = img_copy[:,
1] # blank out the timestamp column... couldn't get delete() or slicing operations to work without issues...
if self.quit_thread_flag:
break
self.signals.newImage.emit(img_copy, frame_timestamp,
time.perf_counter())
else:
time.sleep(0.1) # idle while not connected
self.disconnect()
def request_handler(event_generator):
request = next(event_generator, None)
if not isinstance(request, h11.Request):
return
def gen():
yield request
yield from event_generator
handler = routes.get((request.method, request.target), None)
print(threading.get_native_id(),
"router",
request.method,
request.target,
handler_name(handler),
file=sys.stdout)
if handler is None:
yield h11.Response(
status_code=404,
headers=[
("content-type", "text/plain"),
],
)
yield h11.Data(data=b"404 Not Found\n\nsubscribe to Lulu-chan\n")
else:
yield from handler(gen())
def test_various_ops(self):
# This takes about n/3 seconds to run (about n/3 clumps of tasks,
# times about 1 second per clump).
NUMTASKS = 10
# no more than 3 of the 10 can run at once
sema = threading.BoundedSemaphore(value=3)
mutex = threading.RLock()
numrunning = Counter()
threads = []
for i in range(NUMTASKS):
t = TestThread("<thread %d>" % i, self, sema, mutex, numrunning)
threads.append(t)
self.assertIsNone(t.ident)
self.assertRegex(repr(t), r'^<TestThread\(.*, initial\)>$')
t.start()
native_ids = set(t.native_id
for t in threads) | {threading.get_native_id()}
self.assertNotIn(None, native_ids)
self.assertEqual(len(native_ids), NUMTASKS + 1)
if verbose:
print('waiting for all tasks to complete')
for t in threads:
t.join()
self.assertFalse(t.is_alive())
self.assertNotEqual(t.ident, 0)
self.assertIsNotNone(t.ident)
self.assertRegex(repr(t), r'^<TestThread\(.*, stopped -?\d+\)>$')
if verbose:
print('all tasks done')
self.assertEqual(numrunning.get(), 0)
def rename_file_helper(src_path, des_path, tries, max_tries, sleep_time):
if tries >= max_tries:
return
tries = tries + 1
try:
os.rename(src_path, des_path)
print("Moved ", src_path, " to ", des_path)
except PermissionError as error:
print("PermissionError - Thread: ", str(threading.get_native_id()))
time.sleep(sleep_time)
rename_file_helper(src_path, des_path, tries, max_tries, sleep_time)
except FileNotFoundError as error:
print("Another event must have moved the file because ", src_path,
" does not exist")
except FileExistsError as error:
full_path_and_file_name, decimal, file_extension = des_path.rpartition(
".")[:3]
unique_file_name = "%s%s%s%s" % (full_path_and_file_name, "-1",
decimal, file_extension)
rename_file_helper(src_path, unique_file_name, tries, max_tries,
sleep_time) # possible infinite loop
except Exception as error:
print("Unexpected exception:", error)
raise error
def first_function():
t_name = threading.currentThread().getName()
tid = threading.get_native_id()
mid = threading.main_thread().native_id
print(f"{t_name} is starting, {tid}, {mid}")
time.sleep(2)
print(f"{t_name} is exiting, {tid}, {mid}")
return
def __init__(self, serverName, serverPort):
super(Client, self).__init__()
self.serverName = serverName
self.serverPort = serverPort
self.genericMessage = "Message from Thread: " + str(
threading.get_native_id())
self.stopServerMessage = "BASTA"
def __init__(self, db_name: str):
"""
Create a lock, with the current thread id as the lock id,
and the current time in nanoseconds as the time.
"""
self.id = str(threading.get_native_id())
self.time_ns = time.time_ns()
self.db_name = db_name
self.path = None
def __init__(self, *args, **kwargs):
""" """
super().__init__()
self.path_to_watch = args[0]
print("CameraProcessor main thread id: ", threading.get_native_id())
self.frame_timestamps = np.zeros(1000)
self.cpu_timestamps = np.zeros(1000)
self.ind_stamp = 0
with open("frame_timestamps.bin", "wb") as f:
pass
with open("cpu_timestamps.bin", "wb") as f:
pass
print('Cleared timestamps file')
self.imgProc = image_processor.ImageProcessor()
self.imgProcPlugin = image_processor_plugin.ImageProcessor()
self.camera = sui_camera.SUICamera()
self.ebusReader = ebus_reader.EbusReader(use_mock=False,
camera=self.camera)
self.fileWatcher = QtCore.QFileSystemWatcher(
["image_processor_plugin.py"])
self.fileWatcher.fileChanged.connect(self.fileWatcher_fileChanged)
self.setupUI()
self.widgets_exposure = {
'requestedCounts': self.editExposureCount,
'requestedMS': self.lblExposureMSRequested,
'actualMS': self.lblExposureMSActual,
'actualCounts': self.lblExposureCounts,
}
self.widgets_frameperiod = {
'requestedCounts': self.editFrameCount,
'requestedMS': self.lblFrameMSRequested,
'actualMS': self.lblFrameMSActual,
'actualCounts': self.lblFrameCounts,
}
for w in [
self.editPixelPitch, self.editTargetDistance,
self.editResolution, self.editFocalLength
]:
w.editingFinished.connect(self.updateFOV)
# start polling timer for the serial reads:
self.timer = QtCore.QTimer()
self.timer.timeout.connect(self.pollSerial)
self.timer_period_ms = 100
self.timer.start(self.timer_period_ms)
# start the image reader thread:
# it will actually sit idle until we actually connect and open a stream:
self.startImageReadoutThread()
def ping(dest_addr: str, timeout: int = 4, unit: str = "s", src_addr: str = None, ttl: int = None, seq: int = 0, size: int = 56, interface: str = None) -> float:
"""
Send one ping to destination address with the given timeout.
Args:
dest_addr: The destination address, can be an IP address or a domain name. Ex. "192.168.1.1"/"example.com"
timeout: Time to wait for a response, in seconds. Default is 4s, same as Windows CMD. (default 4)
unit: The unit of returned value. "s" for seconds, "ms" for milliseconds. (default "s")
src_addr: WINDOWS ONLY. The IP address to ping from. This is for multiple network interfaces. Ex. "192.168.1.20". (default None)
interface: LINUX ONLY. The gateway network interface to ping from. Ex. "wlan0". (default None)
ttl: The Time-To-Live of the outgoing packet. Default is None, which means using OS default ttl -- 64 onLinux and macOS, and 128 on Windows. (default None)
seq: ICMP packet sequence, usually increases from 0 in the same process. (default 0)
size: The ICMP packet payload size in bytes. If the input of this is less than the bytes of a double format (usually 8), the size of ICMP packet payload is 8 bytes to hold a time. The max should be the router_MTU(Usually 1480) - IP_Header(20) - ICMP_Header(8). Default is 56, same as in macOS. (default 56)
Returns:
The delay in seconds/milliseconds or None on timeout.
Raises:
PingError: Any PingError will raise again if `ping3.EXCEPTIONS` is True.
"""
with socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP) as sock:
if ttl:
try: # IPPROTO_IP is for Windows and BSD Linux.
if sock.getsockopt(socket.IPPROTO_IP, socket.IP_TTL):
sock.setsockopt(socket.IPPROTO_IP, socket.IP_TTL, ttl)
except OSError as err:
_debug("Set Socket Option `IP_TTL` in `IPPROTO_IP` Failed: {}".format(err))
try:
if sock.getsockopt(socket.SOL_IP, socket.IP_TTL):
sock.setsockopt(socket.SOL_IP, socket.IP_TTL, ttl)
except OSError as err:
_debug("Set Socket Option `IP_TTL` in `SOL_IP` Failed: {}".format(err))
if interface:
sock.setsockopt(socket.SOL_SOCKET, SOCKET_SO_BINDTODEVICE, interface.encode()) # packets will be sent from specified interface.
_debug("Socket Interface Binded:", interface)
if src_addr:
sock.bind((src_addr, 0)) # only packets send to src_addr are received.
_debug("Socket Source Address Binded:", src_addr)
thread_id = threading.get_native_id() if hasattr(threading, 'get_native_id') else threading.currentThread().ident # threading.get_native_id() is supported >= python3.8.
process_id = os.getpid() # If ping() run under different process, thread_id may be identical.
icmp_id = zlib.crc32("{}{}".format(process_id, thread_id).encode()) & 0xffff # to avoid icmp_id collision.
try:
send_one_ping(sock=sock, dest_addr=dest_addr, icmp_id=icmp_id, seq=seq, size=size)
delay = receive_one_ping(sock=sock, icmp_id=icmp_id, seq=seq, timeout=timeout) # in seconds
except errors.HostUnknown as err: # Unsolved
_debug(err)
_raise(err)
return False
except errors.PingError as err:
_debug(err)
_raise(err)
return None
if delay is None:
return None
if unit == "ms":
delay *= 1000 # in milliseconds
return delay
def gen_dataset(p_lock,
t_lock,
num_files,
idx,
stars,
args,
max_retries=3,
time_to_wait=5):
g = github.Github(args["token"])
random.seed(threading.get_native_id())
with open(args["processed_repos_path"],
"a+") as processed_repos_file, tempfile.TemporaryDirectory(
) as tmpdir:
processed_repos_file.seek(0)
processed_repos = set(processed_repos_file.read().split("\n"))
repositories = g.search_repositories(
query=f"stars:<={stars.value} language:rust",
sort="stars",
order="desc")
prev_idx = -1
while num_files.value < args["num_files"]:
prev_idx, repo, repositories = safe_call(p_lock, t_lock, get_repo,
idx, stars, prev_idx, g,
repositories)
full_name = repo.full_name.replace("/", "_")
if full_name in processed_repos:
continue
ct = max_retries + 1
while ct := ct - 1:
try:
r = requests.get(repo.html_url + "/archive/master.zip")
break
except:
time.sleep(time_to_wait)
else:
continue
if not r.ok:
continue
path_zip = os.path.join(tmpdir, full_name + ".zip")
with open(path_zip, "wb") as f:
f.write(r.content)
with zipfile.ZipFile(path_zip, "r") as zip_ref:
zip_ref.extractall(tmpdir)
nfiles = process_project(
os.path.join(tmpdir, repo.name + "-master"), args)
with p_lock, t_lock:
num_files.value += nfiles
print(f"{full_name}: {nfiles}")
rmtree(os.path.join(tmpdir, repo.name + "-master"))
os.remove(path_zip)
processed_repos.add(full_name)
safe_call(p_lock, t_lock, processed_repos_file.write,
full_name + "\n")
processed_repos_file.flush()
def odd(thread_name):
global count2
local_data = threading.local()
while count2 < 10:
if count2 % 2 != 0:
local_data.name = thread_name
print(local_data.name + ": \t", count2, "\t", threading.get_native_id(), "\t",threading.get_ident())
# time.sleep(.5)
count2 = count2 + 1
def even(thread_name):
global count1
local_data = threading.local()
while count1 < 10:
if count1 % 2 == 0:
local_data.name = thread_name
print(local_data.name + ": \t", count1, "\t", threading.get_native_id(), "\t",threading.get_ident())
# time.sleep(.5)
count1 = count1 + 1
def _trackProgress(self, totalDown, currentDown, _totalUp, _currentUp):
part = self.threads[threading.get_native_id()]
# Add the current byte position of each individual part to a dictionary for writing in state.xml file.
self.partPosition[part] = currentDown
if currentDown != 0 and currentDown == totalDown:
self._save(part, "complete")
if self.stop:
return 1
async def count():
print("Start of count()")
print("Thread id:", threading.get_native_id(), "Process id:", os.getpid())
# time.sleep(1)
await asyncio.sleep(1)
print("Middle of count()")
await asyncio.sleep(1)
print("End of count()")
def getThreadId(cls):
try:
if cls.hasNativeTid:
return str(threading.get_native_id())
except:
pass
if sys.platform == 'win32':
return "0"
try:
libc = ctypes.cdll.LoadLibrary('libc.so.6')
tid = libc.syscall(cls.SYS_gettid)
return str(tid)
except:
return "0"
def thread_check():
"""
Check whether this thread is the main thread
Note: we don't want the thread to be main so we return True if it's not
Return False if the check doesn't pass
"""
curr_id = threading.get_native_id()
main_id = threading.main_thread().native_id
if curr_id == main_id:
log.error(f"This functions should only be called from another thread.")
return False
else:
return True
def ping(dest_addr: str, timeout: int = 4, unit: str = "s", src_addr: str = None, ttl: int = 64, seq: int = 0,
size: int = 56, privilege: bool = True) -> float or None:
"""
Send one ping to destination address with the given timeout.
Args:
dest_addr: The destination address, can be an IP address or a domain name. Ex. "192.168.1.1"/"example.com"
timeout: Time to wait for a response, in seconds. Default is 4s, same as Windows CMD. (default 4)
unit: The unit of returned value. "s" for seconds, "ms" for milliseconds. (default "s")
src_addr: The IP address to ping from. This is for multi-interface clients. Ex. "192.168.1.20". (default None)
ttl: The Time-To-Live of the outgoing packet. Default is 64, same as in Linux and macOS. (default 64)
seq: ICMP packet sequence, usually increases from 0 in the same process. (default 0)
size: The ICMP packet payload size in bytes. If the input of this is less than the bytes of a double format (usually 8), the size of ICMP packet payload is 8 bytes to hold a time. The max should be the router_MTU(Usually 1480) - IP_Header(20) - ICMP_Header(8). Default is 56, same as in macOS. (default 56)
privilege: Type of privilege to define type of socket : True for SOCK_RAW False for SOCK_DGRAM (only works on Unix system)
Returns:
The delay in seconds/milliseconds or None on timeout.
Raises:
PingError: Any PingError will raise again if `ping3.EXCEPTIONS` is True.
"""
if not privilege and platform.system() != "Windows":
sock_type = socket.SOCK_DGRAM
else:
sock_type = socket.SOCK_RAW
with socket.socket(socket.AF_INET, sock_type, socket.IPPROTO_ICMP) as sock:
sock.setsockopt(socket.SOL_IP, socket.IP_TTL, ttl)
if src_addr:
sock.bind((src_addr, 0)) # only packets send to src_addr are received.
_debug("Socket Source Address Binded:", src_addr)
thread_id = threading.get_native_id() if hasattr(threading,
'get_native_id') else threading.currentThread().ident # threading.get_native_id() is supported >= python3.8.
icmp_id = checksum(str(thread_id).encode()) # using checksum to avoid icmp_id collision.
try:
send_one_ping(sock=sock, dest_addr=dest_addr, icmp_id=icmp_id, seq=seq, size=size)
delay = receive_one_ping(sock=sock, icmp_id=icmp_id, seq=seq, timeout=timeout) # in seconds
except errors.HostUnknown as e: # Unsolved
_debug(e)
_raise(e)
return False
except errors.PingError as e:
_debug(e)
_raise(e)
return None
if delay is None:
return None
if unit == "ms":
delay *= 1000 # in milliseconds
return delay
def _lockaquire(self, filename):
timeout = 60
t = datetime.datetime.now()
timestart = t.timestamp()
lockfile = "{}.lock".format(filename)
while os.path.isfile(lockfile):
timenow = t.timestamp()
if (timenow - timestart)>timeout:
return False
break
self.debugmsg(6, "waiting for lock on", filename)
time.sleep(0.1)
with open(lockfile, 'w') as f:
f.write("{}".format(threading.get_native_id()))
self.debugmsg(9, "lock aquired on", filename)
return True
def run(self):
randomType = coffeeTypes[random.randint(0,2)]
newCoffee = None
if randomType == "Americano":
newCoffee = Americano()
elif randomType == "Espresso":
newCoffee = Espresso()
elif randomType == "Capuccino":
newCoffee = Capuccino()
print("Producer " + str(threading.get_native_id()) + " produce " + str(newCoffee.get_size()) + " " + newCoffee.get_name())
producerSem.acquire()
self.distribuitor.put_coffee(newCoffee)
consumerSem.release()
def __init__(self, name: str, text: Optional[str] = None) -> None:
"""
Constructs an instance of waterfalls.Timer.
Args:
name: Name of the timer.
text: Text of the first timing block, useful when using `Timer` as decorator or context manager.
"""
self.name: str = name
self.text: Optional[str] = str(text) if text is not None else None
self.blocks: List[Block] = []
self.thread_id: int = get_native_id()
self._start_time: Optional[int] = None
self._start_thread_time: int = 0
self._is_main_process: bool = current_process().name == "MainProcess"
self.__class__.instances.append(self)
def Digits(sValue):
print("\nThread ID : {}[{}]\t Thread Name : {}".format(threading.get_ident(),
threading.get_native_id(),
threading.currentThread().getName()))
iCnt = 0
# Iterate the complete string one-by-one
for char in sValue:
# filter
if(char >= '0' and char <= '9'):
iCnt = iCnt + 1
print("Count of all digits is : ", iCnt)
def __exit__(self, *exc):
"""Release the lock"""
tid = threading.get_native_id()
if self._thread_lock.locked() and self.lock_count.get(tid, 0) > 1:
self.lock_count[tid] -= 1
t = time.time() % 1000.0
LOG.debug(f"{t:.3f} :: release re-entrant %d",
self.lock_count[tid])
return
self.lock_count[tid] -= 1
assert self.lock_count[tid] == 0
LOG.debug(f"%d :: Releasing %s...", time.time() % 1000.0, self.session)
self.session.update([SessionItem.locked.set(False)],
condition=(SessionItem.locked == True))
self._thread_lock.release()
def _downloadRange(self, startRange, endRange, fileNo):
# Get uniquely assigned ID of the current thread by the kernel to identify file part.
self.threads[threading.get_native_id()] = fileNo
path = f"{self.path}/{self.fileName}{fileNo}.part"
try:
# Append bytes to file if it exists (resuming) else only write.
with open(path, "ab" if (os.path.exists(path)) else "wb") as f:
curl = pycurl.Curl()
curl.setopt(curl.URL, self.url)
curl.setopt(curl.FOLLOWLOCATION, True)
curl.setopt(curl.RANGE, f"{startRange}-{endRange}")
curl.setopt(curl.WRITEDATA, f)
curl.setopt(curl.NOPROGRESS, False)
curl.setopt(curl.MAX_RECV_SPEED_LARGE, self.limit)
curl.setopt(curl.XFERINFOFUNCTION, self._trackProgress)
curl.perform()
curl.close()
except pycurl.error:
self._save(fileNo, "incomplete")
def StaticSingleWritePingCSV(stdscr, illust_name: str,
raw_url_class_list: list, index: int,
threads: int):
global downloaded, total
pid = threading.get_native_id()
try:
r = pids.index(pid)
except:
pids.append(pid)
r = pids.index(pid)
finally:
# Rewrite FILE I/O Stream
with open("./PicsDatabase/" + illust_name + "/ping.csv",
"a+",
encoding="utf-8") as ping_stream:
ping_stream.seek(0, 2)
try:
# if .jpg works
stdscr.addstr(
r, 0, raw_url_class_list[index].url.rstrip('.jpg')[58:])
stdscr.refresh()
req.urlopen(raw_url_class_list[index].url)
ping_stream.write(raw_url_class_list[index].url + ',' +
raw_url_class_list[index].is_r18 + '\n')
except er.HTTPError as her:
# then .png works
if her.code == 404:
ping_stream.write(raw_url_class_list[index + 1].url + ',' +
raw_url_class_list[index + 1].is_r18 +
'\n')
else:
raise her
stdscr.addstr(r, 10, 2 * ' ')
ping_stream.close()
downloaded += 1
percent = "%.2f" % (downloaded / total * 100)
stdscr.addstr(
threads, 0,
str(downloaded) + '/' + str(total) + ', ' + percent + '%')
stdscr.refresh()
def __enter__(self):
tid = threading.get_native_id()
# re-lock if this thread has already got one lock
if self._thread_lock.locked() and self.lock_count.get(tid, 0) > 0:
t = time.time() % 1000.0
self.lock_count[tid] += 1
LOG.debug(f"{t:.3f} :: acquire re-entrant %d",
self.lock_count[tid])
return
start = time.time()
while not try_lock(self.session, self._thread_lock):
time.sleep(0.01)
if time.time() - start > self.timeout:
t = time.time() % 1000.0
LOG.debug(f"{t:.3f} :: Timeout getting lock")
raise LockTimeout
self.lock_count[tid] = 1
t = time.time() % 1000.0
LOG.debug(f"{t:.3f} :: Thread {tid} Locked %s", self.session)
