def DbBuildInvert(dbtype,ele,invfunc):
if dbtype not in DBTYPES:
raise Exception('Invalid database type')
# Check ele
if ele not in ELELIST[dbtype]:
raise Exception('Invalid element')
#print '<!-- DbBuildInvert -->\n'
# Target inv db
dbfileinv = ele.upper()+'_INV.db'
# Lock and open inv db
lock = FileLock(dbfileinv,5)
lock.acquire()
#Log('DbBuildInvert open db c %s\n'%dbfileinv)
dbinv = anydbm.open(dbfileinv,'c')
# Clear inv db
dbinv.clear()
# List dir
for dbfile in os.listdir(dbtype):
id = dbfile[:-3]
#Log('DbBuildInvert open db r %s/%s\n'%(dbtype,dbfile))
db = anydbm.open('%s/%s' % (dbtype,dbfile),'r')
if db.has_key(ele):
value = db[ele]
for word in invfunc(value):
if dbinv.has_key(word):
dbinv[word] = dbinv[word] + (',%s' % id)
else:
dbinv[word] = '%s' % id
db.close()
#Log('DbBuildInvert close db r %s/%s\n'%(dbtype,dbfile))
dbinv.close()
#Log('DbBuildInvert close db c %s\n'%dbfileinv)
lock.release()
# Rebuild is no more needed
RearmRebuild(ele)
def DbBuildInvertOld(ele,invfunc):
raise Exception('Deprecated')
# Check ele
if ele not in ELELIST['maps']:
raise Exception('Invalid element')
# Target inv db
dbfileinv = ele.upper()+'_INV.db'
# Lock and open inv db
lock = FileLock(dbfileinv,5)
lock.acquire()
dbinv = anydbm.open(dbfileinv,'c')
# Clear inv db
dbinv.clear()
# List dir
for mapdbfile in os.listdir('maps'):
mapid = mapdbfile[:-3]
dbmap = anydbm.open('maps/%s' % mapdbfile,'r')
if dbmap.has_key(ele):
value = dbmap[ele]
for word in invfunc(value):
if dbinv.has_key(word):
dbinv[word] = dbinv[word] + (',%s' % mapid)
else:
dbinv[word] = '%s' % mapid
dbmap.close()
dbinv.close()
lock.release()
# Rebuild is no more needed
RearmRebuild(ele)
def runjobs(self, verbose = None):
""" Run the jobs """
# The verbose flag
if verbose is not None:
self.verbose = verbose
# Find all 'job_' methods
for (exc_name, exc_value) in inspect.getmembers(self, lambda x: inspect.ismethod(x)):
# The method's name must start with uppercase
if exc_name[0].isupper():
lock = FileLock(os.path.join('/', 'tmp', 'TranPy-%s-%s.lock' % (self.job_class, exc_name)))
# Try to get the lock
if lock.acquire():
if self.verbose:
print >>sys.stderr, 'Running %s %s' % (self.job_class, exc_name)
# Run the job
try:
exc_value()
except:
if self.verbose:
traceback.print_exc(file = sys.stderr)
finally:
# Release the lock
lock.release()
else:
if self.verbose:
print >>sys.stderr, 'Locked %s %s' % (self.job_class, exc_name)
def DbAddComment(mapid,user,comment):
mapfile = 'maps/%s.db' % mapid
if not os.access(mapfile,os.F_OK):
raise Exception('Invalid map id %s' % mapid)
d = getCurrentDate()
lock = FileLock(mapfile,5)
lock.acquire()
#Log('DbAddComment open db r %s\n' % mapfile)
db = anydbm.open(mapfile,'r')
if db.has_key('last_comment_id'):
last_comment_id = int(db['last_comment_id'])
else:
last_comment_id = 0
db.close()
#Log('DbAddComment close db r %s\n' % mapfile)
last_comment_id += 1
if last_comment_id>99999:
lock.release()
raise Exception('Max comments reached')
#Log('DbAddComment open db c %s\n' % mapfile)
db = anydbm.open(mapfile,'c')
db['last_comment_id'] = str(last_comment_id)
db['comment%.5d'%last_comment_id] = '%s,%s,%s' % (d,user,comment)
db.close()
#Log('DbAddComment close db c %s\n' % mapfile)
lock.release()
def run(self):
while True:
lock = FileLock("/var/lock/baseDaemon.lock")
#Just to be safe; pulling data shouldn't take more than 2h
lock.acquire()
wikiDatesToMongo(False)
time.sleep(2*60*60)
lock.release()
def update(self, lock=True):
if lock:
flock = FileLock(self.inifile)
flock.acquire()
try:
inifp = open(self.inifile, 'w')
self.cfg.write(inifp)
inifp.close()
if lock: flock.release()
return True
except:
if lock: flock.release()
return False
def test_wf_in_actor(workflow_start_regular, tmp_path):
fail_flag = tmp_path / "fail"
cnt = tmp_path / "count"
cnt.write_text(str(0))
lock_file = tmp_path / "lock"
@workflow.step
def start_session():
if fail_flag.exists():
raise Exception()
v = int(cnt.read_text()) + 1
cnt.write_text(str(v))
with FileLock(str(lock_file)):
return "UP"
@workflow.virtual_actor
class Session:
def __init__(self):
self._session_status = "DOWN"
@workflow.virtual_actor.readonly
def get_status(self):
return self._session_status
def update_session(self, up):
(ret, err) = up
if err is None:
self._session_status = ret
else:
self._session_status = err
return self._session_status
def session_start(self):
step = start_session.step()
return step
def session_start_with_status(self):
self._session_status = "STARTING"
return self.update_session.step(
start_session.options(catch_exceptions=True).step())
def __getstate__(self):
return self._session_status
def __setstate__(self, state):
self._session_status = state
actor = Session.get_or_create("session_id")
fail_flag.touch()
with pytest.raises(Exception):
actor.session_start.run()
fail_flag.unlink()
ray.get(workflow.resume("session_id"))
# After resume, it'll rerun start_session which will
# generate 1
assert cnt.read_text() == "1"
assert actor.session_start.run() == "UP"
assert cnt.read_text() == "2"
assert actor.session_start_with_status.run() == "UP"
assert cnt.read_text() == "3"
# Now test a new session.
actor = Session.get_or_create("session_id")
fail_flag.touch()
assert isinstance(actor.session_start_with_status.run(), Exception)
assert cnt.read_text() == "3"
lock = FileLock(str(lock_file))
lock.acquire()
fail_flag.unlink()
ret = actor.session_start_with_status.run_async()
for i in range(0, 60):
if cnt.read_text() == "4":
break
time.sleep(1)
assert cnt.read_text() == "4"
# This means when return from session_start_with_status,
# the session got updated
assert actor.get_status.run() == "STARTING"
lock.release()
assert ray.get(ret) == "UP"
class FilelockUtil(object):
"""
文件锁工具类
"""
def __init__(self):
self.flock = FileLock(file_path)
def acquire(self):
"""
文件加锁
"""
self.flock.acquire()
def release(self):
"""
文件锁释放
"""
self.flock.release()
def _get_child_pid(self, ppid):
"""
获取ppid的子进程ID号
:param ppid: 父进程PID号
:return : 子进程PID号
"""
pids = psutil.pids()
for pid in pids:
try:
parent_id = psutil.Process(pid).ppid()
if parent_id == ppid:
return pid
except Exception:
continue
return None
def _get_hwnds_for_pid(self, pid):
"""
获取对应pid进程的所有句柄
:param pid: 进程号
:return : 句柄列表
"""
def callback(hwnd, hwnds):
if win32gui.IsWindowVisible(hwnd) and win32gui.IsWindowEnabled(
hwnd):
_, found_pid = GetWindowThreadProcessId(hwnd)
if found_pid == pid:
hwnds.append(hwnd)
return True
hwnds = []
win32gui.EnumWindows(callback, hwnds)
return hwnds
def get_hwnds(self, ppid):
"""
获取ppid的子进程句柄
:param ppid: 父进程ID
:return : 句柄列表
"""
cpid = self._get_child_pid(ppid)
hwnds = self._get_hwnds_for_pid(cpid)
return hwnds
def hwnd_top_most(self, hwnds):
"""
根据句柄置顶窗口
:param hwnds: 句柄列表
:return : None
"""
for hwnd in hwnds:
win32gui.SetWindowPos(
hwnd, win32con.HWND_TOPMOST, 0, 0, 0, 0,
win32con.SWP_NOSIZE | win32con.SWP_NOMOVE
| win32con.SWP_NOACTIVATE | win32con.SWP_NOOWNERZORDER
| win32con.SWP_SHOWWINDOW)
def hwnd_not_top_most(self, hwnds):
"""
根据句柄取消窗口的置顶
:param hwnds: 句柄列表
:return : None
"""
for hwnd in hwnds:
win32gui.SetWindowPos(
hwnd, win32con.HWND_NOTOPMOST, 0, 0, 0, 0,
win32con.SWP_NOSIZE | win32con.SWP_SHOWWINDOW)
class DiskBufferWriter(object):
def __init__(self, name, filename, path):
self._data = None
self.name: str = name
self._filename: str = filename
self._path: str = path
if not os.path.exists(path):
os.makedirs(path)
self._cache_size: int = -1
self._cache_counter: int = 0
self._file_lock: bool = False
pass
def cleanup_cache(self):
prefix, suffix = self._get_file_components()
if os.path.exists(self._path):
files = [
f for f in os.listdir(self._path)
if re.match(prefix + '(_[0-9]+)?' + suffix + '(.lock)?', f)
]
for f in files:
os.remove(self._path + f)
def allow_cache(self, cache_size: int = 0):
# 0 is unlimited cache
self._cache_size = cache_size
if self._file_lock:
# To force clearing locks. Should this be optional?
self.use_file_lock()
def use_file_lock(self, use: bool = True, clear_locks: bool = True):
self._file_lock = use
try:
if self._cache_size == -1:
os.remove(self._path + self._filename)
else:
files = [
f for f in os.listdir(self._path)
if re.match(r'.*_[0-9]+.*\.lock', f)
]
for f in files:
os.remove(self._path + f)
except:
pass
@property
def data(self):
return self._data
@data.setter
def data(self, data):
self._data = data
# TODO implement file lock
outname = self._make_next_filename()
if self._file_lock:
self._lock = FileLock(self._path + outname + ".lock", timeout=1)
if self._lock.is_locked:
print("Locked " + outname)
self._lock.acquire()
with open(self._path + outname, 'w') as outfile:
json.dump(data, outfile)
for l in self.pserver.listeners:
address = "/__DiskBuffer/" + self.name
# print('sending ' + address)
# print(l.__dict__)
l.send_message(address, outname)
if self._file_lock:
self._lock.release()
def get_filename_for_writing(self):
outname = ''
if self._file_lock:
if self._lock:
print("Error, file is locked")
return ''
outname = self._make_next_filename()
self._lock = FileLock(self._path + outname + ".lock", timeout=1)
if self._lock.is_locked:
print("Locked " + outname)
self._lock.acquire()
else:
outname = self._make_next_filename()
self.outname = outname
return self._path + outname
def done_writing_file(self, filename: str = ''):
if filename.find(self._path) == 0:
filename = filename[len(self._path):]
else:
# TODO more robust checking that we are managing that file.
raise ValueError('Invalid filename')
for l in self.pserver.listeners:
address = "/__DiskBuffer/" + self.name
# print('sending ' + address)
# print(l.__dict__)
# TODO check that we are actually managing this file, as it is a vector for attack in unsecure networks.
l.send_message(address, filename)
if self._file_lock:
self._lock.release()
def expose_to_network(self, pserver: ParameterServer):
self.pserver = pserver
def _make_next_filename(self):
outname = self._filename
if self._cache_size >= 0:
if self._cache_size > 0 and self._cache_counter == self._cache_size:
self._cache_counter = 0
outname = self._make_filename(self._cache_counter)
self._cache_counter += 1
return outname
def _make_filename(self, index):
prefix, suffix = self._get_file_components()
outname = prefix + '_' + str(index) + suffix
return outname
def _get_file_components(self):
outname = self._filename
try:
index_dot = outname.index('.')
prefix = outname[0:index_dot]
suffix = outname[index_dot:]
except:
prefix = outname
suffix = ''
return [prefix, suffix]
def read(
self, file_path: Union[Path, str]
) -> Union[AllennlpDataset, AllennlpLazyDataset]:
"""
Returns an dataset containing all the instances that can be read from the file path.
If `self.lazy` is `False`, this eagerly reads all instances from `self._read()`
and returns an `AllennlpDataset`.
If `self.lazy` is `True`, this returns an `AllennlpLazyDataset`, which internally
relies on the generator created from `self._read()` to lazily produce `Instance`s.
In this case your implementation of `_read()` must also be lazy
(that is, not load all instances into memory at once), otherwise
you will get a `ConfigurationError`.
In either case, the returned `Iterable` can be iterated
over multiple times. It's unlikely you want to override this function,
but if you do your result should likewise be repeatedly iterable.
"""
if not isinstance(file_path, str):
file_path = str(file_path)
lazy = getattr(self, "lazy", None)
if lazy is None:
warnings.warn(
"DatasetReader.lazy is not set, "
"did you forget to call the superclass constructor?",
UserWarning,
)
if lazy:
return AllennlpLazyDataset(self._instance_iterator, file_path)
else:
cache_file: Optional[str] = None
if self._cache_directory:
cache_file = self._get_cache_location_for_file_path(file_path)
if cache_file is not None and os.path.exists(cache_file):
try:
# Try to acquire a lock just to make sure another process isn't in the middle
# of writing to the cache.
cache_file_lock = FileLock(
cache_file + ".lock",
timeout=self.CACHE_FILE_LOCK_TIMEOUT)
cache_file_lock.acquire()
# We make an assumption here that if we can obtain the lock, no one will
# be trying to write to the file anymore, so it should be safe to release the lock
# before reading so that other processes can also read from it.
cache_file_lock.release()
logger.info("Reading instances from cache %s", cache_file)
instances = self._instances_from_cache_file(cache_file)
except Timeout:
logger.warning(
"Failed to acquire lock on dataset cache file within %d seconds. "
"Cannot use cache to read instances.",
self.CACHE_FILE_LOCK_TIMEOUT,
)
instances = self._multi_worker_islice(
self._read(file_path))
else:
instances = self._multi_worker_islice(self._read(file_path))
# Then some validation.
if not isinstance(instances, list):
instances = list(instances)
if not instances:
raise ConfigurationError(
"No instances were read from the given filepath {}. "
"Is the path correct?".format(file_path))
# And finally we try writing to the cache.
if cache_file is not None and not os.path.exists(cache_file):
if self.max_instances is not None:
# But we don't write to the cache when max_instances is specified.
logger.warning(
"Skipping writing to data cache since max_instances was specified."
)
elif util.is_distributed() or (get_worker_info() and
get_worker_info().num_workers):
# We also shouldn't write to the cache if there's more than one process loading
# instances since each worker only receives a partial share of the instances.
logger.warning(
"Can't cache data instances when there are multiple processes loading data"
)
else:
try:
with FileLock(cache_file + ".lock",
timeout=self.CACHE_FILE_LOCK_TIMEOUT):
self._instances_to_cache_file(
cache_file, instances)
except Timeout:
logger.warning(
"Failed to acquire lock on dataset cache file within %d seconds. "
"Cannot write to cache.",
self.CACHE_FILE_LOCK_TIMEOUT,
)
return AllennlpDataset(instances)
def cache_adata(session_ID,
adata=None,
group=None,
store_dir=None,
store_name=None):
if ((store_dir is None) or (store_name is None)):
save_dir = save_analysis_path + str(session_ID) + "/"
filename = save_dir + "adata_cache"
chunk_factors = [150, 3] #faster, hot storage
else:
save_dir = store_dir
filename = save_dir + store_name
chunk_factors = [3, 3] #slower, cold storage
if not (os.path.isdir(save_dir)):
try:
print("[DEBUG] making directory:" + str(save_dir))
os.mkdir(save_dir)
except:
return None
lock_filename = (save_analysis_path + str(session_ID) + "/" + "adata.lock")
lock = FileLock(lock_filename, timeout=lock_timeout)
compressor = Blosc(cname='blosclz', clevel=3, shuffle=Blosc.SHUFFLE)
zarr_cache_dir = filename + ".zarr"
attribute_groups = [
"obs", "var", "obsm", "varm", "obsp", "varp", "layers", "X", "uns",
"raw"
]
extra_attribute_groups = ["X_dense", "layers_dense"]
if (adata is None): # then -> read it from the store
if (os.path.exists(zarr_cache_dir) is True):
store_store = zarr.DirectoryStore(zarr_cache_dir)
store = zarr.open_group(store=store_store, mode='r')
if (group in attribute_groups
): # then -> return only that part of the object (fast)
group_exists = adata_cache_group_exists(session_ID,
group,
store=store)
if (group_exists is True):
ret = read_attribute(store[group])
else:
ret = None
#store_store.close()
return ret
elif (group is
None): # then -> return the whole adata object (slow)
#adata = ad.read_zarr(zarr_cache_dir)
d = {}
for g in attribute_groups:
if (g in store.keys()):
if (adata_cache_group_exists(session_ID,
g,
store=store)):
if (g in ["obs", "var"]):
d[g] = read_dataframe(store[g])
else:
d[g] = read_attribute(store[g])
#store_store.close()
adata = ad.AnnData(**d)
if not (adata is None):
return adata
else:
print("[ERROR] adata object not saved at: " +
str(filename))
return None
else: # then -> update the state dictionary and write adata to the store
if (group is None):
cache_state(session_ID,
key="# cells/obs",
val=len(adata.obs.index))
cache_state(session_ID,
key="# genes/var",
val=len(adata.var.index))
if ("total_counts" in adata.obs):
cache_state(session_ID,
key="# counts",
val=int(np.sum(adata.obs["total_counts"])))
else:
cache_state(session_ID,
key="# counts",
val=int(np.sum(adata.X)))
elif (group == "obs"):
cache_state(session_ID, key="# cells/obs", val=len(adata.index))
elif (group == "var"):
cache_state(session_ID, key="# genes/var", val=len(adata.index))
with lock:
store_store = zarr.DirectoryStore(zarr_cache_dir)
store = zarr.open_group(store=store_store, mode='a')
if (group in attribute_groups
): # then -> write only that part of the object (fast)
if (group == "var"):
if (np.nan in adata.var.index):
adata.var.index = pd.Series(adata.var.index).replace(
np.nan, 'nanchung')
adata.var["gene_ID"] = pd.Series(
adata.var["gene_ID"]).replace(np.nan, 'nanchung')
adata.var["gene_ids"] = pd.Series(
adata.var["gene_ids"]).replace(np.nan, 'nanchung')
write_attribute(
store, group,
adata) # here "adata" is actually just a subset of adata
# write dense copies of X or layers if they're what was passed
if (group == "X"):
dense_name = "X_dense"
write_dense.delay(zarr_cache_dir, "X", dense_name,
chunk_factors)
if (group == "layers"):
for l in list(adata.keys(
)): #layers was passed with parameter name "adata"
dense_name = "layers_dense/" + str(l)
write_dense.delay(zarr_cache_dir, "layers/" + l,
dense_name, chunk_factors)
#store_store.flush()
#store_store.close()
lock.release()
else:
# check that necessary fields are present in adata object
if not ("leiden_n" in adata.obs):
if ("leiden" in adata.obs):
adata.obs["leiden_n"] = pd.to_numeric(
adata.obs["leiden"])
if not ("cell_ID" in adata.obs):
adata.obs["cell_ID"] = adata.obs.index
if not ("cell_numeric_index" in adata.obs):
adata.obs["cell_numeric_index"] = pd.to_numeric(
list(range(0, len(adata.obs.index))))
for i in ["user_" + str(j) for j in range(0, 6)]:
if not (i in adata.obs.columns):
adata.obs[i] = ["0" for k in adata.obs.index.to_list()]
if not ("gene_ID" in adata.var):
adata.var["gene_ID"] = adata.var.index
# make sure that there are no "nan" genes in the var index
if (np.nan in adata.var.index):
adata.var.index = pd.Series(adata.var.index).replace(
np.nan, 'nanchung')
adata.var["gene_ID"] = pd.Series(
adata.var["gene_ID"]).replace(np.nan, 'nanchung')
adata.var["gene_ids"] = pd.Series(
adata.var["gene_ids"]).replace(np.nan, 'nanchung')
# save it all to the cache, but make dense copies of X and layers
write_attribute(store, "obs", adata.obs)
write_attribute(store, "var", adata.var)
write_attribute(store, "obsm", adata.obsm)
write_attribute(store, "varm", adata.varm)
write_attribute(store, "obsp", adata.obsp)
write_attribute(store, "varp", adata.varp)
write_attribute(store, "uns", adata.uns)
write_attribute(store, "raw", adata.raw)
write_attribute(store, "X", adata.X)
write_attribute(store, "layers", adata.layers)
# making dense copies of X and layers (compressed to save disk space)
dense_name = "X_dense"
write_dense.delay(zarr_cache_dir, "X", dense_name,
chunk_factors)
for l in list(adata.layers.keys()):
dense_name = "layers_dense/" + str(l)
write_dense.delay(zarr_cache_dir, "layers/" + l,
dense_name, chunk_factors)
#store_store.flush()
#store_store.close()
lock.release()
# set the file mod and access times to current time
# then return adata as usual
os.utime(zarr_cache_dir)
return adata
class WeblateLock:
"""Wrapper around Redis or file based lock."""
def __init__(
self,
lock_path: str,
scope: str,
key: int,
slug: str,
cache_template: str = "lock:{scope}:{key}",
file_template: Optional[str] = "{slug}-{scope}.lock",
timeout: int = 1,
):
self._timeout = timeout
self._lock_path = lock_path
self._scope = scope
self._key = key
self._slug = slug
self._depth = 0
default_cache = caches["default"]
self.use_redis = isinstance(default_cache, RedisCache)
if self.use_redis:
# Prefer Redis locking as it works distributed
self._lock = Lock(
default_cache.client.get_client(),
name=self._format_template(cache_template),
expire=60,
auto_renewal=True,
)
else:
# Fall back to file based locking
self._lock = FileLock(
os.path.join(lock_path, self._format_template(file_template)),
timeout=self._timeout,
)
def _format_template(self, template: str):
return template.format(
scope=self._scope,
key=self._key,
slug=self._slug,
)
def __enter__(self):
self._depth += 1
if self._depth > 1:
return
if self.use_redis:
try:
if not self._lock.acquire(timeout=self._timeout):
raise WeblateLockTimeout()
except AlreadyAcquired:
pass
else:
# Fall back to file based locking
try:
self._lock.acquire()
except Timeout:
raise WeblateLockTimeout()
def __exit__(self, exc_type, exc_value, traceback):
self._depth -= 1
if self._depth > 0:
return
try:
self._lock.release()
except NotAcquired:
# This can happen in case of overloaded server fails to renew the
# lock before expiry
pass
@property
def is_locked(self):
return bool(self._depth)
class MultiprocessRotatingFileHandler(TimedRotatingFileHandler):
"""
Ref: https://github.com/di/mrfh/blob/master/mrfh/__init__.py
* 修改doRollover逻辑，避免日志分片时删除已分片日志
* 释放日志锁时关闭FileStream，解决文件重命名操作被拒绝的问题
* 频繁reopen FileStream，造成严重的性能损耗
* 使用FileLock替代threading.Lock，单进程条件下存在性能损失
"""
def __init__(self, lock_file, *args, **kwargs):
self.file_lock = FileLock(lock_file)
super(MultiprocessRotatingFileHandler, self).__init__(*args, **kwargs)
def _open_file(self):
self.stream = self._open()
def acquire(self):
self.file_lock.acquire()
if self.stream and self.stream.closed:
self._open_file()
def release(self):
if self.stream and not self.stream.closed:
self.stream.flush()
self.stream.close()
self.file_lock.release()
def close(self):
if self.stream and not self.stream.closed:
self.stream.flush()
self.stream.close()
if self.file_lock.is_locked:
self.file_lock.release()
def doRollover(self):
if self.stream:
self.stream.close()
self.stream = None
# get the time that this sequence started at and make it a TimeTuple
current_time = int(time.time())
dst_now = time.localtime(current_time)[-1]
t = self.rolloverAt - self.interval
if self.utc:
time_tuple = time.gmtime(t)
else:
time_tuple = time.localtime(t)
dst_then = time_tuple[-1]
if dst_now != dst_then:
if dst_now:
addend = 3600
else:
addend = -3600
time_tuple = time.localtime(t + addend)
dfn = self.rotation_filename(self.baseFilename + "." +
time.strftime(self.suffix, time_tuple))
# # Changed part
# if os.path.exists(dfn):
# os.remove(dfn)
if not os.path.exists(dfn) and os.path.exists(self.baseFilename):
self.rotate(self.baseFilename, dfn)
# # Changed part end
if self.backupCount > 0:
for s in self.getFilesToDelete():
os.remove(s)
if not self.delay:
self.stream = self._open()
new_rollover_at = self.computeRollover(current_time)
while new_rollover_at <= current_time:
new_rollover_at = new_rollover_at + self.interval
# If DST changes and midnight or weekly rollover, adjust for this.
if (self.when == 'MIDNIGHT'
or self.when.startswith('W')) and not self.utc:
dst_at_rollover = time.localtime(new_rollover_at)[-1]
if dst_now != dst_at_rollover:
if not dst_now: # DST kicks in before next rollover, so we need to deduct an hour
addend = -3600
else: # DST bows out before next rollover, so we need to add an hour
addend = 3600
new_rollover_at += addend
self.rolloverAt = new_rollover_at
def download_image(url, write_path, lock_path):
"""
download_image: This method is called asynchronously by executing threads. It performs several useful functions:
* Determines if the supplied url has already been downloaded.
* Determines if another thread is currently downloading the supplied URL.
* Handles the locking and unlocking of the shared resource: image-lock files.
* Prints HTTP errors received while attempting the download of an image.
* Stores the downloaded data at the provided write_path if successfully obtained.
:param url: The image URL which is to be downloaded.
:param write_path: The path indicating where the downloaded image is to be stored.
:param lock_path: The path indicating where the .lock file for the corresponding image is to be located.
:return url: The url that this method was tasked with downloading. Upon completion, this method will have performed
the tasks listed above or returned None: (indicating to the controlling ThreadPoolExecutor that this thread is
dead and should be re-allocated with a new URL to download).
"""
time_stamp = time.time()
# Does the file already exist?
if not os.path.isfile(write_path):
# print('Working on URL: %r' % url)
# Does the lock file already exist?
if not os.path.isfile(lock_path):
# Create an empty lockfile:
open(lock_path, 'a').close()
# Lock the lockfile:
file_lock = FileLock(lock_path, timeout=0.1)
# print('Just created lockfile: %s The file is locked: %s' % (lock_path, file_lock.is_locked))
# Try and acquire the file lock to see if another thread is already working on this url:
try:
# print('Attempting to acquire lockfile: %s. The file is now locked: %s' % (lock_path, file_lock.is_locked))
with file_lock.acquire(timeout=0.1):
# If this code executes the lockfile has been acquired and is now locked by this process instance.
# print('Acquired lockfile %s. The file is locked: %s' % (lock_path, file_lock.is_locked))
# Instantiate http object per urllib3:
http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where())
dl_response = http.request('GET', url)
if dl_response.status == 200:
with open(write_path, 'wb') as fp:
fp.write(dl_response.data)
# print('Downloaded file: %s' % write_path)
return url, time_stamp
else:
print(
'Error downloading accessURI %s. Received http response %d'
% (url, dl_response.status))
except Timeout:
print(
'Attempt to acquire the file lock timed out. Perhaps another instance of this application '
'currently holds the lock: %s' % lock_path)
finally:
# NOTE: This code is guaranteed to run before any return statements are executed,
# see: https://stackoverflow.com/questions/11164144/weird-try-except-else-finally-behavior-with-return-statements
'''
NOTE: Exiting from a 'with FileLock.acquire:' block will automatically release the lock. So uncomment the,
following lines of code only if you believe that the file lock should be released by all threads if a
Timeout exception occurs as well as a success:
'''
file_lock.release()
# print('Released file lock: %s. The file is now locked: %s.' % (lock_path, file_lock.is_locked))
else:
print('The requested url: %r has already been downloaded!' %
write_path)
while 1:
dblock = FileLock("db.lock", timeout=1)
dblock.acquire()
try:
db = sqlite3.connect("./user_gamedata.db3")
cursor = db.cursor()
# DELETE timeout
count = 0
cursor.execute("SELECT * FROM lineupPool")
results = cursor.fetchall()
for record in results:
if round(time.time()) - round(record[1]) > 15:
count += 1
cursor.execute("DELETE FROM lineupPool WHERE uid='" +
str(record[0]) + "'")
db.commit()
db.close()
# Print message
print(
str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) + ": " +
str(count) + " user(s) has/have deleted")
except:
print("ERROR occured, please check")
finally:
dblock.release()
time.sleep(10)
class Container(ContainerView):
"""
Containers are sets of {key: values}.
Ideally we want to query not separate Containers
but Unions of Containers instead.
Each Container corresponds to a single RocksDB database.
Containers satisfy Dict properties per run.
"""
def __init__(self,
path: str,
read_only: bool = False,
wait_if_busy: bool = False) -> None:
self.path = Path(path)
self.read_only = read_only
self._db_opts = dict(
create_if_missing=True,
paranoid_checks=False,
keep_log_file_num=10,
skip_stats_update_on_db_open=True,
skip_checking_sst_file_sizes_on_db_open=True,
max_open_files=-1,
write_buffer_size=64 * 1024 * 1024, # 64MB
max_write_buffer_number=3,
target_file_size_base=64 * 1024 * 1024, # 64MB
max_background_compactions=4,
level0_file_num_compaction_trigger=8,
level0_slowdown_writes_trigger=17,
level0_stop_writes_trigger=24,
num_levels=4,
max_bytes_for_level_base=512 * 1024 * 1024, # 512MB
max_bytes_for_level_multiplier=8,
)
# opts.allow_concurrent_memtable_write = False
# opts.memtable_factory = aimrocks.VectorMemtableFactory()
# opts.table_factory = aimrocks.PlainTableFactory()
# opts.table_factory = aimrocks.BlockBasedTableFactory(block_cache=aimrocks.LRUCache(67108864))
# opts.write_buffer_size = 67108864
# opts.arena_block_size = 67108864
self._db = None
self._lock = None
self._wait_if_busy = wait_if_busy # TODO implement
self._lock_path: Optional[Path] = None
self._progress_path: Optional[Path] = None
# TODO check if Containers are reopenable
@property
def db(self) -> aimrocks.DB:
if self._db is not None:
return self._db
logger.debug(f'opening {self.path} as aimrocks db')
self.path.parent.mkdir(parents=True, exist_ok=True)
locks_dir = self.path.parent.parent / 'locks'
locks_dir.mkdir(parents=True, exist_ok=True)
if not self.read_only:
self._lock_path = locks_dir / self.path.name
self._lock = FileLock(str(self._lock_path), timeout=10)
self._lock.acquire()
self._db = aimrocks.DB(str(self.path),
aimrocks.Options(**self._db_opts),
read_only=self.read_only)
return self._db
@property
def writable_db(self) -> aimrocks.DB:
db = self.db
if self._progress_path is None:
progress_dir = self.path.parent.parent / 'progress'
progress_dir.mkdir(parents=True, exist_ok=True)
self._progress_path = progress_dir / self.path.name
self._progress_path.touch(exist_ok=True)
return db
def finalize(self, *, index: ContainerView):
if not self._progress_path:
return
for k, v in self.items():
index[k] = v
self._progress_path.unlink(missing_ok=False)
self._progress_path = None
def close(self):
if self._lock is not None:
self._lock.release()
self._lock = None
if self._db is not None:
# self._db.close()
self._db = None
def __del__(self):
self.close()
def preload(self):
self.db
def tree(self) -> 'TreeView':
return TreeView(self)
def batch_set(self,
key: bytes,
value: bytes,
*,
store_batch: aimrocks.WriteBatch = None):
if store_batch is None:
store_batch = self.writable_db
store_batch.put(key=key, value=value)
def __setitem__(self, key: bytes, value: bytes):
self.writable_db.put(key=key, value=value)
def __getitem__(self, key: bytes) -> bytes:
return self.db.get(key=key)
def __delitem__(self, key: bytes) -> None:
return self.writable_db.delete(key)
def batch_delete(self,
prefix: bytes,
store_batch: aimrocks.WriteBatch = None):
if store_batch is None:
batch = aimrocks.WriteBatch()
else:
batch = store_batch
batch.delete_range((prefix, prefix + b'\xff'))
if not store_batch:
self.writable_db.write(batch)
else:
return batch
def items(self, prefix: bytes = b'') -> Iterator[Tuple[bytes, bytes]]:
# TODO return ValuesView, not iterator
it: Iterator[Tuple[bytes, bytes]] = self.db.iteritems()
it.seek(prefix)
for key, val in it:
if not key.startswith(prefix):
break
yield key, val
def walk(self, prefix: bytes = b''):
it: Iterator[Tuple[bytes, bytes]] = self.db.iteritems()
it.seek(prefix)
while True:
try:
key, val = next(it)
except StopIteration:
yield None
break
jump = yield key
it.seek(jump)
def iterlevel(self, prefix: bytes = b'') -> Iterator[Tuple[bytes, bytes]]:
# TODO return ValuesView, not iterator
# TODO broken right now
it: Iterator[Tuple[bytes, bytes]] = self.db.iteritems()
it.seek(prefix)
key, val = next(it)
while True:
try:
key, val = next(it)
except StopIteration:
break
if not key.startswith(prefix):
break
next_range = key[:-1] + bytes([key[-1] + 1])
it.seek(next_range)
yield key, val
def keys(self, prefix: bytes = b''):
# TODO return KeyView, not iterator
it: Iterator[Tuple[bytes, bytes]] = self.db.iterkeys()
it.seek(prefix)
for key in it:
if not key.startswith(prefix):
break
yield key
def values(self, prefix: bytes):
# not vraz thing to implement
raise NotImplementedError
def update(self) -> None:
raise NotImplementedError
def view(
self, prefix: Union[bytes, Tuple[Union[int, str],
...]]) -> 'ContainerView':
if not isinstance(prefix, bytes):
prefix = E.encode_path(prefix)
return PrefixView(prefix=prefix, container=self)
def commit(self, batch: aimrocks.WriteBatch):
self.writable_db.write(batch)
def next_key(self, prefix: bytes = b'') -> bytes:
it: Iterator[bytes] = self.db.iterkeys()
it.seek(prefix + b'\x00')
key = next(it)
if not key.startswith(prefix):
raise KeyError
return key
def next_value(self, prefix: bytes = b'') -> bytes:
key, value = self.next_key_value(prefix)
return value
def next_key_value(self, prefix: bytes = b'') -> Tuple[bytes, bytes]:
it: Iterator[Tuple[bytes, bytes]] = self.db.iteritems()
it.seek(prefix + b'\x00')
key, value = next(it)
if not key.startswith(prefix):
raise KeyError
return key, value
def prev_key(self, prefix: bytes = b'') -> bytes:
key, value = self.prev_key_value(prefix)
return key
def prev_value(self, prefix: bytes = b'') -> bytes:
key, value = self.prev_key_value(prefix)
return value
def prev_key_value(self, prefix: bytes) -> Tuple[bytes, bytes]:
it: Iterator[Tuple[bytes, bytes]] = self.db.iteritems()
it.seek_for_prev(prefix + b'\xff')
key, value = it.get()
return key, value
if (args.show):
entryPrint(entry)
if (args.execute is not None):
if (entryExecute(entry, args.execute) != True):
ForceExit('Execution failed!')
# 4. Write entries if were changed
# #####################################################33
if (dataIsChanged == True):
jsonWrite(configFile, entries)
if (recipientsIsChanged == True):
jsonWrite(recipientsFile, recipients)
# 5. Finish execution
if (args.execute is not None):
# If report has something
if (ReportsIsAnyting(reportFile)):
# Report also assets
ReportAssets(reportFile)
# Generate HTML & PDF
ReportsToPDF(reportFile)
# Send emails to all recipients
for i in range(len(recipients)):
ReportsMail(recipients[i]['address'], reportFile + '.html')
lockRecipents.release()
lockConfig.release()
exn_log_path = ".."+ os.sep + "logs"+ os.sep +"ib_exn_log.log"
exn_log_lock_path = ".."+ os.sep + "logs"+ os.sep +"ib_exn_log.log.lock"
exn_log_lock = FileLock(exn_log_lock_path)
pos = Position()
#bot = InterfaceBot(interface_pipe_path, lock, log)
bot = InterfaceBot(interface_pipe_path, lock)
try:
while True:
try:
instr = input()
except Exception as e:
sys.stderr.write('error reading input')
outstr = parse_command(instr, bot, pos)
sys.stdout.write(outstr)
sys.stdout.flush()
except Exception as e:
try:
exn_log_lock.acquire()
exn_log = open(exn_log_path, "a+")
exn_log.write(str(e) + "\n")
exn_log.write(str(traceback.format_exc()))
exn_log.close()
finally:
exn_log_lock.release()
#log.write(str(e) + "\n")
#log.write(str(traceback.format_exc()))
#log.flush()
#log.close()
class Pack():
def __init__(self, config):
self.config = config
self.cwd = pathlib.WindowsPath(self.config['cwd'])
self.status = 0
# Prepare Tor
self.password = uuid.uuid4().hex
self.relay = tor.Tor(self.config['tor'], self.config['data'])
# torrc
torrc = pathlib.Path(config['data']) / 'torrc' / 'torrc'
self.torrc = torrc.resolve()
# The Onion Box
self.box = box.TheOnionBox(config)
# Stop signal, to terminate our run_loop
self.stop = threading.Event()
# the Tray icon
self.tray = pystray.Icon('theonionpack', title='The Onion Pack')
self.tray.icon = Image.open(str(self.cwd / 'icons' / 'top16.ico'))
self.tray.menu = pystray.Menu(
pystray.MenuItem('Monitor...',
action=self.on_monitor,
default=True), pystray.Menu.SEPARATOR,
pystray.MenuItem(
'Relay Control',
pystray.Menu(
pystray.MenuItem('Edit configuration file...',
action=self.on_open_torrc),
pystray.MenuItem('Show logfile...',
action=self.on_show_messages),
pystray.Menu.SEPARATOR,
pystray.MenuItem('Reload relay configuration',
action=self.on_reload_config))),
pystray.Menu.SEPARATOR,
pystray.MenuItem('Stop!', action=self.on_quit))
def run(self):
self.lock = FileLock(str(self.cwd / 'theonionpack.lock'))
running = False
try:
with self.lock.acquire(timeout=0):
# run The Onion Box
tob = self.box.run(password=self.password)
# run Tor
self.relay.run(password=self.password)
running = True
# run the Tray icon
# !! This is a blocking call !!
self.tray.run(self.run_loop)
# the block may be released by self.on_quit, issued by an operator via the Tray
# ... or by a system command (like SIGTERM).
except Timeout:
MBox(
"It seems like another instance of The Onion Pack is already running. Aborting launch procedure...",
style=0x10)
finally:
self.lock.release()
if running:
# Stop theonionbox
self.box.stop()
# Tor has OwningControllerProcess defined ... thus will terminate as soon as we're done.
if self.status == 1:
MBox(
"Our instance of TheOnionBox terminated.\r\nThus we have to terminate as well! Sorry...",
style=0x10)
sys.exit(0)
def run_loop(self, icon: pystray.Icon):
icon.visible = True
while self.stop.is_set() is False:
# quit if TheOnionBox died!
if self.box.poll() is not None:
# indicate that the Box terminated!
self.status += 1
self.do_quit()
return
self.relay.collect_messages()
sleep(1)
# Tray menu actions
def on_monitor(self, icon, item):
webbrowser.open_new_tab('http://127.0.0.1:8080/')
def on_quit(self, icon, item):
self.do_quit()
def do_quit(self):
# Stop the run_loop
self.stop.set()
# Stop the Tray
self.tray.stop()
# cleanup is being performed in self.run()
# def get_tor_messages(self):
# while True:
# self.relay.collect_messages()
# sleep(5)
def on_show_messages(self, icon, item):
fd, name = tempfile.mkstemp(prefix="Tor_", suffix='.html', text=True)
with open(fd, 'w') as tmp:
tmp.write('<br>'.join(self.relay.messages))
webbrowser.open_new_tab(name)
def on_open_torrc(self):
def get_default_windows_app(suffix):
class_root = winreg.QueryValue(winreg.HKEY_CLASSES_ROOT, suffix)
with winreg.OpenKey(
winreg.HKEY_CLASSES_ROOT,
r'{}\shell\open\command'.format(class_root)) as key:
command = winreg.QueryValueEx(key, '')[0]
return command.split(' ')[0]
if not self.torrc.exists():
self.torrc.parent.mkdir(parents=True, exist_ok=True)
self.torrc.touch()
path = get_default_windows_app('.txt')
subprocess.Popen([os.path.expandvars(path), str(self.torrc)])
def on_reload_config(self):
controller = None
try:
controller = SimplePort('127.0.0.1', 9051)
except Exception:
MBox('Failed to connect to the local Tor relay.', style=0x10)
if controller is None:
return
ok = ''
try:
ok = controller.msg(f'AUTHENTICATE "{self.password}"')
except:
if ok != '250 OK':
MBox('Failed to authenticate against local Tor relay.',
style=0x10)
controller.shutdown()
return
ok = ''
try:
ok = controller.msg("SIGNAL RELOAD")
except:
if ok != '250 OK':
MBox('Failed to reload the Tor relay configuration.',
style=0x10)
controller.shutdown()
return
def main(argv):
final_gen = False
version = None
start_version = None
fltg_root_dir = None
output_root_dir = None
dump_all_dir = None
global dev_included
global variant_included
optimize = True
dev_included = []
variant_included = []
try:
sdk = os.environ['SDK']
except:
print('environment variable SDK required')
sys.exit(-1)
# Check if there are any input parameters
try:
opts, args = getopt.getopt(argv, "hfnv:s:d:o:g:c:t:", ["help"])
except getopt.GetoptError:
usage()
sys.exit(-1)
for opt, arg in opts:
if opt in ("-h", "--help"):
usage()
sys.exit(0)
elif opt == '-f':
final_gen = True
elif opt == '-v':
version = arg
elif opt == '-s':
start_version = arg
elif opt == '-d':
fltg_root_dir = arg
elif opt == '-o':
output_root_dir = arg
elif opt == '-g':
dump_all_dir = arg
elif opt == '-n':
optimize = False
elif opt == '-c':
if arg.lower() not in dev_included:
dev_included.append(arg.lower())
elif opt == '-t':
if arg.lower() not in variant_included:
variant_included.append(arg.lower())
else:
print('Variant param must be preceed with device param -c')
usage()
sys.exit(-1)
else:
usage()
sys.exit(-1)
if version == None:
print('Error: version must be defined')
usage()
sys.exit(-1)
if output_root_dir == None:
output_root_dir = sdk + ISSU_DB_DIR
clean_python_default_boolean()
lock_file = output_root_dir + '/issu_db.lock'
if final_gen:
if start_version == None:
print('missing -s parameter')
usage()
sys.exit(-1)
print('Generating final LTID DB')
try:
lock = FileLock(lock_file, 1800) # wait up to 30 min
lock.acquire()
except Timeout:
print('Failed to acquire ISSU DB lock')
sys.exit(-1)
rv = prepare_final_db(output_root_dir, start_version)
if rv != 0:
lock.release()
sys.exit(rv)
if start_version != version:
print('Resolving ENUM differences')
rv = prepare_enum_defs(output_root_dir, start_version, version,
optimize)
if rv != 0:
lock.release()
sys.exit(rv)
# Parse component action YAML
rv = parse_comp_actions(sdk, sdk + ISSU_DLL_DIR)
lock.release()
sys.exit(rv)
if version == None:
print('-v required')
usage()
sys.exit(-1)
# Open the output file
v = version.replace('.', '_')
out_f = output_root_dir + '/ver_' + version + '/ltids_db_v' + v + '.ltdb'
output_dir = output_root_dir + '/ver_' + version + '/'
try:
lock = FileLock(lock_file, 1800) # wait up to 30 min
lock.acquire()
except Timeout:
print('Failed to acquire ISSU DB lock')
sys.exit(-1)
try:
fh = open(out_f, "w")
except:
print('Failed to create file ', out_f)
lock.release()
sys.exit(-1)
write_out_header(fh)
if fltg_root_dir == None:
fltg_root_dir = sdk + FLTG_INPUT_DIR
rv = parse_ltid_yml_files(fltg_root_dir, output_dir, dump_all_dir, version,
optimize, fh)
fh.close()
lock.release()
sys.exit(rv)
def handle_client(self, conn, addr):
"""receive request from the client and send response"""
self.print_debug("Connection established with client " + addr[0] + ":" + str(addr[1]) + " at " +
time.strftime("%a, %d %b %Y %I:%M:%S %p %Z", time.gmtime()))
try:
while True:
_receive_data = []
_tmp = conn.recv(2048)
if _tmp:
_tmp = _tmp.decode("utf-8") # convert to text string
_receive_data.append(_tmp) # append to receive array
else:
break
str_received_data = ''
for data in _receive_data:
str_received_data += data # convert received array to string
print(str_received_data)
first_line = str_received_data.split("\r\n", 1)
req_type = first_line[0].split(" ")[0]
uri = first_line[0].split(" ")[1]
headers = {}
header_body = str_received_data.split("\r\n\r\n")
count = 0
for line in header_body[0].split("\r\n"):
if count != 0:
headers[line.split(":")[0].strip()] = line.split(":")[1].strip()
else:
count = count + 1
count = -1
body = ''
if len(header_body) > 1:
if header_body[0].find("Content-Length:") > -1:
last = header_body[0].find("\r\n", header_body[0].find("Content-Length:"))
--last
count = int(header_body[0][header_body[0].find("Content-Length:") + 16:last])
body = ''
for line in header_body[1]:
if count > 0:
body += line
--count
http_object = HTTPObject(req_type, uri, headers, body)
# Sending code
message = 'HTTP/1.1 '
if ".." in uri:
self.print_debug("Access Denied " + uri)
message += "403 Forbidden" + "\r\n"
message += "Content-type: text/plain" + "\r\n"
message += "Content-Disposition: inline" + "\r\n\r\n"
else:
if HTTPObject.get_req_type(http_object).lower() == "get":
try:
if HTTPObject.get_uri(http_object) == "/":
message += "200 OK" + "\r\n"
message += "Date: " + formatdate(timeval=None, localtime=False, usegmt=True) + "\r\n"
message += "Server:" + socket.gethostname() + "\r\n"
self.print_debug("GET DIR " + os.getcwd().replace("\\", "/") + "/" +
getattr(self._server_obj, '_path') + HTTPObject.get_uri(http_object))
working_dir = os.getcwd().replace("\\", "/") + "/" + getattr(self._server_obj, '_path') \
+ HTTPObject.get_uri(http_object)
list_files = os.listdir(working_dir)
str_files = ''
for file in list_files:
str_files += file + "\r\n"
message += "Content-Length: " + str(len("{\r\n" + str_files + "}")) + "\r\n"
message += "Content-Type: text/directory" + "\r\n"
if "Content-Disposition" in HTTPObject.get_headers(http_object):
if HTTPObject.get_headers(http_object)["Content-Disposition"] == "attachment":
message += "Content-Disposition: attachment/output" + "\r\n"
else:
message += "Content-Disposition: " + HTTPObject.get_headers(http_object) \
["Content-Disposition"] + "\r\n"
elif "inline" in HTTPObject.get_uri(http_object):
message += "Content-Disposition: inline" + "\r\n"
else:
message += "Content-Disposition: attachment/output" + "\r\n"
message += "\r\n"
message += "{\r\n" + str_files + "}"
else:
working_file = os.getcwd().replace("\\", "/") + "/" + getattr(self._server_obj, '_path') \
+ HTTPObject.get_uri(http_object)
if "Content-Type" in HTTPObject.get_headers(http_object):
if HTTPObject.get_headers(http_object)["Content-Type"] == "text/plain":
if not working_file.endswith(".txt"):
working_file += ".txt"
elif HTTPObject.get_headers(http_object)["Content-Type"] == "text/html":
if not working_file.endswith(".html"):
working_file += ".html"
elif HTTPObject.get_headers(http_object)["Content-Type"] == "text/html":
if not working_file.endswith(".html"):
working_file += ".html"
elif HTTPObject.get_headers(http_object)["Content-Type"] == "text/xml":
if not working_file.endswith(".xml"):
working_file += ".xml"
elif HTTPObject.get_headers(http_object)["Content-Type"] == "application/json":
if not working_file.endswith(".json"):
working_file += ".json"
self.print_debug("GET File " + working_file)
if not os.path.isfile(working_file):
message += "404 Not Found" + "\r\n"
message += "Date: " + formatdate(timeval=None, localtime=False,
usegmt=True) + "\r\n"
message += "Server: " + socket.gethostname() + "\r\n"
if "Content-Disposition" in HTTPObject.get_headers(http_object):
if HTTPObject.get_headers(http_object)["Content-Disposition"] == "attachment":
message += "Content-Disposition: attachment/output" + "\r\n"
else:
message += "Content-Disposition: " + HTTPObject.get_headers(http_object) \
["Content-Disposition"] + "\r\n"
elif "inline" in HTTPObject.get_uri(http_object):
message += "Content-Disposition: inline" + "\r\n"
else:
message += "Content-Disposition: inline" + "\r\n"
message += "\r\n"
else:
message += "200 OK" + "\r\n"
message += "Date: " + formatdate(timeval=None, localtime=False,
usegmt=True) + "\r\n"
message += "Server: " + socket.gethostname() + "\r\n"
fr = open(working_file, 'r')
file_data = fr.read()
if "Content-Type" in HTTPObject.get_headers(http_object):
message += "Content-Type: " + HTTPObject.get_headers(http_object)[
"Content-Type"] + "\r\n"
else:
if working_file.endswith(".txt"):
message += "Content-Type: text/plain" + "\r\n"
elif working_file.endswith(".html"):
message += "Content-Type: text/html" + "\r\n"
elif working_file.endswith(".xml"):
message += "Content-Type: text/xml" + "\r\n"
elif working_file.endswith(".json"):
message += "Content-Type: application/json" + "\r\n"
else:
message += "Content-Type: text/plain" + "\r\n"
message += "Content-Length: " + str(len(file_data)) + "\r\n"
if "Content-Disposition" in HTTPObject.get_headers(http_object):
if HTTPObject.get_headers(http_object)["Content-Disposition"] == "attachment":
message += "Content-Disposition: attachment/output" + "\r\n"
else:
message += "Content-Disposition: " + HTTPObject.get_headers(http_object) \
["Content-Disposition"] + "\r\n"
elif "inline" in HTTPObject.get_uri(http_object):
message += "Content-Disposition: inline" + "\r\n"
else:
message += "Content-Disposition: inline" + "\r\n"
message += "\r\n"
message += file_data
except OSError as msg:
self.print_debug(msg)
message = "HTTP/1.1 400 Bad Request\r\n\r\n"
message += msg.strerror
elif HTTPObject.get_req_type(http_object).lower() == "post":
try:
if HTTPObject.get_uri(http_object) != "":
working_file = os.getcwd().replace("\\", "/") + "/" + getattr(self._server_obj, '_path') \
+ HTTPObject.get_uri(http_object)
if "Content-Type" in HTTPObject.get_headers(http_object):
if HTTPObject.get_headers(http_object)["Content-Type"] == "text/plain":
if not working_file.endswith(".txt"):
working_file += ".txt"
elif HTTPObject.get_headers(http_object)["Content-Type"] == "text/html":
if not working_file.endswith(".html"):
working_file += ".html"
elif HTTPObject.get_headers(http_object)["Content-Type"] == "text/html":
if not working_file.endswith(".html"):
working_file += ".html"
elif HTTPObject.get_headers(http_object)["Content-Type"] == "text/xml":
if not working_file.endswith(".xml"):
working_file += ".xml"
elif HTTPObject.get_headers(http_object)["Content-Type"] == "application/json":
if not working_file.endswith(".json"):
working_file += ".json"
self.print_debug("POST File " + working_file)
path = pathlib.Path(working_file)
path.parent.mkdir(parents=True, exist_ok=True)
lock_path = working_file + ".lock"
my_lock = FileLock(lock_path, timeout=2)
my_lock.acquire()
try:
open(working_file, "a").write(HTTPObject.get_data(http_object) + "\n")
finally:
my_lock.release()
message += "200 OK" + "\r\n"
message += "Date: " + formatdate(timeval=None, localtime=False, usegmt=True) + "\r\n"
message += "Server: " + socket.gethostname() + "\r\n"
if "Content-Type" in HTTPObject.get_headers(http_object):
message += "Content-Type: " + HTTPObject.get_headers(http_object)[
"Content-Type"] + "\r\n"
else:
if working_file.endswith(".txt"):
message += "Content-Type: text/plain" + "\r\n"
elif working_file.endswith(".html"):
message += "Content-Type: text/html" + "\r\n"
elif working_file.endswith(".json"):
message += "Content-Type: application/json" + "\r\n"
elif working_file.endswith(".xml"):
message += "Content-Type: text/xml" + "\r\n"
else:
message += "Content-Type: text/plain" + "\r\n"
message += "Content-Length: " + str(len(HTTPObject.get_data(http_object))) + "\r\n"
if "Content-Disposition" in HTTPObject.get_headers(http_object):
if HTTPObject.get_headers(http_object)["Content-Disposition"] == "attachment":
message += "Content-Disposition: attachment/output" + "\r\n"
else:
message += "Content-Disposition: " + HTTPObject.get_headers(http_object) \
["Content-Disposition"] + "\r\n"
elif "inline" in HTTPObject.get_uri(http_object):
message += "Content-Disposition: inline" + "\r\n"
else:
message += "Content-Disposition: inline" + "\r\n"
message += "\r\n"
message += HTTPObject.get_data(http_object)
except OSError as msg:
self.print_debug(msg)
message = "HTTP/1.1 400 Bad Request\r\n\r\n"
message += msg.strerror
print("Sending response message to client " + addr[0])
print(message)
conn.sendall(bytes(message, 'utf-8'))
self.print_debug("Connection closed with client " + addr[0] + ":" + str(addr[1]) + " at " +
time.strftime("%a, %d %b %Y %I:%M:%S %p %Z", time.gmtime()))
break
finally:
conn.close()
class CandleConnector():
def __init__(self):
self.lock = FileLock("config.csv.lock")
# make dict here that stores the amount for each coin
self.config = "config.csv"
self.candles = candles.BinaceConnector()
def readConfig(self):
self.lock.acquire()
df = pd.read_csv(self.config,
encoding='utf8',
delimiter=',',
names=[
'coin', 'capital', 'starting', 'limit',
'currentPrice', 'autobought', 'takeprofit',
'updatetime', 'orderid'
])
self.lock.release()
df.set_index('coin', inplace=True)
return df
#get the current config
def getCoinConfigData(self, coin):
df = self.readConfig()
return df.loc[coin]
def getBuyPower(self):
return float(self.candles.getUSD())
#save a new copy of the config
def setCoinConfigData(self, df):
self.lock.acquire()
df.to_csv(f'config.csv', mode='w', header=False, index=True)
self.lock.release()
def getAutoBoughtAmount(self, coin):
return float(self.getCoinConfigData(coin)['autobought'])
# helper for buying a number of coins at current price
def orderNumber(self, coin, number):
return (self.candles.buyMarket(coin, number))
# gives you a quote for a coin
def getQuote(self, coin):
return float(self.candles.getCoinPrice(coin))
# set an order for an number amount
def orderAmount(self, coin, amount):
return (self.candles.order_buy_crypto_by_price(coin, amount))
# write out to a log file
def logit(self, message, destination):
with open(f"logdata/{destination}.txt", "a") as f:
f.write(message)
f.write("\n")
def saveCoinBuyData(self, coin, price, amount, setcap=None):
df = self.readConfig()
if setcap is not None:
df.at[coin, 'capital'] = setcap
df.at[coin, 'starting'] = price
df.at[coin, 'autobought'] = amount
df.at[coin, 'limit'] = price * .98
self.setCoinConfigData(df)
# check to see how much can be purchased with the current capital
# then purchase that amount of coins
def buyNow(self, coin, strat=None):
coinsCapital = self.getCoinConfigData(coin)['capital']
avalFunds = self.getBuyPower()
if (coinsCapital > avalFunds) is True:
return 0
price = self.getQuote(coin)
#TODO add logic that allows for multiple strategies that will
#allow for different allocations of the starting capital
BOUGHT = float(coinsCapital / self.getQuote(coin))
minOrder = None
minNot = None
print(BOUGHT)
#grab the trading rules for the coin
for filt in (self.candles.getCoinInfo(coin)['filters']):
if filt['filterType'] == "LOT_SIZE":
minOrder = float(filt['minQty'])
if filt['filterType'] == 'MIN_NOTIONAL':
minNot = float(filt['minNotional'])
mod = BOUGHT % minOrder
#make sure the amount we are buying is standardized for Binance
if mod:
BOUGHT = BOUGHT - mod
#this needs to get the perciesion from the filter
BOUGHT = round(BOUGHT, 8)
print(BOUGHT)
if (BOUGHT * price) > minNot:
order = self.orderNumber(coin, BOUGHT)
self.saveCoinBuyData(coin, price, BOUGHT)
self.logit(f"BUYING {order}", coin)
else:
BOUGHT = None
self.logit(
f"Failed to buy {BOUGHT}, {coin}. Due minNotional of {minNot}",
coin)
return BOUGHT
#sell an amount at current price
def sellNow(self, coin):
#get the amount the bot bought
amount = self.getAutoBoughtAmount(coin)
if amount > 0:
# self.candles.testOrder(coin, SIDE_SELL, amount)
sellorder = self.candles.sellMarket(coin, amount)
orderID = sellorder['clientOrderId']
status = self.candles.checkStatus(coin, orderID)
timeout = 5
time.sleep(2)
#check a couple of times to make sure we are selling
while status != 'FILLED':
if timeout > 5:
timeout = 0
self.candles.cancelOrder(coin, orderID)
status = self.candles.checkStatus(coin, orderID)
timeout += 1
time.sleep(2)
# save the data for analysis later and reset the bot coin's config
self.logit(f"SELLING DUE TO STRAT {sellorder}", coin)
sellprice = float(sellorder['fills'][0]['price']) * amount
print(sellprice)
self.saveCoinBuyData(coin, 0, 0, setcap=sellprice)
class Db(object):
"""
Basic file storage for a JSON object.
"""
def where_not_in(self, column, *values):
"""
Selects results which do not match the given column/values expression.
Args:
column (str): The named field to test against.
values (str): Vales to search for. A record will not be returned if the field named
in *column* is contained inside of the list of values given.
"""
return [x for x in self.data if x[column] not in values]
def all_jobs(self):
"Retrieve all records."
return self.data
def values(self):
"Synonym for #all_jobs."
return self.data
def find_by_remote_job_id(self, job_id):
"""
Finds a record by the id number for that job on the remote cluster.
Args:
job_id(str): the job id for this job as scheduled or running on the remote.
Returns:
A :class:`dict` object containing various attributes of the job on the local and the remote::
{
local_id: '123123',
local_state: 3,
local_wd: '/var/sge_working_dir',
remote_id: '1234324',
remote_state: 2,
remote_wd: '/var/remote_working_dir',
last_checked: '2017-12-27 16:35:30.898984'
}
"""
return next((x for x in self.values() if x[R_ID] == job_id), None)
def find_by_local_job_id(self, job_id):
return next((x for x in self.values() if x[L_ID] == job_id), None)
def insert(self, local_id, local_state, local_wd, remote_id, remote_state,
remote_wd):
self.data.append(
dict(local_id=local_id,
local_state=local_state,
local_wd=local_wd,
remote_id=remote_id,
remote_state=remote_state,
remote_wd=remote_wd,
last_checked=str(datetime.now())))
def update(self, job):
job['last_checked'] = str(datetime.now())
def delete(self, job):
return self.data.remove(job)
def __init__(self):
self.lock = FileLock(LOCKFILE)
self.data = None
def save(self):
with open(DBFILE, 'w') as file:
file.write(dumps(self.data))
def open(self):
self.lock.acquire(timeout=2)
with open(DBFILE) as file:
self.data = loads(file.read())
def close(self):
self.save()
self.data = None
self.lock.release()
def __enter__(self):
self.open()
return self
def __exit__(self, x, y, z):
"Exit method for resource"
self.close()
def main(argv):
current_ver = ''
start_ver = ''
version_list = []
rev_version_list = [] # Version list in reverse order
fltg_root_dir = None
try:
opts, args = getopt.getopt(argv, "hv:s:d:", ["help"])
except getopt.GetoptError:
usage()
sys.exit(2)
for opt, arg in opts:
if opt in ("-h", "--help"):
usage()
sys.exit(0);
elif opt == '-v':
current_ver = arg
elif opt == '-s':
start_ver = arg
elif opt == '-d':
fltg_root_dir = arg
else:
usage()
sys.exit(0);
if start_ver == '':
start_ver = current_ver
try:
sdk = os.environ['SDK']
except:
print("SDK Environment variable must be defined")
sys.exit(0);
try:
issu_db_dir = os.environ['ISSU_DB'] + '/'
except:
issu_db_dir = sdk + ISSU_DB_DEF_DIR
base_dir = issu_db_dir
ver_list_c = issu_version_list.issu_version_list(base_dir,
start_ver,
current_ver)
rev_version_list = ver_list_c.get_ver_list()
version_list = ver_list_c.get_ver_list(reverse = False)
# Build the base dir
if current_ver == '':
base_dir += 'current'
else:
base_dir += 'ver_' + current_ver
target_dir = base_dir
# Lock the DB to enable multiple simultaneous builds. If the lock is taken
# it means that other build is building the DB. Let it finish first before
# we can jump to the next phase of compiling the DLL
lock_file = issu_db_dir + 'issu_db.lock'
try:
lock = FileLock(lock_file, 1800)
lock.acquire() # wait up to 30 min
except Timeout:
print('Failed to acquire ISSU DB lock')
sys.exit(-1)
db_class = issu_db_gen.global_struct_db(base_dir, None)
if not db_class.generate_c_db(current_ver, version_list):
print('Failed to generate DB for version %s' % current_ver)
base_dir = issu_db_dir + 'ver_'
for ver in rev_version_list:
if ver != current_ver:
db_class.parse_previous_versions(base_dir, ver)
# Final generation and shutdown the DB gen class
db_class.shutdown()
# Save reg file signatures into the current version DB
if current_ver != '':
if fltg_root_dir == None:
fltg_root_dir = sdk + FLTG_INPUT_DIR
reg_file_c = issu_reg_files.reg_file_sig(issu_db_dir)
reg_file_c.save_sig(current_ver, fltg_root_dir)
# Generate the patch list
patch_class = issu_patch_insert.issu_patch(issu_db_dir)
patch_class.read_patch_db(version_list)
patch_class.create_patch_c_file()
patch_class.create_patch_makefile()
lock.release()
print("A template doesn't exist for group " + groupName +
", creating one")
imageTemplate = files[0]
print('!!!!!!!!!!!!!!!!!!!!!!!!!')
template = os.path.join(directory, groupName,
'template' + groupName + '.json')
#tkMessageBox.showinfo("Template", "A template doesn't exist for group "+groupName+", creating one")
timeStart = timeit.default_timer()
textsT, fieldsT, pairsT, samePairsT, horzLinksT, groupsT, transcriptionsT, cornersT, actualCornersT, complete, height, width = labelImage(
os.path.join(directory, groupName, imageTemplate), textsT,
fieldsT, pairsT, samePairsT, horzLinksT, groupsT,
transcriptionsT, None, cornersT, cornersActualT)
timeElapsed = timeit.default_timer() - timeStart + (
labelTime if labelTime is not None else 0)
if (len(textsT) == 0 and len(fieldsT) == 0):
lock.release()
lock = None
exit()
else:
if not complete:
template += '.nf'
with open(template, 'w') as out:
out.write(
json.dumps({
"textBBs": textsT,
"fieldBBs": fieldsT,
"pairs": pairsT,
"samePairs": samePairsT,
"horzLinks": horzLinksT,
"groups": groupsT,
"page_corners": cornersT,
def mirror_file_with_lock(fname, lockfile="/tmp/.lockfile_hk_vectorgen_mirror"):
lock = FileLock(lockfile, timeout=60*10)
lock.lock()
result = mirror_file(fname)
lock.release()
return result
def test_workflow_manager(workflow_start_regular, tmp_path):
# For sync between jobs
tmp_file = str(tmp_path / "lock")
lock = FileLock(tmp_file)
lock.acquire()
# For sync between jobs
flag_file = tmp_path / "flag"
flag_file.touch()
@ray.remote
def long_running(i):
lock = FileLock(tmp_file)
with lock.acquire():
pass
if i % 2 == 0:
if flag_file.exists():
raise ValueError()
return 100
outputs = [
workflow.run_async(long_running.bind(i), workflow_id=str(i))
for i in range(100)
]
# Test list all, it should list all jobs running
all_tasks = workflow.list_all()
assert len(all_tasks) == 100
all_tasks_running = workflow.list_all(workflow.RUNNING)
assert dict(all_tasks) == dict(all_tasks_running)
assert workflow.get_status("0") == "RUNNING"
# Release lock and make sure all tasks finished
lock.release()
for o in outputs:
try:
r = ray.get(o)
except Exception:
continue
assert 100 == r
all_tasks_running = workflow.list_all(workflow.WorkflowStatus.RUNNING)
assert len(all_tasks_running) == 0
# Half of them failed and half succeed
failed_jobs = workflow.list_all("FAILED")
assert len(failed_jobs) == 50
finished_jobs = workflow.list_all("SUCCESSFUL")
assert len(finished_jobs) == 50
all_tasks_status = workflow.list_all({
workflow.WorkflowStatus.SUCCESSFUL,
workflow.WorkflowStatus.FAILED,
workflow.WorkflowStatus.RUNNING,
})
assert len(all_tasks_status) == 100
assert failed_jobs == [(k, v) for (k, v) in all_tasks_status
if v == workflow.WorkflowStatus.FAILED]
assert finished_jobs == [(k, v) for (k, v) in all_tasks_status
if v == workflow.WorkflowStatus.SUCCESSFUL]
# Test get_status
assert workflow.get_status("0") == "FAILED"
assert workflow.get_status("1") == "SUCCESSFUL"
lock.acquire()
r = workflow.resume_async("0")
assert workflow.get_status("0") == workflow.RUNNING
flag_file.unlink()
lock.release()
assert 100 == ray.get(r)
assert workflow.get_status("0") == workflow.SUCCESSFUL
# Test cancel
lock.acquire()
workflow.resume_async("2")
assert workflow.get_status("2") == workflow.RUNNING
workflow.cancel("2")
assert workflow.get_status("2") == workflow.CANCELED
# Now resume_all
resumed = workflow.resume_all(include_failed=True)
assert len(resumed) == 48
lock.release()
assert [ray.get(o) for (_, o) in resumed] == [100] * 48
if new_progress != progress:
progress = new_progress
print('PROGRESS: %d%%' % progress)
sys.stdout.flush()
file_in.close()
result = chsum.hexdigest()
print( result)
obj = dict()
if os.path.isfile( Options.output):
lock = FileLock( Options.output, 100, 1)
file_out = open( Options.output, 'r')
obj = json.load( file_out)
file_out.close()
lock.release()
#print( json.dumps( obj))
file_in_name = os.path.basename( Options.input)
if not 'files' in obj: obj['files'] = dict()
if not file_in_name in obj['files']: obj['files'][file_in_name] = dict()
if not 'checksum' in obj['files'][file_in_name]: obj['files'][file_in_name]['checksum'] = dict()
obj['files'][file_in_name]['checksum'][Options.type] = result
obj['files'][file_in_name]['checksum']['time'] = time.time()
result = json.dumps( obj, indent=1)
lock = FileLock( Options.output, 100, 1)
file_out = open( Options.output, 'w')
file_out.write( result)
file_out.close()
lock.release()
def _instance_iterator(self, file_path: str) -> Iterable[Instance]:
cache_file: Optional[str] = None
if self._cache_directory:
cache_file = self._get_cache_location_for_file_path(file_path)
if cache_file is not None and os.path.exists(cache_file):
cache_file_lock = FileLock(cache_file + ".lock",
timeout=self.CACHE_FILE_LOCK_TIMEOUT)
try:
cache_file_lock.acquire()
# We make an assumption here that if we can obtain the lock, no one will
# be trying to write to the file anymore, so it should be safe to release the lock
# before reading so that other processes can also read from it.
cache_file_lock.release()
logger.info("Reading instances from cache %s", cache_file)
with open(cache_file) as data_file:
yield from self._multi_worker_islice(
data_file, transform=self.deserialize_instance)
except Timeout:
logger.warning(
"Failed to acquire lock on dataset cache file within %d seconds. "
"Cannot use cache to read instances.",
self.CACHE_FILE_LOCK_TIMEOUT,
)
yield from self._multi_worker_islice(self._read(file_path),
ensure_lazy=True)
elif cache_file is not None and not os.path.exists(cache_file):
instances = self._multi_worker_islice(self._read(file_path),
ensure_lazy=True)
# The cache file doesn't exist so we'll try writing to it.
if self.max_instances is not None:
# But we don't write to the cache when max_instances is specified.
logger.warning(
"Skipping writing to data cache since max_instances was specified."
)
yield from instances
elif util.is_distributed() or (get_worker_info()
and get_worker_info().num_workers):
# We also shouldn't write to the cache if there's more than one process loading
# instances since each worker only receives a partial share of the instances.
logger.warning(
"Can't cache data instances when there are multiple processes loading data"
)
yield from instances
else:
try:
with FileLock(cache_file + ".lock",
timeout=self.CACHE_FILE_LOCK_TIMEOUT):
with CacheFile(cache_file, mode="w+") as cache_handle:
logger.info("Caching instances to temp file %s",
cache_handle.name)
for instance in instances:
cache_handle.write(
self.serialize_instance(instance) + "\n")
yield instance
except Timeout:
logger.warning(
"Failed to acquire lock on dataset cache file within %d seconds. "
"Cannot write to cache.",
self.CACHE_FILE_LOCK_TIMEOUT,
)
yield from instances
else:
# No cache.
yield from self._multi_worker_islice(self._read(file_path),
ensure_lazy=True)
def test_workflow_manager(workflow_start_regular, tmp_path):
# For sync between jobs
tmp_file = str(tmp_path / "lock")
lock = FileLock(tmp_file)
lock.acquire()
# For sync between jobs
flag_file = tmp_path / "flag"
flag_file.touch()
@workflow.step
def long_running(i):
lock = FileLock(tmp_file)
with lock.acquire():
pass
if i % 2 == 0:
if flag_file.exists():
raise ValueError()
return 100
outputs = [
long_running.step(i).run_async(workflow_id=str(i)) for i in range(100)
]
# Test list all, it should list all jobs running
all_tasks = workflow.list_all()
assert len(all_tasks) == 100
all_tasks_running = workflow.list_all(workflow.WorkflowStatus.RUNNING)
assert dict(all_tasks) == dict(all_tasks_running)
assert workflow.get_status("0") == workflow.WorkflowStatus.RUNNING
# Release lock and make sure all tasks finished
lock.release()
for o in outputs:
try:
r = ray.get(o)
except Exception:
continue
assert 100 == r
all_tasks_running = workflow.list_all(workflow.WorkflowStatus.RUNNING)
assert len(all_tasks_running) == 0
# Half of them failed and half succeed
failed_jobs = workflow.list_all(workflow.WorkflowStatus.RESUMABLE)
assert len(failed_jobs) == 50
finished_jobs = workflow.list_all(workflow.WorkflowStatus.FINISHED)
assert len(finished_jobs) == 50
all_tasks_status = workflow.list_all({
workflow.WorkflowStatus.FINISHED, workflow.WorkflowStatus.RESUMABLE,
workflow.WorkflowStatus.RUNNING
})
assert len(all_tasks_status) == 100
assert failed_jobs == {
k: v
for (k, v) in all_tasks_status.items()
if v == workflow.WorkflowStatus.RESUMABLE
}
assert finished_jobs == {
k: v
for (k, v) in all_tasks_status.items()
if v == workflow.WorkflowStatus.FINISHED
}
# Test get_status
assert workflow.get_status("0") == workflow.WorkflowStatus.RESUMABLE
assert workflow.get_status("1") == workflow.WorkflowStatus.FINISHED
assert workflow.get_status("X") is None
lock.acquire()
r = workflow.resume("0")
assert workflow.get_status("0") == workflow.WorkflowStatus.RUNNING
flag_file.unlink()
lock.release()
assert 100 == ray.get(r)
assert workflow.get_status("0") == workflow.WorkflowStatus.FINISHED
# Test cancel
lock.acquire()
workflow.resume("2")
assert workflow.get_status("2") == workflow.WorkflowStatus.RUNNING
workflow.cancel("2")
assert workflow.get_status("2") == workflow.WorkflowStatus.CANCELED
# Now resume_all
resumed = workflow.resume_all()
assert len(resumed) == 48
lock.release()
assert [ray.get(o) for o in resumed.values()] == [100] * 48
def test_actor_writer_2(workflow_start_regular, tmp_path):
g_lock = str(Path(tmp_path / "g.lock"))
incr_lock = str(Path(tmp_path / "incr.lock"))
val_lock = str(Path(tmp_path / "val.lock"))
val_err = str(Path(tmp_path / "val.err"))
incr_err = str(Path(tmp_path / "incr.err"))
@workflow.virtual_actor
class SyncCounter:
def __init__(
self,
val_lock: str,
incr_lock: str,
g_lock: str,
val_err: str,
incr_err: str,
):
self.val_lock = val_lock
self.incr_lock = incr_lock
self.g_lock = g_lock
self.val_err = val_err
self.incr_err = incr_err
self.v = 0
if Path(self.val_err).exists():
raise ValueError()
@workflow.virtual_actor.readonly
def val(self):
with FileLock(self.val_lock), FileLock(self.g_lock):
if Path(self.val_err).exists():
raise ValueError()
return self.v
def incr(self, create_incr_err=False):
with FileLock(self.incr_lock), FileLock(self.g_lock):
if Path(self.incr_err).exists():
raise ValueError()
if create_incr_err:
Path(incr_err).touch()
self.v += 1
return self.v
def __getstate__(self):
return (
self.v,
self.val_lock,
self.incr_lock,
self.g_lock,
self.val_err,
self.incr_err,
)
def __setstate__(self, state):
(
self.v,
self.val_lock,
self.incr_lock,
self.g_lock,
self.val_err,
self.incr_err,
) = state
# trigger error in init
Path(val_err).touch()
actor = SyncCounter.get_or_create(
"sync_counter", val_lock, incr_lock, g_lock, val_err, incr_err
)
with pytest.raises(Exception):
actor.incr.run()
Path(val_err).unlink()
assert ray.get([actor.incr.run_async() for _ in range(9)]) == list(range(2, 11))
incr_lock = FileLock(incr_lock)
incr_lock.acquire()
objs = [actor.incr.run_async() for _ in range(10)]
assert 10 == actor.val.run()
Path(val_err).touch()
with pytest.raises(Exception):
actor.val.run()
Path(val_err).unlink()
incr_lock.release()
assert ray.get(objs) == list(range(11, 21))
# test error cases
actor.incr.run_async() # 21
actor.incr.run_async() # 22
actor.incr.run_async(create_incr_err=True) # 23
actor.incr.run_async() # 24
s5 = actor.incr.run_async() # 25
with pytest.raises(Exception):
ray.get(s5)
assert 23 == actor.val.run()
Path(incr_err).unlink()
obj = workflow.resume("sync_counter")
assert 25 == ray.get(obj)[0]
assert 25 == actor.val.run()
class ObdIO(object):
"""Create a obd server"""
def __init__(self, port):
self.port = port
self.ser = None
self.__lock = FileLock(gettempdir() + "/yacm-" +
self.port.replace("/", "") + ".lock",
timeout=1)
self.__lock.acquire()
def __enter__(self):
self.ser = serial.Serial(self.port,
parity=serial.PARITY_NONE,
stopbits=1,
bytesize=8)
self.ser.baudrate = 500000
# self.__write("at", "ws") # Reset device
self.__write("at", "d0") # Set defaults
self.__write("at", "l0") # Disable line feed
self.__write("at", "e0") # Disable echo
self.__write("at", "h0") # Disable headers
self.__write("at", "sp0") # Auto set protocol
return self
def __exit__(self, exception_type, exception_value, exception_traceback):
self.__write("at", "ws") # Reset device
self.ser.close()
self.__lock.release()
os.remove(self.__lock.lock_file)
def query(self, mode: str, code: str) -> Union[str, Tuple]:
"""Query obd requests"""
self.__write(mode, code)
return self.__read()
def __write(self, mode: str, code: str) -> None:
self.ser.flushInput()
self.ser.write(f"{mode}{code}\r".encode())
logging.info(f"Mode: {mode} Code: {code}")
self.ser.flush()
if mode == "at":
if code == "ws":
self.ser.readline()
else:
self.ser.read_until(b'>') # Discard the "OK" message
def __read(self) -> Union[str, Tuple]:
raw_data = self.ser.read_until(b'\r>')
while raw_data == 0:
raw_data = self.ser.read_until(b'\r>')
logging.info(f"raw_data: {raw_data}")
if raw_data not in {b'\r?\r>', b'?\r\r'}:
if raw_data[0] == 13 and raw_data[-3] != 13: # Emulator
raw_data = raw_data[1:-2]
if raw_data[0] != 13 and raw_data[-3] == 13: # Car
raw_data = raw_data[:-3]
if raw_data == b"NO DATA":
result = "NO DATA"
else:
result = raw_data.decode("ascii").lower().split(' ')[2:]
else:
result = "?"
logging.info(f"result: {result}\n")
return result
def supported_pids(self) -> List[str]:
"""Return supported pids"""
hex2bin_map = {
"0": "0000",
"1": "0001",
"2": "0010",
"3": "0011",
"4": "0100",
"5": "0101",
"6": "0110",
"7": "0111",
"8": "1000",
"9": "1001",
"a": "1010",
"b": "1011",
"c": "1100",
"d": "1101",
"e": "1110",
"f": "1111",
}
supported_pids: List[str] = []
for pid in ["00", "20", "40", "60", "80"]:
pids = ''.join(self.query("01", pid))
if pids not in {"?", "NO DATA"}:
binary_pids = ''.join(hex2bin_map[nibble] for nibble in pids)
pid_code = int(pid)
for bit in binary_pids:
pid_code += 1
if bit == "1":
supported_pids.append(hex(pid_code)[2:])
return supported_pids
class Metric(MetricInfoMixin):
"""A Metrics is the base class and common API for all metrics.
Args:
config_name (``str``): This is used to define a hash specific to a metrics computation script and prevents the metric's data
to be overridden when the metric loading script is modified.
keep_in_memory (``bool``): keep all predictions and references in memory. Not possible in distributed settings.
cache_dir (``str``): Path to a directory in which temporary prediction/references data will be stored.
The data directory should be located on a shared file-system in distributed setups.
num_process (``int``): specify the total number of nodes in a distributed settings.
This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1).
process_id (``int``): specify the id of the current process in a distributed setup (between 0 and num_process-1)
This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1).
seed (Optional ``int``): If specified, this will temporarily set numpy's random seed when :func:`datasets.Metric.compute` is run.
experiment_id (``str``): A specific experiment id. This is used if several distributed evaluations share the same file system.
This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1).
max_concurrent_cache_files (``int``): Max number of concurrent metrics cache files (default 10000).
timeout (``Union[int, float]``): Timeout in second for distributed setting synchronization.
"""
def __init__(
self,
config_name: Optional[str] = None,
keep_in_memory: bool = False,
cache_dir: Optional[str] = None,
num_process: int = 1,
process_id: int = 0,
seed: Optional[int] = None,
experiment_id: Optional[str] = None,
max_concurrent_cache_files: int = 10000,
timeout: Union[int, float] = 100,
**kwargs,
):
# prepare info
self.config_name = config_name or "default"
info = self._info()
info.metric_name = camelcase_to_snakecase(self.__class__.__name__)
info.config_name = self.config_name
info.experiment_id = experiment_id or "default_experiment"
MetricInfoMixin.__init__(self, info) # For easy access on low level
# Safety checks on num_process and process_id
assert isinstance(process_id, int) and process_id >= 0, "'process_id' should be a number greater than 0"
assert (
isinstance(num_process, int) and num_process > process_id
), "'num_process' should be a number greater than process_id"
assert (
num_process == 1 or not keep_in_memory
), "Using 'keep_in_memory' is not possible in distributed setting (num_process > 1)."
self.num_process = num_process
self.process_id = process_id
self.max_concurrent_cache_files = max_concurrent_cache_files
self.keep_in_memory = keep_in_memory
self._data_dir_root = os.path.expanduser(cache_dir or HF_METRICS_CACHE)
self.data_dir = self._build_data_dir()
self.seed: int = seed or np.random.get_state()[1][0]
self.timeout: Union[int, float] = timeout
# Update 'compute' and 'add' docstring
# methods need to be copied otherwise it changes the docstrings of every instance
self.compute = types.MethodType(copyfunc(self.compute), self)
self.add_batch = types.MethodType(copyfunc(self.add_batch), self)
self.add = types.MethodType(copyfunc(self.add), self)
self.compute.__func__.__doc__ += self.info.inputs_description
self.add_batch.__func__.__doc__ += self.info.inputs_description
self.add.__func__.__doc__ += self.info.inputs_description
# self.arrow_schema = pa.schema(field for field in self.info.features.type)
self.buf_writer = None
self.writer = None
self.writer_batch_size = None
self.data = None
# This is the cache file we store our predictions/references in
# Keep it None for now so we can (cloud)pickle the object
self.cache_file_name = None
self.filelock = None
self.rendez_vous_lock = None
# This is all the cache files on which we have a lock when we are in a distributed setting
self.file_paths = None
self.filelocks = None
def __len__(self):
"""Return the number of examples (predictions or predictions/references pair)
currently stored in the metric's cache.
"""
return 0 if self.writer is None else len(self.writer)
def __repr__(self):
return (
f'Metric(name: "{self.name}", features: {self.features}, '
f'usage: """{self.inputs_description}""", '
f"stored examples: {len(self)})"
)
def _build_data_dir(self):
"""Path of this metric in cache_dir:
Will be:
self._data_dir_root/self.name/self.config_name/self.hash (if not none)/
If any of these element is missing or if ``with_version=False`` the corresponding subfolders are dropped.
"""
builder_data_dir = self._data_dir_root
builder_data_dir = os.path.join(builder_data_dir, self.name, self.config_name)
os.makedirs(builder_data_dir, exist_ok=True)
return builder_data_dir
def _create_cache_file(self, timeout=1) -> Tuple[str, FileLock]:
""" Create a new cache file. If the default cache file is used, we generated a new hash. """
file_path = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{self.process_id}.arrow")
filelock = None
for i in range(self.max_concurrent_cache_files):
filelock = FileLock(file_path + ".lock")
try:
filelock.acquire(timeout=timeout)
except Timeout:
# If we have reached the max number of attempts or we are not allow to find a free name (distributed setup)
# We raise an error
if self.num_process != 1:
raise ValueError(
f"Error in _create_cache_file: another metric instance is already using the local cache file at {file_path}. "
f"Please specify an experiment_id (currently: {self.experiment_id}) to avoid colision "
f"between distributed metric instances."
)
if i == self.max_concurrent_cache_files - 1:
raise ValueError(
f"Cannot acquire lock, too many metric instance are operating concurrently on this file system."
f"You should set a larger value of max_concurrent_cache_files when creating the metric "
f"(current value is {self.max_concurrent_cache_files})."
)
# In other cases (allow to find new file name + not yet at max num of attempts) we can try to sample a new hashing name.
file_uuid = str(uuid.uuid4())
file_path = os.path.join(
self.data_dir, f"{self.experiment_id}-{file_uuid}-{self.num_process}-{self.process_id}.arrow"
)
else:
break
return file_path, filelock
def _get_all_cache_files(self) -> Tuple[List[str], List[FileLock]]:
"""Get a lock on all the cache files in a distributed setup.
We wait for timeout second to let all the distributed node finish their tasks (default is 100 seconds).
"""
if self.num_process == 1:
if self.cache_file_name is None:
raise ValueError(
"Metric cache file doesn't exist. Please make sure that you call `add` or `add_batch` "
"at least once before calling `compute`."
)
file_paths = [self.cache_file_name]
else:
file_paths = [
os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{process_id}.arrow")
for process_id in range(self.num_process)
]
# Let's acquire a lock on each process files to be sure they are finished writing
filelocks = []
for process_id, file_path in enumerate(file_paths):
filelock = FileLock(file_path + ".lock")
try:
filelock.acquire(timeout=self.timeout)
except Timeout:
raise ValueError(f"Cannot acquire lock on cached file {file_path} for process {process_id}.")
else:
filelocks.append(filelock)
return file_paths, filelocks
def _check_all_processes_locks(self):
expected_lock_file_names = [
os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{process_id}.arrow.lock")
for process_id in range(self.num_process)
]
for expected_lock_file_name in expected_lock_file_names:
nofilelock = FileFreeLock(expected_lock_file_name)
try:
nofilelock.acquire(timeout=self.timeout)
except Timeout:
raise ValueError(
f"Expected to find locked file {expected_lock_file_name} from process {self.process_id} but it doesn't exist."
)
else:
nofilelock.release()
def _check_rendez_vous(self):
expected_lock_file_name = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-0.arrow.lock")
nofilelock = FileFreeLock(expected_lock_file_name)
try:
nofilelock.acquire(timeout=self.timeout)
except Timeout:
raise ValueError(
f"Expected to find locked file {expected_lock_file_name} from process {self.process_id} but it doesn't exist."
)
else:
nofilelock.release()
lock_file_name = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-rdv.lock")
rendez_vous_lock = FileLock(lock_file_name)
try:
rendez_vous_lock.acquire(timeout=self.timeout)
except Timeout:
raise ValueError(f"Couldn't acquire lock on {lock_file_name} from process {self.process_id}.")
else:
rendez_vous_lock.release()
def _finalize(self):
"""Close all the writing process and load/gather the data
from all the nodes if main node or all_process is True.
"""
if self.writer is not None:
self.writer.finalize()
self.writer = None
if self.filelock is not None:
self.filelock.release()
if self.keep_in_memory:
# Read the predictions and references
reader = ArrowReader(path=self.data_dir, info=DatasetInfo(features=self.features))
self.data = Dataset.from_buffer(self.buf_writer.getvalue())
elif self.process_id == 0:
# Let's acquire a lock on each node files to be sure they are finished writing
file_paths, filelocks = self._get_all_cache_files()
# Read the predictions and references
try:
reader = ArrowReader(path="", info=DatasetInfo(features=self.features))
self.data = Dataset(**reader.read_files([{"filename": f} for f in file_paths]))
except FileNotFoundError:
raise ValueError(
"Error in finalize: another metric instance is already using the local cache file. "
"Please specify an experiment_id to avoid colision between distributed metric instances."
)
# Store file paths and locks and we will release/delete them after the computation.
self.file_paths = file_paths
self.filelocks = filelocks
def compute(self, *args, **kwargs) -> Optional[dict]:
"""Compute the metrics.
Args:
We disallow the usage of positional arguments to prevent mistakes
`predictions` (Optional list/array/tensor): predictions
`references` (Optional list/array/tensor): references
`**kwargs` (Optional other kwargs): will be forwared to the metrics :func:`_compute` method (see details in the docstring)
Return:
Dictionnary with the metrics if this metric is run on the main process (process_id == 0)
None if the metric is not run on the main process (process_id != 0)
"""
if args:
raise ValueError("Please call `compute` using keyword arguments.")
predictions = kwargs.pop("predictions", None)
references = kwargs.pop("references", None)
if predictions is not None:
self.add_batch(predictions=predictions, references=references)
self._finalize()
self.cache_file_name = None
self.filelock = None
if self.process_id == 0:
self.data.set_format(type=self.info.format)
predictions = self.data["predictions"]
references = self.data["references"]
with temp_seed(self.seed):
output = self._compute(predictions=predictions, references=references, **kwargs)
if self.buf_writer is not None:
self.buf_writer = None
del self.data
self.data = None
else:
# Release locks and delete all the cache files
for filelock, file_path in zip(self.filelocks, self.file_paths):
logger.info(f"Removing {file_path}")
del self.data
self.data = None
del self.writer
self.writer = None
os.remove(file_path)
filelock.release()
return output
else:
return None
def add_batch(self, *, predictions=None, references=None):
"""
Add a batch of predictions and references for the metric's stack.
"""
batch = {"predictions": predictions, "references": references}
batch = self.info.features.encode_batch(batch)
if self.writer is None:
self._init_writer()
try:
self.writer.write_batch(batch)
except pa.ArrowInvalid:
raise ValueError(
f"Predictions and/or references don't match the expected format.\n"
f"Expected format: {self.features},\n"
f"Input predictions: {predictions},\n"
f"Input references: {references}"
)
def add(self, *, prediction=None, reference=None):
"""Add one prediction and reference for the metric's stack."""
example = {"predictions": prediction, "references": reference}
example = self.info.features.encode_example(example)
if self.writer is None:
self._init_writer()
try:
self.writer.write(example)
except pa.ArrowInvalid:
raise ValueError(
f"Prediction and/or reference don't match the expected format.\n"
f"Expected format: {self.features},\n"
f"Input predictions: {prediction},\n"
f"Input references: {reference}"
)
def _init_writer(self, timeout=1):
if self.num_process > 1:
if self.process_id == 0:
file_path = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-rdv.lock")
self.rendez_vous_lock = FileLock(file_path)
try:
self.rendez_vous_lock.acquire(timeout=timeout)
except TimeoutError:
raise ValueError(
f"Error in _init_writer: another metric instance is already using the local cache file at {file_path}. "
f"Please specify an experiment_id (currently: {self.experiment_id}) to avoid colision "
f"between distributed metric instances."
)
if self.keep_in_memory:
self.buf_writer = pa.BufferOutputStream()
self.writer = ArrowWriter(
features=self.info.features, stream=self.buf_writer, writer_batch_size=self.writer_batch_size
)
else:
self.buf_writer = None
# Get cache file name and lock it
if self.cache_file_name is None or self.filelock is None:
cache_file_name, filelock = self._create_cache_file() # get ready
self.cache_file_name = cache_file_name
self.filelock = filelock
self.writer = ArrowWriter(
features=self.info.features, path=self.cache_file_name, writer_batch_size=self.writer_batch_size
)
# Setup rendez-vous here if
if self.num_process > 1:
if self.process_id == 0:
self._check_all_processes_locks() # wait for everyone to be ready
self.rendez_vous_lock.release() # let everyone go
else:
self._check_rendez_vous() # wait for master to be ready and to let everyone go
def _info(self) -> MetricInfo:
"""Construct the MetricInfo object. See `MetricInfo` for details.
Warning: This function is only called once and the result is cached for all
following .info() calls.
Returns:
info: (MetricInfo) The metrics information
"""
raise NotImplementedError
def download_and_prepare(
self,
download_config: Optional[DownloadConfig] = None,
dl_manager: Optional[DownloadManager] = None,
**download_and_prepare_kwargs,
):
"""Downloads and prepares dataset for reading.
Args:
download_config (Optional ``datasets.DownloadConfig``: specific download configuration parameters.
dl_manager (Optional ``datasets.DownloadManager``): specific Download Manger to use
"""
if dl_manager is None:
if download_config is None:
download_config = DownloadConfig()
download_config.cache_dir = os.path.join(self.data_dir, "downloads")
download_config.force_download = False
dl_manager = DownloadManager(
dataset_name=self.name, download_config=download_config, data_dir=self.data_dir
)
self._download_and_prepare(dl_manager)
def _download_and_prepare(self, dl_manager):
"""Downloads and prepares resources for the metric.
This is the internal implementation to overwrite called when user calls
`download_and_prepare`. It should download all required resources for the metric.
Args:
dl_manager: (DownloadManager) `DownloadManager` used to download and cache
data..
"""
return None
def _compute(self, *, predictions=None, references=None, **kwargs) -> Dict[str, Any]:
""" This method defines the common API for all the metrics in the library """
raise NotImplementedError
def __del__(self):
if self.filelock is not None:
self.filelock.release()
if self.rendez_vous_lock is not None:
self.rendez_vous_lock.release()
del self.writer
del self.data
class Cam():
"""Class to control the camera device
"""
params_dict = {
'brightness': cv2.CAP_PROP_BRIGHTNESS,
'gamma': cv2.CAP_PROP_GAMMA,
'gain': cv2.CAP_PROP_GAIN,
'exposure': cv2.CAP_PROP_EXPOSURE,
}
def __init__(self):
self._id = 0
self._lock = FileLock("/tmp/cam.lock")
def lock_acquire(self, timeout=10) -> None:
"""Acquire a lock to use the camera device
:param timeout: Time to wait for the lock; defaults to 10 seconds.
"""
self._lock.acquire(timeout=timeout)
def lock_release(self) -> None:
"""Release the lock
"""
self._lock.release()
def read(self) -> (int, numpy.ndarray):
"""Returns the cv2 camera object
:return: (ret, frame)
"""
cam = cv2.VideoCapture(self._id)
cam.set(cv2.CAP_PROP_BUFFERSIZE, 1)
cam.set(cv2.CAP_PROP_AUTO_EXPOSURE, 1.0)
ret, frame = cam.read()
return ret, frame
def get_camera_params(self) -> {}:
"""Returns the parameters read from the camera
:return: Dictionary with the parameters' values
"""
cam = cv2.VideoCapture(self._id)
response = {
'brightness': cam.get(cv2.CAP_PROP_BRIGHTNESS),
'gamma': cam.get(cv2.CAP_PROP_GAMMA),
'gain': cam.get(cv2.CAP_PROP_GAIN),
'exposure': cam.get(cv2.CAP_PROP_EXPOSURE),
}
return response
def set_camera_params(self, params) -> None:
"""Set camera parameters
:param params: Dictionary with the parameters to set
"""
cam = cv2.VideoCapture(self._id)
cam.set(cv2.CAP_PROP_BUFFERSIZE, 1)
cam.set(cv2.CAP_PROP_AUTO_EXPOSURE, 1.0)
for param, value in params.items():
if value is not None:
cam.set(self.params_dict[param], value)
class Metric(object):
def __init__(
self,
name: str = None,
experiment_id: Optional[str] = None,
process_id: int = 0,
num_process: int = 1,
data_dir: Optional[str] = None,
in_memory: bool = False,
**kwargs,
):
""" A Metrics is the base class and common API for all metrics.
Args:
process_id (int): specify the id of the node in a distributed settings between 0 and num_nodes-1
This can be used, to compute metrics on distributed setups
(in particular non-additive metrics like F1).
data_dir (str): path to a directory in which temporary data will be stored.
This should be a shared file-system for distributed setups.
experiment_id (str): Should be used if you perform several concurrent experiments using
the same caching directory (will be indicated in the raise error)
in_memory (bool): keep all predictions and references in memory. Not possible in distributed settings.
"""
# Safety checks
assert isinstance(
process_id, int
) and process_id >= 0, "'process_id' should be a number greater than 0"
assert (isinstance(num_process, int) and num_process > process_id
), "'num_process' should be a number greater than process_id"
assert (
process_id == 0 or not in_memory
), "Using 'in_memory' is not possible in distributed setting (process_id > 0)."
# Metric name
self.name = camelcase_to_snakecase(self.__class__.__name__)
# Configuration name
self.config_name: str = name or "default"
self.process_id = process_id
self.num_process = num_process
self.in_memory = in_memory
self.experiment_id = experiment_id if experiment_id is not None else "cache"
self._version = "1.0.0"
self._data_dir_root = os.path.expanduser(data_dir or HF_METRICS_CACHE)
self.data_dir = self._build_data_dir()
# prepare info
info = self._info()
info.metric_name = self.name
info.config_name = self.config_name
info.version = self._version
self.info = info
# Update 'compute' and 'add' docstring
self.compute.__func__.__doc__ += self.info.inputs_description
self.add_batch.__func__.__doc__ += self.info.inputs_description
self.add.__func__.__doc__ += self.info.inputs_description
self.arrow_schema = pa.schema(field
for field in self.info.features.type)
self.buf_writer = None
self.writer = None
self.writer_batch_size = None
self.data = None
# Check we can write on the cache file without competitors
self.cache_file_name = self._get_cache_path(self.process_id)
self.filelock = FileLock(self.cache_file_name + ".lock")
try:
self.filelock.acquire(timeout=1)
except Timeout:
raise ValueError(
"Cannot acquire lock, caching file might be used by another process, "
"you should setup a unique 'experiment_id' for this run.")
def _relative_data_dir(self, with_version=True):
"""Relative path of this dataset in data_dir."""
builder_data_dir = os.path.join(self.name, self.config_name)
if not with_version:
return builder_data_dir
version = self._version
version_data_dir = os.path.join(builder_data_dir, str(version))
return version_data_dir
def _build_data_dir(self):
""" Return the directory for the current version.
"""
builder_data_dir = os.path.join(
self._data_dir_root, self._relative_data_dir(with_version=False))
version_data_dir = os.path.join(
self._data_dir_root, self._relative_data_dir(with_version=True))
def _other_versions_on_disk():
"""Returns previous versions on disk."""
if not os.path.exists(builder_data_dir):
return []
version_dirnames = []
for dir_name in os.listdir(builder_data_dir):
try:
version_dirnames.append((Version(dir_name), dir_name))
except ValueError: # Invalid version (ex: incomplete data dir)
pass
version_dirnames.sort(reverse=True)
return version_dirnames
# Check and warn if other versions exist on disk
version_dirs = _other_versions_on_disk()
if version_dirs:
other_version = version_dirs[0][0]
if other_version != self._version:
warn_msg = (
"Found a different version {other_version} of metric {name} in "
"data_dir {data_dir}. Using currently defined version "
"{cur_version}.".format(
other_version=str(other_version),
name=self.name,
data_dir=self._data_dir_root,
cur_version=str(self._version),
))
logger.warning(warn_msg)
os.makedirs(version_data_dir, exist_ok=True)
return version_data_dir
def _get_cache_path(self, node_id):
return os.path.join(
self.data_dir, f"{self.experiment_id}-{self.name}-{node_id}.arrow")
def finalize(self, timeout=120):
""" Close all the writing process and load/gather the data
from all the nodes if main node or all_process is True.
"""
self.writer.finalize()
self.writer = None
self.buf_writer = None
self.filelock.release()
if self.process_id == 0:
# Let's acquire a lock on each node files to be sure they are finished writing
node_files = []
locks = []
for node_id in range(self.num_process):
node_file = self._get_cache_path(node_id)
filelock = FileLock(node_file + ".lock")
filelock.acquire(timeout=timeout)
node_files.append({"filename": node_file})
locks.append(filelock)
# Read the predictions and references
reader = ArrowReader(path=self.data_dir, info=None)
self.data = reader.read_files(node_files)
# Release all of our locks
for lock in locks:
lock.release()
def compute(self,
predictions=None,
references=None,
timeout=120,
**metrics_kwargs):
""" Compute the metrics.
"""
if predictions is not None:
self.add_batch(predictions=predictions, references=references)
self.finalize(timeout=timeout)
self.data.set_format(type=self.info.format)
predictions = self.data["predictions"]
references = self.data["references"]
output = self._compute(predictions=predictions,
references=references,
**metrics_kwargs)
return output
def add_batch(self, predictions=None, references=None, **kwargs):
""" Add a batch of predictions and references for the metric's stack.
"""
batch = {"predictions": predictions, "references": references}
if self.writer is None:
self._init_writer()
self.writer.write_batch(batch)
def add(self, prediction=None, reference=None, **kwargs):
""" Add one prediction and reference for the metric's stack.
"""
example = {"predictions": prediction, "references": reference}
example = self.info.features.encode_example(example)
if self.writer is None:
self._init_writer()
self.writer.write(example)
def _init_writer(self):
if self.in_memory:
self.buf_writer = pa.BufferOutputStream()
self.writer = ArrowWriter(schema=self.arrow_schema,
stream=self.buf_writer,
writer_batch_size=self.writer_batch_size)
else:
self.buf_writer = None
self.writer = ArrowWriter(schema=self.arrow_schema,
path=self.cache_file_name,
writer_batch_size=self.writer_batch_size)
def _info(self) -> MetricInfo:
"""Construct the MetricInfo object. See `MetricInfo` for details.
Warning: This function is only called once and the result is cached for all
following .info() calls.
Returns:
info: (MetricInfo) The metrics information
"""
raise NotImplementedError
def _compute(self,
predictions=None,
references=None,
**kwargs) -> Dict[str, Any]:
""" This method defines the common API for all the metrics in the library """
raise NotImplementedError
class CandleConnector():
def __init__(self):
self.lock = FileLock("config.csv.lock")
# make dict here that stores the amount for each coin
self.config = "config.csv"
self.candles = candles.BinaceConnector()
self.masterTicker = -60
def readConfig(self):
self.lock.acquire()
df = pd.read_csv(self.config,encoding='utf8', delimiter=',' ,
names= ['coin', 'capital', 'starting', 'limit', 'currentPrice', 'autobought', 'takeprofit', 'updatetime', 'orderid', 'takeProfitAmount', 'takeProfitOrder', 'delta'])
self.lock.release()
df.set_index('coin', inplace=True)
return df
#get the current config
def getCoinConfigData(self, coin):
df = self.readConfig()
return df.loc[coin]
#save a new copy of the config
def setCoinConfigData(self, df):
self.lock.acquire()
df.to_csv(f'config.csv', mode='w', header=False, index=True)
self.lock.release()
def getAutoBoughtAmount(self, coin):
return float(self.getCoinConfigData(coin)['autobought'])
# helper for buying a number of coins at current price
def orderNumber(self, coin, number):
return (self.candles.buyMarket(coin, number))
# gives you a quote for a coin
def getQuote(self, coin):
return float(self.candles.getCoinPrice(coin))
# write out to a log file
def logit(self, message, destination):
with open(f"testData/{destination}.txt", "a") as f:
f.write(message)
f.write("\n")
def saveCoinBuyData(self, coin, price, amount, setcap=0.0, setupdatetime=180, order="none"):
df = self.readConfig()
if setcap > 0:
df.at[coin, 'capital'] = setcap
df.at[coin, 'starting'] = price
df.at[coin, 'autobought'] = amount
df.at[coin, 'limit'] = price * df.at[coin, 'takeprofit']
df.at[coin, 'updatetime'] = setupdatetime
df.at[coin, 'orderid'] = order
self.setCoinConfigData(df)
def saveCoinLimitData(self, coin, price, limit, setupdatetime=180,):
df = self.readConfig()
df.at[coin, 'currentPrice'] = price
df.at[coin, 'limit'] = limit
self.setCoinConfigData(df)
def updateDelta(self, coin, delta, price):
df = self.readConfig()
df.at[coin, 'currentPrice'] = price
df.at[coin, 'delta'] = delta -180
self.setCoinConfigData(df)
def updateOrder(self, coin, order):
df = self.readConfig()
df.at[coin, 'orderid'] = order
self.setCoinConfigData(df)
#sell an amount at current price
def sellNow(self, coin):
#get the amount the bot bought
amount = self.getAutoBoughtAmount(coin)
print(f"found {amount}")
if amount > 0:
print(f"selling")
sellorder = self.candles.sellMarket(coin, amount)
time.sleep(1)
# save the data for analysis later and reset the bot coin's config
self.logit(f"SELLING DUE TO TAKEPROFIT {sellorder}", "logger")
#need to check to make sure we did sell before we save this
sellprice = float(sellorder['fills'][0]['price']) * amount
print(sellprice)
self.saveCoinBuyData(coin, 0, 0, setcap=sellprice)
def echoCurrentTick(self):
with open('tick', 'w') as f:
f.write(f"{self.masterTicker}")
def runForever(self):
while 1:
self.masterTicker += 60
df = self.readConfig()
# loop over the contents of our config file
tickers = self.candles.getBook()
for coin, row in df.iterrows():
# check to see if the bot has made a purchase
position = float(row['autobought'])
if position > 0:
for x in tickers:
if coin == x['symbol']:
currentPrice = float(x['bidPrice'])
# if the bot has bought, check the update time
currentOrder = row['orderid']
updatetime = int(row['updatetime'])
delta = self.masterTicker % updatetime
if delta== 0:
#get the current price and check if it's above our current limit
currentLimit = float(row['limit'])
if currentPrice < currentLimit:
print("checking order")
if "none" not in currentOrder:
print("cancelOrder")
status = self.candles.checkStatus(coin, currentOrder)
if status != 'FILLED':
self.candles.cancelOrder(coin, currentOrder)
time.sleep(.3)
print(f"selling because price {currentPrice} < limit {currentLimit}")
self.sellNow(coin)
else:
self.sellNow(coin)
else:
# calculate a new limit based on our coin's config profile
newlimit = currentPrice*float(row['takeprofit'])
print(f"new limit {newlimit}")
if newlimit > currentLimit:
print(f"new limit > {currentLimit}")
self.saveCoinLimitData(coin, currentPrice, newlimit)
starting = float(row['starting'])
# check to see if a stop loss order has been placed
if "none" not in currentOrder:
print(f"order {currentOrder} is", end = " ")
status = self.candles.checkStatus(coin, currentOrder)
if status == 'FILLED':
print("FILLED so close")
sellprice = float(status['fills'][0]['price']) * row['autobought']
self.saveCoinBuyData(coin, 0, 0, setcap=sellprice)
print("open")
# if stop loss has not been placed, and we are in profit attempt to atleast cover our fees
elif currentPrice > starting + (starting * 0.005):
print("made our money back placing limit order")
#save this order and save to config`
order = connector.candles.stopLoss(coin,
stop=(starting + (starting * (2 * .0008))),
limit=(starting + (starting * (2 * .00076))),
position=position)['clientOrderId']
self.updateOrder(coin, order)
self.updateDelta(coin, delta, currentPrice)
self.logit(f"{self.masterTicker}, {row.starting}, {currentPrice}, {row.limit}", coin)
self.echoCurrentTick()
time.sleep(60)
class Store(object):
auto_sync = True
auto_mem_resync_counter = 0
@staticmethod
def is_valid_schema(self):
return all([len(x) == 3 and x[1] in DUMPERS for x in self])
def __init__(self, db_name, schema, log_level=1):
"""
schema example = (
(name, type, len),
...
)
int
str
"""
self._log_lvl = log_level
db_name = os.path.join('db', db_name)
self.lock = FileLock(db_name)
if not self.is_valid_schema(schema):
self.error("Invalid DB schema")
raise Exception('Schema is bead!')
all = self.init_mem(db_name, schema)
self._memory = all
self._db_name = db_name
self._schema = schema
def info(self, msg):
if self._log_lvl > 3:
print(msg)
def error(self, msg):
if self._log_lvl > 0:
print(msg)
def _insert(self, object_dict_):
assert type(object_dict_) == dict, 'Wow! Not dict!'
self._memory.append(object_dict_)
def sync(self):
self.lock.acquire()
f = open(self._db_name, 'wb')
for d in self._memory:
for name_, type_, len_ in self._schema:
bytes_bytes = DUMPERS[type_](d[name_], len_)
f.write(bytes_bytes)
f.write('\n')
f.close()
self.lock.release()
self._memory = self.init_mem(self._db_name, self._schema)
def execute(self, row):
"""
Execute SQL query. Support: select, delete, insert.
"""
# row = "select where value[1] == 'd' and id >= 0 limit 2"
# row = "insert into ... values (2, 'awdwa')"
# row = "delete where ... limit k"
try:
method, tail = row.split(' ', 1)
method = method.lower()
tail = tail.strip(' ')
rez = None
self.info(u'-- SQL {0} {1} --'.format(method, tail))
if method == 'insert':
r = re.compile(r'^.*?values?[ ]*(\(.*?\))$', re.M)
z = r.match(tail)
if z:
rez = [self.insert(*z.groups())]
if self.auto_sync:
self.sync()
elif method in ['select', 'delete']:
r = re.compile(r'^.*?(?:(?:where)[ ]*(.*?))?[ ]*(?:limit[ ]*(\d+))?[ ]*([dD][Ee][ScCs][ckCK])?[ ]*$')
z = r.match(tail)
if z:
rez = self.__getattribute__('go_go')(method, z.groups())
else:
rez = self.__getattribute__(method)()
elif method == 'last':
rez = [self.last()]
if hasattr(rez, '__len__') and rez.__len__() == 1:
return rez[0]
return rez
except Exception as e:
self.error("Invalid SQL syntax detected: {0!r} by {1}".format(row, e))
raise Exception('Invalid SQL syntax!!')
def go_go(self, method, args):
return self.__getattribute__(method)(*args)
def delete(self, where=None, limit=None, desk=None):
limit = int(limit.strip()) if limit else 0
where = 'True' if not where else where
where = self.fix_where(where)
rez = 0
del_indexes = []
l = locals()
i = 0
mem = self._memory if not desk else reversed(self._memory)
for d in mem:
for name_, type_, len_ in self._schema:
l[name_] = d[name_]
st = parser.expr(where)
is_ok = eval(st.compile())
if is_ok:
rez += 1
del_indexes.append(i)
i += 1
if limit and rez >= limit:
break
z = 0
for x in sorted(del_indexes):
self._delete_dy_index(x - z)
z += 1
return rez
def _delete_dy_index(self, index):
if 0 <= index < len(self._memory):
del self._memory[index]
return 1
return 0
def _delete_dy_indexes(self, *indexes):
del_counter = 0
for index in sorted(indexes):
deleted = self._delete_dy_index(index - del_counter)
del_counter += deleted
return del_counter
def _memory_dump(self):
print '\n-- dump --'
for d in self._memory:
print d.values()
print '-- |--| --\n'
def select(self, where=None, limit=None, desk=None):
limit = int(limit.strip()) if limit else 0
where = 'True' if not where else where
where = self.fix_where(where)
rez = []
l = locals()
mem = self._memory if not desk else reversed(self._memory)
for d in mem:
for name_, type_, len_ in self._schema:
l[name_] = d[name_]
st = parser.expr(where)
is_ok = eval(st.compile())
if limit and len(rez) >= limit:
return rez
if is_ok:
rez.append(d)
return rez
def insert(self, insert_obj_row):
if not insert_obj_row.startswith('(') or not insert_obj_row.endswith(')'):
return
insert_obj_row = insert_obj_row.replace("'", "'''")
insert_obj_row = insert_obj_row.replace("\"", "'''")
try:
st = parser.expr(insert_obj_row)
obj = eval(st.compile())
if type(obj) != tuple or len(obj) != len(self._schema):
return
d = {}
i = 0
for name_, type_, len_ in self._schema:
d[name_] = obj[i]
i += 1
_ck = CHECKERS[type_]
if _ck(d[name_]):
return
self._insert(d)
return d
except Exception as e:
self.error('Insertion error!', e)
return
def fix_where(self, where):
z = where.replace(' = ', ' == ')
z = z.replace('__', '')
return z.replace('import', '')
def last(self):
pass
def init_mem(self, db_name, schema):
i = 0
all = []
data = {}
self.lock.acquire()
def read_tail(f):
while True:
c = f.read(1)
if not c or c == '\n':
return
try:
_r = open(db_name, 'rb')
except:
self.info("Create New DB")
try:
t = open(db_name, 'a')
t.close()
except Exception as e:
self.error("Can not create NEW DB", e)
raise Exception('Can not create new DB')
try:
_r = open(db_name, 'rb')
except:
self.error("I/0 Error #1! Can not open!")
raise Exception("I/0 Error #1! Can not open!")
while 1:
if i == len(schema):
all.append(data)
read_tail(_r)
i = 0
data = {}
name_, type_, len_ = schema[i]
d = _r.read(len_).replace('\0', '')
if not d: break
zero = _r.read(1)
if not zero or ord(zero) != 0:
read_tail(_r)
i = 0
data = {}
continue
try:
data[name_] = type_(d)
except:
read_tail(_r)
i = 0
data = {}
else:
i += 1
_r.close()
self.lock.release()
return all
class EngineConnector:
def __init__(self, otherbot_path, is_first):
import uuid
import subprocess
from filelock import FileLock
self.str_uuid = str(uuid.uuid1())
self.interface_pipe_path = os.path.abspath("pipes" + os.sep +
self.str_uuid + ".pipe")
self.interface_lock_path = self.interface_pipe_path + ".lock"
self.lock = FileLock(self.interface_lock_path)
self.is_first = is_first
#create pipe file
#os.mknod(os.path.abspath(self.interface_pipe_path))
pipe = open(self.interface_pipe_path, "w+")
pipe.close()
#run engine
ibot_launch_str = "python3 .." + os.sep + "EngineInterface" + os.sep + "main.py " + self.interface_pipe_path
if is_first:
engine_launch_str = "java -cp bin com.theaigames.tictactoe.Tictactoe \"" + ibot_launch_str + "\" \"" + otherbot_path + "\""
else:
engine_launch_str = "java -cp bin com.theaigames.tictactoe.Tictactoe \"" + otherbot_path + "\" \"" + ibot_launch_str + "\""
self.engine_process = subprocess.Popen(engine_launch_str,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=".." + os.sep +
"ultimatetictactoe-engine")
#avoid blocking by setting flags
flags = fcntl(self.engine_process.stdout, F_GETFL)
fcntl(self.engine_process.stdout, F_SETFL, flags | os.O_NONBLOCK)
fcntl(self.engine_process.stderr, F_SETFL, flags | os.O_NONBLOCK)
def read_state(self):
#print("reading state")
received = ""
received_selection = False
while (not received_selection):
#try to read selection
self.lock.acquire()
try:
pipe = open(self.interface_pipe_path, "r")
#check that file has been updated
first_line = pipe.readline()
expected = "state"
if (expected in first_line):
received_selection = True
received = pipe.read()
pipe.close()
pipe.close()
finally:
self.lock.release()
#sleep for a duration to give time for selection to be made
time.sleep(0.001)
#process received into a tuple of board, macroboard, moves
lines = received.splitlines(keepends=False)
#print("state: " + str((lines[1], lines[3], lines[5])))
return (self.process_board(lines[1]),
self.process_macroboard(lines[3]), lines[5])
#attempts a single state read
def try_read_state(self):
received = ""
received_selection = False
self.lock.acquire()
try:
pipe = open(self.interface_pipe_path, "r")
#check that file has been updated
first_line = pipe.readline()
expected = "state"
if (expected in first_line):
received_selection = True
received = pipe.read()
pipe.close()
pipe.close()
finally:
self.lock.release()
if received_selection:
lines = received.splitlines(keepends=False)
#print("state: " + str((lines[1], lines[3], lines[5])))
return (self.process_board(lines[1]),
self.process_macroboard(lines[3]), lines[5])
else:
return None
def send_move(self, selection):
self.lock.acquire()
try:
pipe = open(self.interface_pipe_path, "w+")
pipe.write("move\n")
pipe.write(selection)
pipe.close()
finally:
self.lock.release()
def process_board(self, pre_board):
if self.is_first:
return pre_board.replace("-1", "a").replace("1", "m").replace(
"2", "y").replace("a", "-1")
else:
return pre_board.replace("-1", "a").replace("2", "m").replace(
"1", "y").replace("a", "-1")
def process_macroboard(self, pre_macroboard):
if self.is_first:
return pre_macroboard.replace("-1", "a").replace("1", "m").replace(
"2", "y").replace("a", "-1")
else:
return pre_macroboard.replace("-1", "a").replace("2", "m").replace(
"1", "y").replace("a", "-1")
# return 0 if draw, 1 if player 1 wins, 2 if player 2 wins, -1 if active game
def win_status(self):
l = self.engine_process.stdout.readline().decode(
sys.getdefaultencoding())
while l != "":
#print("l:" + l)
if "Draw" in l:
return 0
if "winner: player1" in l:
return 1
if "winner: player2" in l:
return 2
l = self.engine_process.stdout.readline().decode(
sys.getdefaultencoding())
return -1
def close(self):
if os.path.exists(self.interface_lock_path):
os.remove(self.interface_lock_path)
os.remove(self.interface_pipe_path)
